diff --git a/moonep/tierbprobe/manifest.txt b/moonep/tierbprobe/manifest.txt index c222e026900055f35167a52c932b6cde7cb34f33..91e05003b8676f414f777beedac57a4c3dc6b299 100644 --- a/moonep/tierbprobe/manifest.txt +++ b/moonep/tierbprobe/manifest.txt @@ -1,9 +1,9 @@ -timestamp: 2026-08-26T16:52:04+00:00 +timestamp: 2026-08-26T18:04:12+00:00 host: sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 gpu: NVIDIA B200 x8 driver: 580.95.05 study_scripts_branch: granularity-moonep -study_scripts_commit: cc725d8 +study_scripts_commit: ef3305e campaign: moonep backend: moonep phase: tierbprobe diff --git a/moonep/tierbprobe/provenance.txt b/moonep/tierbprobe/provenance.txt index cd4a78eb8c810d5f7b1a18ca1e01a19e9ebc3b0d..b14a27ada5821024d58eeae87f3c13489117a4e7 100644 --- a/moonep/tierbprobe/provenance.txt +++ b/moonep/tierbprobe/provenance.txt @@ -1,7 +1,7 @@ ============================================================== PROVENANCE: moonep__tierbprobe__gbs8 ============================================================== -captured_at : 2026-08-26T16:51:49+00:00 +captured_at : 2026-08-26T18:03:56+00:00 hostname : sp-aiinfra-interactive-aghosh-b5xzj-8xb200-moe-cg-0 kernel : 6.8.0-1043-nvidia cwd : /workspace/blackwell_run_scripts @@ -18,13 +18,13 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ 7, NVIDIA B200, 10.0, 183359 MiB, 580.95.05 persistence/clocks: 0, 120 MHz, 1965 MHz, 1000.00 W, 29 - 1, 150 MHz, 1965 MHz, 1000.00 W, 34 + 1, 120 MHz, 1965 MHz, 1000.00 W, 34 2, 120 MHz, 1965 MHz, 1000.00 W, 34 3, 120 MHz, 1965 MHz, 1000.00 W, 30 4, 120 MHz, 1965 MHz, 1000.00 W, 30 5, 120 MHz, 1965 MHz, 1000.00 W, 34 6, 120 MHz, 1965 MHz, 1000.00 W, 34 - 7, 120 MHz, 1965 MHz, 1000.00 W, 30 + 7, 120 MHz, 1965 MHz, 1000.00 W, 29 NVLink topology: GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID @@ -64,7 +64,7 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb_ UltraEP main 94cab099b44f DeepEP HEAD 34152ae28f80 (+3 untracked: build output) MoonEP megatron-integration c27d35da15f2 - blackwell_run_scripts granularity-moonep cc725d8fcc79 + blackwell_run_scripts granularity-moonep ef3305ebfabd ---- environment --------------------------------------------- CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv index 1eeb1d380836d650767420989755b1df019e8b2e..ab669e8bdf1bb657bae8bb3838cc812d19919c6e 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv @@ -46,22 +46,14 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 629 -2, 629 -3, 629 -4, 629 -5, 629 -6, 629 -7, 629 -0, 943 -1, 961 -2, 987 -3, 959 -4, 953 -5, 943 -6, 957 -7, 949 +0, 821 +1, 821 +2, 821 +3, 821 +4, 821 +5, 821 +6, 821 +7, 821 0, 1955 1, 1955 2, 1955 @@ -70,38 +62,38 @@ 5, 1955 6, 1955 7, 1955 -0, 3955 -1, 3977 -2, 3953 -3, 3957 -4, 3977 -5, 3967 -6, 3961 -7, 3979 -0, 34565 -1, 34441 -2, 34493 -3, 34631 -4, 34417 -5, 34419 -6, 34585 -7, 34441 +0, 3867 +1, 3857 +2, 3875 +3, 3885 +4, 3867 +5, 3881 +6, 3879 +7, 3869 +0, 15439 +1, 15439 +2, 15439 +3, 15439 +4, 15439 +5, 15439 +6, 15439 +7, 15439 0, 35867 -1, 35991 -2, 35861 -3, 35921 -4, 35873 -5, 35921 -6, 35909 -7, 35991 -0, 45899 -1, 46019 -2, 45893 -3, 45951 -4, 45903 -5, 45949 -6, 45941 -7, 46019 +1, 35879 +2, 35753 +3, 35785 +4, 35755 +5, 35767 +6, 35835 +7, 35875 +0, 45733 +1, 45871 +2, 45733 +3, 45783 +4, 45725 +5, 45799 +6, 45775 +7, 45847 0, 46289 1, 46409 2, 46283 @@ -110,7 +102,7 @@ 5, 46339 6, 46331 7, 46409 -0, 46293 +0, 46291 1, 46415 2, 46287 3, 46347 @@ -118,30 +110,30 @@ 5, 46345 6, 46335 7, 46415 -0, 116085 -1, 116077 -2, 116079 -3, 116073 -4, 116089 -5, 116071 -6, 116063 -7, 116063 -0, 139413 -1, 139405 -2, 139407 -3, 139401 -4, 139417 -5, 139399 -6, 140177 -7, 139405 -0, 139769 -1, 139733 -2, 139721 -3, 139721 -4, 139755 -5, 139761 -6, 140477 -7, 139727 +0, 116051 +1, 116041 +2, 116047 +3, 116047 +4, 116057 +5, 116041 +6, 116033 +7, 116049 +0, 139025 +1, 139017 +2, 139019 +3, 139013 +4, 139029 +5, 139011 +6, 139789 +7, 139017 +0, 139707 +1, 139723 +2, 139695 +3, 139709 +4, 139733 +5, 139715 +6, 140475 +7, 139735 0, 139801 1, 139793 2, 139795 @@ -150,22 +142,22 @@ 5, 139787 6, 140565 7, 139793 -0, 142165 -1, 142187 -2, 142189 -3, 142185 -4, 142191 -5, 142181 -6, 142969 -7, 142197 -0, 143123 -1, 143133 -2, 143151 -3, 143095 -4, 143125 -5, 143139 -6, 143885 -7, 143133 +0, 142035 +1, 142051 +2, 142025 +3, 142025 +4, 142041 +5, 142049 +6, 142807 +7, 142039 +0, 143117 +1, 143159 +2, 143191 +3, 143157 +4, 143153 +5, 143115 +6, 143941 +7, 143139 0, 157149 1, 157133 2, 157143 @@ -198,35 +190,43 @@ 5, 157127 6, 156339 7, 157133 -0, 157085 -1, 156871 -2, 157143 -3, 156809 +0, 156823 +1, 156487 +2, 156497 +3, 156489 4, 156569 -5, 156807 -6, 156339 +5, 157127 +6, 156077 7, 157133 -0, 155267 -1, 154989 -2, 155261 -3, 154927 -4, 154687 -5, 155245 -6, 154519 -7, 154669 -0, 153587 -1, 153245 -2, 152935 -3, 152927 -4, 152555 -5, 152973 -6, 152385 -7, 153259 -0, 143044 +0, 154621 +1, 154217 +2, 154615 +3, 154607 +4, 154623 +5, 154987 +6, 153811 +7, 154993 +0, 152873 +1, 152139 +2, 152547 +3, 152539 +4, 153007 +5, 153565 +6, 152515 +7, 153571 +0, 4 1, 4 2, 4 3, 4 4, 4 -5, 4 +5, 143027 6, 4 -7, 4 +7, 143033 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log index 145a112d6469ac5e2a071adbc5c1bdc2f44b8437..4107347d45cbaae5154ef9a24f07e9f7248b9097 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/train.log @@ -1,5 +1,5 @@ -[16:54:29] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:54:29] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[18:06:36] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:06:36] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -8,10 +8,21 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 16:54:30.684000 284717 torch/distributed/run.py:766] -W0826 16:54:30.684000 284717 torch/distributed/run.py:766] ***************************************** -W0826 16:54:30.684000 284717 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:54:30.684000 284717 torch/distributed/run.py:766] ***************************************** +W0826 18:06:38.328000 457134 torch/distributed/run.py:766] +W0826 18:06:38.328000 457134 torch/distributed/run.py:766] ***************************************** +W0826 18:06:38.328000 457134 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:06:38.328000 457134 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 @@ -564,67 +575,56 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4[Gloo] Rank is connected to 27 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 46 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : [Gloo] Rank 77 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. [Gloo] Rank Expected number of connected peer ranks is : 75 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 77 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. +Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. [Gloo] Rank Expected number of connected peer ranks is : Expected number of connected peer ranks is : [Gloo] Rank 00 -00 is connected to - is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank6]:[W826 16:54:49.157344522 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:54:49.158782559 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:54:49.158973773 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:54:49.159053957 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:54:49.159860450 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:54:49.159958602 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:54:49.160277730 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank2]:[W826 18:06:56.225602079 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:06:56.228269888 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:06:56.228329316 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:06:56.228669225 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:06:56.231104086 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:06:56.231631034 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:06:56.234868977 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:54:49.316974956 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.783 seconds -time to initialize megatron (seconds): 7.057 -[after megatron is initialized] datetime: 2026-08-26 16:54:55 +[rank0]:[W826 18:06:56.389066321 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.873 seconds +time to initialize megatron (seconds): 7.227 +[after megatron is initialized] datetime: 2026-08-26 18:07:02 building GPT model ... /workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1542: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -664,217 +664,217 @@ INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradi Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.mlp.router.weight module.decoder.layers.23.pre_mlp_layernorm.weight + module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight - module.decoder.final_layernorm.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight module.decoder.layers.23.self_attention.linear_q_proj.weight - module.decoder.layers.23.input_layernorm.weight Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.pre_mlp_layernorm.weight - module.decoder.layers.21.mlp.router.weight - module.decoder.layers.21.pre_mlp_layernorm.weight - module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.21.self_attention.linear_q_proj.weight - module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.22.self_attention.linear_kv_up_proj.weight - module.decoder.layers.22.self_attention.linear_kv_down_proj.weight - module.decoder.layers.22.self_attention.linear_q_proj.weight module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.22.input_layernorm.weight module.decoder.layers.21.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.weight + module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.self_attention.linear_q_proj.weight + module.decoder.layers.21.mlp.router.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_q_proj.weight Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_kv_down_proj.weight + module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.20.input_layernorm.weight module.decoder.layers.21.self_attention.linear_proj.weight module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.input_layernorm.weight module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.20.pre_mlp_layernorm.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.20.self_attention.linear_kv_down_proj.weight - module.decoder.layers.20.mlp.router.weight - module.decoder.layers.20.pre_mlp_layernorm.weight - module.decoder.layers.20.self_attention.linear_q_proj.weight - module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight Params for bucket 5 (42734080 elements, 42734080 padded size): + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.19.mlp.router.weight - module.decoder.layers.19.self_attention.linear_q_proj.weight module.decoder.layers.19.self_attention.linear_proj.weight + module.decoder.layers.19.input_layernorm.weight module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.self_attention.linear_kv_down_proj.weight - module.decoder.layers.19.input_layernorm.weight + module.decoder.layers.19.self_attention.linear_q_proj.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.weight module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.weight module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.18.input_layernorm.weight module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight module.decoder.layers.17.self_attention.linear_q_proj.weight Params for bucket 7 (41163264 elements, 41163264 padded size): + module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.self_attention.linear_q_proj.weight - module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.weight + module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.self_attention.linear_proj.weight module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.17.input_layernorm.weight module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.16.self_attention.linear_kv_up_proj.weight - module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.16.pre_mlp_layernorm.weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight - module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.input_layernorm.weight Params for bucket 8 (42734080 elements, 42734080 padded size): module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.15.self_attention.linear_kv_up_proj.weight - module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.pre_mlp_layernorm.weight module.decoder.layers.15.self_attention.linear_q_proj.weight - module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.15.input_layernorm.weight module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.weight module.decoder.layers.15.mlp.router.weight - module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.15.self_attention.linear_proj.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_kv_up_proj.weight module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_proj.weight - module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.13.self_attention.linear_kv_up_proj.weight - module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight module.decoder.layers.14.mlp.router.weight module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.14.self_attention.linear_q_proj.weight module.decoder.layers.14.input_layernorm.weight module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.self_attention.linear_q_proj.weight + module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.weight + module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.13.self_attention.linear_kv_down_proj.weight Params for bucket 10 (41163264 elements, 41163264 padded size): - module.decoder.layers.13.self_attention.linear_proj.weight - module.decoder.layers.12.mlp.router.weight - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.12.self_attention.linear_kv_down_proj.weight module.decoder.layers.13.input_layernorm.weight - module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.12.self_attention.linear_q_proj.weight - module.decoder.layers.12.self_attention.linear_proj.weight module.decoder.layers.12.input_layernorm.weight module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight Params for bucket 11 (42734080 elements, 42734080 padded size): module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.weight module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight module.decoder.layers.11.self_attention.linear_kv_down_proj.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_q_proj.weight module.decoder.layers.11.input_layernorm.weight module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.linear_proj.weight module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.10.mlp.router.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.input_layernorm.weight module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.weight module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.9.self_attention.linear_q_proj.weight module.decoder.layers.10.self_attention.linear_kv_up_proj.weight module.decoder.layers.10.self_attention.linear_kv_down_proj.weight module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.9.mlp.router.weight module.decoder.layers.9.self_attention.linear_kv_down_proj.weight + module.decoder.layers.9.self_attention.linear_q_proj.weight Params for bucket 13 (41163264 elements, 41163264 padded size): - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.9.input_layernorm.weight - module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.weight module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.8.self_attention.linear_kv_down_proj.weight - module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_q_proj.weight - module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.8.input_layernorm.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.7.self_attention.linear_kv_down_proj.weight module.decoder.layers.7.self_attention.linear_q_proj.weight module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.self_attention.linear_kv_down_proj.weight module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 15 (40901632 elements, 40901632 padded size): module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_q_proj.weight module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_q_proj.weight module.decoder.layers.6.input_layernorm.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.5.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_q_proj.weight Params for bucket 16 (41163264 elements, 41163264 padded size): - module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_kv_up_proj.weight module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.4.self_attention.linear_kv_down_proj.weight module.decoder.layers.4.self_attention.linear_q_proj.weight @@ -884,8 +884,8 @@ Params for bucket 17 (42734080 elements, 42734080 padded size): module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_up_proj.weight + module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.3.self_attention.linear_proj.weight @@ -894,135 +894,135 @@ Params for bucket 17 (42734080 elements, 42734080 padded size): module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (40901632 elements, 40901632 padded size): module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.2.self_attention.linear_q_proj.weight module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.2.input_layernorm.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.1.self_attention.linear_q_proj.weight module.decoder.layers.2.self_attention.linear_kv_up_proj.weight module.decoder.layers.2.self_attention.linear_kv_down_proj.weight module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.1.mlp.router.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight module.decoder.layers.1.self_attention.linear_kv_down_proj.weight - module.decoder.layers.1.self_attention.linear_q_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.0.mlp.linear_fc1.weight - module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc2.weight + module.decoder.layers.1.input_layernorm.weight Params for bucket 20 (116789760 elements, 116789760 padded size): module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_kv_up_proj.weight module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.0.self_attention.linear_q_proj.weight - module.decoder.layers.0.input_layernorm.weight module.decoder.layers.0.self_attention.linear_proj.weight + module.decoder.layers.0.input_layernorm.weight module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight7 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 module.decoder.layers.23.mlp.experts.linear_fc1.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 Params for bucket 2 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.22.mlp.experts.linear_fc2.weight7 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 Params for bucket 3 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc1.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 - module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 Params for bucket 4 (40370176 elements, 40370176 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc2.weight7 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 - module.decoder.layers.21.mlp.experts.linear_fc2.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 + module.decoder.layers.21.mlp.experts.linear_fc2.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 Params for bucket 5 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc1.weight3 - module.decoder.layers.21.mlp.experts.linear_fc1.weight6 - module.decoder.layers.21.mlp.experts.linear_fc1.weight4 module.decoder.layers.21.mlp.experts.linear_fc1.weight7 module.decoder.layers.21.mlp.experts.linear_fc1.weight5 module.decoder.layers.21.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.21.mlp.experts.linear_fc1.weight4 + module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 Params for bucket 6 (40370176 elements, 40370176 padded size): - module.decoder.layers.21.mlp.experts.linear_fc1.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc2.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight0 module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 - module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 - module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 Params for bucket 7 (40370176 elements, 40370176 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 + module.decoder.layers.20.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight5 module.decoder.layers.20.mlp.experts.linear_fc1.weight2 module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 - module.decoder.layers.20.mlp.experts.linear_fc1.weight4 - module.decoder.layers.20.mlp.experts.linear_fc1.weight0 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 Params for bucket 8 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight5 module.decoder.layers.19.mlp.experts.linear_fc2.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 - module.decoder.layers.19.mlp.experts.linear_fc2.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 Params for bucket 9 (40370176 elements, 40370176 padded size): + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight6 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 module.decoder.layers.19.mlp.experts.linear_fc1.weight3 - module.decoder.layers.18.mlp.experts.linear_fc2.weight7 module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 - module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight2 Params for bucket 10 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc2.weight1 - module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 module.decoder.layers.18.mlp.experts.linear_fc1.weight6 @@ -1030,15 +1030,15 @@ Params for bucket 10 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight3 Params for bucket 11 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight7 - module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.17.mlp.experts.linear_fc2.weight3 - module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 module.decoder.layers.17.mlp.experts.linear_fc2.weight4 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.17.mlp.experts.linear_fc2.weight7 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight0 module.decoder.layers.17.mlp.experts.linear_fc1.weight7 Params for bucket 12 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight6 @@ -1050,30 +1050,30 @@ Params for bucket 12 (40370176 elements, 40370176 padded size): module.decoder.layers.17.mlp.experts.linear_fc1.weight1 Params for bucket 13 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc2.weight7 - module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc2.weight0 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc2.weight6 - module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight7 module.decoder.layers.16.mlp.experts.linear_fc1.weight5 Params for bucket 14 (40370176 elements, 40370176 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight3 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight6 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 Params for bucket 15 (40370176 elements, 40370176 padded size): module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight5 module.decoder.layers.15.mlp.experts.linear_fc2.weight3 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 @@ -1087,8 +1087,8 @@ Params for bucket 16 (40370176 elements, 40370176 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight4 module.decoder.layers.14.mlp.experts.linear_fc2.weight2 module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 module.decoder.layers.14.mlp.experts.linear_fc2.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 @@ -1104,103 +1104,103 @@ Params for bucket 18 (40370176 elements, 40370176 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 module.decoder.layers.14.mlp.experts.linear_fc1.weight0 - module.decoder.layers.13.mlp.experts.linear_fc2.weight4 module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc2.weight4 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 Params for bucket 19 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight7 Params for bucket 20 (40370176 elements, 40370176 padded size): + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight4 - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight7 - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 Params for bucket 21 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 - module.decoder.layers.12.mlp.experts.linear_fc1.weight0 - module.decoder.layers.11.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc2.weight5 - module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight1 module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.11.mlp.experts.linear_fc2.weight4 module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 Params for bucket 22 (40370176 elements, 40370176 padded size): - module.decoder.layers.11.mlp.experts.linear_fc1.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 Params for bucket 23 (40370176 elements, 40370176 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 24 (40370176 elements, 40370176 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 Params for bucket 26 (40370176 elements, 40370176 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight4 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 - module.decoder.layers.9.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 @@ -1210,71 +1210,71 @@ Params for bucket 28 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 30 (40370176 elements, 40370176 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 32 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight6 - module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 Params for bucket 34 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 Params for bucket 35 (40370176 elements, 40370176 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight5 @@ -1298,8 +1298,8 @@ Params for bucket 36 (40370176 elements, 40370176 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 37 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc1.weight7 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 @@ -1319,22 +1319,22 @@ Params for bucket 38 (40370176 elements, 40370176 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 39 (40370176 elements, 40370176 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight7 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight5 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:54:58 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:07:06 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1342,7 +1342,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1355,14 +1355,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:54:58 +[after dataloaders are built] datetime: 2026-08-26 18:07:06 done with setup ... training ... Overwriting rerun_state_machine.current_iteration from -1 to 0... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3531.24, 3542.05) - train/valid/test-data-iterators-setup ..........: (62.16, 68.64) -[before the start of training step] datetime: 2026-08-26 16:54:58 + model-and-optimizer-setup ......................: (3889.30, 3918.67) + train/valid/test-data-iterators-setup ..........: (60.87, 69.40) +[before the start of training step] datetime: 2026-08-26 18:07:06 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 sm_count : num_sms=32, num_sms_dedup=148 @@ -1371,5636 +1371,5635 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 2.01 MiB total_size : 204.01 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 16:55:21] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22531.6 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092206E+01 | loss scale: 1.0 | grad norm: 30.650 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 - + [2026-08-26 18:07:28] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22506.5 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092206E+01 | loss scale: 1.0 | grad norm: 30.650 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 13.53 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 Number of parameters in most loaded shard in billions: 13.7356 Theoretical memory footprints: weight and optimizer=98244.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 66080.0703125 | max allocated: 124949.189453125 | reserved: 127502.0 | max reserved: 127502.0 - [2026-08-26 16:55:22] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 801.1 | throughput per GPU (TFLOP/s/GPU): 159.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091986E+01 | loss scale: 1.0 | grad norm: 30.565 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:07:29] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 797.7 | throughput per GPU (TFLOP/s/GPU): 159.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091986E+01 | loss scale: 1.0 | grad norm: 30.565 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75044 payload_rows_remote=64192 dup_rows=23260 dup_rows_remote=19654 dedup_ratio=0.236613 router_tokens_per_expert=[1386, 677, 582, 589, 473, 947, 1297, 233, 925, 1183, 1308, 460, 118, 1110, 420, 1020, 1073, 799, 826, 837, 581, 495, 859, 573, 811, 878, 343, 480, 791, 1055, 454, 886, 954, 443, 763, 882, 420, 364, 850, 975, 501, 479, 376, 512, 1515, 1543, 776, 326, 610, 801, 755, 822, 1153, 1633, 808, 501, 606, 857, 1360, 746, 499, 434, 1153, 266] payload_rows_per_dst=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] dup_rows_per_dst=[2278, 1260, 2734, 2732, 3766, 2680, 4204, 3606] combine_rows_per_src=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=64782 dup_rows=23670 dup_rows_remote=19798 dedup_ratio=0.240784 router_tokens_per_expert=[1384, 896, 557, 585, 534, 959, 1303, 231, 941, 1164, 1301, 605, 153, 1233, 420, 1045, 1012, 867, 772, 829, 573, 411, 1007, 644, 849, 830, 339, 480, 878, 751, 431, 901, 890, 511, 820, 837, 504, 378, 857, 788, 459, 544, 318, 513, 1416, 1590, 771, 319, 522, 737, 881, 822, 1130, 1686, 727, 555, 523, 807, 1378, 767, 476, 434, 1049, 258] payload_rows_per_dst=[9594, 9852, 9414, 10074, 8426, 9420, 8962, 8892] dup_rows_per_dst=[3304, 3872, 2816, 4216, 2744, 2440, 1786, 2492] combine_rows_per_src=[9594, 9852, 9414, 10074, 8426, 9420, 8962, 8892] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64898 dup_rows=23938 dup_rows_remote=20786 dedup_ratio=0.243510 router_tokens_per_expert=[1467, 723, 585, 512, 602, 980, 1268, 241, 683, 1252, 1204, 362, 177, 936, 442, 1032, 1304, 830, 742, 849, 594, 473, 1008, 535, 895, 766, 325, 452, 831, 1020, 430, 905, 771, 434, 803, 797, 431, 400, 922, 913, 403, 517, 406, 838, 1400, 1547, 841, 358, 620, 628, 787, 759, 1071, 2002, 695, 508, 489, 835, 1443, 716, 557, 420, 1165, 251] payload_rows_per_dst=[9414, 9156, 9718, 8528, 8712, 9468, 10176, 9194] dup_rows_per_dst=[2844, 3020, 2952, 2720, 2728, 3152, 3964, 2558] combine_rows_per_src=[9414, 9156, 9718, 8528, 8712, 9468, 10176, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75044 payload_rows_remote=64192 dup_rows=23260 dup_rows_remote=19654 dedup_ratio=0.236613 router_tokens_per_expert=[1386, 677, 582, 589, 473, 947, 1297, 233, 925, 1183, 1308, 460, 118, 1110, 420, 1020, 1073, 799, 826, 837, 581, 495, 859, 573, 811, 878, 343, 480, 791, 1055, 454, 886, 954, 443, 763, 882, 420, 364, 850, 975, 501, 479, 376, 512, 1515, 1543, 776, 326, 610, 801, 755, 822, 1153, 1633, 808, 501, 606, 857, 1360, 746, 499, 434, 1153, 266] payload_rows_per_dst=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] dup_rows_per_dst=[2278, 1260, 2734, 2732, 3766, 2680, 4204, 3606] combine_rows_per_src=[7318, 8438, 9352, 8664, 10308, 9376, 10736, 10852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73764 payload_rows_remote=65216 dup_rows=24540 dup_rows_remote=21874 dedup_ratio=0.249634 router_tokens_per_expert=[1280, 745, 587, 641, 619, 930, 1346, 162, 823, 781, 1301, 442, 124, 1034, 442, 950, 954, 841, 696, 803, 617, 492, 1080, 552, 810, 811, 323, 365, 866, 1025, 475, 932, 1232, 438, 1015, 785, 488, 393, 877, 941, 500, 553, 361, 780, 1386, 1520, 810, 336, 474, 770, 696, 850, 1182, 1727, 824, 512, 559, 929, 1297, 761, 569, 383, 1049, 306] payload_rows_per_dst=[9388, 8920, 9266, 8548, 8946, 10022, 9684, 8990] dup_rows_per_dst=[3232, 2874, 2804, 2666, 3392, 3614, 3242, 2716] combine_rows_per_src=[9388, 8920, 9266, 8548, 8946, 10022, 9684, 8990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64254 dup_rows=23924 dup_rows_remote=20304 dedup_ratio=0.243368 router_tokens_per_expert=[1466, 712, 583, 604, 507, 937, 1337, 190, 898, 1179, 1285, 467, 169, 927, 462, 993, 994, 874, 762, 778, 645, 461, 1037, 581, 799, 838, 286, 492, 863, 942, 360, 926, 1051, 438, 894, 841, 541, 392, 939, 924, 443, 464, 329, 498, 1523, 1546, 833, 382, 564, 748, 770, 777, 1181, 1525, 813, 495, 477, 953, 1446, 751, 540, 407, 1050, 263] payload_rows_per_dst=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] dup_rows_per_dst=[2280, 2992, 2770, 2576, 4488, 2566, 3620, 2632] combine_rows_per_src=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=65746 dup_rows=24152 dup_rows_remote=21160 dedup_ratio=0.245687 router_tokens_per_expert=[1471, 694, 544, 633, 638, 995, 1377, 184, 768, 1244, 1175, 448, 142, 1022, 472, 993, 1069, 871, 752, 873, 577, 481, 957, 617, 796, 696, 336, 403, 834, 1006, 385, 937, 1113, 425, 761, 834, 441, 374, 965, 786, 451, 519, 369, 479, 1386, 1486, 765, 370, 611, 773, 735, 786, 1012, 1751, 798, 509, 547, 971, 1510, 839, 504, 391, 1177, 324] payload_rows_per_dst=[9744, 9302, 9600, 8196, 8406, 9170, 10158, 9576] dup_rows_per_dst=[3328, 3226, 2794, 2590, 2992, 2480, 3792, 2950] combine_rows_per_src=[9744, 9302, 9600, 8196, 8406, 9170, 10158, 9576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=65326 dup_rows=23432 dup_rows_remote=20484 dedup_ratio=0.238363 router_tokens_per_expert=[1425, 764, 612, 536, 673, 847, 1422, 267, 911, 1081, 1310, 401, 145, 1132, 496, 952, 916, 809, 807, 789, 638, 488, 1125, 675, 731, 870, 428, 487, 881, 909, 406, 907, 798, 487, 777, 720, 540, 377, 877, 924, 513, 510, 371, 683, 1437, 1456, 624, 352, 641, 695, 741, 878, 967, 1658, 693, 436, 561, 913, 1442, 765, 648, 468, 1065, 295] payload_rows_per_dst=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] dup_rows_per_dst=[3236, 3190, 2948, 4180, 2512, 2848, 1522, 2996] combine_rows_per_src=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=63572 dup_rows=24140 dup_rows_remote=19704 dedup_ratio=0.245565 router_tokens_per_expert=[1368, 800, 687, 608, 583, 912, 1307, 180, 827, 878, 1254, 626, 159, 1029, 512, 1030, 1115, 891, 708, 849, 582, 479, 1066, 534, 704, 830, 315, 415, 878, 959, 312, 962, 999, 506, 801, 868, 466, 385, 988, 943, 486, 499, 374, 520, 1580, 1567, 806, 370, 463, 645, 745, 830, 1104, 1602, 677, 481, 459, 1032, 1468, 789, 531, 421, 1071, 317] payload_rows_per_dst=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] dup_rows_per_dst=[4436, 3376, 2790, 3758, 3152, 2698, 1016, 2914] combine_rows_per_src=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73764 payload_rows_remote=65216 dup_rows=24540 dup_rows_remote=21874 dedup_ratio=0.249634 router_tokens_per_expert=[1280, 745, 587, 641, 619, 930, 1346, 162, 823, 781, 1301, 442, 124, 1034, 442, 950, 954, 841, 696, 803, 617, 492, 1080, 552, 810, 811, 323, 365, 866, 1025, 475, 932, 1232, 438, 1015, 785, 488, 393, 877, 941, 500, 553, 361, 780, 1386, 1520, 810, 336, 474, 770, 696, 850, 1182, 1727, 824, 512, 559, 929, 1297, 761, 569, 383, 1049, 306] payload_rows_per_dst=[9388, 8920, 9266, 8548, 8946, 10022, 9684, 8990] dup_rows_per_dst=[3232, 2874, 2804, 2666, 3392, 3614, 3242, 2716] combine_rows_per_src=[9388, 8920, 9266, 8548, 8946, 10022, 9684, 8990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262930432 dispatch_meta_bytes=335384 combine_payload_bytes=262930432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263265816 ingress_bytes=297533440 link_bytes=297533440 token_bytes=526196248 weight_movement_bytes=34603008 total_nvlink_bytes=560799256 weight_to_token_ratio=0.065761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119594381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=342736 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266164944 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987152 weight_movement_bytes=34603008 total_nvlink_bytes=566590160 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=338320 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265685392 ingress_bytes=369156096 link_bytes=369156096 token_bytes=531032464 weight_movement_bytes=103809024 total_nvlink_bytes=634841488 weight_to_token_ratio=0.195485 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389104670e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262930432 dispatch_meta_bytes=335384 combine_payload_bytes=262930432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263265816 ingress_bytes=297533440 link_bytes=297533440 token_bytes=526196248 weight_movement_bytes=34603008 total_nvlink_bytes=560799256 weight_to_token_ratio=0.065761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119594381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=348360 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267473096 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534597832 weight_movement_bytes=34603008 total_nvlink_bytes=569200840 weight_to_token_ratio=0.064727 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=338232 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=263522616 ingress_bytes=401596416 link_bytes=401596416 token_bytes=526707000 weight_movement_bytes=138412032 total_nvlink_bytes=665119032 weight_to_token_ratio=0.262788 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511174982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=63572 dup_rows=24140 dup_rows_remote=19704 dedup_ratio=0.245565 router_tokens_per_expert=[1368, 800, 687, 608, 583, 912, 1307, 180, 827, 878, 1254, 626, 159, 1029, 512, 1030, 1115, 891, 708, 849, 582, 479, 1066, 534, 704, 830, 315, 415, 878, 959, 312, 962, 999, 506, 801, 868, 466, 385, 988, 943, 486, 499, 374, 520, 1580, 1567, 806, 370, 463, 645, 745, 830, 1104, 1602, 677, 481, 459, 1032, 1468, 789, 531, 421, 1071, 317] payload_rows_per_dst=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] dup_rows_per_dst=[4436, 3376, 2790, 3758, 3152, 2698, 1016, 2914] combine_rows_per_src=[10592, 9606, 9306, 10196, 8760, 9706, 6736, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=65326 dup_rows=23432 dup_rows_remote=20484 dedup_ratio=0.238363 router_tokens_per_expert=[1425, 764, 612, 536, 673, 847, 1422, 267, 911, 1081, 1310, 401, 145, 1132, 496, 952, 916, 809, 807, 789, 638, 488, 1125, 675, 731, 870, 428, 487, 881, 909, 406, 907, 798, 487, 777, 720, 540, 377, 877, 924, 513, 510, 371, 683, 1437, 1456, 624, 352, 641, 695, 741, 878, 967, 1658, 693, 436, 561, 913, 1442, 765, 648, 468, 1065, 295] payload_rows_per_dst=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] dup_rows_per_dst=[3236, 3190, 2948, 4180, 2512, 2848, 1522, 2996] combine_rows_per_src=[9856, 9666, 9546, 10224, 8488, 9194, 8580, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=342736 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266164944 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987152 weight_movement_bytes=34603008 total_nvlink_bytes=566590160 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269295616 dispatch_meta_bytes=347624 combine_payload_bytes=269295616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269643240 ingress_bytes=303898624 link_bytes=303898624 token_bytes=538938856 weight_movement_bytes=34603008 total_nvlink_bytes=573541864 weight_to_token_ratio=0.064206 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143546056e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267575296 dispatch_meta_bytes=343240 combine_payload_bytes=267575296 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267918536 ingress_bytes=302178304 link_bytes=302178304 token_bytes=535493832 weight_movement_bytes=34603008 total_nvlink_bytes=570096840 weight_to_token_ratio=0.064619 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137072631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=348360 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267473096 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534597832 weight_movement_bytes=34603008 total_nvlink_bytes=569200840 weight_to_token_ratio=0.064727 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260390912 dispatch_meta_bytes=333104 combine_payload_bytes=260390912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260724016 ingress_bytes=329596928 link_bytes=329596928 token_bytes=521114928 weight_movement_bytes=69206016 total_nvlink_bytes=590320944 weight_to_token_ratio=0.132804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240246705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=65498 dup_rows=22786 dup_rows_remote=20038 dedup_ratio=0.231791 router_tokens_per_expert=[216, 392, 782, 558, 1226, 590, 936, 534, 281, 284, 900, 272, 1167, 470, 912, 2235, 833, 1289, 938, 865, 905, 944, 1311, 589, 631, 764, 902, 1910, 257, 914, 470, 589, 294, 351, 454, 594, 997, 223, 340, 1049, 1664, 494, 22, 823, 391, 498, 38, 1143, 3276, 316, 1294, 70, 541, 1084, 768, 751, 511, 1035, 415, 603, 1747, 421, 637, 442] payload_rows_per_dst=[8100, 10200, 9680, 9670, 7364, 10020, 11216, 9268] dup_rows_per_dst=[2368, 2842, 3046, 3204, 1240, 2748, 4984, 2354] combine_rows_per_src=[8100, 10200, 9680, 9670, 7364, 10020, 11216, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64254 dup_rows=23924 dup_rows_remote=20304 dedup_ratio=0.243368 router_tokens_per_expert=[1466, 712, 583, 604, 507, 937, 1337, 190, 898, 1179, 1285, 467, 169, 927, 462, 993, 994, 874, 762, 778, 645, 461, 1037, 581, 799, 838, 286, 492, 863, 942, 360, 926, 1051, 438, 894, 841, 541, 392, 939, 924, 443, 464, 329, 498, 1523, 1546, 833, 382, 564, 748, 770, 777, 1181, 1525, 813, 495, 477, 953, 1446, 751, 540, 407, 1050, 263] payload_rows_per_dst=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] dup_rows_per_dst=[2280, 2992, 2770, 2576, 4488, 2566, 3620, 2632] combine_rows_per_src=[7460, 9554, 9494, 8436, 10484, 9470, 10126, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267575296 dispatch_meta_bytes=343240 combine_payload_bytes=267575296 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267918536 ingress_bytes=302178304 link_bytes=302178304 token_bytes=535493832 weight_movement_bytes=34603008 total_nvlink_bytes=570096840 weight_to_token_ratio=0.064619 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137072631e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75560 payload_rows_remote=63084 dup_rows=22744 dup_rows_remote=17350 dedup_ratio=0.231364 router_tokens_per_expert=[224, 455, 380, 625, 1305, 715, 835, 614, 265, 328, 1198, 232, 1111, 544, 932, 2049, 924, 1210, 967, 882, 925, 973, 1479, 599, 605, 750, 954, 1823, 295, 844, 426, 645, 342, 316, 413, 522, 875, 313, 393, 982, 1702, 455, 50, 750, 409, 469, 32, 929, 3133, 358, 1396, 118, 697, 1012, 782, 789, 544, 966, 588, 527, 1613, 413, 688, 463] payload_rows_per_dst=[8118, 10212, 9862, 8198, 7078, 9868, 9748, 12476] dup_rows_per_dst=[2188, 3106, 3098, 2320, 1234, 2682, 2722, 5394] combine_rows_per_src=[8118, 10212, 9862, 8198, 7078, 9868, 9748, 12476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76362 payload_rows_remote=66832 dup_rows=21942 dup_rows_remote=18460 dedup_ratio=0.223206 router_tokens_per_expert=[232, 547, 838, 562, 1273, 651, 970, 647, 300, 395, 1076, 286, 983, 550, 753, 1869, 873, 1219, 847, 309, 963, 1229, 1398, 517, 655, 797, 1001, 1804, 242, 871, 453, 683, 266, 350, 397, 712, 1105, 281, 317, 1041, 1510, 828, 42, 673, 264, 511, 62, 864, 3209, 505, 1422, 131, 789, 1087, 778, 880, 369, 860, 614, 592, 1516, 398, 532, 454] payload_rows_per_dst=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] dup_rows_per_dst=[3494, 1662, 2884, 3482, 3336, 2934, 2316, 1834] combine_rows_per_src=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74792 payload_rows_remote=64026 dup_rows=23512 dup_rows_remote=19016 dedup_ratio=0.239176 router_tokens_per_expert=[175, 439, 370, 540, 1293, 632, 926, 591, 254, 261, 1255, 322, 1115, 573, 852, 2293, 765, 1140, 863, 678, 944, 1055, 1453, 681, 594, 701, 826, 1903, 182, 917, 465, 700, 280, 328, 483, 567, 939, 250, 400, 1008, 1586, 724, 40, 740, 344, 457, 37, 1120, 3238, 379, 1297, 110, 764, 1081, 844, 725, 596, 963, 453, 517, 1674, 437, 544, 469] payload_rows_per_dst=[7790, 10442, 9266, 9492, 7248, 9842, 10766, 9946] dup_rows_per_dst=[2142, 3408, 2986, 3084, 1262, 3160, 4496, 2974] combine_rows_per_src=[7790, 10442, 9266, 9492, 7248, 9842, 10766, 9946] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76064 payload_rows_remote=66752 dup_rows=22240 dup_rows_remote=19652 dedup_ratio=0.226237 router_tokens_per_expert=[183, 456, 527, 500, 1258, 793, 922, 592, 236, 335, 1226, 187, 1239, 623, 904, 2214, 828, 1200, 885, 563, 944, 957, 1212, 611, 560, 807, 918, 1891, 203, 873, 447, 563, 382, 407, 479, 624, 1051, 357, 393, 981, 1545, 748, 45, 794, 346, 484, 46, 1014, 3069, 283, 1315, 99, 662, 1067, 789, 822, 479, 963, 369, 667, 1802, 462, 560, 391] payload_rows_per_dst=[10928, 9312, 7614, 9344, 12096, 9408, 8140, 9222] dup_rows_per_dst=[3962, 2588, 1962, 3180, 3390, 3060, 1934, 2164] combine_rows_per_src=[10928, 9312, 7614, 9344, 12096, 9408, 8140, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=65560 dup_rows=22768 dup_rows_remote=20384 dedup_ratio=0.231608 router_tokens_per_expert=[156, 613, 212, 596, 1342, 713, 881, 628, 296, 293, 1175, 218, 1057, 524, 887, 2243, 930, 1239, 852, 485, 994, 1066, 1289, 675, 596, 764, 777, 1873, 186, 797, 425, 729, 336, 331, 418, 573, 1030, 281, 396, 880, 1601, 795, 32, 852, 453, 516, 49, 877, 3321, 370, 1252, 91, 686, 1104, 821, 833, 540, 968, 561, 501, 1814, 387, 494, 478] payload_rows_per_dst=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] dup_rows_per_dst=[2034, 3214, 3110, 3124, 2384, 3196, 3458, 2248] combine_rows_per_src=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76358 payload_rows_remote=65448 dup_rows=21946 dup_rows_remote=18306 dedup_ratio=0.223246 router_tokens_per_expert=[264, 540, 419, 543, 1246, 601, 922, 480, 259, 332, 1306, 233, 1166, 587, 869, 2260, 818, 1374, 911, 587, 1038, 1015, 1433, 670, 650, 835, 924, 1742, 244, 904, 437, 671, 350, 331, 478, 659, 982, 334, 327, 981, 1492, 747, 39, 747, 324, 452, 45, 1121, 3000, 367, 1144, 158, 539, 1033, 811, 946, 499, 936, 334, 500, 1812, 385, 548, 451] payload_rows_per_dst=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] dup_rows_per_dst=[3640, 2748, 3278, 2878, 3158, 2388, 1890, 1966] combine_rows_per_src=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=67166 dup_rows=21336 dup_rows_remote=17952 dedup_ratio=0.217041 router_tokens_per_expert=[211, 433, 519, 609, 1305, 543, 780, 515, 302, 386, 1248, 331, 1173, 600, 869, 2040, 992, 1263, 989, 1043, 1057, 1091, 1289, 578, 578, 783, 858, 2071, 238, 875, 455, 485, 368, 358, 431, 541, 974, 256, 397, 1040, 1628, 554, 58, 773, 321, 435, 23, 729, 3031, 338, 1366, 136, 619, 961, 718, 878, 567, 965, 578, 510, 1533, 407, 639, 511] payload_rows_per_dst=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] dup_rows_per_dst=[3054, 2290, 3384, 3198, 3026, 2290, 1914, 2180] combine_rows_per_src=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268279808 dispatch_meta_bytes=342144 combine_payload_bytes=268279808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268621952 ingress_bytes=302882816 link_bytes=302882816 token_bytes=536901760 weight_movement_bytes=34603008 total_nvlink_bytes=571504768 weight_to_token_ratio=0.064449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139723653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=65498 dup_rows=22786 dup_rows_remote=20038 dedup_ratio=0.231791 router_tokens_per_expert=[216, 392, 782, 558, 1226, 590, 936, 534, 281, 284, 900, 272, 1167, 470, 912, 2235, 833, 1289, 938, 865, 905, 944, 1311, 589, 631, 764, 902, 1910, 257, 914, 470, 589, 294, 351, 454, 594, 997, 223, 340, 1049, 1664, 494, 22, 823, 391, 498, 38, 1143, 3276, 316, 1294, 70, 541, 1084, 768, 751, 511, 1035, 415, 603, 1747, 421, 637, 442] payload_rows_per_dst=[8100, 10200, 9680, 9670, 7364, 10020, 11216, 9268] dup_rows_per_dst=[2368, 2842, 3046, 3204, 1240, 2748, 4984, 2354] combine_rows_per_src=[8100, 10200, 9680, 9670, 7364, 10020, 11216, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=258392064 dispatch_meta_bytes=321736 combine_payload_bytes=258392064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258713800 ingress_bytes=292995072 link_bytes=292995072 token_bytes=517105864 weight_movement_bytes=34603008 total_nvlink_bytes=551708872 weight_to_token_ratio=0.066917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102516868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273743872 dispatch_meta_bytes=341168 combine_payload_bytes=273743872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274085040 ingress_bytes=342949888 link_bytes=342949888 token_bytes=547828912 weight_movement_bytes=69206016 total_nvlink_bytes=617034928 weight_to_token_ratio=0.126328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290492819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262250496 dispatch_meta_bytes=332168 combine_payload_bytes=262250496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262582664 ingress_bytes=366059520 link_bytes=366059520 token_bytes=524833160 weight_movement_bytes=103809024 total_nvlink_bytes=628642184 weight_to_token_ratio=0.197794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=65560 dup_rows=22768 dup_rows_remote=20384 dedup_ratio=0.231608 router_tokens_per_expert=[156, 613, 212, 596, 1342, 713, 881, 628, 296, 293, 1175, 218, 1057, 524, 887, 2243, 930, 1239, 852, 485, 994, 1066, 1289, 675, 596, 764, 777, 1873, 186, 797, 425, 729, 336, 331, 418, 573, 1030, 281, 396, 880, 1601, 795, 32, 852, 453, 516, 49, 877, 3321, 370, 1252, 91, 686, 1104, 821, 833, 540, 968, 561, 501, 1814, 387, 494, 478] payload_rows_per_dst=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] dup_rows_per_dst=[2034, 3214, 3110, 3124, 2384, 3196, 3458, 2248] combine_rows_per_src=[8248, 10172, 9372, 9170, 9976, 9732, 9628, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76362 payload_rows_remote=66832 dup_rows=21942 dup_rows_remote=18460 dedup_ratio=0.223206 router_tokens_per_expert=[232, 547, 838, 562, 1273, 651, 970, 647, 300, 395, 1076, 286, 983, 550, 753, 1869, 873, 1219, 847, 309, 963, 1229, 1398, 517, 655, 797, 1001, 1804, 242, 871, 453, 683, 266, 350, 397, 712, 1105, 281, 317, 1041, 1510, 828, 42, 673, 264, 511, 62, 864, 3209, 505, 1422, 131, 789, 1087, 778, 880, 369, 860, 614, 592, 1516, 398, 532, 454] payload_rows_per_dst=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] dup_rows_per_dst=[3494, 1662, 2884, 3482, 3336, 2934, 2316, 1834] combine_rows_per_src=[10472, 8236, 9030, 9530, 12020, 9370, 8868, 8836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=338232 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=263522616 ingress_bytes=401596416 link_bytes=401596416 token_bytes=526707000 weight_movement_bytes=138412032 total_nvlink_bytes=665119032 weight_to_token_ratio=0.262788 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511174982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273416192 dispatch_meta_bytes=345616 combine_payload_bytes=273416192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273761808 ingress_bytes=342622208 link_bytes=342622208 token_bytes=547178000 weight_movement_bytes=69206016 total_nvlink_bytes=616384016 weight_to_token_ratio=0.126478 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289259785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76358 payload_rows_remote=65448 dup_rows=21946 dup_rows_remote=18306 dedup_ratio=0.223246 router_tokens_per_expert=[264, 540, 419, 543, 1246, 601, 922, 480, 259, 332, 1306, 233, 1166, 587, 869, 2260, 818, 1374, 911, 587, 1038, 1015, 1433, 670, 650, 835, 924, 1742, 244, 904, 437, 671, 350, 331, 478, 659, 982, 334, 327, 981, 1492, 747, 39, 747, 324, 452, 45, 1121, 3000, 367, 1144, 158, 539, 1033, 811, 946, 499, 936, 334, 500, 1812, 385, 548, 451] payload_rows_per_dst=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] dup_rows_per_dst=[3640, 2748, 3278, 2878, 3158, 2388, 1890, 1966] combine_rows_per_src=[10910, 9504, 9930, 8942, 11726, 8276, 8106, 8964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268279808 dispatch_meta_bytes=342144 combine_payload_bytes=268279808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268621952 ingress_bytes=302882816 link_bytes=302882816 token_bytes=536901760 weight_movement_bytes=34603008 total_nvlink_bytes=571504768 weight_to_token_ratio=0.064449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139723653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268533760 dispatch_meta_bytes=343776 combine_payload_bytes=268533760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268877536 ingress_bytes=303136768 link_bytes=303136768 token_bytes=537411296 weight_movement_bytes=34603008 total_nvlink_bytes=572014304 weight_to_token_ratio=0.064388 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140679253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268075008 dispatch_meta_bytes=335016 combine_payload_bytes=268075008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268410024 ingress_bytes=302678016 link_bytes=302678016 token_bytes=536485032 weight_movement_bytes=34603008 total_nvlink_bytes=571088040 weight_to_token_ratio=0.064499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138953007e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=340472 combine_payload_bytes=275111936 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275452408 ingress_bytes=378920960 link_bytes=378920960 token_bytes=550564344 weight_movement_bytes=103809024 total_nvlink_bytes=654373368 weight_to_token_ratio=0.188550 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425849067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75090 payload_rows_remote=64300 dup_rows=23214 dup_rows_remote=20144 dedup_ratio=0.236145 router_tokens_per_expert=[211, 803, 108, 380, 269, 756, 220, 1773, 26, 1139, 220, 1896, 694, 1379, 343, 625, 1936, 461, 613, 1128, 442, 313, 1934, 165, 471, 1552, 2629, 1072, 751, 672, 218, 108, 878, 647, 1932, 1223, 1875, 895, 1814, 433, 332, 685, 107, 152, 1301, 1616, 108, 693, 929, 804, 633, 305, 505, 404, 190, 210, 1477, 671, 293, 408, 293, 752, 545, 735] payload_rows_per_dst=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] dup_rows_per_dst=[2556, 2704, 2072, 3392, 2490, 3070, 2038, 4892] combine_rows_per_src=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74894 payload_rows_remote=65120 dup_rows=23410 dup_rows_remote=21118 dedup_ratio=0.238139 router_tokens_per_expert=[126, 739, 92, 457, 310, 513, 226, 1946, 26, 1203, 208, 2030, 776, 1246, 416, 712, 2183, 482, 519, 1031, 557, 318, 1818, 156, 541, 1675, 2852, 1251, 733, 628, 439, 92, 581, 596, 1793, 1269, 1574, 670, 1879, 437, 327, 738, 125, 204, 1125, 1822, 146, 425, 714, 1025, 588, 374, 544, 565, 195, 149, 1386, 641, 261, 422, 300, 743, 544, 689] payload_rows_per_dst=[7222, 9172, 8722, 11210, 8358, 11230, 9774, 9206] dup_rows_per_dst=[1596, 2368, 2734, 5212, 1896, 2960, 2292, 4352] combine_rows_per_src=[7222, 9172, 8722, 11210, 8358, 11230, 9774, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273743872 dispatch_meta_bytes=341168 combine_payload_bytes=273743872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274085040 ingress_bytes=342949888 link_bytes=342949888 token_bytes=547828912 weight_movement_bytes=69206016 total_nvlink_bytes=617034928 weight_to_token_ratio=0.126328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290492819e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=67166 dup_rows=21336 dup_rows_remote=17952 dedup_ratio=0.217041 router_tokens_per_expert=[211, 433, 519, 609, 1305, 543, 780, 515, 302, 386, 1248, 331, 1173, 600, 869, 2040, 992, 1263, 989, 1043, 1057, 1091, 1289, 578, 578, 783, 858, 2071, 238, 875, 455, 485, 368, 358, 431, 541, 974, 256, 397, 1040, 1628, 554, 58, 773, 321, 435, 23, 729, 3031, 338, 1366, 136, 619, 961, 718, 878, 567, 965, 578, 510, 1533, 407, 639, 511] payload_rows_per_dst=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] dup_rows_per_dst=[3054, 2290, 3384, 3198, 3026, 2290, 1914, 2180] combine_rows_per_src=[10856, 8368, 9802, 9488, 11766, 9330, 8118, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268075008 dispatch_meta_bytes=335016 combine_payload_bytes=268075008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268410024 ingress_bytes=302678016 link_bytes=302678016 token_bytes=536485032 weight_movement_bytes=34603008 total_nvlink_bytes=571088040 weight_to_token_ratio=0.064499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138953007e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65754 dup_rows=23860 dup_rows_remote=20054 dedup_ratio=0.242716 router_tokens_per_expert=[179, 818, 117, 454, 275, 457, 252, 1837, 37, 1216, 245, 1755, 964, 1246, 344, 729, 2021, 568, 654, 830, 562, 400, 1492, 234, 534, 1701, 2698, 1151, 753, 646, 476, 92, 609, 733, 1485, 1337, 1610, 904, 1755, 415, 332, 897, 117, 149, 1583, 1651, 96, 526, 957, 1023, 728, 324, 590, 510, 178, 143, 1331, 579, 288, 317, 266, 737, 589, 656] payload_rows_per_dst=[7260, 7874, 9168, 11156, 8364, 11482, 10450, 8690] dup_rows_per_dst=[1518, 1688, 3362, 4946, 2320, 3262, 2958, 3806] combine_rows_per_src=[7260, 7874, 9168, 11156, 8364, 11482, 10450, 8690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=340472 combine_payload_bytes=275111936 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275452408 ingress_bytes=378920960 link_bytes=378920960 token_bytes=550564344 weight_movement_bytes=103809024 total_nvlink_bytes=654373368 weight_to_token_ratio=0.188550 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425849067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76464 payload_rows_remote=66790 dup_rows=21840 dup_rows_remote=19244 dedup_ratio=0.222168 router_tokens_per_expert=[189, 833, 81, 454, 272, 483, 223, 2000, 40, 1283, 254, 1721, 702, 1114, 413, 608, 2626, 436, 552, 1040, 480, 266, 1699, 161, 572, 1617, 2877, 1247, 816, 508, 463, 126, 552, 742, 1721, 1388, 1462, 834, 1977, 442, 330, 827, 113, 180, 1421, 1608, 109, 410, 782, 1082, 701, 321, 578, 442, 198, 183, 1184, 580, 256, 297, 378, 763, 549, 586] payload_rows_per_dst=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] dup_rows_per_dst=[3916, 2596, 3774, 2028, 3304, 1740, 2304, 2178] combine_rows_per_src=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269328384 dispatch_meta_bytes=343232 combine_payload_bytes=269328384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269671616 ingress_bytes=303931392 link_bytes=303931392 token_bytes=539000000 weight_movement_bytes=34603008 total_nvlink_bytes=573603008 weight_to_token_ratio=0.064199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143669360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74792 payload_rows_remote=64026 dup_rows=23512 dup_rows_remote=19016 dedup_ratio=0.239176 router_tokens_per_expert=[175, 439, 370, 540, 1293, 632, 926, 591, 254, 261, 1255, 322, 1115, 573, 852, 2293, 765, 1140, 863, 678, 944, 1055, 1453, 681, 594, 701, 826, 1903, 182, 917, 465, 700, 280, 328, 483, 567, 939, 250, 400, 1008, 1586, 724, 40, 740, 344, 457, 37, 1120, 3238, 379, 1297, 110, 764, 1081, 844, 725, 596, 963, 453, 517, 1674, 437, 544, 469] payload_rows_per_dst=[7790, 10442, 9266, 9492, 7248, 9842, 10766, 9946] dup_rows_per_dst=[2142, 3408, 2986, 3084, 1262, 3160, 4496, 2974] combine_rows_per_src=[7790, 10442, 9266, 9492, 7248, 9842, 10766, 9946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76802 payload_rows_remote=67520 dup_rows=21502 dup_rows_remote=19120 dedup_ratio=0.218730 router_tokens_per_expert=[110, 714, 86, 456, 271, 465, 256, 1833, 29, 985, 244, 2027, 664, 1330, 370, 633, 1974, 529, 674, 944, 369, 318, 1804, 132, 541, 1813, 2881, 1149, 711, 502, 399, 99, 770, 785, 1563, 1245, 1927, 643, 1796, 462, 259, 990, 152, 155, 1417, 1829, 98, 463, 906, 996, 790, 309, 487, 409, 160, 116, 1525, 578, 260, 429, 319, 766, 564, 672] payload_rows_per_dst=[10818, 9892, 7950, 8702, 9282, 11072, 9792, 9294] dup_rows_per_dst=[3326, 2672, 1910, 1726, 2382, 3282, 2146, 4058] combine_rows_per_src=[10818, 9892, 7950, 8702, 9282, 11072, 9792, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76398 payload_rows_remote=68220 dup_rows=21906 dup_rows_remote=20388 dedup_ratio=0.222839 router_tokens_per_expert=[113, 762, 145, 465, 323, 405, 240, 1340, 43, 994, 249, 1801, 573, 1672, 392, 914, 1996, 372, 716, 1061, 366, 293, 1684, 210, 496, 1425, 2594, 1034, 673, 681, 376, 163, 577, 689, 1952, 1210, 1778, 855, 1875, 572, 250, 876, 169, 126, 1693, 1599, 157, 409, 955, 909, 641, 475, 666, 448, 212, 227, 1632, 486, 294, 335, 402, 677, 718, 717] payload_rows_per_dst=[10084, 10362, 10078, 8178, 9170, 8608, 10160, 9758] dup_rows_per_dst=[2690, 2914, 3318, 1518, 2192, 1950, 2656, 4668] combine_rows_per_src=[10084, 10362, 10078, 8178, 9170, 8608, 10160, 9758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75090 payload_rows_remote=64300 dup_rows=23214 dup_rows_remote=20144 dedup_ratio=0.236145 router_tokens_per_expert=[211, 803, 108, 380, 269, 756, 220, 1773, 26, 1139, 220, 1896, 694, 1379, 343, 625, 1936, 461, 613, 1128, 442, 313, 1934, 165, 471, 1552, 2629, 1072, 751, 672, 218, 108, 878, 647, 1932, 1223, 1875, 895, 1814, 433, 332, 685, 107, 152, 1301, 1616, 108, 693, 929, 804, 633, 305, 505, 404, 190, 210, 1477, 671, 293, 408, 293, 752, 545, 735] payload_rows_per_dst=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] dup_rows_per_dst=[2556, 2704, 2072, 3392, 2490, 3070, 2038, 4892] combine_rows_per_src=[8606, 9940, 8040, 9432, 9412, 10790, 9550, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273571840 dispatch_meta_bytes=344136 combine_payload_bytes=273571840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273915976 ingress_bytes=342777856 link_bytes=342777856 token_bytes=547487816 weight_movement_bytes=69206016 total_nvlink_bytes=616693832 weight_to_token_ratio=0.126406 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289845476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276561920 dispatch_meta_bytes=346560 combine_payload_bytes=276561920 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=276908480 ingress_bytes=449576960 link_bytes=449576960 token_bytes=553470400 weight_movement_bytes=173015040 total_nvlink_bytes=726485440 weight_to_token_ratio=0.312600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.691721907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279429120 dispatch_meta_bytes=354432 combine_payload_bytes=279429120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279783552 ingress_bytes=348635136 link_bytes=348635136 token_bytes=559212672 weight_movement_bytes=69206016 total_nvlink_bytes=628418688 weight_to_token_ratio=0.123756 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311885949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=337776 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263710576 ingress_bytes=297975808 link_bytes=297975808 token_bytes=527083376 weight_movement_bytes=34603008 total_nvlink_bytes=561686384 weight_to_token_ratio=0.065650 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121258976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76172 payload_rows_remote=65818 dup_rows=22132 dup_rows_remote=18792 dedup_ratio=0.225138 router_tokens_per_expert=[124, 934, 134, 454, 318, 434, 262, 1446, 60, 1171, 192, 1853, 588, 1418, 400, 755, 1904, 493, 624, 953, 566, 322, 1438, 170, 506, 1671, 2741, 1071, 923, 565, 388, 110, 922, 785, 1862, 1145, 1767, 749, 1881, 559, 316, 882, 120, 209, 1313, 1583, 113, 399, 834, 922, 648, 472, 528, 487, 197, 172, 1379, 564, 296, 490, 469, 711, 584, 806] payload_rows_per_dst=[10354, 10150, 9642, 9220, 10546, 8044, 10170, 8046] dup_rows_per_dst=[3340, 2724, 3298, 2610, 3670, 1826, 2112, 2552] combine_rows_per_src=[10354, 10150, 9642, 9220, 10546, 8044, 10170, 8046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76802 payload_rows_remote=67520 dup_rows=21502 dup_rows_remote=19120 dedup_ratio=0.218730 router_tokens_per_expert=[110, 714, 86, 456, 271, 465, 256, 1833, 29, 985, 244, 2027, 664, 1330, 370, 633, 1974, 529, 674, 944, 369, 318, 1804, 132, 541, 1813, 2881, 1149, 711, 502, 399, 99, 770, 785, 1563, 1245, 1927, 643, 1796, 462, 259, 990, 152, 155, 1417, 1829, 98, 463, 906, 996, 790, 309, 487, 409, 160, 116, 1525, 578, 260, 429, 319, 766, 564, 672] payload_rows_per_dst=[10818, 9892, 7950, 8702, 9282, 11072, 9792, 9294] dup_rows_per_dst=[3326, 2672, 1910, 1726, 2382, 3282, 2146, 4058] combine_rows_per_src=[10818, 9892, 7950, 8702, 9282, 11072, 9792, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76464 payload_rows_remote=66790 dup_rows=21840 dup_rows_remote=19244 dedup_ratio=0.222168 router_tokens_per_expert=[189, 833, 81, 454, 272, 483, 223, 2000, 40, 1283, 254, 1721, 702, 1114, 413, 608, 2626, 436, 552, 1040, 480, 266, 1699, 161, 572, 1617, 2877, 1247, 816, 508, 463, 126, 552, 742, 1721, 1388, 1462, 834, 1977, 442, 330, 827, 113, 180, 1421, 1608, 109, 410, 782, 1082, 701, 321, 578, 442, 198, 183, 1184, 580, 256, 297, 378, 763, 549, 586] payload_rows_per_dst=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] dup_rows_per_dst=[3916, 2596, 3774, 2028, 3304, 1740, 2304, 2178] combine_rows_per_src=[10908, 9674, 10746, 8670, 10978, 8256, 10224, 7008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262250496 dispatch_meta_bytes=332168 combine_payload_bytes=262250496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262582664 ingress_bytes=366059520 link_bytes=366059520 token_bytes=524833160 weight_movement_bytes=103809024 total_nvlink_bytes=628642184 weight_to_token_ratio=0.197794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76288 payload_rows_remote=66450 dup_rows=22016 dup_rows_remote=18364 dedup_ratio=0.223958 router_tokens_per_expert=[165, 868, 136, 394, 262, 523, 231, 1665, 36, 1445, 284, 1841, 636, 1247, 382, 916, 2171, 511, 652, 981, 528, 318, 1373, 211, 553, 1525, 2811, 1256, 815, 465, 456, 82, 664, 706, 1668, 1154, 1737, 947, 1698, 513, 318, 879, 124, 154, 1580, 1738, 125, 448, 746, 1005, 762, 362, 561, 312, 177, 216, 1286, 620, 256, 375, 360, 757, 537, 628] payload_rows_per_dst=[10732, 10348, 9838, 8588, 10560, 8840, 9604, 7778] dup_rows_per_dst=[3378, 3226, 3652, 1716, 3380, 1892, 2074, 2698] combine_rows_per_src=[10732, 10348, 9838, 8588, 10560, 8840, 9604, 7778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=337776 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263710576 ingress_bytes=297975808 link_bytes=297975808 token_bytes=527083376 weight_movement_bytes=34603008 total_nvlink_bytes=561686384 weight_to_token_ratio=0.065650 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121258976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65208 dup_rows=23866 dup_rows_remote=20878 dedup_ratio=0.242778 router_tokens_per_expert=[2051, 689, 726, 68, 121, 286, 1857, 1489, 128, 238, 763, 308, 771, 631, 353, 1938, 2296, 348, 542, 1373, 433, 1053, 85, 1064, 278, 333, 837, 214, 766, 1229, 2536, 793, 1844, 781, 250, 140, 671, 665, 294, 362, 1269, 2851, 374, 520, 191, 452, 483, 505, 1071, 890, 1003, 352, 79, 964, 379, 2352, 861, 38, 186, 256, 883, 633, 692, 264] payload_rows_per_dst=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] dup_rows_per_dst=[1282, 1754, 1690, 3406, 3894, 1758, 7094, 2988] combine_rows_per_src=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344952 combine_payload_bytes=266731520 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=267076472 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807992 weight_movement_bytes=69206016 total_nvlink_bytes=603014008 weight_to_token_ratio=0.129646 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75002 payload_rows_remote=63774 dup_rows=23302 dup_rows_remote=18916 dedup_ratio=0.237040 router_tokens_per_expert=[1874, 787, 963, 46, 88, 182, 1773, 1905, 89, 238, 713, 304, 924, 537, 384, 1583, 2999, 251, 461, 1452, 433, 1224, 108, 1582, 131, 290, 563, 217, 850, 1145, 3035, 582, 1904, 880, 198, 93, 502, 823, 212, 354, 1165, 2352, 331, 654, 294, 332, 448, 539, 1205, 846, 622, 354, 99, 1231, 169, 2164, 901, 8, 178, 245, 1060, 544, 497, 235] payload_rows_per_dst=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] dup_rows_per_dst=[3728, 4386, 4594, 870, 2086, 2710, 1376, 3552] combine_rows_per_src=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269590528 dispatch_meta_bytes=338440 combine_payload_bytes=269590528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269928968 ingress_bytes=304193536 link_bytes=304193536 token_bytes=539519496 weight_movement_bytes=34603008 total_nvlink_bytes=574122504 weight_to_token_ratio=0.064137 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269328384 dispatch_meta_bytes=343232 combine_payload_bytes=269328384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269671616 ingress_bytes=303931392 link_bytes=303931392 token_bytes=539000000 weight_movement_bytes=34603008 total_nvlink_bytes=573603008 weight_to_token_ratio=0.064199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143669360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=279429120 dispatch_meta_bytes=354432 combine_payload_bytes=279429120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279783552 ingress_bytes=348635136 link_bytes=348635136 token_bytes=559212672 weight_movement_bytes=69206016 total_nvlink_bytes=628418688 weight_to_token_ratio=0.123756 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311885949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276561920 dispatch_meta_bytes=346560 combine_payload_bytes=276561920 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=276908480 ingress_bytes=449576960 link_bytes=449576960 token_bytes=553470400 weight_movement_bytes=173015040 total_nvlink_bytes=726485440 weight_to_token_ratio=0.312600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.691721907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273571840 dispatch_meta_bytes=344136 combine_payload_bytes=273571840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273915976 ingress_bytes=342777856 link_bytes=342777856 token_bytes=547487816 weight_movement_bytes=69206016 total_nvlink_bytes=616693832 weight_to_token_ratio=0.126406 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289845476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=339256 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272518456 ingress_bytes=341385216 link_bytes=341385216 token_bytes=544697656 weight_movement_bytes=69206016 total_nvlink_bytes=613903672 weight_to_token_ratio=0.127054 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=66090 dup_rows=22258 dup_rows_remote=19046 dedup_ratio=0.226420 router_tokens_per_expert=[1714, 697, 655, 60, 86, 243, 1768, 1772, 146, 253, 784, 295, 1005, 677, 265, 1825, 2163, 371, 630, 1435, 550, 1035, 76, 1333, 175, 274, 975, 194, 700, 1224, 2827, 712, 1989, 1227, 201, 99, 649, 647, 172, 355, 1135, 2812, 505, 686, 166, 400, 478, 402, 745, 1071, 580, 271, 101, 1306, 283, 2454, 832, 22, 234, 213, 737, 768, 514, 179] payload_rows_per_dst=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] dup_rows_per_dst=[1530, 2052, 2356, 3540, 4112, 3212, 3096, 2360] combine_rows_per_src=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76324 payload_rows_remote=66044 dup_rows=21980 dup_rows_remote=18404 dedup_ratio=0.223592 router_tokens_per_expert=[1999, 880, 695, 71, 87, 257, 2001, 1555, 123, 304, 792, 313, 874, 508, 341, 1694, 2351, 259, 731, 1377, 519, 974, 65, 1244, 223, 383, 729, 218, 796, 1218, 2972, 608, 1623, 745, 234, 145, 649, 721, 146, 443, 1368, 2453, 475, 581, 274, 393, 442, 687, 1164, 893, 647, 269, 123, 1327, 291, 2214, 838, 25, 233, 270, 949, 615, 478, 276] payload_rows_per_dst=[9238, 8230, 8382, 10962, 10072, 9974, 10280, 9186] dup_rows_per_dst=[1854, 1668, 1956, 3332, 3338, 3372, 3576, 2884] combine_rows_per_src=[9238, 8230, 8382, 10962, 10072, 9974, 10280, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=63322 dup_rows=24012 dup_rows_remote=20532 dedup_ratio=0.244263 router_tokens_per_expert=[2144, 753, 587, 85, 101, 282, 1638, 1635, 108, 231, 671, 396, 1111, 600, 245, 1938, 2182, 321, 533, 1443, 474, 1157, 98, 1365, 186, 382, 811, 216, 724, 1200, 2701, 685, 1794, 839, 153, 178, 380, 615, 264, 466, 1250, 2609, 587, 605, 231, 598, 502, 446, 1020, 956, 494, 300, 140, 1126, 418, 2049, 1127, 37, 213, 258, 992, 709, 584, 209] payload_rows_per_dst=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] dup_rows_per_dst=[3480, 4752, 4808, 2664, 1990, 2292, 1376, 2650] combine_rows_per_src=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65208 dup_rows=23866 dup_rows_remote=20878 dedup_ratio=0.242778 router_tokens_per_expert=[2051, 689, 726, 68, 121, 286, 1857, 1489, 128, 238, 763, 308, 771, 631, 353, 1938, 2296, 348, 542, 1373, 433, 1053, 85, 1064, 278, 333, 837, 214, 766, 1229, 2536, 793, 1844, 781, 250, 140, 671, 665, 294, 362, 1269, 2851, 374, 520, 191, 452, 483, 505, 1071, 890, 1003, 352, 79, 964, 379, 2352, 861, 38, 186, 256, 883, 633, 692, 264] payload_rows_per_dst=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] dup_rows_per_dst=[1282, 1754, 1690, 3406, 3894, 1758, 7094, 2988] combine_rows_per_src=[7854, 8506, 8106, 10566, 10222, 7680, 12274, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344344 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436312 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528280 weight_movement_bytes=34603008 total_nvlink_bytes=569131288 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76480 payload_rows_remote=66906 dup_rows=21824 dup_rows_remote=19416 dedup_ratio=0.222005 router_tokens_per_expert=[1663, 868, 948, 37, 53, 301, 1864, 1992, 140, 319, 619, 354, 912, 642, 286, 1765, 2300, 206, 833, 1072, 509, 1000, 144, 1445, 181, 319, 605, 336, 583, 1087, 2673, 582, 2039, 921, 182, 138, 669, 757, 137, 489, 1304, 2701, 486, 721, 246, 430, 550, 586, 1213, 735, 564, 336, 137, 1133, 286, 1922, 766, 76, 229, 398, 977, 649, 500, 237] payload_rows_per_dst=[9902, 8856, 9202, 9574, 10094, 10448, 9028, 9376] dup_rows_per_dst=[2224, 1968, 2206, 2408, 3896, 3600, 2634, 2888] combine_rows_per_src=[9902, 8856, 9202, 9574, 10094, 10448, 9028, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=340544 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271045184 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541749824 weight_movement_bytes=103809024 total_nvlink_bytes=645558848 weight_to_token_ratio=0.191618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=66898 dup_rows=21984 dup_rows_remote=18812 dedup_ratio=0.223633 router_tokens_per_expert=[1704, 956, 695, 39, 175, 284, 1762, 1867, 137, 334, 725, 259, 1142, 521, 286, 1538, 2099, 336, 387, 1403, 466, 1072, 41, 1164, 166, 386, 941, 194, 820, 1261, 2959, 963, 1708, 842, 210, 174, 397, 589, 233, 440, 1709, 2600, 409, 618, 228, 442, 376, 375, 813, 947, 774, 231, 114, 984, 298, 2404, 832, 22, 359, 278, 966, 915, 572, 211] payload_rows_per_dst=[9628, 8288, 8104, 11220, 9422, 10250, 9952, 9456] dup_rows_per_dst=[1928, 1596, 1634, 4160, 3172, 3264, 3178, 3052] combine_rows_per_src=[9628, 8288, 8104, 11220, 9422, 10250, 9952, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75002 payload_rows_remote=63774 dup_rows=23302 dup_rows_remote=18916 dedup_ratio=0.237040 router_tokens_per_expert=[1874, 787, 963, 46, 88, 182, 1773, 1905, 89, 238, 713, 304, 924, 537, 384, 1583, 2999, 251, 461, 1452, 433, 1224, 108, 1582, 131, 290, 563, 217, 850, 1145, 3035, 582, 1904, 880, 198, 93, 502, 823, 212, 354, 1165, 2352, 331, 654, 294, 332, 448, 539, 1205, 846, 622, 354, 99, 1231, 169, 2164, 901, 8, 178, 245, 1060, 544, 497, 235] payload_rows_per_dst=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] dup_rows_per_dst=[3728, 4386, 4594, 870, 2086, 2710, 1376, 3552] combine_rows_per_src=[11508, 11228, 10756, 6686, 7846, 9520, 7676, 9782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75554 payload_rows_remote=64782 dup_rows=22750 dup_rows_remote=18824 dedup_ratio=0.231425 router_tokens_per_expert=[1888, 805, 797, 48, 106, 293, 1803, 1881, 118, 292, 483, 323, 1108, 654, 525, 1518, 2594, 303, 554, 1387, 448, 1274, 101, 1178, 194, 423, 778, 239, 796, 1289, 2640, 911, 1412, 883, 222, 219, 632, 575, 249, 649, 1302, 2217, 252, 655, 335, 371, 459, 506, 1145, 823, 593, 230, 147, 1104, 311, 2104, 812, 81, 190, 327, 1078, 664, 550, 304] payload_rows_per_dst=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] dup_rows_per_dst=[3310, 4330, 3926, 1548, 2170, 2574, 1244, 3648] combine_rows_per_src=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=66090 dup_rows=22258 dup_rows_remote=19046 dedup_ratio=0.226420 router_tokens_per_expert=[1714, 697, 655, 60, 86, 243, 1768, 1772, 146, 253, 784, 295, 1005, 677, 265, 1825, 2163, 371, 630, 1435, 550, 1035, 76, 1333, 175, 274, 975, 194, 700, 1224, 2827, 712, 1989, 1227, 201, 99, 649, 647, 172, 355, 1135, 2812, 505, 686, 166, 400, 478, 402, 745, 1071, 580, 271, 101, 1306, 283, 2454, 832, 22, 234, 213, 737, 768, 514, 179] payload_rows_per_dst=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] dup_rows_per_dst=[1530, 2052, 2356, 3540, 4112, 3212, 3096, 2360] combine_rows_per_src=[9032, 8448, 8504, 10622, 9994, 9956, 10526, 8964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261218304 dispatch_meta_bytes=330760 combine_payload_bytes=261218304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261549064 ingress_bytes=295821312 link_bytes=295821312 token_bytes=522767368 weight_movement_bytes=34603008 total_nvlink_bytes=557370376 weight_to_token_ratio=0.066192 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113151781e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270516224 dispatch_meta_bytes=337792 combine_payload_bytes=270516224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270854016 ingress_bytes=339722240 link_bytes=339722240 token_bytes=541370240 weight_movement_bytes=69206016 total_nvlink_bytes=610576256 weight_to_token_ratio=0.127835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278347439e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259366912 dispatch_meta_bytes=335416 combine_payload_bytes=259366912 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=259702328 ingress_bytes=328572928 link_bytes=328572928 token_bytes=519069240 weight_movement_bytes=69206016 total_nvlink_bytes=588275256 weight_to_token_ratio=0.133327 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236393476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344344 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436312 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528280 weight_movement_bytes=34603008 total_nvlink_bytes=569131288 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74894 payload_rows_remote=65120 dup_rows=23410 dup_rows_remote=21118 dedup_ratio=0.238139 router_tokens_per_expert=[126, 739, 92, 457, 310, 513, 226, 1946, 26, 1203, 208, 2030, 776, 1246, 416, 712, 2183, 482, 519, 1031, 557, 318, 1818, 156, 541, 1675, 2852, 1251, 733, 628, 439, 92, 581, 596, 1793, 1269, 1574, 670, 1879, 437, 327, 738, 125, 204, 1125, 1822, 146, 425, 714, 1025, 588, 374, 544, 565, 195, 149, 1386, 641, 261, 422, 300, 743, 544, 689] payload_rows_per_dst=[7222, 9172, 8722, 11210, 8358, 11230, 9774, 9206] dup_rows_per_dst=[1596, 2368, 2734, 5212, 1896, 2960, 2292, 4352] combine_rows_per_src=[7222, 9172, 8722, 11210, 8358, 11230, 9774, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274046976 dispatch_meta_bytes=345288 combine_payload_bytes=274046976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274392264 ingress_bytes=343252992 link_bytes=343252992 token_bytes=548439240 weight_movement_bytes=69206016 total_nvlink_bytes=617645256 weight_to_token_ratio=0.126187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291633375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274014208 dispatch_meta_bytes=342840 combine_payload_bytes=274014208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274357048 ingress_bytes=308617216 link_bytes=308617216 token_bytes=548371256 weight_movement_bytes=34603008 total_nvlink_bytes=582974264 weight_to_token_ratio=0.063101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161301738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261218304 dispatch_meta_bytes=330760 combine_payload_bytes=261218304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261549064 ingress_bytes=295821312 link_bytes=295821312 token_bytes=522767368 weight_movement_bytes=34603008 total_nvlink_bytes=557370376 weight_to_token_ratio=0.066192 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113151781e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=334424 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265681496 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531028568 weight_movement_bytes=69206016 total_nvlink_bytes=600234584 weight_to_token_ratio=0.130324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77140 payload_rows_remote=67954 dup_rows=21164 dup_rows_remote=19254 dedup_ratio=0.215291 router_tokens_per_expert=[432, 346, 1473, 516, 309, 351, 396, 1193, 112, 1129, 871, 463, 210, 1198, 2297, 166, 219, 4086, 33, 871, 57, 757, 3178, 226, 558, 435, 2354, 1750, 1359, 2478, 648, 0, 55, 106, 8, 1652, 287, 392, 571, 604, 190, 155, 151, 1264, 2029, 16, 1136, 607, 240, 347, 629, 1325, 1063, 424, 548, 590, 87, 1394, 72, 97, 1956, 29, 41, 616] payload_rows_per_dst=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] dup_rows_per_dst=[1624, 3386, 1226, 3916, 4100, 1910, 2222, 2780] combine_rows_per_src=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75016 payload_rows_remote=63248 dup_rows=23288 dup_rows_remote=19176 dedup_ratio=0.236898 router_tokens_per_expert=[462, 522, 1135, 413, 773, 532, 547, 1091, 143, 523, 916, 553, 257, 1323, 2148, 193, 255, 3925, 41, 956, 32, 427, 3432, 270, 521, 353, 2465, 1645, 1623, 2153, 594, 0, 43, 112, 6, 2216, 419, 424, 532, 424, 228, 130, 149, 1267, 2024, 28, 1156, 794, 333, 484, 641, 1678, 952, 518, 331, 706, 116, 1214, 53, 170, 1208, 85, 79, 409] payload_rows_per_dst=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] dup_rows_per_dst=[4112, 3374, 6560, 1794, 1510, 1834, 2678, 1426] combine_rows_per_src=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278339584 dispatch_meta_bytes=348832 combine_payload_bytes=278339584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278688416 ingress_bytes=347545600 link_bytes=347545600 token_bytes=557028000 weight_movement_bytes=69206016 total_nvlink_bytes=626234016 weight_to_token_ratio=0.124242 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307786113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74728 payload_rows_remote=66432 dup_rows=23576 dup_rows_remote=20762 dedup_ratio=0.239827 router_tokens_per_expert=[446, 750, 1004, 518, 624, 402, 418, 1101, 159, 715, 504, 551, 298, 1220, 2470, 164, 286, 3887, 41, 811, 56, 512, 2609, 281, 623, 312, 1890, 1954, 1393, 2460, 810, 0, 33, 155, 5, 2151, 357, 412, 490, 473, 236, 154, 150, 1320, 1681, 36, 1814, 1243, 247, 377, 897, 1229, 714, 674, 467, 925, 129, 1396, 44, 139, 1258, 85, 80, 512] payload_rows_per_dst=[8892, 9200, 4818, 12076, 11714, 9730, 8296, 10002] dup_rows_per_dst=[1634, 2962, 484, 5728, 4212, 3538, 2814, 2204] combine_rows_per_src=[8892, 9200, 4818, 12076, 11714, 9730, 8296, 10002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=340544 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271045184 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541749824 weight_movement_bytes=103809024 total_nvlink_bytes=645558848 weight_to_token_ratio=0.191618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75554 payload_rows_remote=64782 dup_rows=22750 dup_rows_remote=18824 dedup_ratio=0.231425 router_tokens_per_expert=[1888, 805, 797, 48, 106, 293, 1803, 1881, 118, 292, 483, 323, 1108, 654, 525, 1518, 2594, 303, 554, 1387, 448, 1274, 101, 1178, 194, 423, 778, 239, 796, 1289, 2640, 911, 1412, 883, 222, 219, 632, 575, 249, 649, 1302, 2217, 252, 655, 335, 371, 459, 506, 1145, 823, 593, 230, 147, 1104, 311, 2104, 812, 81, 190, 327, 1078, 664, 550, 304] payload_rows_per_dst=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] dup_rows_per_dst=[3310, 4330, 3926, 1548, 2170, 2574, 1244, 3648] combine_rows_per_src=[10836, 10992, 10772, 7712, 8608, 9620, 7462, 9552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=63322 dup_rows=24012 dup_rows_remote=20532 dedup_ratio=0.244263 router_tokens_per_expert=[2144, 753, 587, 85, 101, 282, 1638, 1635, 108, 231, 671, 396, 1111, 600, 245, 1938, 2182, 321, 533, 1443, 474, 1157, 98, 1365, 186, 382, 811, 216, 724, 1200, 2701, 685, 1794, 839, 153, 178, 380, 615, 264, 466, 1250, 2609, 587, 605, 231, 598, 502, 446, 1020, 956, 494, 300, 140, 1126, 418, 2049, 1127, 37, 213, 258, 992, 709, 584, 209] payload_rows_per_dst=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] dup_rows_per_dst=[3480, 4752, 4808, 2664, 1990, 2292, 1376, 2650] combine_rows_per_src=[10970, 11250, 11308, 9034, 7388, 8074, 7532, 8736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344952 combine_payload_bytes=266731520 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=267076472 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807992 weight_movement_bytes=69206016 total_nvlink_bytes=603014008 weight_to_token_ratio=0.129646 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75030 payload_rows_remote=65036 dup_rows=23274 dup_rows_remote=20562 dedup_ratio=0.236755 router_tokens_per_expert=[489, 570, 1209, 302, 543, 388, 418, 983, 117, 956, 730, 488, 313, 1320, 2421, 150, 208, 3738, 26, 1022, 60, 449, 3816, 458, 452, 84, 2225, 1817, 1093, 2158, 519, 0, 45, 147, 10, 1948, 314, 429, 444, 545, 200, 155, 134, 1568, 1925, 34, 1706, 880, 319, 297, 514, 1737, 1019, 583, 567, 755, 226, 1077, 78, 181, 1205, 30, 101, 457] payload_rows_per_dst=[8368, 9536, 4142, 8642, 11542, 9962, 12844, 9994] dup_rows_per_dst=[1436, 3454, 304, 2058, 3698, 3242, 6370, 2712] combine_rows_per_src=[8368, 9536, 4142, 8642, 11542, 9962, 12844, 9994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75684 payload_rows_remote=66024 dup_rows=22620 dup_rows_remote=19538 dedup_ratio=0.230103 router_tokens_per_expert=[1023, 876, 1152, 364, 576, 558, 786, 613, 95, 714, 814, 650, 534, 1257, 1913, 206, 276, 3434, 68, 964, 37, 817, 2544, 259, 400, 290, 2158, 1519, 1268, 2031, 735, 1, 64, 159, 21, 1682, 327, 376, 555, 659, 273, 184, 232, 1158, 2240, 22, 1005, 1049, 323, 414, 783, 1298, 1051, 840, 512, 763, 153, 982, 77, 261, 2130, 84, 110, 433] payload_rows_per_dst=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] dup_rows_per_dst=[2268, 3364, 2980, 3082, 2732, 2608, 3206, 2380] combine_rows_per_src=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75958 payload_rows_remote=64204 dup_rows=22346 dup_rows_remote=18220 dedup_ratio=0.227315 router_tokens_per_expert=[391, 628, 1289, 445, 355, 645, 437, 1027, 174, 929, 779, 493, 332, 1157, 1893, 128, 233, 4099, 37, 842, 60, 584, 2593, 463, 475, 113, 2341, 2077, 1335, 2307, 524, 2, 36, 107, 23, 1980, 335, 404, 402, 554, 128, 126, 265, 1533, 2027, 29, 1327, 1156, 353, 363, 835, 1146, 1115, 749, 369, 584, 95, 1486, 119, 240, 1281, 94, 76, 628] payload_rows_per_dst=[8700, 8614, 8078, 10662, 11754, 9816, 8204, 10130] dup_rows_per_dst=[1734, 3156, 1546, 3072, 4126, 3366, 2824, 2522] combine_rows_per_src=[8700, 8614, 8078, 10662, 11754, 9816, 8204, 10130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=334424 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265681496 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531028568 weight_movement_bytes=69206016 total_nvlink_bytes=600234584 weight_to_token_ratio=0.130324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=259366912 dispatch_meta_bytes=335416 combine_payload_bytes=259366912 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=259702328 ingress_bytes=328572928 link_bytes=328572928 token_bytes=519069240 weight_movement_bytes=69206016 total_nvlink_bytes=588275256 weight_to_token_ratio=0.133327 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236393476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76450 payload_rows_remote=67494 dup_rows=21854 dup_rows_remote=18872 dedup_ratio=0.222310 router_tokens_per_expert=[352, 654, 1390, 567, 381, 399, 359, 1181, 126, 849, 673, 485, 326, 1203, 2190, 117, 244, 4238, 27, 993, 55, 508, 2819, 301, 385, 376, 2348, 1941, 1292, 2555, 760, 0, 46, 93, 11, 2028, 282, 326, 534, 469, 169, 122, 210, 1297, 2144, 20, 1102, 670, 274, 367, 651, 1762, 1057, 494, 494, 639, 67, 1520, 71, 391, 1340, 30, 45, 333] payload_rows_per_dst=[11514, 8956, 12988, 7760, 6378, 9500, 8800, 10554] dup_rows_per_dst=[3748, 2982, 5382, 1748, 1200, 1968, 2676, 2150] combine_rows_per_src=[11514, 8956, 12988, 7760, 6378, 9500, 8800, 10554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259063808 dispatch_meta_bytes=329696 combine_payload_bytes=259063808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259393504 ingress_bytes=293666816 link_bytes=293666816 token_bytes=518457312 weight_movement_bytes=34603008 total_nvlink_bytes=553060320 weight_to_token_ratio=0.066742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.105044586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=272105472 dispatch_meta_bytes=348776 combine_payload_bytes=272105472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272454248 ingress_bytes=306708480 link_bytes=306708480 token_bytes=544559720 weight_movement_bytes=34603008 total_nvlink_bytes=579162728 weight_to_token_ratio=0.063543 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154119318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75956 payload_rows_remote=63834 dup_rows=22348 dup_rows_remote=17856 dedup_ratio=0.227336 router_tokens_per_expert=[390, 565, 1247, 429, 372, 477, 550, 928, 127, 903, 728, 457, 293, 1423, 1797, 146, 230, 3444, 30, 1091, 49, 422, 2733, 308, 386, 70, 2354, 2010, 1892, 2249, 632, 1, 71, 110, 9, 2349, 382, 317, 450, 599, 236, 198, 139, 1923, 1798, 42, 1501, 1099, 364, 359, 660, 1289, 899, 1088, 571, 715, 165, 752, 154, 208, 1464, 53, 55, 400] payload_rows_per_dst=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] dup_rows_per_dst=[3192, 2950, 4492, 1542, 1504, 3768, 3132, 1768] combine_rows_per_src=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342392 combine_payload_bytes=266387456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=266729848 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117304 weight_movement_bytes=69206016 total_nvlink_bytes=602323320 weight_to_token_ratio=0.129814 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75684 payload_rows_remote=66024 dup_rows=22620 dup_rows_remote=19538 dedup_ratio=0.230103 router_tokens_per_expert=[1023, 876, 1152, 364, 576, 558, 786, 613, 95, 714, 814, 650, 534, 1257, 1913, 206, 276, 3434, 68, 964, 37, 817, 2544, 259, 400, 290, 2158, 1519, 1268, 2031, 735, 1, 64, 159, 21, 1682, 327, 376, 555, 659, 273, 184, 232, 1158, 2240, 22, 1005, 1049, 323, 414, 783, 1298, 1051, 840, 512, 763, 153, 982, 77, 261, 2130, 84, 110, 433] payload_rows_per_dst=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] dup_rows_per_dst=[2268, 3364, 2980, 3082, 2732, 2608, 3206, 2380] combine_rows_per_src=[9628, 9002, 10272, 9660, 8500, 9718, 8762, 10142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75958 payload_rows_remote=64204 dup_rows=22346 dup_rows_remote=18220 dedup_ratio=0.227315 router_tokens_per_expert=[391, 628, 1289, 445, 355, 645, 437, 1027, 174, 929, 779, 493, 332, 1157, 1893, 128, 233, 4099, 37, 842, 60, 584, 2593, 463, 475, 113, 2341, 2077, 1335, 2307, 524, 2, 36, 107, 23, 1980, 335, 404, 402, 554, 128, 126, 265, 1533, 2027, 29, 1327, 1156, 353, 363, 835, 1146, 1115, 749, 369, 584, 95, 1486, 119, 240, 1281, 94, 76, 628] payload_rows_per_dst=[8700, 8614, 8078, 10662, 11754, 9816, 8204, 10130] dup_rows_per_dst=[1734, 3156, 1546, 3072, 4126, 3366, 2824, 2522] combine_rows_per_src=[8700, 8614, 8078, 10662, 11754, 9816, 8204, 10130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276455424 dispatch_meta_bytes=345464 combine_payload_bytes=276455424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276800888 ingress_bytes=311058432 link_bytes=311058432 token_bytes=553256312 weight_movement_bytes=34603008 total_nvlink_bytes=587859320 weight_to_token_ratio=0.062544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170487837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=342248 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270776552 ingress_bytes=408846336 link_bytes=408846336 token_bytes=541210856 weight_movement_bytes=138412032 total_nvlink_bytes=679622888 weight_to_token_ratio=0.255745 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538455848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77140 payload_rows_remote=67954 dup_rows=21164 dup_rows_remote=19254 dedup_ratio=0.215291 router_tokens_per_expert=[432, 346, 1473, 516, 309, 351, 396, 1193, 112, 1129, 871, 463, 210, 1198, 2297, 166, 219, 4086, 33, 871, 57, 757, 3178, 226, 558, 435, 2354, 1750, 1359, 2478, 648, 0, 55, 106, 8, 1652, 287, 392, 571, 604, 190, 155, 151, 1264, 2029, 16, 1136, 607, 240, 347, 629, 1325, 1063, 424, 548, 590, 87, 1394, 72, 97, 1956, 29, 41, 616] payload_rows_per_dst=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] dup_rows_per_dst=[1624, 3386, 1226, 3916, 4100, 1910, 2222, 2780] combine_rows_per_src=[8408, 9506, 9456, 10292, 11422, 9186, 8110, 10760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76324 payload_rows_remote=66044 dup_rows=21980 dup_rows_remote=18404 dedup_ratio=0.223592 router_tokens_per_expert=[1999, 880, 695, 71, 87, 257, 2001, 1555, 123, 304, 792, 313, 874, 508, 341, 1694, 2351, 259, 731, 1377, 519, 974, 65, 1244, 223, 383, 729, 218, 796, 1218, 2972, 608, 1623, 745, 234, 145, 649, 721, 146, 443, 1368, 2453, 475, 581, 274, 393, 442, 687, 1164, 893, 647, 269, 123, 1327, 291, 2214, 838, 25, 233, 270, 949, 615, 478, 276] payload_rows_per_dst=[9238, 8230, 8382, 10962, 10072, 9974, 10280, 9186] dup_rows_per_dst=[1854, 1668, 1956, 3332, 3338, 3372, 3576, 2884] combine_rows_per_src=[9238, 8230, 8382, 10962, 10072, 9974, 10280, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262979584 dispatch_meta_bytes=329696 combine_payload_bytes=262979584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263309280 ingress_bytes=297582592 link_bytes=297582592 token_bytes=526288864 weight_movement_bytes=34603008 total_nvlink_bytes=560891872 weight_to_token_ratio=0.065749 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119779336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75956 payload_rows_remote=63834 dup_rows=22348 dup_rows_remote=17856 dedup_ratio=0.227336 router_tokens_per_expert=[390, 565, 1247, 429, 372, 477, 550, 928, 127, 903, 728, 457, 293, 1423, 1797, 146, 230, 3444, 30, 1091, 49, 422, 2733, 308, 386, 70, 2354, 2010, 1892, 2249, 632, 1, 71, 110, 9, 2349, 382, 317, 450, 599, 236, 198, 139, 1923, 1798, 42, 1501, 1099, 364, 359, 660, 1289, 899, 1088, 571, 715, 165, 752, 154, 208, 1464, 53, 55, 400] payload_rows_per_dst=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] dup_rows_per_dst=[3192, 2950, 4492, 1542, 1504, 3768, 3132, 1768] combine_rows_per_src=[11278, 8798, 12122, 8594, 7070, 10104, 8758, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75016 payload_rows_remote=63248 dup_rows=23288 dup_rows_remote=19176 dedup_ratio=0.236898 router_tokens_per_expert=[462, 522, 1135, 413, 773, 532, 547, 1091, 143, 523, 916, 553, 257, 1323, 2148, 193, 255, 3925, 41, 956, 32, 427, 3432, 270, 521, 353, 2465, 1645, 1623, 2153, 594, 0, 43, 112, 6, 2216, 419, 424, 532, 424, 228, 130, 149, 1267, 2024, 28, 1156, 794, 333, 484, 641, 1678, 952, 518, 331, 706, 116, 1214, 53, 170, 1208, 85, 79, 409] payload_rows_per_dst=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] dup_rows_per_dst=[4112, 3374, 6560, 1794, 1510, 1834, 2678, 1426] combine_rows_per_src=[11768, 9064, 13454, 7678, 6842, 8054, 8608, 9548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=278339584 dispatch_meta_bytes=348832 combine_payload_bytes=278339584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278688416 ingress_bytes=347545600 link_bytes=347545600 token_bytes=557028000 weight_movement_bytes=69206016 total_nvlink_bytes=626234016 weight_to_token_ratio=0.124242 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307786113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270516224 dispatch_meta_bytes=337792 combine_payload_bytes=270516224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270854016 ingress_bytes=339722240 link_bytes=339722240 token_bytes=541370240 weight_movement_bytes=69206016 total_nvlink_bytes=610576256 weight_to_token_ratio=0.127835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278347439e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276455424 dispatch_meta_bytes=345464 combine_payload_bytes=276455424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276800888 ingress_bytes=311058432 link_bytes=311058432 token_bytes=553256312 weight_movement_bytes=34603008 total_nvlink_bytes=587859320 weight_to_token_ratio=0.062544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170487837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75572 payload_rows_remote=65304 dup_rows=22732 dup_rows_remote=20090 dedup_ratio=0.231242 router_tokens_per_expert=[111, 821, 938, 75, 582, 972, 291, 3833, 563, 164, 194, 2259, 412, 1099, 1655, 510, 3384, 2074, 40, 271, 1654, 63, 1430, 62, 83, 132, 771, 760, 3, 461, 1708, 77, 713, 35, 212, 568, 22, 98, 3774, 2125, 1677, 176, 5, 627, 5, 468, 1137, 293, 296, 335, 1479, 2520, 639, 899, 120, 406, 147, 475, 747, 59, 69, 728, 9, 837] payload_rows_per_dst=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] dup_rows_per_dst=[650, 2182, 1568, 1152, 6136, 4564, 3838, 2642] combine_rows_per_src=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=66466 dup_rows=23080 dup_rows_remote=20362 dedup_ratio=0.234782 router_tokens_per_expert=[213, 983, 935, 29, 639, 1007, 88, 4140, 488, 147, 152, 1895, 321, 933, 1309, 493, 3356, 2557, 59, 225, 1430, 43, 1811, 46, 146, 56, 700, 950, 1, 501, 1314, 69, 771, 26, 409, 686, 31, 93, 3567, 2341, 1582, 127, 17, 676, 3, 603, 842, 159, 170, 543, 1820, 2672, 451, 699, 108, 591, 96, 688, 888, 44, 34, 599, 17, 763] payload_rows_per_dst=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] dup_rows_per_dst=[7748, 2718, 3108, 3612, 1322, 964, 1078, 2530] combine_rows_per_src=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261464064 dispatch_meta_bytes=326760 combine_payload_bytes=261464064 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261790824 ingress_bytes=399876096 link_bytes=399876096 token_bytes=523254888 weight_movement_bytes=138412032 total_nvlink_bytes=661666920 weight_to_token_ratio=0.264521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504701557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76782 payload_rows_remote=65172 dup_rows=21522 dup_rows_remote=17546 dedup_ratio=0.218933 router_tokens_per_expert=[249, 981, 692, 57, 496, 720, 166, 3890, 604, 182, 237, 2199, 288, 1099, 1198, 259, 3629, 3486, 41, 354, 1212, 38, 1627, 15, 113, 90, 563, 428, 0, 623, 1879, 55, 670, 24, 141, 881, 5, 89, 3399, 2586, 1702, 147, 2, 367, 4, 584, 871, 226, 117, 397, 1645, 2661, 648, 799, 41, 213, 140, 665, 1049, 34, 29, 636, 6, 904] payload_rows_per_dst=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] dup_rows_per_dst=[592, 2944, 3304, 726, 3662, 3976, 2974, 3344] combine_rows_per_src=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75042 payload_rows_remote=64602 dup_rows=23262 dup_rows_remote=20418 dedup_ratio=0.236633 router_tokens_per_expert=[277, 894, 748, 55, 443, 704, 163, 4166, 517, 126, 241, 2190, 390, 1157, 1172, 583, 3225, 2412, 53, 245, 1205, 25, 1744, 50, 93, 42, 730, 840, 3, 898, 1543, 38, 699, 62, 367, 783, 14, 47, 3765, 2789, 1921, 158, 22, 510, 14, 443, 1088, 172, 149, 630, 1826, 2156, 655, 861, 50, 315, 143, 461, 728, 69, 59, 574, 19, 631] payload_rows_per_dst=[5942, 9234, 7052, 6998, 13506, 12122, 10440, 9748] dup_rows_per_dst=[626, 3518, 1058, 1376, 6904, 4866, 2844, 2070] combine_rows_per_src=[5942, 9234, 7052, 6998, 13506, 12122, 10440, 9748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=330872 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267275384 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534219896 weight_movement_bytes=34603008 total_nvlink_bytes=568822904 weight_to_token_ratio=0.064773 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73916 payload_rows_remote=60546 dup_rows=24388 dup_rows_remote=17550 dedup_ratio=0.248088 router_tokens_per_expert=[132, 897, 699, 65, 759, 749, 243, 4203, 453, 209, 157, 2241, 368, 584, 1786, 513, 3430, 2697, 42, 182, 1468, 61, 1829, 52, 133, 67, 652, 483, 0, 912, 1942, 41, 729, 41, 271, 685, 16, 99, 3597, 2496, 1630, 231, 19, 423, 24, 410, 869, 156, 191, 217, 1744, 2357, 440, 853, 109, 332, 144, 512, 1148, 106, 31, 612, 11, 600] payload_rows_per_dst=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] dup_rows_per_dst=[6838, 3054, 6786, 3928, 1208, 792, 834, 948] combine_rows_per_src=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259063808 dispatch_meta_bytes=329696 combine_payload_bytes=259063808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259393504 ingress_bytes=293666816 link_bytes=293666816 token_bytes=518457312 weight_movement_bytes=34603008 total_nvlink_bytes=553060320 weight_to_token_ratio=0.066742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.105044586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77162 payload_rows_remote=65274 dup_rows=21142 dup_rows_remote=17468 dedup_ratio=0.215068 router_tokens_per_expert=[285, 1214, 630, 30, 491, 795, 217, 3428, 685, 145, 202, 2253, 433, 730, 1702, 447, 2953, 2665, 57, 180, 1523, 42, 1378, 46, 87, 54, 575, 833, 0, 653, 1662, 58, 652, 38, 388, 863, 9, 65, 3859, 2271, 1618, 157, 34, 300, 16, 510, 970, 261, 120, 214, 1916, 2586, 644, 936, 67, 505, 115, 754, 1179, 62, 61, 670, 7, 852] payload_rows_per_dst=[10044, 9854, 9816, 11888, 7340, 7778, 10376, 10066] dup_rows_per_dst=[2654, 3340, 1966, 3674, 1232, 1436, 3600, 3240] combine_rows_per_src=[10044, 9854, 9816, 11888, 7340, 7778, 10376, 10066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77222 payload_rows_remote=69356 dup_rows=21082 dup_rows_remote=19642 dedup_ratio=0.214457 router_tokens_per_expert=[209, 998, 1020, 37, 606, 926, 231, 4048, 296, 112, 250, 2317, 277, 1493, 1476, 473, 3455, 2155, 34, 141, 1047, 61, 1791, 72, 88, 73, 614, 480, 8, 797, 1522, 49, 840, 38, 254, 510, 27, 110, 3679, 2508, 1610, 120, 30, 360, 16, 375, 763, 301, 163, 659, 1622, 2453, 448, 772, 73, 415, 137, 639, 1223, 54, 43, 715, 10, 1029] payload_rows_per_dst=[7242, 9694, 8708, 10908, 7866, 11460, 10306, 11038] dup_rows_per_dst=[812, 3694, 1894, 2980, 1440, 3786, 2904, 3572] combine_rows_per_src=[7242, 9694, 8708, 10908, 7866, 11460, 10306, 11038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75572 payload_rows_remote=65304 dup_rows=22732 dup_rows_remote=20090 dedup_ratio=0.231242 router_tokens_per_expert=[111, 821, 938, 75, 582, 972, 291, 3833, 563, 164, 194, 2259, 412, 1099, 1655, 510, 3384, 2074, 40, 271, 1654, 63, 1430, 62, 83, 132, 771, 760, 3, 461, 1708, 77, 713, 35, 212, 568, 22, 98, 3774, 2125, 1677, 176, 5, 627, 5, 468, 1137, 293, 296, 335, 1479, 2520, 639, 899, 120, 406, 147, 475, 747, 59, 69, 728, 9, 837] payload_rows_per_dst=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] dup_rows_per_dst=[650, 2182, 1568, 1152, 6136, 4564, 3838, 2642] combine_rows_per_src=[6930, 8254, 7360, 6838, 13106, 11878, 10938, 10268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=341576 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267826760 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535311944 weight_movement_bytes=34603008 total_nvlink_bytes=569914952 weight_to_token_ratio=0.064641 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=340080 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264949872 ingress_bytes=333815808 link_bytes=333815808 token_bytes=529559664 weight_movement_bytes=69206016 total_nvlink_bytes=598765680 weight_to_token_ratio=0.130686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256122011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=66466 dup_rows=23080 dup_rows_remote=20362 dedup_ratio=0.234782 router_tokens_per_expert=[213, 983, 935, 29, 639, 1007, 88, 4140, 488, 147, 152, 1895, 321, 933, 1309, 493, 3356, 2557, 59, 225, 1430, 43, 1811, 46, 146, 56, 700, 950, 1, 501, 1314, 69, 771, 26, 409, 686, 31, 93, 3567, 2341, 1582, 127, 17, 676, 3, 603, 842, 159, 170, 543, 1820, 2672, 451, 699, 108, 591, 96, 688, 888, 44, 34, 599, 17, 763] payload_rows_per_dst=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] dup_rows_per_dst=[7748, 2718, 3108, 3612, 1322, 964, 1078, 2530] combine_rows_per_src=[13664, 8758, 10056, 10996, 7392, 7054, 7686, 9618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=247996416 dispatch_meta_bytes=312384 combine_payload_bytes=247996416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248308800 ingress_bytes=317202432 link_bytes=317202432 token_bytes=496305216 weight_movement_bytes=69206016 total_nvlink_bytes=565511232 weight_to_token_ratio=0.139442 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193607215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=347312 combine_payload_bytes=272244736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272592048 ingress_bytes=341450752 link_bytes=341450752 token_bytes=544836784 weight_movement_bytes=69206016 total_nvlink_bytes=614042800 weight_to_token_ratio=0.127022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284851691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77222 payload_rows_remote=69356 dup_rows=21082 dup_rows_remote=19642 dedup_ratio=0.214457 router_tokens_per_expert=[209, 998, 1020, 37, 606, 926, 231, 4048, 296, 112, 250, 2317, 277, 1493, 1476, 473, 3455, 2155, 34, 141, 1047, 61, 1791, 72, 88, 73, 614, 480, 8, 797, 1522, 49, 840, 38, 254, 510, 27, 110, 3679, 2508, 1610, 120, 30, 360, 16, 375, 763, 301, 163, 659, 1622, 2453, 448, 772, 73, 415, 137, 639, 1223, 54, 43, 715, 10, 1029] payload_rows_per_dst=[7242, 9694, 8708, 10908, 7866, 11460, 10306, 11038] dup_rows_per_dst=[812, 3694, 1894, 2980, 1440, 3786, 2904, 3572] combine_rows_per_src=[7242, 9694, 8708, 10908, 7866, 11460, 10306, 11038] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=330968 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267693272 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535055576 weight_movement_bytes=34603008 total_nvlink_bytes=569658584 weight_to_token_ratio=0.064672 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77448 payload_rows_remote=67358 dup_rows=20856 dup_rows_remote=18296 dedup_ratio=0.212158 router_tokens_per_expert=[264, 898, 850, 67, 374, 854, 283, 3892, 511, 163, 102, 1827, 368, 1011, 1382, 788, 3558, 2079, 54, 406, 2109, 52, 1489, 136, 83, 81, 820, 743, 11, 626, 1286, 134, 815, 121, 349, 519, 19, 155, 3579, 1920, 1674, 124, 28, 604, 31, 703, 935, 544, 397, 430, 1274, 2910, 290, 785, 135, 568, 78, 699, 634, 58, 34, 822, 26, 591] payload_rows_per_dst=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] dup_rows_per_dst=[3644, 3000, 2560, 3670, 1012, 1608, 2928, 2434] combine_rows_per_src=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=341576 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267826760 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535311944 weight_movement_bytes=34603008 total_nvlink_bytes=569914952 weight_to_token_ratio=0.064641 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76782 payload_rows_remote=65172 dup_rows=21522 dup_rows_remote=17546 dedup_ratio=0.218933 router_tokens_per_expert=[249, 981, 692, 57, 496, 720, 166, 3890, 604, 182, 237, 2199, 288, 1099, 1198, 259, 3629, 3486, 41, 354, 1212, 38, 1627, 15, 113, 90, 563, 428, 0, 623, 1879, 55, 670, 24, 141, 881, 5, 89, 3399, 2586, 1702, 147, 2, 367, 4, 584, 871, 226, 117, 397, 1645, 2661, 648, 799, 41, 213, 140, 665, 1049, 34, 29, 636, 6, 904] payload_rows_per_dst=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] dup_rows_per_dst=[592, 2944, 3304, 726, 3662, 3976, 2974, 3344] combine_rows_per_src=[6130, 9188, 10242, 6776, 11928, 11610, 10068, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74728 payload_rows_remote=66432 dup_rows=23576 dup_rows_remote=20762 dedup_ratio=0.239827 router_tokens_per_expert=[446, 750, 1004, 518, 624, 402, 418, 1101, 159, 715, 504, 551, 298, 1220, 2470, 164, 286, 3887, 41, 811, 56, 512, 2609, 281, 623, 312, 1890, 1954, 1393, 2460, 810, 0, 33, 155, 5, 2151, 357, 412, 490, 473, 236, 154, 150, 1320, 1681, 36, 1814, 1243, 247, 377, 897, 1229, 714, 674, 467, 925, 129, 1396, 44, 139, 1258, 85, 80, 512] payload_rows_per_dst=[8892, 9200, 4818, 12076, 11714, 9730, 8296, 10002] dup_rows_per_dst=[1634, 2962, 484, 5728, 4212, 3538, 2814, 2204] combine_rows_per_src=[8892, 9200, 4818, 12076, 11714, 9730, 8296, 10002] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=355992 combine_payload_bytes=284082176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284438168 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568520344 weight_movement_bytes=69206016 total_nvlink_bytes=637726360 weight_to_token_ratio=0.121730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77448 payload_rows_remote=67358 dup_rows=20856 dup_rows_remote=18296 dedup_ratio=0.212158 router_tokens_per_expert=[264, 898, 850, 67, 374, 854, 283, 3892, 511, 163, 102, 1827, 368, 1011, 1382, 788, 3558, 2079, 54, 406, 2109, 52, 1489, 136, 83, 81, 820, 743, 11, 626, 1286, 134, 815, 121, 349, 519, 19, 155, 3579, 1920, 1674, 124, 28, 604, 31, 703, 935, 544, 397, 430, 1274, 2910, 290, 785, 135, 568, 78, 699, 634, 58, 34, 822, 26, 591] payload_rows_per_dst=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] dup_rows_per_dst=[3644, 3000, 2560, 3670, 1012, 1608, 2928, 2434] combine_rows_per_src=[11418, 9304, 10090, 11056, 6784, 7678, 10552, 10566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73916 payload_rows_remote=60546 dup_rows=24388 dup_rows_remote=17550 dedup_ratio=0.248088 router_tokens_per_expert=[132, 897, 699, 65, 759, 749, 243, 4203, 453, 209, 157, 2241, 368, 584, 1786, 513, 3430, 2697, 42, 182, 1468, 61, 1829, 52, 133, 67, 652, 483, 0, 912, 1942, 41, 729, 41, 271, 685, 16, 99, 3597, 2496, 1630, 231, 19, 423, 24, 410, 869, 156, 191, 217, 1744, 2357, 440, 853, 109, 332, 144, 512, 1148, 106, 31, 612, 11, 600] payload_rows_per_dst=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] dup_rows_per_dst=[6838, 3054, 6786, 3928, 1208, 792, 834, 948] combine_rows_per_src=[13370, 9568, 12736, 11726, 7466, 6732, 6938, 5380] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80474 payload_rows_remote=68954 dup_rows=17830 dup_rows_remote=14704 dedup_ratio=0.181376 router_tokens_per_expert=[1660, 41, 90, 4853, 863, 5, 206, 1008, 499, 694, 775, 424, 4684, 2493, 7, 1476, 654, 14, 42, 936, 34, 530, 1292, 794, 21, 433, 280, 961, 600, 17, 42, 9, 11, 257, 908, 159, 644, 0, 29, 485, 849, 5603, 5, 2031, 2962, 7, 1691, 247, 85, 627, 173, 816, 2714, 798, 328, 7, 234, 36, 30, 17, 40, 1, 897, 24] payload_rows_per_dst=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] dup_rows_per_dst=[776, 946, 5186, 2512, 1744, 3126, 1784, 1756] combine_rows_per_src=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=347312 combine_payload_bytes=272244736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272592048 ingress_bytes=341450752 link_bytes=341450752 token_bytes=544836784 weight_movement_bytes=69206016 total_nvlink_bytes=614042800 weight_to_token_ratio=0.127022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284851691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275898368 dispatch_meta_bytes=342616 combine_payload_bytes=275898368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276240984 ingress_bytes=379707392 link_bytes=379707392 token_bytes=552139352 weight_movement_bytes=103809024 total_nvlink_bytes=655948376 weight_to_token_ratio=0.188012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428808347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=282435584 dispatch_meta_bytes=334632 combine_payload_bytes=282435584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282770216 ingress_bytes=420847616 link_bytes=420847616 token_bytes=565205800 weight_movement_bytes=138412032 total_nvlink_bytes=703617832 weight_to_token_ratio=0.244888 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583615698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=68438 dup_rows=19380 dup_rows_remote=15908 dedup_ratio=0.197144 router_tokens_per_expert=[1733, 39, 277, 3757, 681, 2, 246, 1114, 398, 634, 865, 298, 4796, 1635, 18, 883, 607, 18, 23, 1021, 30, 369, 1261, 1303, 10, 769, 428, 862, 339, 42, 55, 46, 23, 502, 1522, 166, 753, 0, 25, 579, 664, 5584, 10, 2028, 2304, 28, 2001, 330, 182, 1611, 172, 684, 2384, 1258, 652, 36, 225, 56, 9, 29, 16, 5, 736, 19] payload_rows_per_dst=[7014, 5656, 12388, 11708, 9454, 10352, 10486, 11866] dup_rows_per_dst=[1170, 536, 4390, 2986, 2294, 3040, 3472, 1492] combine_rows_per_src=[7014, 5656, 12388, 11708, 9454, 10352, 10486, 11866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72932 payload_rows_remote=60382 dup_rows=25372 dup_rows_remote=19796 dedup_ratio=0.258097 router_tokens_per_expert=[1440, 91, 171, 4561, 809, 27, 119, 888, 217, 619, 891, 602, 4550, 2557, 48, 1793, 699, 25, 33, 1438, 50, 381, 986, 800, 43, 757, 606, 1252, 480, 62, 65, 51, 41, 386, 885, 86, 765, 2, 72, 355, 285, 4913, 14, 2351, 2080, 50, 1788, 267, 123, 867, 57, 627, 1902, 2357, 774, 26, 192, 147, 10, 66, 52, 2, 485, 14] payload_rows_per_dst=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] dup_rows_per_dst=[5576, 8874, 1502, 868, 3286, 2846, 2376, 44] combine_rows_per_src=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79708 payload_rows_remote=69356 dup_rows=18596 dup_rows_remote=16838 dedup_ratio=0.189168 router_tokens_per_expert=[1173, 56, 214, 4705, 281, 3, 125, 1160, 555, 799, 1014, 346, 4109, 1783, 33, 1348, 640, 10, 20, 903, 34, 353, 1352, 726, 15, 444, 230, 707, 314, 109, 21, 106, 20, 321, 1073, 402, 812, 1, 144, 465, 411, 5377, 29, 2235, 2192, 25, 2350, 299, 243, 1702, 148, 960, 2553, 1810, 888, 65, 253, 49, 19, 20, 28, 2, 530, 38] payload_rows_per_dst=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] dup_rows_per_dst=[3576, 2162, 1222, 1758, 1874, 1952, 4932, 1120] combine_rows_per_src=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=330872 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267275384 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534219896 weight_movement_bytes=34603008 total_nvlink_bytes=568822904 weight_to_token_ratio=0.064773 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78932 payload_rows_remote=67704 dup_rows=19372 dup_rows_remote=17382 dedup_ratio=0.197062 router_tokens_per_expert=[1373, 72, 233, 4340, 1093, 4, 213, 1024, 516, 645, 883, 516, 3938, 1670, 59, 1583, 759, 28, 90, 1037, 26, 766, 1213, 931, 34, 776, 477, 752, 298, 26, 49, 31, 79, 365, 864, 249, 929, 2, 166, 313, 559, 5099, 19, 2649, 2380, 73, 2041, 265, 250, 832, 140, 845, 2130, 1189, 695, 54, 257, 76, 11, 22, 75, 6, 1042, 21] payload_rows_per_dst=[7084, 7524, 13216, 10344, 8786, 11070, 9680, 11228] dup_rows_per_dst=[940, 880, 5164, 2418, 1908, 3482, 2590, 1990] combine_rows_per_src=[7084, 7524, 13216, 10344, 8786, 11070, 9680, 11228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78410 payload_rows_remote=70082 dup_rows=19894 dup_rows_remote=18304 dedup_ratio=0.202372 router_tokens_per_expert=[1338, 73, 159, 4215, 693, 11, 154, 1228, 350, 635, 839, 380, 5028, 2209, 16, 1427, 710, 27, 15, 1435, 53, 954, 766, 942, 17, 752, 453, 859, 278, 30, 34, 36, 32, 390, 815, 160, 701, 0, 98, 441, 587, 6091, 18, 2026, 2322, 24, 2111, 242, 133, 349, 88, 962, 1803, 1831, 252, 30, 202, 117, 13, 46, 65, 2, 1071, 14] payload_rows_per_dst=[8078, 9872, 7752, 11960, 8328, 10826, 8630, 12964] dup_rows_per_dst=[1648, 3438, 2052, 3014, 1590, 3608, 2266, 2278] combine_rows_per_src=[8078, 9872, 7752, 11960, 8328, 10826, 8630, 12964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280322048 dispatch_meta_bytes=337384 combine_payload_bytes=280322048 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=280659432 ingress_bytes=314925056 link_bytes=314925056 token_bytes=560981480 weight_movement_bytes=34603008 total_nvlink_bytes=595584488 weight_to_token_ratio=0.061683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185037632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=247324672 dispatch_meta_bytes=320712 combine_payload_bytes=247324672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247645384 ingress_bytes=351133696 link_bytes=351133696 token_bytes=494970056 weight_movement_bytes=103809024 total_nvlink_bytes=598779080 weight_to_token_ratio=0.209728 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321287829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=272105472 dispatch_meta_bytes=348776 combine_payload_bytes=272105472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272454248 ingress_bytes=306708480 link_bytes=306708480 token_bytes=544559720 weight_movement_bytes=34603008 total_nvlink_bytes=579162728 weight_to_token_ratio=0.063543 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154119318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77144 payload_rows_remote=66090 dup_rows=21160 dup_rows_remote=16268 dedup_ratio=0.215251 router_tokens_per_expert=[1193, 66, 313, 4387, 759, 3, 161, 885, 350, 1158, 797, 430, 4068, 1917, 21, 1693, 672, 56, 33, 930, 25, 590, 1247, 1173, 31, 515, 418, 1003, 448, 23, 45, 50, 30, 348, 1311, 323, 913, 6, 86, 467, 691, 5405, 29, 1919, 2875, 26, 1799, 228, 164, 729, 115, 709, 2548, 629, 842, 18, 238, 30, 29, 36, 54, 2, 1071, 22] payload_rows_per_dst=[11842, 11054, 7906, 8284, 11816, 6004, 9412, 10826] dup_rows_per_dst=[3692, 4892, 1546, 1704, 4740, 540, 2096, 1950] combine_rows_per_src=[11842, 11054, 7906, 8284, 11816, 6004, 9412, 10826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79708 payload_rows_remote=69356 dup_rows=18596 dup_rows_remote=16838 dedup_ratio=0.189168 router_tokens_per_expert=[1173, 56, 214, 4705, 281, 3, 125, 1160, 555, 799, 1014, 346, 4109, 1783, 33, 1348, 640, 10, 20, 903, 34, 353, 1352, 726, 15, 444, 230, 707, 314, 109, 21, 106, 20, 321, 1073, 402, 812, 1, 144, 465, 411, 5377, 29, 2235, 2192, 25, 2350, 299, 243, 1702, 148, 960, 2553, 1810, 888, 65, 253, 49, 19, 20, 28, 2, 530, 38] payload_rows_per_dst=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] dup_rows_per_dst=[3576, 2162, 1222, 1758, 1874, 1952, 4932, 1120] combine_rows_per_src=[11858, 9018, 6854, 10352, 8986, 9322, 11806, 11512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275898368 dispatch_meta_bytes=342616 combine_payload_bytes=275898368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276240984 ingress_bytes=379707392 link_bytes=379707392 token_bytes=552139352 weight_movement_bytes=103809024 total_nvlink_bytes=655948376 weight_to_token_ratio=0.188012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428808347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=247996416 dispatch_meta_bytes=312384 combine_payload_bytes=247996416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248308800 ingress_bytes=317202432 link_bytes=317202432 token_bytes=496305216 weight_movement_bytes=69206016 total_nvlink_bytes=565511232 weight_to_token_ratio=0.139442 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193607215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277315584 dispatch_meta_bytes=340344 combine_payload_bytes=277315584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277655928 ingress_bytes=311918592 link_bytes=311918592 token_bytes=554971512 weight_movement_bytes=34603008 total_nvlink_bytes=589574520 weight_to_token_ratio=0.062351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173724550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=329432 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271034072 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541738712 weight_movement_bytes=103809024 total_nvlink_bytes=645547736 weight_to_token_ratio=0.191622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78410 payload_rows_remote=70082 dup_rows=19894 dup_rows_remote=18304 dedup_ratio=0.202372 router_tokens_per_expert=[1338, 73, 159, 4215, 693, 11, 154, 1228, 350, 635, 839, 380, 5028, 2209, 16, 1427, 710, 27, 15, 1435, 53, 954, 766, 942, 17, 752, 453, 859, 278, 30, 34, 36, 32, 390, 815, 160, 701, 0, 98, 441, 587, 6091, 18, 2026, 2322, 24, 2111, 242, 133, 349, 88, 962, 1803, 1831, 252, 30, 202, 117, 13, 46, 65, 2, 1071, 14] payload_rows_per_dst=[8078, 9872, 7752, 11960, 8328, 10826, 8630, 12964] dup_rows_per_dst=[1648, 3438, 2052, 3014, 1590, 3608, 2266, 2278] combine_rows_per_src=[8078, 9872, 7752, 11960, 8328, 10826, 8630, 12964] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=344776 combine_payload_bytes=284082176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284426952 ingress_bytes=318685184 link_bytes=318685184 token_bytes=568509128 weight_movement_bytes=34603008 total_nvlink_bytes=603112136 weight_to_token_ratio=0.060866 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199186691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79142 payload_rows_remote=71626 dup_rows=19162 dup_rows_remote=17574 dedup_ratio=0.194926 router_tokens_per_expert=[1361, 99, 334, 3704, 1109, 27, 151, 980, 303, 1103, 852, 372, 3924, 1772, 33, 1843, 618, 62, 126, 866, 32, 651, 1080, 1117, 15, 544, 493, 1004, 382, 60, 40, 63, 60, 212, 830, 477, 829, 4, 114, 977, 754, 6016, 13, 2535, 2678, 29, 1407, 281, 145, 659, 224, 801, 2147, 800, 334, 33, 305, 73, 3, 16, 70, 3, 1165, 38] payload_rows_per_dst=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] dup_rows_per_dst=[3600, 3060, 1588, 2522, 2994, 1142, 1758, 2498] combine_rows_per_src=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75042 payload_rows_remote=64602 dup_rows=23262 dup_rows_remote=20418 dedup_ratio=0.236633 router_tokens_per_expert=[277, 894, 748, 55, 443, 704, 163, 4166, 517, 126, 241, 2190, 390, 1157, 1172, 583, 3225, 2412, 53, 245, 1205, 25, 1744, 50, 93, 42, 730, 840, 3, 898, 1543, 38, 699, 62, 367, 783, 14, 47, 3765, 2789, 1921, 158, 22, 510, 14, 443, 1088, 172, 149, 630, 1826, 2156, 655, 861, 50, 315, 143, 461, 728, 69, 59, 574, 19, 631] payload_rows_per_dst=[5942, 9234, 7052, 6998, 13506, 12122, 10440, 9748] dup_rows_per_dst=[626, 3518, 1058, 1376, 6904, 4866, 2844, 2070] combine_rows_per_src=[5942, 9234, 7052, 6998, 13506, 12122, 10440, 9748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80474 payload_rows_remote=68954 dup_rows=17830 dup_rows_remote=14704 dedup_ratio=0.181376 router_tokens_per_expert=[1660, 41, 90, 4853, 863, 5, 206, 1008, 499, 694, 775, 424, 4684, 2493, 7, 1476, 654, 14, 42, 936, 34, 530, 1292, 794, 21, 433, 280, 961, 600, 17, 42, 9, 11, 257, 908, 159, 644, 0, 29, 485, 849, 5603, 5, 2031, 2962, 7, 1691, 247, 85, 627, 173, 816, 2714, 798, 328, 7, 234, 36, 30, 17, 40, 1, 897, 24] payload_rows_per_dst=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] dup_rows_per_dst=[776, 946, 5186, 2512, 1744, 3126, 1784, 1756] combine_rows_per_src=[6970, 7216, 12700, 11582, 9166, 11520, 9312, 12008] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287055872 dispatch_meta_bytes=353544 combine_payload_bytes=287055872 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287409416 ingress_bytes=356261888 link_bytes=356261888 token_bytes=574465288 weight_movement_bytes=69206016 total_nvlink_bytes=643671304 weight_to_token_ratio=0.120470 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340584803e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75732 payload_rows_remote=65364 dup_rows=22572 dup_rows_remote=19480 dedup_ratio=0.229614 router_tokens_per_expert=[1265, 727, 74, 27, 2706, 18, 258, 273, 18, 191, 5136, 44, 147, 1654, 693, 1309, 2687, 79, 58, 97, 285, 305, 7, 571, 311, 922, 387, 718, 332, 7252, 2087, 16, 361, 55, 1658, 427, 1780, 228, 72, 1, 1356, 711, 0, 40, 97, 757, 121, 8, 2068, 343, 6, 129, 261, 319, 3047, 1200, 161, 513, 11, 557, 2, 625, 1246, 338] payload_rows_per_dst=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] dup_rows_per_dst=[2528, 1302, 5310, 1150, 3260, 3092, 2452, 3478] combine_rows_per_src=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270704640 dispatch_meta_bytes=329432 combine_payload_bytes=270704640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271034072 ingress_bytes=374513664 link_bytes=374513664 token_bytes=541738712 weight_movement_bytes=103809024 total_nvlink_bytes=645547736 weight_to_token_ratio=0.191622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409264767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293380096 dispatch_meta_bytes=356800 combine_payload_bytes=293380096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293736896 ingress_bytes=327983104 link_bytes=327983104 token_bytes=587116992 weight_movement_bytes=34603008 total_nvlink_bytes=621720000 weight_to_token_ratio=0.058937 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234174016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=339376 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070320 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535801264 weight_movement_bytes=34603008 total_nvlink_bytes=570404272 weight_to_token_ratio=0.064582 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=61900 dup_rows=24154 dup_rows_remote=18718 dedup_ratio=0.245707 router_tokens_per_expert=[1620, 589, 18, 76, 2469, 34, 200, 482, 10, 205, 5471, 13, 59, 1495, 840, 1562, 2390, 60, 76, 50, 284, 645, 16, 409, 401, 960, 522, 757, 165, 6829, 1707, 12, 586, 52, 1392, 535, 1689, 235, 72, 9, 1280, 700, 0, 46, 152, 616, 95, 4, 2139, 308, 18, 50, 264, 325, 2811, 1221, 111, 806, 44, 770, 5, 544, 1275, 572] payload_rows_per_dst=[8192, 7156, 13506, 4872, 7962, 5082, 12250, 15130] dup_rows_per_dst=[2784, 1212, 5296, 762, 1178, 704, 5436, 6782] combine_rows_per_src=[8192, 7156, 13506, 4872, 7962, 5082, 12250, 15130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79142 payload_rows_remote=71626 dup_rows=19162 dup_rows_remote=17574 dedup_ratio=0.194926 router_tokens_per_expert=[1361, 99, 334, 3704, 1109, 27, 151, 980, 303, 1103, 852, 372, 3924, 1772, 33, 1843, 618, 62, 126, 866, 32, 651, 1080, 1117, 15, 544, 493, 1004, 382, 60, 40, 63, 60, 212, 830, 477, 829, 4, 114, 977, 754, 6016, 13, 2535, 2678, 29, 1407, 281, 145, 659, 224, 801, 2147, 800, 334, 33, 305, 73, 3, 16, 70, 3, 1165, 38] payload_rows_per_dst=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] dup_rows_per_dst=[3600, 3060, 1588, 2522, 2994, 1142, 1758, 2498] combine_rows_per_src=[11930, 9496, 7516, 10528, 10578, 7686, 8528, 12880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=340080 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264949872 ingress_bytes=333815808 link_bytes=333815808 token_bytes=529559664 weight_movement_bytes=69206016 total_nvlink_bytes=598765680 weight_to_token_ratio=0.130686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256122011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72932 payload_rows_remote=60382 dup_rows=25372 dup_rows_remote=19796 dedup_ratio=0.258097 router_tokens_per_expert=[1440, 91, 171, 4561, 809, 27, 119, 888, 217, 619, 891, 602, 4550, 2557, 48, 1793, 699, 25, 33, 1438, 50, 381, 986, 800, 43, 757, 606, 1252, 480, 62, 65, 51, 41, 386, 885, 86, 765, 2, 72, 355, 285, 4913, 14, 2351, 2080, 50, 1788, 267, 123, 867, 57, 627, 1902, 2357, 774, 26, 192, 147, 10, 66, 52, 2, 485, 14] payload_rows_per_dst=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] dup_rows_per_dst=[5576, 8874, 1502, 868, 3286, 2846, 2376, 44] combine_rows_per_src=[12550, 13680, 7322, 5764, 10760, 11788, 9176, 1892] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=59798 dup_rows=24098 dup_rows_remote=17732 dedup_ratio=0.245138 router_tokens_per_expert=[1845, 624, 18, 54, 2480, 41, 466, 149, 41, 370, 4656, 50, 141, 1509, 848, 974, 1938, 51, 105, 104, 155, 753, 6, 655, 266, 1449, 621, 682, 271, 6551, 2393, 4, 512, 56, 1509, 302, 1873, 251, 189, 0, 1360, 571, 3, 70, 120, 565, 120, 15, 2350, 290, 19, 135, 324, 308, 3263, 841, 127, 713, 18, 555, 12, 760, 1122, 529] payload_rows_per_dst=[8568, 6614, 12302, 5736, 8216, 5156, 13206, 14408] dup_rows_per_dst=[2786, 1252, 4544, 850, 1168, 492, 6640, 6366] combine_rows_per_src=[8568, 6614, 12302, 5736, 8216, 5156, 13206, 14408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=282435584 dispatch_meta_bytes=334632 combine_payload_bytes=282435584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282770216 ingress_bytes=420847616 link_bytes=420847616 token_bytes=565205800 weight_movement_bytes=138412032 total_nvlink_bytes=703617832 weight_to_token_ratio=0.244888 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583615698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74904 payload_rows_remote=63212 dup_rows=23400 dup_rows_remote=19532 dedup_ratio=0.238037 router_tokens_per_expert=[1727, 711, 60, 58, 1743, 55, 202, 248, 14, 272, 5733, 32, 38, 1527, 783, 1574, 1770, 78, 150, 86, 230, 647, 12, 675, 249, 1192, 627, 578, 270, 7173, 1847, 12, 492, 46, 1267, 392, 2018, 257, 44, 4, 1093, 918, 0, 140, 99, 812, 105, 18, 2484, 271, 11, 43, 151, 380, 2442, 1311, 93, 720, 16, 968, 10, 486, 1167, 521] payload_rows_per_dst=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] dup_rows_per_dst=[3910, 3868, 762, 8516, 3086, 964, 1204, 1090] combine_rows_per_src=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76652 payload_rows_remote=69734 dup_rows=21652 dup_rows_remote=20384 dedup_ratio=0.220256 router_tokens_per_expert=[1352, 545, 19, 60, 2648, 41, 372, 548, 60, 309, 4768, 43, 167, 1681, 820, 1127, 2070, 85, 162, 50, 271, 920, 13, 309, 151, 1069, 388, 513, 445, 7118, 1509, 18, 809, 35, 1135, 382, 1315, 261, 90, 17, 1424, 922, 0, 84, 134, 628, 255, 23, 2465, 276, 39, 78, 583, 259, 2306, 1628, 236, 808, 38, 479, 12, 617, 1484, 679] payload_rows_per_dst=[8250, 13334, 6572, 6918, 9796, 15334, 9232, 7216] dup_rows_per_dst=[2920, 4616, 1188, 1268, 2904, 5842, 1424, 1490] combine_rows_per_src=[8250, 13334, 6572, 6918, 9796, 15334, 9232, 7216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78096 payload_rows_remote=68494 dup_rows=20208 dup_rows_remote=18384 dedup_ratio=0.205566 router_tokens_per_expert=[2341, 775, 22, 83, 1978, 25, 554, 203, 14, 202, 5231, 18, 38, 1476, 863, 1222, 2424, 121, 64, 60, 383, 366, 6, 167, 340, 1229, 320, 472, 144, 7305, 2121, 5, 419, 53, 854, 375, 1343, 242, 251, 16, 1199, 935, 4, 77, 100, 499, 588, 16, 2979, 286, 4, 28, 135, 280, 2160, 1167, 112, 834, 14, 1026, 19, 543, 1632, 390] payload_rows_per_dst=[9036, 9718, 10114, 7896, 9602, 15586, 8498, 7646] dup_rows_per_dst=[2926, 2584, 2894, 1366, 1824, 5860, 1260, 1494] combine_rows_per_src=[9036, 9718, 10114, 7896, 9602, 15586, 8498, 7646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74832 payload_rows_remote=61372 dup_rows=23472 dup_rows_remote=17326 dedup_ratio=0.238770 router_tokens_per_expert=[1767, 474, 28, 55, 1389, 31, 602, 147, 16, 294, 5310, 20, 126, 1374, 1082, 1299, 3180, 78, 74, 119, 198, 649, 9, 262, 186, 1251, 304, 780, 234, 7069, 2143, 14, 481, 36, 1402, 359, 1348, 341, 149, 7, 1114, 1641, 2, 78, 47, 630, 74, 8, 2068, 438, 34, 33, 299, 269, 2708, 1121, 133, 860, 21, 525, 5, 627, 1332, 398] payload_rows_per_dst=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] dup_rows_per_dst=[6146, 2262, 1100, 7464, 3182, 1308, 886, 1124] combine_rows_per_src=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253542400 dispatch_meta_bytes=322472 combine_payload_bytes=253542400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253864872 ingress_bytes=322748416 link_bytes=322748416 token_bytes=507407272 weight_movement_bytes=69206016 total_nvlink_bytes=576613288 weight_to_token_ratio=0.136391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214476306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293380096 dispatch_meta_bytes=356800 combine_payload_bytes=293380096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293736896 ingress_bytes=327983104 link_bytes=327983104 token_bytes=587116992 weight_movement_bytes=34603008 total_nvlink_bytes=621720000 weight_to_token_ratio=0.058937 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234174016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=247324672 dispatch_meta_bytes=320712 combine_payload_bytes=247324672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247645384 ingress_bytes=351133696 link_bytes=351133696 token_bytes=494970056 weight_movement_bytes=103809024 total_nvlink_bytes=598779080 weight_to_token_ratio=0.209728 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321287829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=244932608 dispatch_meta_bytes=310120 combine_payload_bytes=244932608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245242728 ingress_bytes=279535616 link_bytes=279535616 token_bytes=490175336 weight_movement_bytes=34603008 total_nvlink_bytes=524778344 weight_to_token_ratio=0.070593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051870019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74904 payload_rows_remote=63212 dup_rows=23400 dup_rows_remote=19532 dedup_ratio=0.238037 router_tokens_per_expert=[1727, 711, 60, 58, 1743, 55, 202, 248, 14, 272, 5733, 32, 38, 1527, 783, 1574, 1770, 78, 150, 86, 230, 647, 12, 675, 249, 1192, 627, 578, 270, 7173, 1847, 12, 492, 46, 1267, 392, 2018, 257, 44, 4, 1093, 918, 0, 140, 99, 812, 105, 18, 2484, 271, 11, 43, 151, 380, 2442, 1311, 93, 720, 16, 968, 10, 486, 1167, 521] payload_rows_per_dst=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] dup_rows_per_dst=[3910, 3868, 762, 8516, 3086, 964, 1204, 1090] combine_rows_per_src=[10084, 11692, 6534, 15380, 10838, 5406, 8098, 6872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=258916352 dispatch_meta_bytes=330976 combine_payload_bytes=258916352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259247328 ingress_bytes=362725376 link_bytes=362725376 token_bytes=518163680 weight_movement_bytes=103809024 total_nvlink_bytes=621972704 weight_to_token_ratio=0.200340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364906388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78096 payload_rows_remote=68494 dup_rows=20208 dup_rows_remote=18384 dedup_ratio=0.205566 router_tokens_per_expert=[2341, 775, 22, 83, 1978, 25, 554, 203, 14, 202, 5231, 18, 38, 1476, 863, 1222, 2424, 121, 64, 60, 383, 366, 6, 167, 340, 1229, 320, 472, 144, 7305, 2121, 5, 419, 53, 854, 375, 1343, 242, 251, 16, 1199, 935, 4, 77, 100, 499, 588, 16, 2979, 286, 4, 28, 135, 280, 2160, 1167, 112, 834, 14, 1026, 19, 543, 1632, 390] payload_rows_per_dst=[9036, 9718, 10114, 7896, 9602, 15586, 8498, 7646] dup_rows_per_dst=[2926, 2584, 2894, 1366, 1824, 5860, 1260, 1494] combine_rows_per_src=[9036, 9718, 10114, 7896, 9602, 15586, 8498, 7646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=285630464 dispatch_meta_bytes=360472 combine_payload_bytes=285630464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=285990936 ingress_bytes=424042496 link_bytes=424042496 token_bytes=571621400 weight_movement_bytes=138412032 total_nvlink_bytes=710033432 weight_to_token_ratio=0.242139 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595637774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76344 payload_rows_remote=70294 dup_rows=21960 dup_rows_remote=21006 dedup_ratio=0.223389 router_tokens_per_expert=[1433, 709, 56, 50, 1499, 42, 362, 122, 39, 431, 4571, 17, 28, 1933, 755, 969, 1915, 137, 123, 111, 354, 406, 7, 449, 366, 1215, 254, 737, 402, 7236, 2595, 83, 508, 275, 1184, 319, 1625, 421, 278, 11, 2026, 780, 6, 30, 174, 353, 60, 31, 3088, 546, 35, 105, 227, 410, 2410, 905, 191, 766, 10, 446, 6, 513, 1415, 592] payload_rows_per_dst=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] dup_rows_per_dst=[1898, 4536, 954, 4534, 3190, 4254, 1354, 1240] combine_rows_per_src=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=68438 dup_rows=19380 dup_rows_remote=15908 dedup_ratio=0.197144 router_tokens_per_expert=[1733, 39, 277, 3757, 681, 2, 246, 1114, 398, 634, 865, 298, 4796, 1635, 18, 883, 607, 18, 23, 1021, 30, 369, 1261, 1303, 10, 769, 428, 862, 339, 42, 55, 46, 23, 502, 1522, 166, 753, 0, 25, 579, 664, 5584, 10, 2028, 2304, 28, 2001, 330, 182, 1611, 172, 684, 2384, 1258, 652, 36, 225, 56, 9, 29, 16, 5, 736, 19] payload_rows_per_dst=[7014, 5656, 12388, 11708, 9454, 10352, 10486, 11866] dup_rows_per_dst=[1170, 536, 4390, 2986, 2294, 3040, 3472, 1492] combine_rows_per_src=[7014, 5656, 12388, 11708, 9454, 10352, 10486, 11866] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280551424 dispatch_meta_bytes=347512 combine_payload_bytes=280551424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280898936 ingress_bytes=315154432 link_bytes=315154432 token_bytes=561450360 weight_movement_bytes=34603008 total_nvlink_bytes=596053368 weight_to_token_ratio=0.061631 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185900756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=251379712 dispatch_meta_bytes=314792 combine_payload_bytes=251379712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251694504 ingress_bytes=285982720 link_bytes=285982720 token_bytes=503074216 weight_movement_bytes=34603008 total_nvlink_bytes=537677224 weight_to_token_ratio=0.068783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076129952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78010 payload_rows_remote=68660 dup_rows=20294 dup_rows_remote=18624 dedup_ratio=0.206441 router_tokens_per_expert=[4315, 279, 116, 616, 2, 671, 624, 1769, 41, 0, 0, 1847, 2, 1897, 0, 801, 33, 3, 1607, 8, 268, 50, 53, 113, 999, 40, 2354, 6, 4, 4221, 469, 25, 87, 922, 22, 2145, 22, 445, 754, 0, 50, 3617, 36, 7, 4159, 262, 6, 1532, 138, 15, 19, 218, 213, 832, 6, 2, 105, 2026, 416, 574, 1, 986, 5734, 568] payload_rows_per_dst=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] dup_rows_per_dst=[1064, 1844, 2706, 3706, 5690, 1670, 1520, 2094] combine_rows_per_src=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=258916352 dispatch_meta_bytes=330976 combine_payload_bytes=258916352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259247328 ingress_bytes=362725376 link_bytes=362725376 token_bytes=518163680 weight_movement_bytes=103809024 total_nvlink_bytes=621972704 weight_to_token_ratio=0.200340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364906388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287924224 dispatch_meta_bytes=365200 combine_payload_bytes=287924224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288289424 ingress_bytes=322527232 link_bytes=322527232 token_bytes=576213648 weight_movement_bytes=34603008 total_nvlink_bytes=610816656 weight_to_token_ratio=0.060052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213644008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75296 payload_rows_remote=64744 dup_rows=23008 dup_rows_remote=21100 dedup_ratio=0.234049 router_tokens_per_expert=[4029, 344, 267, 564, 16, 257, 906, 1165, 29, 0, 0, 1711, 0, 1595, 2, 939, 70, 0, 1176, 30, 428, 52, 41, 268, 680, 121, 2930, 2, 6, 3605, 1057, 72, 103, 996, 42, 1623, 163, 830, 575, 0, 14, 4614, 26, 7, 4120, 140, 8, 1929, 146, 12, 7, 256, 485, 1176, 25, 3, 183, 1939, 234, 723, 0, 692, 4828, 891] payload_rows_per_dst=[6282, 7224, 3926, 12224, 9970, 12374, 10552, 12744] dup_rows_per_dst=[756, 1328, 204, 4722, 3300, 4554, 1908, 6236] combine_rows_per_src=[6282, 7224, 3926, 12224, 9970, 12374, 10552, 12744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281231360 dispatch_meta_bytes=349136 combine_payload_bytes=281231360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281580496 ingress_bytes=385040384 link_bytes=385040384 token_bytes=562811856 weight_movement_bytes=103809024 total_nvlink_bytes=666620880 weight_to_token_ratio=0.184447 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448875967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75732 payload_rows_remote=65364 dup_rows=22572 dup_rows_remote=19480 dedup_ratio=0.229614 router_tokens_per_expert=[1265, 727, 74, 27, 2706, 18, 258, 273, 18, 191, 5136, 44, 147, 1654, 693, 1309, 2687, 79, 58, 97, 285, 305, 7, 571, 311, 922, 387, 718, 332, 7252, 2087, 16, 361, 55, 1658, 427, 1780, 228, 72, 1, 1356, 711, 0, 40, 97, 757, 121, 8, 2068, 343, 6, 129, 261, 319, 3047, 1200, 161, 513, 11, 557, 2, 625, 1246, 338] payload_rows_per_dst=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] dup_rows_per_dst=[2528, 1302, 5310, 1150, 3260, 3092, 2452, 3478] combine_rows_per_src=[8168, 6810, 13140, 6400, 10746, 10368, 9448, 10652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76344 payload_rows_remote=70294 dup_rows=21960 dup_rows_remote=21006 dedup_ratio=0.223389 router_tokens_per_expert=[1433, 709, 56, 50, 1499, 42, 362, 122, 39, 431, 4571, 17, 28, 1933, 755, 969, 1915, 137, 123, 111, 354, 406, 7, 449, 366, 1215, 254, 737, 402, 7236, 2595, 83, 508, 275, 1184, 319, 1625, 421, 278, 11, 2026, 780, 6, 30, 174, 353, 60, 31, 3088, 546, 35, 105, 227, 410, 2410, 905, 191, 766, 10, 446, 6, 513, 1415, 592] payload_rows_per_dst=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] dup_rows_per_dst=[1898, 4536, 954, 4534, 3190, 4254, 1354, 1240] combine_rows_per_src=[6648, 12950, 6050, 10932, 10872, 12976, 9278, 6638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280322048 dispatch_meta_bytes=337384 combine_payload_bytes=280322048 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=280659432 ingress_bytes=314925056 link_bytes=314925056 token_bytes=560981480 weight_movement_bytes=34603008 total_nvlink_bytes=595584488 weight_to_token_ratio=0.061683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185037632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74832 payload_rows_remote=61372 dup_rows=23472 dup_rows_remote=17326 dedup_ratio=0.238770 router_tokens_per_expert=[1767, 474, 28, 55, 1389, 31, 602, 147, 16, 294, 5310, 20, 126, 1374, 1082, 1299, 3180, 78, 74, 119, 198, 649, 9, 262, 186, 1251, 304, 780, 234, 7069, 2143, 14, 481, 36, 1402, 359, 1348, 341, 149, 7, 1114, 1641, 2, 78, 47, 630, 74, 8, 2068, 438, 34, 33, 299, 269, 2708, 1121, 133, 860, 21, 525, 5, 627, 1332, 398] payload_rows_per_dst=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] dup_rows_per_dst=[6146, 2262, 1100, 7464, 3182, 1308, 886, 1124] combine_rows_per_src=[13460, 8398, 8038, 14260, 10480, 5880, 7638, 6678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=69214 dup_rows=17760 dup_rows_remote=14950 dedup_ratio=0.180664 router_tokens_per_expert=[4101, 409, 463, 514, 26, 330, 750, 1910, 16, 0, 0, 1136, 2, 581, 2, 712, 16, 5, 975, 36, 450, 53, 56, 149, 925, 79, 2930, 0, 7, 4621, 861, 24, 84, 1073, 24, 1609, 212, 1262, 753, 0, 57, 3661, 75, 21, 3919, 191, 26, 1625, 180, 10, 17, 373, 708, 1033, 35, 2, 231, 1268, 370, 579, 0, 833, 5850, 932] payload_rows_per_dst=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] dup_rows_per_dst=[4876, 2810, 2352, 1212, 1450, 1856, 1938, 1266] combine_rows_per_src=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72270 payload_rows_remote=57412 dup_rows=26034 dup_rows_remote=14946 dedup_ratio=0.264832 router_tokens_per_expert=[4273, 279, 531, 653, 4, 405, 679, 816, 12, 0, 0, 1620, 3, 1378, 3, 795, 14, 11, 1191, 10, 390, 91, 101, 92, 759, 194, 3026, 3, 5, 4076, 728, 35, 190, 1016, 70, 1494, 102, 1256, 893, 0, 45, 3230, 39, 25, 3616, 209, 16, 2052, 216, 18, 8, 451, 556, 770, 8, 4, 259, 1782, 516, 533, 2, 785, 5862, 952] payload_rows_per_dst=[6018, 6530, 3536, 12796, 8324, 8572, 11636, 14858] dup_rows_per_dst=[716, 1092, 264, 4856, 1718, 2670, 3630, 11088] combine_rows_per_src=[6018, 6530, 3536, 12796, 8324, 8572, 11636, 14858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=339376 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070320 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535801264 weight_movement_bytes=34603008 total_nvlink_bytes=570404272 weight_to_token_ratio=0.064582 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79218 payload_rows_remote=70800 dup_rows=19086 dup_rows_remote=17806 dedup_ratio=0.194153 router_tokens_per_expert=[3818, 299, 352, 456, 14, 427, 1438, 1170, 85, 0, 0, 1819, 2, 791, 0, 442, 58, 7, 1103, 78, 228, 29, 71, 173, 781, 198, 3145, 5, 11, 4056, 681, 28, 197, 1335, 99, 2111, 331, 698, 784, 0, 37, 3250, 58, 5, 3500, 307, 23, 2473, 381, 48, 16, 438, 537, 1002, 18, 12, 159, 2160, 590, 806, 14, 530, 5241, 227] payload_rows_per_dst=[11848, 10852, 11904, 8418, 9236, 9424, 9936, 7600] dup_rows_per_dst=[4100, 3538, 2072, 1280, 1874, 2882, 1968, 1372] combine_rows_per_src=[11848, 10852, 11904, 8418, 9236, 9424, 9936, 7600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78486 payload_rows_remote=67752 dup_rows=19818 dup_rows_remote=14920 dedup_ratio=0.201599 router_tokens_per_expert=[4285, 387, 106, 660, 40, 342, 1443, 1944, 42, 0, 0, 2189, 2, 1159, 3, 656, 50, 15, 559, 21, 575, 30, 81, 168, 570, 73, 2665, 2, 1, 3460, 797, 37, 174, 1854, 64, 1511, 92, 580, 829, 1, 34, 4223, 29, 20, 3846, 151, 10, 1724, 135, 8, 27, 288, 386, 779, 110, 1, 183, 1674, 319, 375, 3, 642, 5863, 855] payload_rows_per_dst=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] dup_rows_per_dst=[1710, 3718, 2150, 2048, 4898, 2220, 1418, 1656] combine_rows_per_src=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343376 combine_payload_bytes=265191424 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=265534800 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530726224 weight_movement_bytes=69206016 total_nvlink_bytes=599932240 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80116 payload_rows_remote=68624 dup_rows=18188 dup_rows_remote=14304 dedup_ratio=0.185018 router_tokens_per_expert=[4124, 244, 166, 409, 22, 378, 786, 1559, 52, 0, 0, 1886, 3, 964, 1, 390, 62, 22, 556, 10, 400, 61, 83, 201, 569, 279, 2916, 4, 7, 4519, 620, 33, 327, 1393, 118, 1619, 270, 413, 803, 0, 18, 4114, 42, 22, 3997, 243, 5, 2130, 250, 19, 74, 257, 715, 716, 139, 8, 236, 1496, 628, 625, 1, 283, 6106, 759] payload_rows_per_dst=[11492, 11874, 13080, 7916, 8358, 10304, 10542, 6550] dup_rows_per_dst=[3884, 3756, 1922, 940, 1528, 2844, 1808, 1506] combine_rows_per_src=[11492, 11874, 13080, 7916, 8358, 10304, 10542, 6550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287924224 dispatch_meta_bytes=365200 combine_payload_bytes=287924224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288289424 ingress_bytes=322527232 link_bytes=322527232 token_bytes=576213648 weight_movement_bytes=34603008 total_nvlink_bytes=610816656 weight_to_token_ratio=0.060052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213644008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=251379712 dispatch_meta_bytes=314792 combine_payload_bytes=251379712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251694504 ingress_bytes=285982720 link_bytes=285982720 token_bytes=503074216 weight_movement_bytes=34603008 total_nvlink_bytes=537677224 weight_to_token_ratio=0.068783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076129952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283500544 dispatch_meta_bytes=336656 combine_payload_bytes=283500544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283837200 ingress_bytes=318103552 link_bytes=318103552 token_bytes=567337744 weight_movement_bytes=34603008 total_nvlink_bytes=601940752 weight_to_token_ratio=0.060992 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196998057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78486 payload_rows_remote=67752 dup_rows=19818 dup_rows_remote=14920 dedup_ratio=0.201599 router_tokens_per_expert=[4285, 387, 106, 660, 40, 342, 1443, 1944, 42, 0, 0, 2189, 2, 1159, 3, 656, 50, 15, 559, 21, 575, 30, 81, 168, 570, 73, 2665, 2, 1, 3460, 797, 37, 174, 1854, 64, 1511, 92, 580, 829, 1, 34, 4223, 29, 20, 3846, 151, 10, 1724, 135, 8, 27, 288, 386, 779, 110, 1, 183, 1674, 319, 375, 3, 642, 5863, 855] payload_rows_per_dst=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] dup_rows_per_dst=[1710, 3718, 2150, 2048, 4898, 2220, 1418, 1656] combine_rows_per_src=[8176, 11304, 12574, 9348, 10734, 10162, 9742, 6446] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=235159552 dispatch_meta_bytes=289432 combine_payload_bytes=235159552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235448984 ingress_bytes=304365568 link_bytes=304365568 token_bytes=470608536 weight_movement_bytes=69206016 total_nvlink_bytes=539814552 weight_to_token_ratio=0.147056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145303129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=69214 dup_rows=17760 dup_rows_remote=14950 dedup_ratio=0.180664 router_tokens_per_expert=[4101, 409, 463, 514, 26, 330, 750, 1910, 16, 0, 0, 1136, 2, 581, 2, 712, 16, 5, 975, 36, 450, 53, 56, 149, 925, 79, 2930, 0, 7, 4621, 861, 24, 84, 1073, 24, 1609, 212, 1262, 753, 0, 57, 3661, 75, 21, 3919, 191, 26, 1625, 180, 10, 17, 373, 708, 1033, 35, 2, 231, 1268, 370, 579, 0, 833, 5850, 932] payload_rows_per_dst=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] dup_rows_per_dst=[4876, 2810, 2352, 1212, 1450, 1856, 1938, 1266] combine_rows_per_src=[12130, 11330, 12828, 8440, 8584, 9456, 10616, 7160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78094 payload_rows_remote=65516 dup_rows=20210 dup_rows_remote=16348 dedup_ratio=0.205587 router_tokens_per_expert=[4030, 323, 325, 631, 12, 280, 1006, 2474, 54, 5, 2, 885, 42, 762, 8, 542, 13, 17, 1944, 29, 551, 62, 68, 149, 952, 315, 2239, 3, 2, 4065, 919, 25, 223, 1037, 214, 2005, 178, 992, 797, 7, 66, 2869, 155, 9, 3527, 249, 10, 1625, 171, 28, 82, 419, 526, 827, 114, 9, 353, 2135, 546, 498, 7, 386, 5387, 967] payload_rows_per_dst=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] dup_rows_per_dst=[5596, 2356, 3862, 1050, 2058, 1464, 1686, 2138] combine_rows_per_src=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289996800 dispatch_meta_bytes=354424 combine_payload_bytes=289996800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290351224 ingress_bytes=359202816 link_bytes=359202816 token_bytes=580348024 weight_movement_bytes=69206016 total_nvlink_bytes=649554040 weight_to_token_ratio=0.119249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351651278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=61900 dup_rows=24154 dup_rows_remote=18718 dedup_ratio=0.245707 router_tokens_per_expert=[1620, 589, 18, 76, 2469, 34, 200, 482, 10, 205, 5471, 13, 59, 1495, 840, 1562, 2390, 60, 76, 50, 284, 645, 16, 409, 401, 960, 522, 757, 165, 6829, 1707, 12, 586, 52, 1392, 535, 1689, 235, 72, 9, 1280, 700, 0, 46, 152, 616, 95, 4, 2139, 308, 18, 50, 264, 325, 2811, 1221, 111, 806, 44, 770, 5, 544, 1275, 572] payload_rows_per_dst=[8192, 7156, 13506, 4872, 7962, 5082, 12250, 15130] dup_rows_per_dst=[2784, 1212, 5296, 762, 1178, 704, 5436, 6782] combine_rows_per_src=[8192, 7156, 13506, 4872, 7962, 5082, 12250, 15130] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277512192 dispatch_meta_bytes=330688 combine_payload_bytes=277512192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277842880 ingress_bytes=312115200 link_bytes=312115200 token_bytes=555355072 weight_movement_bytes=34603008 total_nvlink_bytes=589958080 weight_to_token_ratio=0.062308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174464370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281083904 dispatch_meta_bytes=331712 combine_payload_bytes=281083904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=281415616 ingress_bytes=419495936 link_bytes=419495936 token_bytes=562499520 weight_movement_bytes=138412032 total_nvlink_bytes=700911552 weight_to_token_ratio=0.246066 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578529435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77384 payload_rows_remote=65928 dup_rows=20920 dup_rows_remote=18366 dedup_ratio=0.212809 router_tokens_per_expert=[226, 139, 151, 1755, 150, 908, 1102, 9, 4, 348, 736, 71, 63, 701, 37, 269, 48, 8, 1, 619, 9, 6, 286, 1251, 1019, 4076, 468, 817, 2097, 4437, 3306, 173, 17, 1850, 785, 284, 2010, 147, 25, 94, 66, 1748, 41, 0, 712, 0, 0, 1, 5918, 455, 34, 2854, 55, 15, 6, 205, 4, 2739, 207, 84, 3426, 0, 0, 80] payload_rows_per_dst=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] dup_rows_per_dst=[1468, 1822, 158, 4842, 1504, 2554, 5738, 2834] combine_rows_per_src=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283500544 dispatch_meta_bytes=336656 combine_payload_bytes=283500544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283837200 ingress_bytes=318103552 link_bytes=318103552 token_bytes=567337744 weight_movement_bytes=34603008 total_nvlink_bytes=601940752 weight_to_token_ratio=0.060992 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196998057e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268353536 dispatch_meta_bytes=327456 combine_payload_bytes=268353536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268680992 ingress_bytes=302956544 link_bytes=302956544 token_bytes=537034528 weight_movement_bytes=34603008 total_nvlink_bytes=571637536 weight_to_token_ratio=0.064433 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140001085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77184 payload_rows_remote=64526 dup_rows=21120 dup_rows_remote=15256 dedup_ratio=0.214844 router_tokens_per_expert=[542, 191, 296, 669, 174, 2034, 848, 14, 48, 295, 497, 4, 45, 1450, 64, 283, 46, 17, 4, 971, 5, 0, 443, 1888, 820, 4124, 651, 497, 2698, 2984, 3089, 327, 72, 1544, 952, 254, 2199, 617, 24, 120, 111, 2174, 80, 2, 501, 7, 0, 0, 4597, 351, 77, 3067, 70, 30, 22, 216, 14, 3017, 347, 257, 2303, 0, 0, 109] payload_rows_per_dst=[7988, 11092, 6438, 7780, 11594, 9692, 12658, 9942] dup_rows_per_dst=[1548, 2528, 310, 1942, 4750, 2026, 5864, 2152] combine_rows_per_src=[7988, 11092, 6438, 7780, 11594, 9692, 12658, 9942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=270041088 dispatch_meta_bytes=337176 combine_payload_bytes=270041088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270378264 ingress_bytes=304644096 link_bytes=304644096 token_bytes=540419352 weight_movement_bytes=34603008 total_nvlink_bytes=575022360 weight_to_token_ratio=0.064030 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146351207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78010 payload_rows_remote=68660 dup_rows=20294 dup_rows_remote=18624 dedup_ratio=0.206441 router_tokens_per_expert=[4315, 279, 116, 616, 2, 671, 624, 1769, 41, 0, 0, 1847, 2, 1897, 0, 801, 33, 3, 1607, 8, 268, 50, 53, 113, 999, 40, 2354, 6, 4, 4221, 469, 25, 87, 922, 22, 2145, 22, 445, 754, 0, 50, 3617, 36, 7, 4159, 262, 6, 1532, 138, 15, 19, 218, 213, 832, 6, 2, 105, 2026, 416, 574, 1, 986, 5734, 568] payload_rows_per_dst=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] dup_rows_per_dst=[1064, 1844, 2706, 3706, 5690, 1670, 1520, 2094] combine_rows_per_src=[7090, 8374, 12346, 11488, 11734, 9350, 9684, 7944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78094 payload_rows_remote=65516 dup_rows=20210 dup_rows_remote=16348 dedup_ratio=0.205587 router_tokens_per_expert=[4030, 323, 325, 631, 12, 280, 1006, 2474, 54, 5, 2, 885, 42, 762, 8, 542, 13, 17, 1944, 29, 551, 62, 68, 149, 952, 315, 2239, 3, 2, 4065, 919, 25, 223, 1037, 214, 2005, 178, 992, 797, 7, 66, 2869, 155, 9, 3527, 249, 10, 1625, 171, 28, 82, 419, 526, 827, 114, 9, 353, 2135, 546, 498, 7, 386, 5387, 967] payload_rows_per_dst=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] dup_rows_per_dst=[5596, 2356, 3862, 1050, 2058, 1464, 1686, 2138] combine_rows_per_src=[12566, 10374, 12578, 7860, 8848, 8502, 9720, 7646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=253542400 dispatch_meta_bytes=322472 combine_payload_bytes=253542400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253864872 ingress_bytes=322748416 link_bytes=322748416 token_bytes=507407272 weight_movement_bytes=69206016 total_nvlink_bytes=576613288 weight_to_token_ratio=0.136391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214476306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66320 dup_rows=22084 dup_rows_remote=20112 dedup_ratio=0.224650 router_tokens_per_expert=[806, 194, 236, 1346, 177, 903, 1118, 37, 29, 597, 586, 37, 42, 1101, 57, 197, 143, 22, 3, 869, 8, 13, 813, 1589, 1176, 4126, 847, 374, 1705, 3696, 2373, 163, 36, 2174, 938, 246, 1505, 1117, 100, 100, 152, 1810, 37, 15, 862, 0, 0, 0, 5757, 411, 49, 2387, 51, 67, 34, 244, 10, 2753, 165, 87, 2536, 3, 2, 121] payload_rows_per_dst=[8232, 11342, 6560, 4530, 10962, 10530, 14164, 9900] dup_rows_per_dst=[1402, 2202, 360, 590, 4880, 2614, 8064, 1972] combine_rows_per_src=[8232, 11342, 6560, 4530, 10962, 10530, 14164, 9900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79600 payload_rows_remote=69356 dup_rows=18704 dup_rows_remote=17184 dedup_ratio=0.190267 router_tokens_per_expert=[680, 128, 204, 540, 130, 1052, 763, 22, 13, 357, 640, 18, 68, 927, 60, 165, 16, 14, 1, 1019, 24, 8, 517, 1827, 1982, 3617, 1138, 283, 2204, 3620, 2824, 306, 48, 1807, 1104, 264, 2503, 284, 27, 73, 101, 2730, 20, 4, 622, 0, 0, 2, 4850, 481, 21, 2412, 28, 2, 38, 213, 16, 2660, 310, 131, 3079, 1, 0, 154] payload_rows_per_dst=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] dup_rows_per_dst=[2324, 1520, 4330, 2602, 2266, 2802, 554, 2306] combine_rows_per_src=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80116 payload_rows_remote=68624 dup_rows=18188 dup_rows_remote=14304 dedup_ratio=0.185018 router_tokens_per_expert=[4124, 244, 166, 409, 22, 378, 786, 1559, 52, 0, 0, 1886, 3, 964, 1, 390, 62, 22, 556, 10, 400, 61, 83, 201, 569, 279, 2916, 4, 7, 4519, 620, 33, 327, 1393, 118, 1619, 270, 413, 803, 0, 18, 4114, 42, 22, 3997, 243, 5, 2130, 250, 19, 74, 257, 715, 716, 139, 8, 236, 1496, 628, 625, 1, 283, 6106, 759] payload_rows_per_dst=[11492, 11874, 13080, 7916, 8358, 10304, 10542, 6550] dup_rows_per_dst=[3884, 3756, 1922, 940, 1528, 2844, 1808, 1506] combine_rows_per_src=[11492, 11874, 13080, 7916, 8358, 10304, 10542, 6550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281231360 dispatch_meta_bytes=349136 combine_payload_bytes=281231360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281580496 ingress_bytes=385040384 link_bytes=385040384 token_bytes=562811856 weight_movement_bytes=103809024 total_nvlink_bytes=666620880 weight_to_token_ratio=0.184447 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448875967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78190 payload_rows_remote=67756 dup_rows=20114 dup_rows_remote=15690 dedup_ratio=0.204610 router_tokens_per_expert=[528, 209, 149, 1920, 356, 1485, 927, 44, 17, 258, 753, 14, 28, 284, 45, 297, 32, 11, 3, 737, 4, 3, 363, 1305, 706, 3734, 814, 671, 2096, 3280, 3237, 152, 33, 1202, 1211, 240, 1656, 1060, 46, 179, 94, 1966, 91, 3, 986, 3, 1, 4, 5343, 401, 20, 2620, 55, 21, 25, 281, 11, 3179, 35, 575, 3229, 0, 0, 120] payload_rows_per_dst=[9014, 9594, 12418, 10434, 9066, 10332, 6186, 11146] dup_rows_per_dst=[2716, 1266, 3184, 4424, 2188, 2524, 660, 3152] combine_rows_per_src=[9014, 9594, 12418, 10434, 9066, 10332, 6186, 11146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264298496 dispatch_meta_bytes=319128 combine_payload_bytes=264298496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264617624 ingress_bytes=333504512 link_bytes=333504512 token_bytes=528916120 weight_movement_bytes=69206016 total_nvlink_bytes=598122136 weight_to_token_ratio=0.130845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254950629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77976 payload_rows_remote=68926 dup_rows=20328 dup_rows_remote=18598 dedup_ratio=0.206787 router_tokens_per_expert=[619, 76, 505, 1221, 190, 1991, 672, 42, 90, 393, 579, 13, 139, 929, 118, 89, 63, 19, 8, 1124, 23, 2, 357, 1294, 1399, 3558, 1600, 197, 2574, 2570, 3158, 115, 40, 1892, 922, 258, 1875, 245, 42, 116, 126, 1943, 88, 6, 663, 0, 0, 0, 5594, 571, 33, 2418, 112, 39, 41, 203, 12, 3044, 211, 304, 2494, 2, 1, 130] payload_rows_per_dst=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] dup_rows_per_dst=[2150, 1884, 2218, 5596, 1730, 1778, 2968, 2004] combine_rows_per_src=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78038 payload_rows_remote=65178 dup_rows=20266 dup_rows_remote=14706 dedup_ratio=0.206156 router_tokens_per_expert=[434, 233, 228, 2496, 569, 1581, 932, 52, 30, 53, 584, 19, 53, 225, 56, 146, 60, 13, 3, 1248, 7, 6, 325, 1791, 1271, 3241, 912, 724, 1288, 3859, 2685, 275, 147, 1442, 573, 240, 3067, 257, 54, 132, 215, 1897, 42, 25, 1168, 0, 1, 0, 5161, 474, 86, 3168, 53, 18, 20, 163, 12, 2672, 149, 45, 2374, 7, 0, 91] payload_rows_per_dst=[12860, 9408, 12712, 8980, 9580, 8206, 7152, 9140] dup_rows_per_dst=[5560, 1982, 4516, 2104, 2244, 1488, 812, 1560] combine_rows_per_src=[12860, 9408, 12712, 8980, 9580, 8206, 7152, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268353536 dispatch_meta_bytes=327456 combine_payload_bytes=268353536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268680992 ingress_bytes=302956544 link_bytes=302956544 token_bytes=537034528 weight_movement_bytes=34603008 total_nvlink_bytes=571637536 weight_to_token_ratio=0.064433 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140001085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=7 export_slots=7 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79600 payload_rows_remote=69356 dup_rows=18704 dup_rows_remote=17184 dedup_ratio=0.190267 router_tokens_per_expert=[680, 128, 204, 540, 130, 1052, 763, 22, 13, 357, 640, 18, 68, 927, 60, 165, 16, 14, 1, 1019, 24, 8, 517, 1827, 1982, 3617, 1138, 283, 2204, 3620, 2824, 306, 48, 1807, 1104, 264, 2503, 284, 27, 73, 101, 2730, 20, 4, 622, 0, 0, 2, 4850, 481, 21, 2412, 28, 2, 38, 213, 16, 2660, 310, 131, 3079, 1, 0, 154] payload_rows_per_dst=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] dup_rows_per_dst=[2324, 1520, 4330, 2602, 2266, 2802, 554, 2306] combine_rows_per_src=[10362, 10244, 12222, 9190, 9954, 11396, 5836, 10396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271646720 dispatch_meta_bytes=345728 combine_payload_bytes=271646720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271992448 ingress_bytes=306249728 link_bytes=306249728 token_bytes=543639168 weight_movement_bytes=34603008 total_nvlink_bytes=578242176 weight_to_token_ratio=0.063651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152393071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77608 payload_rows_remote=64340 dup_rows=20696 dup_rows_remote=14702 dedup_ratio=0.210531 router_tokens_per_expert=[893, 82, 420, 1508, 245, 1659, 925, 23, 20, 714, 1040, 47, 45, 829, 55, 120, 125, 105, 4, 1520, 90, 6, 514, 1983, 1221, 3480, 1274, 672, 2389, 3027, 2638, 385, 74, 1800, 1125, 243, 1216, 218, 34, 98, 157, 1924, 32, 44, 730, 2, 27, 3, 5284, 564, 112, 1423, 70, 18, 44, 271, 12, 2938, 175, 394, 1941, 3, 2, 116] payload_rows_per_dst=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] dup_rows_per_dst=[4500, 1988, 5994, 2494, 1552, 2022, 484, 1662] combine_rows_per_src=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77976 payload_rows_remote=68926 dup_rows=20328 dup_rows_remote=18598 dedup_ratio=0.206787 router_tokens_per_expert=[619, 76, 505, 1221, 190, 1991, 672, 42, 90, 393, 579, 13, 139, 929, 118, 89, 63, 19, 8, 1124, 23, 2, 357, 1294, 1399, 3558, 1600, 197, 2574, 2570, 3158, 115, 40, 1892, 922, 258, 1875, 245, 42, 116, 126, 1943, 88, 6, 663, 0, 0, 0, 5594, 571, 33, 2418, 112, 39, 41, 203, 12, 3044, 211, 304, 2494, 2, 1, 130] payload_rows_per_dst=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] dup_rows_per_dst=[2150, 1884, 2218, 5596, 1730, 1778, 2968, 2004] combine_rows_per_src=[8482, 9932, 9318, 12490, 9050, 9014, 9298, 10392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=346160 combine_payload_bytes=284082176 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=284428336 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568510512 weight_movement_bytes=69206016 total_nvlink_bytes=637716528 weight_to_token_ratio=0.121732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281083904 dispatch_meta_bytes=331712 combine_payload_bytes=281083904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=281415616 ingress_bytes=419495936 link_bytes=419495936 token_bytes=562499520 weight_movement_bytes=138412032 total_nvlink_bytes=700911552 weight_to_token_ratio=0.246066 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578529435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=277528576 dispatch_meta_bytes=333784 combine_payload_bytes=277528576 prefetch_bytes=0 reduce_bytes=242221056 egress_bytes=277862360 ingress_bytes=519749632 link_bytes=519749632 token_bytes=555390936 weight_movement_bytes=242221056 total_nvlink_bytes=797611992 weight_to_token_ratio=0.436127 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.955776022e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79216 payload_rows_remote=70646 dup_rows=19088 dup_rows_remote=16910 dedup_ratio=0.194173 router_tokens_per_expert=[72, 776, 303, 281, 2194, 3843, 57, 9, 109, 1247, 205, 17, 218, 549, 2214, 57, 42, 49, 44, 4158, 0, 104, 279, 0, 306, 20, 147, 316, 5703, 508, 27, 2726, 61, 96, 73, 15, 410, 823, 0, 0, 1486, 1352, 2, 4806, 1289, 8, 24, 1213, 3, 4306, 28, 637, 34, 0, 22, 2183, 90, 23, 1254, 1932, 277, 6, 93, 26] payload_rows_per_dst=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] dup_rows_per_dst=[548, 1868, 4672, 702, 1550, 2178, 2552, 5018] combine_rows_per_src=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75296 payload_rows_remote=64744 dup_rows=23008 dup_rows_remote=21100 dedup_ratio=0.234049 router_tokens_per_expert=[4029, 344, 267, 564, 16, 257, 906, 1165, 29, 0, 0, 1711, 0, 1595, 2, 939, 70, 0, 1176, 30, 428, 52, 41, 268, 680, 121, 2930, 2, 6, 3605, 1057, 72, 103, 996, 42, 1623, 163, 830, 575, 0, 14, 4614, 26, 7, 4120, 140, 8, 1929, 146, 12, 7, 256, 485, 1176, 25, 3, 183, 1939, 234, 723, 0, 692, 4828, 891] payload_rows_per_dst=[6282, 7224, 3926, 12224, 9970, 12374, 10552, 12744] dup_rows_per_dst=[756, 1328, 204, 4722, 3300, 4554, 1908, 6236] combine_rows_per_src=[6282, 7224, 3926, 12224, 9970, 12374, 10552, 12744] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=350096 combine_payload_bytes=282320896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282670992 ingress_bytes=316923904 link_bytes=316923904 token_bytes=564991888 weight_movement_bytes=34603008 total_nvlink_bytes=599594896 weight_to_token_ratio=0.061245 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192559136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=319536 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288624 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534257712 weight_movement_bytes=34603008 total_nvlink_bytes=568860720 weight_to_token_ratio=0.064768 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=284082176 dispatch_meta_bytes=346160 combine_payload_bytes=284082176 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=284428336 ingress_bytes=353288192 link_bytes=353288192 token_bytes=568510512 weight_movement_bytes=69206016 total_nvlink_bytes=637716528 weight_to_token_ratio=0.121732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329395024e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263536640 dispatch_meta_bytes=316168 combine_payload_bytes=263536640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263852808 ingress_bytes=298139648 link_bytes=298139648 token_bytes=527389448 weight_movement_bytes=34603008 total_nvlink_bytes=561992456 weight_to_token_ratio=0.065612 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121875493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=289366016 dispatch_meta_bytes=350224 combine_payload_bytes=289366016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=289716240 ingress_bytes=393175040 link_bytes=393175040 token_bytes=579082256 weight_movement_bytes=103809024 total_nvlink_bytes=682891280 weight_to_token_ratio=0.179265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479486022e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76542 payload_rows_remote=65390 dup_rows=21762 dup_rows_remote=19288 dedup_ratio=0.221375 router_tokens_per_expert=[288, 658, 328, 288, 1619, 4132, 102, 6, 289, 1680, 556, 20, 597, 789, 1481, 109, 34, 13, 206, 2653, 0, 246, 366, 1, 333, 117, 114, 231, 5227, 471, 9, 2913, 37, 160, 123, 53, 598, 1004, 0, 0, 1380, 1623, 2, 5402, 1021, 21, 96, 1596, 1, 4062, 22, 601, 34, 1, 160, 1932, 62, 51, 891, 2078, 180, 35, 32, 18] payload_rows_per_dst=[6002, 8184, 11210, 6876, 8164, 11872, 11152, 13082] dup_rows_per_dst=[576, 2858, 4092, 698, 1372, 5626, 2474, 4066] combine_rows_per_src=[6002, 8184, 11210, 6876, 8164, 11872, 11152, 13082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77384 payload_rows_remote=65928 dup_rows=20920 dup_rows_remote=18366 dedup_ratio=0.212809 router_tokens_per_expert=[226, 139, 151, 1755, 150, 908, 1102, 9, 4, 348, 736, 71, 63, 701, 37, 269, 48, 8, 1, 619, 9, 6, 286, 1251, 1019, 4076, 468, 817, 2097, 4437, 3306, 173, 17, 1850, 785, 284, 2010, 147, 25, 94, 66, 1748, 41, 0, 712, 0, 0, 1, 5918, 455, 34, 2854, 55, 15, 6, 205, 4, 2739, 207, 84, 3426, 0, 0, 80] payload_rows_per_dst=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] dup_rows_per_dst=[1468, 1822, 158, 4842, 1504, 2554, 5738, 2834] combine_rows_per_src=[7412, 10788, 4298, 10918, 8920, 11456, 13346, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77608 payload_rows_remote=64340 dup_rows=20696 dup_rows_remote=14702 dedup_ratio=0.210531 router_tokens_per_expert=[893, 82, 420, 1508, 245, 1659, 925, 23, 20, 714, 1040, 47, 45, 829, 55, 120, 125, 105, 4, 1520, 90, 6, 514, 1983, 1221, 3480, 1274, 672, 2389, 3027, 2638, 385, 74, 1800, 1125, 243, 1216, 218, 34, 98, 157, 1924, 32, 44, 730, 2, 27, 3, 5284, 564, 112, 1423, 70, 18, 44, 271, 12, 2938, 175, 394, 1941, 3, 2, 116] payload_rows_per_dst=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] dup_rows_per_dst=[4500, 1988, 5994, 2494, 1552, 2022, 484, 1662] combine_rows_per_src=[12286, 10712, 13268, 9388, 8064, 9870, 4520, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343376 combine_payload_bytes=265191424 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=265534800 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530726224 weight_movement_bytes=69206016 total_nvlink_bytes=599932240 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71548 payload_rows_remote=57722 dup_rows=26756 dup_rows_remote=22126 dedup_ratio=0.272176 router_tokens_per_expert=[129, 590, 259, 330, 1841, 3899, 77, 5, 162, 723, 573, 12, 409, 386, 2062, 38, 108, 116, 136, 4285, 0, 305, 325, 0, 171, 144, 206, 333, 5908, 232, 32, 2176, 32, 155, 62, 13, 281, 711, 0, 0, 1163, 1460, 0, 5558, 1022, 11, 275, 1889, 0, 3673, 34, 509, 22, 0, 184, 2806, 25, 31, 650, 2272, 254, 9, 51, 28] payload_rows_per_dst=[5916, 7070, 12810, 2172, 2450, 15084, 12220, 13826] dup_rows_per_dst=[546, 1660, 5538, 64, 58, 10258, 4002, 4630] combine_rows_per_src=[5916, 7070, 12810, 2172, 2450, 15084, 12220, 13826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267837440 dispatch_meta_bytes=338712 combine_payload_bytes=267837440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268176152 ingress_bytes=337043456 link_bytes=337043456 token_bytes=536013592 weight_movement_bytes=69206016 total_nvlink_bytes=605219608 weight_to_token_ratio=0.129112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268267391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77282 payload_rows_remote=63606 dup_rows=21022 dup_rows_remote=14930 dedup_ratio=0.213847 router_tokens_per_expert=[81, 1110, 198, 297, 1992, 4457, 9, 3, 284, 1674, 656, 19, 253, 545, 2006, 168, 92, 27, 138, 2401, 0, 217, 712, 1, 261, 122, 105, 243, 5262, 330, 7, 3358, 9, 111, 50, 2, 823, 884, 0, 0, 1498, 1240, 2, 5012, 815, 51, 85, 998, 0, 4279, 26, 325, 227, 1, 118, 1979, 37, 84, 727, 2517, 184, 9, 25, 6] payload_rows_per_dst=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] dup_rows_per_dst=[5400, 6092, 502, 5350, 2392, 616, 168, 502] combine_rows_per_src=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78038 payload_rows_remote=65178 dup_rows=20266 dup_rows_remote=14706 dedup_ratio=0.206156 router_tokens_per_expert=[434, 233, 228, 2496, 569, 1581, 932, 52, 30, 53, 584, 19, 53, 225, 56, 146, 60, 13, 3, 1248, 7, 6, 325, 1791, 1271, 3241, 912, 724, 1288, 3859, 2685, 275, 147, 1442, 573, 240, 3067, 257, 54, 132, 215, 1897, 42, 25, 1168, 0, 1, 0, 5161, 474, 86, 3168, 53, 18, 20, 163, 12, 2672, 149, 45, 2374, 7, 0, 91] payload_rows_per_dst=[12860, 9408, 12712, 8980, 9580, 8206, 7152, 9140] dup_rows_per_dst=[5560, 1982, 4516, 2104, 2244, 1488, 812, 1560] combine_rows_per_src=[12860, 9408, 12712, 8980, 9580, 8206, 7152, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=270041088 dispatch_meta_bytes=337176 combine_payload_bytes=270041088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270378264 ingress_bytes=304644096 link_bytes=304644096 token_bytes=540419352 weight_movement_bytes=34603008 total_nvlink_bytes=575022360 weight_to_token_ratio=0.064030 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146351207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79182 payload_rows_remote=65848 dup_rows=19122 dup_rows_remote=13950 dedup_ratio=0.194519 router_tokens_per_expert=[184, 1095, 338, 281, 1101, 4578, 12, 26, 483, 723, 274, 9, 253, 724, 2338, 101, 22, 84, 146, 2050, 0, 323, 345, 6, 351, 187, 280, 280, 4945, 465, 109, 2636, 70, 183, 102, 59, 331, 835, 2, 3, 1849, 2251, 4, 4954, 1132, 14, 42, 1980, 0, 3921, 26, 654, 34, 0, 53, 1591, 115, 105, 1607, 2264, 122, 34, 27, 39] payload_rows_per_dst=[10090, 7766, 8028, 13334, 11184, 10290, 10700, 7790] dup_rows_per_dst=[2476, 2044, 1622, 5172, 1894, 3220, 1858, 836] combine_rows_per_src=[10090, 7766, 8028, 13334, 11184, 10290, 10700, 7790] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78648 payload_rows_remote=66872 dup_rows=19656 dup_rows_remote=16918 dedup_ratio=0.199951 router_tokens_per_expert=[258, 784, 455, 287, 1870, 4250, 33, 23, 121, 984, 587, 18, 476, 553, 1687, 142, 126, 25, 237, 2649, 0, 373, 167, 14, 414, 219, 130, 397, 5216, 597, 5, 2272, 23, 144, 94, 0, 982, 665, 1, 0, 2006, 1384, 1, 5348, 849, 47, 69, 1665, 1, 3831, 19, 786, 39, 4, 119, 2500, 64, 8, 816, 2009, 239, 10, 46, 14] payload_rows_per_dst=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] dup_rows_per_dst=[590, 1986, 3870, 1334, 2738, 2458, 3102, 3578] combine_rows_per_src=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77058 payload_rows_remote=64056 dup_rows=21246 dup_rows_remote=15960 dedup_ratio=0.216125 router_tokens_per_expert=[131, 803, 181, 279, 2535, 3767, 22, 2, 218, 1667, 1301, 41, 213, 301, 1913, 58, 36, 75, 191, 3370, 0, 274, 193, 3, 498, 202, 225, 119, 4935, 186, 5, 2696, 12, 112, 13, 52, 558, 2128, 0, 0, 1424, 964, 1, 5534, 760, 30, 22, 941, 0, 4034, 23, 664, 18, 0, 65, 1833, 103, 14, 792, 2361, 184, 4, 29, 37] payload_rows_per_dst=[13002, 13610, 7746, 13438, 12830, 4932, 4976, 6524] dup_rows_per_dst=[5286, 5882, 538, 4294, 3988, 504, 230, 524] combine_rows_per_src=[13002, 13610, 7746, 13438, 12830, 4932, 4976, 6524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263536640 dispatch_meta_bytes=316168 combine_payload_bytes=263536640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263852808 ingress_bytes=298139648 link_bytes=298139648 token_bytes=527389448 weight_movement_bytes=34603008 total_nvlink_bytes=561992456 weight_to_token_ratio=0.065612 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121875493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78648 payload_rows_remote=66872 dup_rows=19656 dup_rows_remote=16918 dedup_ratio=0.199951 router_tokens_per_expert=[258, 784, 455, 287, 1870, 4250, 33, 23, 121, 984, 587, 18, 476, 553, 1687, 142, 126, 25, 237, 2649, 0, 373, 167, 14, 414, 219, 130, 397, 5216, 597, 5, 2272, 23, 144, 94, 0, 982, 665, 1, 0, 2006, 1384, 1, 5348, 849, 47, 69, 1665, 1, 3831, 19, 786, 39, 4, 119, 2500, 64, 8, 816, 2009, 239, 10, 46, 14] payload_rows_per_dst=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] dup_rows_per_dst=[590, 1986, 3870, 1334, 2738, 2458, 3102, 3578] combine_rows_per_src=[6830, 7150, 11812, 8098, 11776, 9584, 11496, 11902] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=236429312 dispatch_meta_bytes=319392 combine_payload_bytes=236429312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=236748704 ingress_bytes=271032320 link_bytes=271032320 token_bytes=473178016 weight_movement_bytes=34603008 total_nvlink_bytes=507781024 weight_to_token_ratio=0.073129 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019872800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77282 payload_rows_remote=63606 dup_rows=21022 dup_rows_remote=14930 dedup_ratio=0.213847 router_tokens_per_expert=[81, 1110, 198, 297, 1992, 4457, 9, 3, 284, 1674, 656, 19, 253, 545, 2006, 168, 92, 27, 138, 2401, 0, 217, 712, 1, 261, 122, 105, 243, 5262, 330, 7, 3358, 9, 111, 50, 2, 823, 884, 0, 0, 1498, 1240, 2, 5012, 815, 51, 85, 998, 0, 4279, 26, 325, 227, 1, 118, 1979, 37, 84, 727, 2517, 184, 9, 25, 6] payload_rows_per_dst=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] dup_rows_per_dst=[5400, 6092, 502, 5350, 2392, 616, 168, 502] combine_rows_per_src=[13890, 13676, 6674, 14026, 11390, 5766, 5184, 6676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76346 payload_rows_remote=70090 dup_rows=21958 dup_rows_remote=21390 dedup_ratio=0.223368 router_tokens_per_expert=[109, 1083, 712, 498, 2366, 4362, 40, 13, 380, 1706, 629, 65, 322, 551, 1404, 281, 334, 70, 228, 2308, 18, 178, 276, 0, 337, 177, 105, 330, 3093, 455, 7, 3170, 24, 173, 54, 16, 1057, 1641, 2, 37, 975, 1083, 0, 4746, 1070, 6, 124, 1390, 0, 4287, 20, 1187, 372, 2, 52, 2447, 114, 30, 604, 1648, 307, 3, 61, 13] payload_rows_per_dst=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] dup_rows_per_dst=[6580, 2214, 568, 3678, 3698, 912, 3972, 336] combine_rows_per_src=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79006 payload_rows_remote=68672 dup_rows=19298 dup_rows_remote=17760 dedup_ratio=0.196309 router_tokens_per_expert=[2, 2236, 841, 4213, 110, 36, 340, 1, 92, 13, 79, 78, 370, 0, 1408, 41, 1280, 3, 568, 23, 653, 297, 60, 1024, 229, 2231, 14, 79, 53, 1366, 336, 457, 4405, 107, 224, 446, 380, 1, 884, 1827, 157, 7, 1062, 102, 0, 2, 3, 173, 451, 328, 5255, 1291, 6709, 0, 1020, 0, 509, 64, 0, 3659, 435, 41, 303, 774] payload_rows_per_dst=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] dup_rows_per_dst=[460, 2178, 980, 5246, 4572, 1538, 2194, 2130] combine_rows_per_src=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=260530176 dispatch_meta_bytes=314144 combine_payload_bytes=260530176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260844320 ingress_bytes=295133184 link_bytes=295133184 token_bytes=521374496 weight_movement_bytes=34603008 total_nvlink_bytes=555977504 weight_to_token_ratio=0.066369 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110562411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=319536 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288624 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534257712 weight_movement_bytes=34603008 total_nvlink_bytes=568860720 weight_to_token_ratio=0.064768 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=319192 combine_payload_bytes=269713408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270032600 ingress_bytes=373522432 link_bytes=373522432 token_bytes=539746008 weight_movement_bytes=103809024 total_nvlink_bytes=643555032 weight_to_token_ratio=0.192329 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405534841e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77184 payload_rows_remote=64526 dup_rows=21120 dup_rows_remote=15256 dedup_ratio=0.214844 router_tokens_per_expert=[542, 191, 296, 669, 174, 2034, 848, 14, 48, 295, 497, 4, 45, 1450, 64, 283, 46, 17, 4, 971, 5, 0, 443, 1888, 820, 4124, 651, 497, 2698, 2984, 3089, 327, 72, 1544, 952, 254, 2199, 617, 24, 120, 111, 2174, 80, 2, 501, 7, 0, 0, 4597, 351, 77, 3067, 70, 30, 22, 216, 14, 3017, 347, 257, 2303, 0, 0, 109] payload_rows_per_dst=[7988, 11092, 6438, 7780, 11594, 9692, 12658, 9942] dup_rows_per_dst=[1548, 2528, 310, 1942, 4750, 2026, 5864, 2152] combine_rows_per_src=[7988, 11092, 6438, 7780, 11594, 9692, 12658, 9942] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273907712 dispatch_meta_bytes=335160 combine_payload_bytes=273907712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274242872 ingress_bytes=308510720 link_bytes=308510720 token_bytes=548150584 weight_movement_bytes=34603008 total_nvlink_bytes=582753592 weight_to_token_ratio=0.063127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160901002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=320064 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262693440 ingress_bytes=331579392 link_bytes=331579392 token_bytes=525066816 weight_movement_bytes=69206016 total_nvlink_bytes=594272832 weight_to_token_ratio=0.131804 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247706558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79216 payload_rows_remote=70646 dup_rows=19088 dup_rows_remote=16910 dedup_ratio=0.194173 router_tokens_per_expert=[72, 776, 303, 281, 2194, 3843, 57, 9, 109, 1247, 205, 17, 218, 549, 2214, 57, 42, 49, 44, 4158, 0, 104, 279, 0, 306, 20, 147, 316, 5703, 508, 27, 2726, 61, 96, 73, 15, 410, 823, 0, 0, 1486, 1352, 2, 4806, 1289, 8, 24, 1213, 3, 4306, 28, 637, 34, 0, 22, 2183, 90, 23, 1254, 1932, 277, 6, 93, 26] payload_rows_per_dst=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] dup_rows_per_dst=[548, 1868, 4672, 702, 1550, 2178, 2552, 5018] combine_rows_per_src=[6836, 7364, 12366, 7398, 11018, 8570, 11874, 13790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264298496 dispatch_meta_bytes=319128 combine_payload_bytes=264298496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264617624 ingress_bytes=333504512 link_bytes=333504512 token_bytes=528916120 weight_movement_bytes=69206016 total_nvlink_bytes=598122136 weight_to_token_ratio=0.130845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254950629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76346 payload_rows_remote=70090 dup_rows=21958 dup_rows_remote=21390 dedup_ratio=0.223368 router_tokens_per_expert=[109, 1083, 712, 498, 2366, 4362, 40, 13, 380, 1706, 629, 65, 322, 551, 1404, 281, 334, 70, 228, 2308, 18, 178, 276, 0, 337, 177, 105, 330, 3093, 455, 7, 3170, 24, 173, 54, 16, 1057, 1641, 2, 37, 975, 1083, 0, 4746, 1070, 6, 124, 1390, 0, 4287, 20, 1187, 372, 2, 52, 2447, 114, 30, 604, 1648, 307, 3, 61, 13] payload_rows_per_dst=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] dup_rows_per_dst=[6580, 2214, 568, 3678, 3698, 912, 3972, 336] combine_rows_per_src=[13736, 8880, 6256, 11670, 11802, 6434, 12344, 5224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=260530176 dispatch_meta_bytes=314144 combine_payload_bytes=260530176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260844320 ingress_bytes=295133184 link_bytes=295133184 token_bytes=521374496 weight_movement_bytes=34603008 total_nvlink_bytes=555977504 weight_to_token_ratio=0.066369 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110562411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287088640 dispatch_meta_bytes=365920 combine_payload_bytes=287088640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287454560 ingress_bytes=321691648 link_bytes=321691648 token_bytes=574543200 weight_movement_bytes=34603008 total_nvlink_bytes=609146208 weight_to_token_ratio=0.060227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210499773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=281280512 dispatch_meta_bytes=345728 combine_payload_bytes=281280512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281626240 ingress_bytes=315883520 link_bytes=315883520 token_bytes=562906752 weight_movement_bytes=34603008 total_nvlink_bytes=597509760 weight_to_token_ratio=0.061472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188644255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72768 payload_rows_remote=57666 dup_rows=25536 dup_rows_remote=14762 dedup_ratio=0.259766 router_tokens_per_expert=[0, 1879, 776, 4577, 73, 36, 278, 18, 164, 6, 77, 114, 865, 0, 1986, 22, 1315, 10, 775, 26, 419, 1003, 106, 2374, 239, 1351, 7, 145, 402, 1800, 220, 371, 2994, 128, 78, 366, 300, 8, 833, 2151, 290, 14, 862, 151, 9, 4, 1, 172, 470, 131, 2979, 2011, 6503, 4, 932, 0, 418, 66, 0, 3534, 475, 89, 378, 1367] payload_rows_per_dst=[5710, 6248, 10012, 12636, 10220, 2884, 15102, 9956] dup_rows_per_dst=[410, 404, 2044, 5588, 3496, 122, 10774, 2698] combine_rows_per_src=[5710, 6248, 10012, 12636, 10220, 2884, 15102, 9956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=68174 dup_rows=18358 dup_rows_remote=16060 dedup_ratio=0.186747 router_tokens_per_expert=[6, 1832, 128, 4949, 151, 44, 198, 20, 214, 31, 236, 159, 934, 0, 1206, 110, 1059, 3, 391, 39, 377, 458, 48, 2837, 235, 1053, 6, 75, 149, 1366, 259, 350, 3230, 145, 242, 262, 295, 7, 2173, 2451, 104, 9, 1240, 336, 4, 7, 0, 142, 959, 187, 4145, 1527, 6795, 0, 626, 4, 910, 33, 0, 2517, 435, 137, 246, 1061] payload_rows_per_dst=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] dup_rows_per_dst=[2252, 2298, 4786, 978, 2280, 2880, 864, 2020] combine_rows_per_src=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73380 payload_rows_remote=62750 dup_rows=24924 dup_rows_remote=21070 dedup_ratio=0.253540 router_tokens_per_expert=[10, 3425, 595, 4543, 183, 101, 267, 13, 146, 12, 85, 113, 1098, 0, 1466, 16, 934, 0, 701, 58, 658, 338, 116, 2089, 681, 597, 49, 60, 61, 1911, 364, 423, 4296, 187, 213, 455, 457, 4, 1035, 737, 130, 3, 1312, 125, 6, 13, 7, 267, 333, 669, 3689, 1804, 5786, 1, 1208, 7, 387, 23, 4, 2394, 425, 84, 496, 1482] payload_rows_per_dst=[8386, 5506, 8496, 12830, 8774, 3574, 15184, 10630] dup_rows_per_dst=[802, 366, 1292, 4548, 2100, 152, 11810, 3854] combine_rows_per_src=[8386, 5506, 8496, 12830, 8774, 3574, 15184, 10630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=236199936 dispatch_meta_bytes=289712 combine_payload_bytes=236199936 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=236489648 ingress_bytes=340008960 link_bytes=340008960 token_bytes=472689584 weight_movement_bytes=103809024 total_nvlink_bytes=576498608 weight_to_token_ratio=0.219614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77058 payload_rows_remote=64056 dup_rows=21246 dup_rows_remote=15960 dedup_ratio=0.216125 router_tokens_per_expert=[131, 803, 181, 279, 2535, 3767, 22, 2, 218, 1667, 1301, 41, 213, 301, 1913, 58, 36, 75, 191, 3370, 0, 274, 193, 3, 498, 202, 225, 119, 4935, 186, 5, 2696, 12, 112, 13, 52, 558, 2128, 0, 0, 1424, 964, 1, 5534, 760, 30, 22, 941, 0, 4034, 23, 664, 18, 0, 65, 1833, 103, 14, 792, 2361, 184, 4, 29, 37] payload_rows_per_dst=[13002, 13610, 7746, 13438, 12830, 4932, 4976, 6524] dup_rows_per_dst=[5286, 5882, 538, 4294, 3988, 504, 230, 524] combine_rows_per_src=[13002, 13610, 7746, 13438, 12830, 4932, 4976, 6524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=289366016 dispatch_meta_bytes=350224 combine_payload_bytes=289366016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=289716240 ingress_bytes=393175040 link_bytes=393175040 token_bytes=579082256 weight_movement_bytes=103809024 total_nvlink_bytes=682891280 weight_to_token_ratio=0.179265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479486022e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287088640 dispatch_meta_bytes=365920 combine_payload_bytes=287088640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287454560 ingress_bytes=321691648 link_bytes=321691648 token_bytes=574543200 weight_movement_bytes=34603008 total_nvlink_bytes=609146208 weight_to_token_ratio=0.060227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210499773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80598 payload_rows_remote=73222 dup_rows=17706 dup_rows_remote=16060 dedup_ratio=0.180115 router_tokens_per_expert=[3, 2646, 494, 4233, 119, 48, 237, 16, 88, 8, 120, 17, 337, 4, 1271, 54, 658, 0, 444, 15, 540, 441, 34, 1928, 223, 1752, 17, 142, 45, 1623, 388, 321, 3595, 180, 276, 448, 589, 1, 471, 1195, 371, 14, 632, 348, 6, 49, 2, 160, 288, 250, 4015, 1578, 7045, 30, 1812, 0, 996, 51, 0, 4278, 603, 135, 386, 1082] payload_rows_per_dst=[12478, 10380, 8838, 7376, 8888, 14626, 6692, 11320] dup_rows_per_dst=[3114, 1448, 1740, 1646, 2164, 2628, 1224, 3742] combine_rows_per_src=[12478, 10380, 8838, 7376, 8888, 14626, 6692, 11320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=279240704 dispatch_meta_bytes=336936 combine_payload_bytes=279240704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279577640 ingress_bytes=313843712 link_bytes=313843712 token_bytes=558818344 weight_movement_bytes=34603008 total_nvlink_bytes=593421352 weight_to_token_ratio=0.061922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180968622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78408 payload_rows_remote=66828 dup_rows=19896 dup_rows_remote=15450 dedup_ratio=0.202393 router_tokens_per_expert=[9, 2152, 548, 3778, 98, 136, 72, 4, 136, 6, 327, 63, 526, 15, 2180, 12, 1950, 2, 775, 114, 432, 479, 21, 2515, 323, 913, 12, 69, 431, 2162, 133, 640, 3986, 99, 198, 228, 248, 14, 1485, 1755, 137, 13, 1407, 118, 6, 26, 29, 152, 170, 114, 3965, 1116, 5875, 1, 1282, 0, 463, 74, 0, 2742, 421, 138, 148, 1709] payload_rows_per_dst=[8718, 11588, 9998, 9674, 11580, 12910, 4662, 9278] dup_rows_per_dst=[1358, 2872, 2578, 3210, 4446, 2616, 704, 2112] combine_rows_per_src=[8718, 11588, 9998, 9674, 11580, 12910, 4662, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=62544 dup_rows=21632 dup_rows_remote=13186 dedup_ratio=0.220052 router_tokens_per_expert=[5, 3637, 788, 4335, 116, 53, 79, 21, 161, 12, 105, 54, 571, 0, 2142, 91, 666, 0, 512, 78, 335, 353, 24, 2572, 502, 921, 21, 54, 141, 1561, 314, 806, 2775, 122, 84, 344, 345, 1, 904, 1451, 110, 5, 1189, 239, 2, 34, 16, 97, 256, 221, 4691, 1841, 6054, 3, 1199, 3, 559, 164, 1, 3478, 477, 217, 156, 1084] payload_rows_per_dst=[14128, 12322, 8622, 7154, 6698, 11922, 6132, 9694] dup_rows_per_dst=[8446, 3332, 1502, 1486, 1340, 2034, 914, 2578] combine_rows_per_src=[14128, 12322, 8622, 7154, 6698, 11922, 6132, 9694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=335280 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257359280 ingress_bytes=291627008 link_bytes=291627008 token_bytes=514383280 weight_movement_bytes=34603008 total_nvlink_bytes=548986288 weight_to_token_ratio=0.067271 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097368953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=320064 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262693440 ingress_bytes=331579392 link_bytes=331579392 token_bytes=525066816 weight_movement_bytes=69206016 total_nvlink_bytes=594272832 weight_to_token_ratio=0.131804 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247706558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299917312 dispatch_meta_bytes=357128 combine_payload_bytes=299917312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300274440 ingress_bytes=334520320 link_bytes=334520320 token_bytes=600191752 weight_movement_bytes=34603008 total_nvlink_bytes=634794760 weight_to_token_ratio=0.057653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=68174 dup_rows=18358 dup_rows_remote=16060 dedup_ratio=0.186747 router_tokens_per_expert=[6, 1832, 128, 4949, 151, 44, 198, 20, 214, 31, 236, 159, 934, 0, 1206, 110, 1059, 3, 391, 39, 377, 458, 48, 2837, 235, 1053, 6, 75, 149, 1366, 259, 350, 3230, 145, 242, 262, 295, 7, 2173, 2451, 104, 9, 1240, 336, 4, 7, 0, 142, 959, 187, 4145, 1527, 6795, 0, 626, 4, 910, 33, 0, 2517, 435, 137, 246, 1061] payload_rows_per_dst=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] dup_rows_per_dst=[2252, 2298, 4786, 978, 2280, 2880, 864, 2020] combine_rows_per_src=[12404, 11772, 12098, 6008, 8870, 14394, 5742, 8658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78690 payload_rows_remote=66246 dup_rows=19614 dup_rows_remote=13576 dedup_ratio=0.199524 router_tokens_per_expert=[40, 2359, 259, 3350, 151, 114, 176, 34, 165, 13, 1074, 94, 395, 4, 2073, 90, 585, 33, 688, 60, 843, 773, 59, 2500, 401, 639, 45, 50, 182, 1418, 413, 459, 3700, 280, 221, 390, 434, 2, 1705, 2418, 114, 10, 501, 323, 6, 14, 17, 413, 823, 271, 3640, 1764, 6447, 5, 1017, 1, 531, 52, 0, 2379, 382, 158, 149, 1446] payload_rows_per_dst=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] dup_rows_per_dst=[1948, 3220, 6038, 1054, 2346, 2026, 1100, 1882] combine_rows_per_src=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=60460 dup_rows=23094 dup_rows_remote=13490 dedup_ratio=0.234924 router_tokens_per_expert=[1259, 43, 1, 0, 46, 806, 0, 57, 2355, 919, 2, 0, 748, 3360, 352, 612, 2259, 3663, 0, 250, 2086, 1895, 47, 379, 45, 0, 5965, 77, 121, 361, 29, 1169, 335, 273, 457, 59, 16, 0, 1181, 19, 997, 424, 97, 53, 15, 5622, 68, 6, 4, 2246, 0, 29, 565, 1259, 1714, 0, 28, 108, 4300, 122, 54, 170, 0, 25] payload_rows_per_dst=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] dup_rows_per_dst=[1436, 1616, 3436, 432, 482, 9604, 2114, 3974] combine_rows_per_src=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76542 payload_rows_remote=65390 dup_rows=21762 dup_rows_remote=19288 dedup_ratio=0.221375 router_tokens_per_expert=[288, 658, 328, 288, 1619, 4132, 102, 6, 289, 1680, 556, 20, 597, 789, 1481, 109, 34, 13, 206, 2653, 0, 246, 366, 1, 333, 117, 114, 231, 5227, 471, 9, 2913, 37, 160, 123, 53, 598, 1004, 0, 0, 1380, 1623, 2, 5402, 1021, 21, 96, 1596, 1, 4062, 22, 601, 34, 1, 160, 1932, 62, 51, 891, 2078, 180, 35, 32, 18] payload_rows_per_dst=[6002, 8184, 11210, 6876, 8164, 11872, 11152, 13082] dup_rows_per_dst=[576, 2858, 4092, 698, 1372, 5626, 2474, 4066] combine_rows_per_src=[6002, 8184, 11210, 6876, 8164, 11872, 11152, 13082] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=273727488 dispatch_meta_bytes=329112 combine_payload_bytes=273727488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274056600 ingress_bytes=412139520 link_bytes=412139520 token_bytes=547784088 weight_movement_bytes=138412032 total_nvlink_bytes=686196120 weight_to_token_ratio=0.252676 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.550847834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=256180224 dispatch_meta_bytes=302920 combine_payload_bytes=256180224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256483144 ingress_bytes=325386240 link_bytes=325386240 token_bytes=512663368 weight_movement_bytes=69206016 total_nvlink_bytes=581869384 weight_to_token_ratio=0.134993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224402225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267837440 dispatch_meta_bytes=338712 combine_payload_bytes=267837440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268176152 ingress_bytes=337043456 link_bytes=337043456 token_bytes=536013592 weight_movement_bytes=69206016 total_nvlink_bytes=605219608 weight_to_token_ratio=0.129112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268267391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79006 payload_rows_remote=68672 dup_rows=19298 dup_rows_remote=17760 dedup_ratio=0.196309 router_tokens_per_expert=[2, 2236, 841, 4213, 110, 36, 340, 1, 92, 13, 79, 78, 370, 0, 1408, 41, 1280, 3, 568, 23, 653, 297, 60, 1024, 229, 2231, 14, 79, 53, 1366, 336, 457, 4405, 107, 224, 446, 380, 1, 884, 1827, 157, 7, 1062, 102, 0, 2, 3, 173, 451, 328, 5255, 1291, 6709, 0, 1020, 0, 509, 64, 0, 3659, 435, 41, 303, 774] payload_rows_per_dst=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] dup_rows_per_dst=[460, 2178, 980, 5246, 4572, 1538, 2194, 2130] combine_rows_per_src=[6672, 12494, 6836, 12710, 11976, 10334, 8544, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78690 payload_rows_remote=66246 dup_rows=19614 dup_rows_remote=13576 dedup_ratio=0.199524 router_tokens_per_expert=[40, 2359, 259, 3350, 151, 114, 176, 34, 165, 13, 1074, 94, 395, 4, 2073, 90, 585, 33, 688, 60, 843, 773, 59, 2500, 401, 639, 45, 50, 182, 1418, 413, 459, 3700, 280, 221, 390, 434, 2, 1705, 2418, 114, 10, 501, 323, 6, 14, 17, 413, 823, 271, 3640, 1764, 6447, 5, 1017, 1, 531, 52, 0, 2379, 382, 158, 149, 1446] payload_rows_per_dst=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] dup_rows_per_dst=[1948, 3220, 6038, 1054, 2346, 2026, 1100, 1882] combine_rows_per_src=[11018, 11876, 12444, 6160, 8554, 13664, 6662, 8312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80642 payload_rows_remote=68814 dup_rows=17662 dup_rows_remote=12150 dedup_ratio=0.179667 router_tokens_per_expert=[1054, 385, 7, 1, 48, 614, 0, 23, 2368, 1129, 23, 0, 650, 3204, 732, 564, 2116, 3702, 0, 269, 997, 1400, 70, 952, 94, 2, 5854, 55, 307, 474, 18, 1340, 530, 202, 377, 25, 59, 0, 1172, 7, 1344, 379, 108, 59, 26, 6805, 206, 45, 119, 1588, 0, 68, 82, 1053, 1766, 0, 58, 93, 3977, 148, 129, 256, 4, 15] payload_rows_per_dst=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] dup_rows_per_dst=[1542, 5512, 2394, 2624, 3614, 212, 1310, 454] combine_rows_per_src=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=279240704 dispatch_meta_bytes=336936 combine_payload_bytes=279240704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279577640 ingress_bytes=313843712 link_bytes=313843712 token_bytes=558818344 weight_movement_bytes=34603008 total_nvlink_bytes=593421352 weight_to_token_ratio=0.061922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180968622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73568 payload_rows_remote=62770 dup_rows=24736 dup_rows_remote=21092 dedup_ratio=0.251628 router_tokens_per_expert=[842, 280, 0, 4, 64, 949, 0, 14, 2698, 1481, 30, 0, 1161, 2932, 776, 652, 2733, 3392, 0, 282, 943, 1164, 49, 282, 212, 0, 6032, 112, 112, 545, 59, 1068, 517, 161, 273, 174, 72, 0, 789, 36, 1107, 561, 531, 38, 19, 5528, 70, 22, 56, 1809, 1, 74, 514, 1034, 2304, 5, 21, 90, 3939, 132, 88, 307, 12, 0] payload_rows_per_dst=[9558, 9028, 9818, 4044, 3596, 14924, 10798, 11802] dup_rows_per_dst=[1532, 2794, 2862, 172, 448, 10044, 3644, 3240] combine_rows_per_src=[9558, 9028, 9818, 4044, 3596, 14924, 10798, 11802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=271343616 dispatch_meta_bytes=319288 combine_payload_bytes=271343616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271662904 ingress_bytes=305946624 link_bytes=305946624 token_bytes=543006520 weight_movement_bytes=34603008 total_nvlink_bytes=577609528 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151252515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=247644160 dispatch_meta_bytes=295800 combine_payload_bytes=247644160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=247939960 ingress_bytes=316850176 link_bytes=316850176 token_bytes=495584120 weight_movement_bytes=69206016 total_nvlink_bytes=564790136 weight_to_token_ratio=0.139645 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192281704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75970 payload_rows_remote=64056 dup_rows=22334 dup_rows_remote=18668 dedup_ratio=0.227193 router_tokens_per_expert=[877, 36, 9, 4, 61, 480, 0, 29, 2791, 1056, 2, 0, 676, 3229, 763, 474, 1211, 3948, 0, 419, 1226, 1321, 135, 501, 197, 0, 4881, 186, 261, 237, 95, 968, 389, 495, 463, 177, 87, 0, 1714, 43, 1349, 1023, 179, 44, 1, 5318, 833, 57, 11, 2004, 1, 118, 360, 1290, 2562, 0, 58, 162, 3850, 179, 62, 224, 2, 24] payload_rows_per_dst=[9894, 5424, 8834, 6908, 5912, 12882, 14202, 11914] dup_rows_per_dst=[994, 518, 2340, 1014, 824, 4726, 8252, 3666] combine_rows_per_src=[9894, 5424, 8834, 6908, 5912, 12882, 14202, 11914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=271343616 dispatch_meta_bytes=319288 combine_payload_bytes=271343616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271662904 ingress_bytes=305946624 link_bytes=305946624 token_bytes=543006520 weight_movement_bytes=34603008 total_nvlink_bytes=577609528 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151252515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=281280512 dispatch_meta_bytes=345728 combine_payload_bytes=281280512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281626240 ingress_bytes=315883520 link_bytes=315883520 token_bytes=562906752 weight_movement_bytes=34603008 total_nvlink_bytes=597509760 weight_to_token_ratio=0.061472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188644255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=62544 dup_rows=21632 dup_rows_remote=13186 dedup_ratio=0.220052 router_tokens_per_expert=[5, 3637, 788, 4335, 116, 53, 79, 21, 161, 12, 105, 54, 571, 0, 2142, 91, 666, 0, 512, 78, 335, 353, 24, 2572, 502, 921, 21, 54, 141, 1561, 314, 806, 2775, 122, 84, 344, 345, 1, 904, 1451, 110, 5, 1189, 239, 2, 34, 16, 97, 256, 221, 4691, 1841, 6054, 3, 1199, 3, 559, 164, 1, 3478, 477, 217, 156, 1084] payload_rows_per_dst=[14128, 12322, 8622, 7154, 6698, 11922, 6132, 9694] dup_rows_per_dst=[8446, 3332, 1502, 1486, 1340, 2034, 914, 2578] combine_rows_per_src=[14128, 12322, 8622, 7154, 6698, 11922, 6132, 9694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81030 payload_rows_remote=67560 dup_rows=17274 dup_rows_remote=14252 dedup_ratio=0.175720 router_tokens_per_expert=[1764, 145, 9, 1, 22, 783, 0, 26, 1980, 880, 51, 0, 617, 2407, 542, 677, 1407, 2920, 0, 844, 2204, 2023, 47, 519, 78, 0, 5573, 55, 116, 685, 81, 1658, 230, 133, 595, 144, 139, 0, 814, 29, 1894, 805, 93, 164, 37, 5885, 33, 26, 18, 1980, 2, 106, 348, 786, 1881, 1, 61, 61, 4209, 112, 128, 306, 10, 8] payload_rows_per_dst=[9816, 10598, 10206, 13470, 13296, 5548, 8744, 9352] dup_rows_per_dst=[1524, 3710, 3882, 3022, 2642, 556, 1500, 438] combine_rows_per_src=[9816, 10598, 10206, 13470, 13296, 5548, 8744, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=335448 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257441368 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514547288 weight_movement_bytes=34603008 total_nvlink_bytes=549150296 weight_to_token_ratio=0.067249 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81392 payload_rows_remote=70370 dup_rows=16912 dup_rows_remote=14210 dedup_ratio=0.172038 router_tokens_per_expert=[724, 219, 32, 2, 53, 616, 0, 21, 2378, 880, 14, 0, 597, 2789, 772, 212, 2664, 5573, 0, 378, 1701, 936, 68, 430, 166, 0, 6110, 117, 64, 535, 57, 971, 550, 512, 221, 74, 20, 1, 737, 93, 1574, 579, 347, 54, 42, 5976, 109, 26, 164, 1907, 0, 47, 476, 1034, 1382, 0, 42, 78, 3577, 124, 41, 245, 37, 4] payload_rows_per_dst=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] dup_rows_per_dst=[1734, 3034, 2486, 2144, 2702, 1248, 1910, 1654] combine_rows_per_src=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=281862144 dispatch_meta_bytes=323856 combine_payload_bytes=281862144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=282186000 ingress_bytes=385671168 link_bytes=385671168 token_bytes=564048144 weight_movement_bytes=103809024 total_nvlink_bytes=667857168 weight_to_token_ratio=0.184043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451249556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81392 payload_rows_remote=70370 dup_rows=16912 dup_rows_remote=14210 dedup_ratio=0.172038 router_tokens_per_expert=[724, 219, 32, 2, 53, 616, 0, 21, 2378, 880, 14, 0, 597, 2789, 772, 212, 2664, 5573, 0, 378, 1701, 936, 68, 430, 166, 0, 6110, 117, 64, 535, 57, 971, 550, 512, 221, 74, 20, 1, 737, 93, 1574, 579, 347, 54, 42, 5976, 109, 26, 164, 1907, 0, 47, 476, 1034, 1382, 0, 42, 78, 3577, 124, 41, 245, 37, 4] payload_rows_per_dst=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] dup_rows_per_dst=[1734, 3034, 2486, 2144, 2702, 1248, 1910, 1654] combine_rows_per_src=[12746, 9656, 9868, 13896, 11022, 6858, 8110, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72768 payload_rows_remote=57666 dup_rows=25536 dup_rows_remote=14762 dedup_ratio=0.259766 router_tokens_per_expert=[0, 1879, 776, 4577, 73, 36, 278, 18, 164, 6, 77, 114, 865, 0, 1986, 22, 1315, 10, 775, 26, 419, 1003, 106, 2374, 239, 1351, 7, 145, 402, 1800, 220, 371, 2994, 128, 78, 366, 300, 8, 833, 2151, 290, 14, 862, 151, 9, 4, 1, 172, 470, 131, 2979, 2011, 6503, 4, 932, 0, 418, 66, 0, 3534, 475, 89, 378, 1367] payload_rows_per_dst=[5710, 6248, 10012, 12636, 10220, 2884, 15102, 9956] dup_rows_per_dst=[410, 404, 2044, 5588, 3496, 122, 10774, 2698] combine_rows_per_src=[5710, 6248, 10012, 12636, 10220, 2884, 15102, 9956] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=262373376 dispatch_meta_bytes=330896 combine_payload_bytes=262373376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262704272 ingress_bytes=296976384 link_bytes=296976384 token_bytes=525077648 weight_movement_bytes=34603008 total_nvlink_bytes=559680656 weight_to_token_ratio=0.065901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117498224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82602 payload_rows_remote=72026 dup_rows=15702 dup_rows_remote=13306 dedup_ratio=0.159729 router_tokens_per_expert=[951, 98, 11, 0, 4, 965, 0, 29, 890, 623, 15, 0, 462, 2507, 741, 275, 3362, 3604, 0, 483, 1091, 990, 55, 399, 131, 0, 6397, 131, 159, 550, 40, 1066, 117, 360, 584, 29, 39, 0, 1434, 37, 2315, 387, 228, 109, 29, 6162, 352, 26, 406, 2087, 1, 40, 63, 1045, 1486, 0, 28, 143, 4917, 146, 234, 300, 10, 9] payload_rows_per_dst=[10576, 8762, 8940, 14520, 14172, 6356, 8420, 10856] dup_rows_per_dst=[2396, 2264, 2172, 2428, 3352, 536, 1836, 718] combine_rows_per_src=[10576, 8762, 8940, 14520, 14172, 6356, 8420, 10856] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276725760 dispatch_meta_bytes=327248 combine_payload_bytes=276725760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277053008 ingress_bytes=380534784 link_bytes=380534784 token_bytes=553778768 weight_movement_bytes=103809024 total_nvlink_bytes=657587792 weight_to_token_ratio=0.187456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431921757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80642 payload_rows_remote=68814 dup_rows=17662 dup_rows_remote=12150 dedup_ratio=0.179667 router_tokens_per_expert=[1054, 385, 7, 1, 48, 614, 0, 23, 2368, 1129, 23, 0, 650, 3204, 732, 564, 2116, 3702, 0, 269, 997, 1400, 70, 952, 94, 2, 5854, 55, 307, 474, 18, 1340, 530, 202, 377, 25, 59, 0, 1172, 7, 1344, 379, 108, 59, 26, 6805, 206, 45, 119, 1588, 0, 68, 82, 1053, 1766, 0, 58, 93, 3977, 148, 129, 256, 4, 15] payload_rows_per_dst=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] dup_rows_per_dst=[1542, 5512, 2394, 2624, 3614, 212, 1310, 454] combine_rows_per_src=[10126, 11828, 9214, 13664, 14740, 4122, 8042, 8906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82368 payload_rows_remote=73224 dup_rows=15936 dup_rows_remote=13624 dedup_ratio=0.162109 router_tokens_per_expert=[955, 226, 44, 4, 83, 864, 0, 10, 1723, 616, 160, 3, 564, 2759, 938, 837, 2265, 4872, 25, 310, 1348, 723, 53, 1004, 105, 14, 5828, 155, 200, 230, 34, 761, 314, 210, 77, 252, 47, 12, 1198, 33, 1993, 452, 86, 47, 50, 6509, 266, 75, 217, 1493, 4, 21, 66, 747, 2400, 3, 103, 140, 3896, 200, 171, 311, 20, 26] payload_rows_per_dst=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] dup_rows_per_dst=[2044, 4306, 2312, 1532, 3068, 386, 1608, 680] combine_rows_per_src=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78878 payload_rows_remote=71336 dup_rows=19426 dup_rows_remote=18944 dedup_ratio=0.197611 router_tokens_per_expert=[21, 9, 15, 0, 1, 3371, 617, 966, 129, 0, 244, 75, 1, 33, 1, 0, 1153, 77, 969, 2047, 6520, 377, 80, 193, 1896, 0, 113, 1214, 1, 4207, 59, 449, 6, 416, 1074, 0, 0, 1232, 48, 0, 28, 14, 3031, 83, 0, 151, 705, 0, 890, 442, 4681, 445, 5419, 164, 114, 624, 3450, 33, 0, 169, 1012, 1, 81, 1] payload_rows_per_dst=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] dup_rows_per_dst=[1504, 644, 1214, 2484, 762, 482, 10094, 2242] combine_rows_per_src=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=256180224 dispatch_meta_bytes=302920 combine_payload_bytes=256180224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256483144 ingress_bytes=325386240 link_bytes=325386240 token_bytes=512663368 weight_movement_bytes=69206016 total_nvlink_bytes=581869384 weight_to_token_ratio=0.134993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224402225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288235520 dispatch_meta_bytes=338320 combine_payload_bytes=288235520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288573840 ingress_bytes=322838528 link_bytes=322838528 token_bytes=576809360 weight_movement_bytes=34603008 total_nvlink_bytes=611412368 weight_to_token_ratio=0.059990 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214815390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=295018496 dispatch_meta_bytes=341328 combine_payload_bytes=295018496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=295359824 ingress_bytes=364224512 link_bytes=364224512 token_bytes=590378320 weight_movement_bytes=69206016 total_nvlink_bytes=659584336 weight_to_token_ratio=0.117223 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370547516e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=236199936 dispatch_meta_bytes=289712 combine_payload_bytes=236199936 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=236489648 ingress_bytes=340008960 link_bytes=340008960 token_bytes=472689584 weight_movement_bytes=103809024 total_nvlink_bytes=576498608 weight_to_token_ratio=0.219614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=60460 dup_rows=23094 dup_rows_remote=13490 dedup_ratio=0.234924 router_tokens_per_expert=[1259, 43, 1, 0, 46, 806, 0, 57, 2355, 919, 2, 0, 748, 3360, 352, 612, 2259, 3663, 0, 250, 2086, 1895, 47, 379, 45, 0, 5965, 77, 121, 361, 29, 1169, 335, 273, 457, 59, 16, 0, 1181, 19, 997, 424, 97, 53, 15, 5622, 68, 6, 4, 2246, 0, 29, 565, 1259, 1714, 0, 28, 108, 4300, 122, 54, 170, 0, 25] payload_rows_per_dst=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] dup_rows_per_dst=[1436, 1616, 3436, 432, 482, 9604, 2114, 3974] combine_rows_per_src=[10314, 8360, 10396, 5312, 4198, 14750, 9520, 12360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82368 payload_rows_remote=73224 dup_rows=15936 dup_rows_remote=13624 dedup_ratio=0.162109 router_tokens_per_expert=[955, 226, 44, 4, 83, 864, 0, 10, 1723, 616, 160, 3, 564, 2759, 938, 837, 2265, 4872, 25, 310, 1348, 723, 53, 1004, 105, 14, 5828, 155, 200, 230, 34, 761, 314, 210, 77, 252, 47, 12, 1198, 33, 1993, 452, 86, 47, 50, 6509, 266, 75, 217, 1493, 4, 21, 66, 747, 2400, 3, 103, 140, 3896, 200, 171, 311, 20, 26] payload_rows_per_dst=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] dup_rows_per_dst=[2044, 4306, 2312, 1532, 3068, 386, 1608, 680] combine_rows_per_src=[12072, 10894, 9144, 13122, 14236, 5552, 8294, 9054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80530 payload_rows_remote=67778 dup_rows=17774 dup_rows_remote=17402 dedup_ratio=0.180806 router_tokens_per_expert=[26, 28, 3, 0, 0, 2841, 1178, 1398, 33, 7, 136, 94, 1, 47, 20, 1, 1024, 24, 1238, 3015, 6223, 678, 127, 63, 1752, 0, 75, 1077, 1, 4225, 33, 922, 12, 685, 1549, 1, 1, 567, 57, 0, 82, 12, 2627, 159, 0, 67, 1466, 0, 641, 337, 4552, 363, 4520, 184, 27, 389, 2786, 7, 0, 145, 1552, 11, 62, 1] payload_rows_per_dst=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] dup_rows_per_dst=[4824, 372, 2166, 1706, 2998, 4444, 416, 848] combine_rows_per_src=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=281862144 dispatch_meta_bytes=323856 combine_payload_bytes=281862144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=282186000 ingress_bytes=385671168 link_bytes=385671168 token_bytes=564048144 weight_movement_bytes=103809024 total_nvlink_bytes=667857168 weight_to_token_ratio=0.184043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451249556e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78520 payload_rows_remote=63554 dup_rows=19784 dup_rows_remote=12116 dedup_ratio=0.201253 router_tokens_per_expert=[78, 17, 12, 2, 6, 2481, 989, 640, 84, 3, 169, 77, 1, 265, 8, 0, 977, 17, 968, 2759, 6536, 257, 61, 32, 2579, 1, 86, 1060, 1, 4924, 48, 574, 6, 477, 1945, 2, 1, 1840, 119, 0, 88, 15, 1635, 35, 1, 38, 1055, 2, 851, 229, 4844, 557, 4140, 210, 62, 424, 3475, 8, 0, 194, 1129, 0, 58, 0] payload_rows_per_dst=[7198, 13516, 7468, 9006, 7278, 5502, 14966, 13586] dup_rows_per_dst=[1252, 770, 900, 1466, 1502, 236, 7668, 5990] combine_rows_per_src=[7198, 13516, 7468, 9006, 7278, 5502, 14966, 13586] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299925504 dispatch_meta_bytes=347392 combine_payload_bytes=299925504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300272896 ingress_bytes=403734528 link_bytes=403734528 token_bytes=600198400 weight_movement_bytes=103809024 total_nvlink_bytes=704007424 weight_to_token_ratio=0.172958 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519220526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=292192256 dispatch_meta_bytes=361120 combine_payload_bytes=292192256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292553376 ingress_bytes=326795264 link_bytes=326795264 token_bytes=584745632 weight_movement_bytes=34603008 total_nvlink_bytes=619348640 weight_to_token_ratio=0.059176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229704269e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=247644160 dispatch_meta_bytes=295800 combine_payload_bytes=247644160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=247939960 ingress_bytes=316850176 link_bytes=316850176 token_bytes=495584120 weight_movement_bytes=69206016 total_nvlink_bytes=564790136 weight_to_token_ratio=0.139645 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192281704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76162 payload_rows_remote=63596 dup_rows=22142 dup_rows_remote=17754 dedup_ratio=0.225240 router_tokens_per_expert=[123, 5, 42, 4, 1, 2698, 629, 1981, 52, 4, 252, 259, 1, 29, 44, 2, 1729, 29, 1003, 3130, 5800, 781, 323, 72, 957, 1, 188, 1253, 17, 3192, 71, 828, 12, 741, 1542, 13, 0, 419, 248, 1, 279, 10, 2705, 236, 3, 123, 1048, 0, 621, 71, 4564, 292, 4434, 234, 314, 457, 3585, 17, 1, 358, 1176, 3, 144, 1] payload_rows_per_dst=[9122, 4574, 12010, 10466, 5126, 7538, 14760, 12566] dup_rows_per_dst=[1844, 170, 4006, 2424, 826, 1270, 7214, 4388] combine_rows_per_src=[9122, 4574, 12010, 10466, 5126, 7538, 14760, 12566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299925504 dispatch_meta_bytes=347392 combine_payload_bytes=299925504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300272896 ingress_bytes=403734528 link_bytes=403734528 token_bytes=600198400 weight_movement_bytes=103809024 total_nvlink_bytes=704007424 weight_to_token_ratio=0.172958 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519220526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82602 payload_rows_remote=72026 dup_rows=15702 dup_rows_remote=13306 dedup_ratio=0.159729 router_tokens_per_expert=[951, 98, 11, 0, 4, 965, 0, 29, 890, 623, 15, 0, 462, 2507, 741, 275, 3362, 3604, 0, 483, 1091, 990, 55, 399, 131, 0, 6397, 131, 159, 550, 40, 1066, 117, 360, 584, 29, 39, 0, 1434, 37, 2315, 387, 228, 109, 29, 6162, 352, 26, 406, 2087, 1, 40, 63, 1045, 1486, 0, 28, 143, 4917, 146, 234, 300, 10, 9] payload_rows_per_dst=[10576, 8762, 8940, 14520, 14172, 6356, 8420, 10856] dup_rows_per_dst=[2396, 2264, 2172, 2428, 3352, 536, 1836, 718] combine_rows_per_src=[10576, 8762, 8940, 14520, 14172, 6356, 8420, 10856] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81312 payload_rows_remote=67132 dup_rows=16992 dup_rows_remote=13332 dedup_ratio=0.172852 router_tokens_per_expert=[69, 17, 30, 3, 1, 2355, 1388, 487, 16, 0, 306, 138, 1, 30, 21, 0, 1160, 12, 1574, 2124, 5071, 732, 19, 12, 1207, 0, 295, 1124, 3, 5700, 80, 511, 12, 1437, 1891, 8, 0, 838, 35, 0, 25, 20, 2075, 36, 2, 62, 1042, 0, 436, 153, 4793, 660, 4205, 175, 48, 796, 4185, 24, 2, 148, 1342, 2, 213, 1] payload_rows_per_dst=[7264, 10656, 9256, 14180, 13056, 12066, 4276, 10558] dup_rows_per_dst=[1436, 510, 2010, 3660, 4972, 2868, 260, 1276] combine_rows_per_src=[7264, 10656, 9256, 14180, 13056, 12066, 4276, 10558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260317184 dispatch_meta_bytes=302680 combine_payload_bytes=260317184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=260619864 ingress_bytes=364126208 link_bytes=364126208 token_bytes=520937048 weight_movement_bytes=103809024 total_nvlink_bytes=624746072 weight_to_token_ratio=0.199274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370177606e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81282 payload_rows_remote=69890 dup_rows=17022 dup_rows_remote=12640 dedup_ratio=0.173157 router_tokens_per_expert=[16, 26, 13, 0, 0, 3314, 910, 908, 2, 8, 343, 155, 4, 68, 70, 1, 1127, 7, 1195, 2160, 6247, 1148, 117, 29, 881, 1, 27, 1291, 2, 4806, 36, 799, 66, 1205, 1230, 1, 0, 620, 299, 5, 67, 30, 2706, 95, 1, 315, 898, 0, 556, 273, 6042, 360, 3305, 173, 116, 669, 3389, 42, 1, 135, 778, 1, 62, 1] payload_rows_per_dst=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] dup_rows_per_dst=[1804, 722, 1992, 2708, 4382, 2698, 2216, 500] combine_rows_per_src=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277618688 dispatch_meta_bytes=340720 combine_payload_bytes=277618688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277959408 ingress_bytes=312221696 link_bytes=312221696 token_bytes=555578096 weight_movement_bytes=34603008 total_nvlink_bytes=590181104 weight_to_token_ratio=0.062283 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174865106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73568 payload_rows_remote=62770 dup_rows=24736 dup_rows_remote=21092 dedup_ratio=0.251628 router_tokens_per_expert=[842, 280, 0, 4, 64, 949, 0, 14, 2698, 1481, 30, 0, 1161, 2932, 776, 652, 2733, 3392, 0, 282, 943, 1164, 49, 282, 212, 0, 6032, 112, 112, 545, 59, 1068, 517, 161, 273, 174, 72, 0, 789, 36, 1107, 561, 531, 38, 19, 5528, 70, 22, 56, 1809, 1, 74, 514, 1034, 2304, 5, 21, 90, 3939, 132, 88, 307, 12, 0] payload_rows_per_dst=[9558, 9028, 9818, 4044, 3596, 14924, 10798, 11802] dup_rows_per_dst=[1532, 2794, 2862, 172, 448, 10044, 3644, 3240] combine_rows_per_src=[9558, 9028, 9818, 4044, 3596, 14924, 10798, 11802] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260489216 dispatch_meta_bytes=325400 combine_payload_bytes=260489216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260814616 ingress_bytes=295092224 link_bytes=295092224 token_bytes=521303832 weight_movement_bytes=34603008 total_nvlink_bytes=555906840 weight_to_token_ratio=0.066378 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110408282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81282 payload_rows_remote=69890 dup_rows=17022 dup_rows_remote=12640 dedup_ratio=0.173157 router_tokens_per_expert=[16, 26, 13, 0, 0, 3314, 910, 908, 2, 8, 343, 155, 4, 68, 70, 1, 1127, 7, 1195, 2160, 6247, 1148, 117, 29, 881, 1, 27, 1291, 2, 4806, 36, 799, 66, 1205, 1230, 1, 0, 620, 299, 5, 67, 30, 2706, 95, 1, 315, 898, 0, 556, 273, 6042, 360, 3305, 173, 116, 669, 3389, 42, 1, 135, 778, 1, 62, 1] payload_rows_per_dst=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] dup_rows_per_dst=[1804, 722, 1992, 2708, 4382, 2698, 2216, 500] combine_rows_per_src=[8570, 13074, 9574, 12978, 11392, 10526, 6850, 8318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78754 payload_rows_remote=65022 dup_rows=19550 dup_rows_remote=13114 dedup_ratio=0.198873 router_tokens_per_expert=[22, 14, 17, 0, 3, 2253, 980, 1197, 11, 3, 300, 40, 10, 118, 12, 0, 2082, 10, 2193, 1851, 6304, 1123, 184, 28, 968, 0, 200, 1152, 3, 5598, 58, 402, 27, 494, 952, 2, 2, 514, 404, 0, 31, 30, 2283, 161, 5, 122, 836, 0, 729, 236, 4899, 501, 4235, 181, 207, 278, 3262, 11, 6, 134, 1256, 155, 63, 0] payload_rows_per_dst=[13732, 12946, 12494, 5268, 11936, 9896, 3718, 8764] dup_rows_per_dst=[6436, 650, 5622, 298, 2652, 2336, 546, 1010] combine_rows_per_src=[13732, 12946, 12494, 5268, 11936, 9896, 3718, 8764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=295018496 dispatch_meta_bytes=341328 combine_payload_bytes=295018496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=295359824 ingress_bytes=364224512 link_bytes=364224512 token_bytes=590378320 weight_movement_bytes=69206016 total_nvlink_bytes=659584336 weight_to_token_ratio=0.117223 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370547516e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=321856 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275294528 ingress_bytes=378781696 link_bytes=378781696 token_bytes=550267200 weight_movement_bytes=103809024 total_nvlink_bytes=654076224 weight_to_token_ratio=0.188652 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425325028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80530 payload_rows_remote=67778 dup_rows=17774 dup_rows_remote=17402 dedup_ratio=0.180806 router_tokens_per_expert=[26, 28, 3, 0, 0, 2841, 1178, 1398, 33, 7, 136, 94, 1, 47, 20, 1, 1024, 24, 1238, 3015, 6223, 678, 127, 63, 1752, 0, 75, 1077, 1, 4225, 33, 922, 12, 685, 1549, 1, 1, 567, 57, 0, 82, 12, 2627, 159, 0, 67, 1466, 0, 641, 337, 4552, 363, 4520, 184, 27, 389, 2786, 7, 0, 145, 1552, 11, 62, 1] payload_rows_per_dst=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] dup_rows_per_dst=[4824, 372, 2166, 1706, 2998, 4444, 416, 848] combine_rows_per_src=[11236, 12752, 10172, 9352, 11850, 13422, 3466, 8280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82020 payload_rows_remote=72634 dup_rows=16284 dup_rows_remote=14828 dedup_ratio=0.165649 router_tokens_per_expert=[31, 270, 12, 6, 4, 4185, 870, 1415, 47, 149, 281, 133, 2, 96, 2, 7, 1031, 46, 2261, 1327, 6076, 595, 118, 43, 1215, 2, 53, 817, 26, 3099, 116, 559, 51, 420, 563, 5, 13, 1360, 208, 1, 66, 226, 2190, 373, 109, 58, 680, 0, 911, 425, 4770, 739, 5037, 230, 398, 492, 3063, 10, 1, 320, 1443, 22, 71, 3] payload_rows_per_dst=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] dup_rows_per_dst=[2564, 848, 1456, 1720, 3288, 4298, 1020, 1090] combine_rows_per_src=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=69122 dup_rows=19380 dup_rows_remote=16936 dedup_ratio=0.197144 router_tokens_per_expert=[23, 193, 0, 0, 578, 313, 62, 0, 57, 72, 3251, 0, 1405, 2078, 1221, 0, 153, 0, 354, 31, 1, 0, 0, 2665, 795, 718, 4326, 0, 0, 53, 1, 13, 245, 0, 228, 665, 0, 0, 1849, 9, 2079, 2134, 300, 7292, 35, 0, 1575, 4023, 1594, 12, 263, 104, 343, 4, 574, 1729, 1392, 2135, 59, 1830, 5, 0, 238, 73] payload_rows_per_dst=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] dup_rows_per_dst=[724, 1222, 4334, 1494, 2982, 2444, 3840, 2340] combine_rows_per_src=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286269440 dispatch_meta_bytes=330120 combine_payload_bytes=286269440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286599560 ingress_bytes=320872448 link_bytes=320872448 token_bytes=572869000 weight_movement_bytes=34603008 total_nvlink_bytes=607472008 weight_to_token_ratio=0.060403 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207417189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=312544 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266642656 ingress_bytes=300933120 link_bytes=300933120 token_bytes=532972768 weight_movement_bytes=34603008 total_nvlink_bytes=567575776 weight_to_token_ratio=0.064925 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78878 payload_rows_remote=71336 dup_rows=19426 dup_rows_remote=18944 dedup_ratio=0.197611 router_tokens_per_expert=[21, 9, 15, 0, 1, 3371, 617, 966, 129, 0, 244, 75, 1, 33, 1, 0, 1153, 77, 969, 2047, 6520, 377, 80, 193, 1896, 0, 113, 1214, 1, 4207, 59, 449, 6, 416, 1074, 0, 0, 1232, 48, 0, 28, 14, 3031, 83, 0, 151, 705, 0, 890, 442, 4681, 445, 5419, 164, 114, 624, 3450, 33, 0, 169, 1012, 1, 81, 1] payload_rows_per_dst=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] dup_rows_per_dst=[1504, 644, 1214, 2484, 762, 482, 10094, 2242] combine_rows_per_src=[8496, 13362, 8578, 10534, 4790, 7542, 15464, 10112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=335448 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257441368 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514547288 weight_movement_bytes=34603008 total_nvlink_bytes=549150296 weight_to_token_ratio=0.067249 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82020 payload_rows_remote=72634 dup_rows=16284 dup_rows_remote=14828 dedup_ratio=0.165649 router_tokens_per_expert=[31, 270, 12, 6, 4, 4185, 870, 1415, 47, 149, 281, 133, 2, 96, 2, 7, 1031, 46, 2261, 1327, 6076, 595, 118, 43, 1215, 2, 53, 817, 26, 3099, 116, 559, 51, 420, 563, 5, 13, 1360, 208, 1, 66, 226, 2190, 373, 109, 58, 680, 0, 911, 425, 4770, 739, 5037, 230, 398, 492, 3063, 10, 1, 320, 1443, 22, 71, 3] payload_rows_per_dst=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] dup_rows_per_dst=[2564, 848, 1456, 1720, 3288, 4298, 1020, 1090] combine_rows_per_src=[11022, 12738, 9386, 10054, 11494, 13180, 5370, 8776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=277618688 dispatch_meta_bytes=340720 combine_payload_bytes=277618688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277959408 ingress_bytes=312221696 link_bytes=312221696 token_bytes=555578096 weight_movement_bytes=34603008 total_nvlink_bytes=590181104 weight_to_token_ratio=0.062283 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174865106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75556 payload_rows_remote=63936 dup_rows=22748 dup_rows_remote=19276 dedup_ratio=0.231405 router_tokens_per_expert=[50, 242, 0, 2, 628, 143, 404, 4, 15, 106, 3110, 0, 1459, 1934, 1959, 0, 154, 1, 68, 13, 3, 1, 0, 2276, 703, 747, 5712, 0, 1, 8, 7, 23, 234, 0, 693, 516, 0, 1, 1526, 5, 1126, 2212, 547, 7283, 25, 0, 1569, 4579, 1028, 25, 51, 181, 332, 0, 684, 2135, 1030, 835, 104, 2320, 4, 1, 170, 163] payload_rows_per_dst=[2850, 9104, 15298, 3388, 6118, 12894, 11620, 14284] dup_rows_per_dst=[96, 1842, 4300, 224, 502, 6552, 3472, 5760] combine_rows_per_src=[2850, 9104, 15298, 3388, 6118, 12894, 11620, 14284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=65954 dup_rows=18000 dup_rows_remote=11466 dedup_ratio=0.183105 router_tokens_per_expert=[14, 451, 0, 0, 483, 87, 88, 2, 20, 31, 3952, 0, 1322, 3294, 1823, 0, 91, 2, 355, 55, 1, 0, 0, 2011, 580, 706, 5179, 0, 6, 34, 0, 19, 281, 0, 259, 545, 0, 1, 1042, 38, 1357, 1486, 178, 7112, 35, 0, 1341, 5726, 641, 11, 143, 202, 138, 1, 262, 2659, 1436, 821, 53, 2148, 8, 0, 390, 232] payload_rows_per_dst=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] dup_rows_per_dst=[2000, 6534, 396, 3408, 2982, 434, 604, 1642] combine_rows_per_src=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=292192256 dispatch_meta_bytes=361120 combine_payload_bytes=292192256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292553376 ingress_bytes=326795264 link_bytes=326795264 token_bytes=584745632 weight_movement_bytes=34603008 total_nvlink_bytes=619348640 weight_to_token_ratio=0.059176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229704269e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=297508864 dispatch_meta_bytes=349848 combine_payload_bytes=297508864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=297858712 ingress_bytes=401317888 link_bytes=401317888 token_bytes=595367576 weight_movement_bytes=103809024 total_nvlink_bytes=699176600 weight_to_token_ratio=0.174361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.510126904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283123712 dispatch_meta_bytes=344232 combine_payload_bytes=283123712 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=283467944 ingress_bytes=490741760 link_bytes=490741760 token_bytes=566591656 weight_movement_bytes=207618048 total_nvlink_bytes=774209704 weight_to_token_ratio=0.366433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846621735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78754 payload_rows_remote=65022 dup_rows=19550 dup_rows_remote=13114 dedup_ratio=0.198873 router_tokens_per_expert=[22, 14, 17, 0, 3, 2253, 980, 1197, 11, 3, 300, 40, 10, 118, 12, 0, 2082, 10, 2193, 1851, 6304, 1123, 184, 28, 968, 0, 200, 1152, 3, 5598, 58, 402, 27, 494, 952, 2, 2, 514, 404, 0, 31, 30, 2283, 161, 5, 122, 836, 0, 729, 236, 4899, 501, 4235, 181, 207, 278, 3262, 11, 6, 134, 1256, 155, 63, 0] payload_rows_per_dst=[13732, 12946, 12494, 5268, 11936, 9896, 3718, 8764] dup_rows_per_dst=[6436, 650, 5622, 298, 2652, 2336, 546, 1010] combine_rows_per_src=[13732, 12946, 12494, 5268, 11936, 9896, 3718, 8764] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70530 payload_rows_remote=55686 dup_rows=27774 dup_rows_remote=20098 dedup_ratio=0.282532 router_tokens_per_expert=[77, 54, 0, 8, 451, 102, 67, 6, 19, 184, 2611, 1, 1520, 2216, 1265, 0, 102, 10, 238, 39, 17, 2, 0, 3152, 425, 558, 5967, 2, 2, 22, 9, 4, 243, 0, 338, 730, 0, 21, 1122, 177, 1787, 2716, 560, 6874, 7, 0, 973, 5275, 1001, 13, 266, 168, 385, 2, 166, 1905, 1615, 1221, 238, 1877, 3, 0, 113, 226] payload_rows_per_dst=[1518, 7100, 15050, 2004, 5038, 14506, 10470, 14844] dup_rows_per_dst=[12, 990, 5818, 40, 224, 10450, 2564, 7676] combine_rows_per_src=[1518, 7100, 15050, 2004, 5038, 14506, 10470, 14844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83090 payload_rows_remote=71202 dup_rows=15214 dup_rows_remote=14146 dedup_ratio=0.154765 router_tokens_per_expert=[119, 127, 0, 0, 578, 227, 190, 0, 63, 158, 3023, 8, 1110, 2579, 808, 0, 219, 7, 775, 119, 0, 3, 0, 3452, 420, 396, 5639, 0, 0, 21, 0, 2, 216, 4, 380, 560, 0, 0, 1132, 19, 2366, 1699, 151, 7409, 21, 0, 810, 4103, 1710, 7, 162, 145, 164, 4, 756, 3088, 880, 1154, 39, 1899, 4, 1, 157, 69] payload_rows_per_dst=[15038, 9828, 8030, 11888, 10554, 8632, 12100, 7020] dup_rows_per_dst=[2262, 1804, 1120, 1068, 2274, 1462, 3838, 1386] combine_rows_per_src=[15038, 9828, 8030, 11888, 10554, 8632, 12100, 7020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261881856 dispatch_meta_bytes=332848 combine_payload_bytes=261881856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262214704 ingress_bytes=296484864 link_bytes=296484864 token_bytes=524096560 weight_movement_bytes=34603008 total_nvlink_bytes=558699568 weight_to_token_ratio=0.066024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115648674e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81460 payload_rows_remote=69546 dup_rows=16844 dup_rows_remote=13108 dedup_ratio=0.171346 router_tokens_per_expert=[8, 170, 0, 0, 789, 639, 207, 5, 25, 68, 4377, 0, 903, 1502, 1118, 0, 237, 7, 301, 55, 13, 0, 0, 2419, 349, 582, 6457, 0, 0, 7, 1, 6, 286, 0, 869, 779, 0, 40, 1262, 152, 1238, 1604, 797, 7372, 28, 0, 1222, 4437, 816, 5, 68, 46, 313, 2, 228, 2566, 782, 1432, 183, 1830, 83, 0, 189, 278] payload_rows_per_dst=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] dup_rows_per_dst=[3178, 624, 334, 1484, 3736, 1560, 4256, 1672] combine_rows_per_src=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=270147584 dispatch_meta_bytes=309680 combine_payload_bytes=270147584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270457264 ingress_bytes=339353600 link_bytes=339353600 token_bytes=540604848 weight_movement_bytes=69206016 total_nvlink_bytes=609810864 weight_to_token_ratio=0.128016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276960277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=312544 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266642656 ingress_bytes=300933120 link_bytes=300933120 token_bytes=532972768 weight_movement_bytes=34603008 total_nvlink_bytes=567575776 weight_to_token_ratio=0.064925 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=228089856 dispatch_meta_bytes=303136 combine_payload_bytes=228089856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228392992 ingress_bytes=262692864 link_bytes=262692864 token_bytes=456482848 weight_movement_bytes=34603008 total_nvlink_bytes=491085856 weight_to_token_ratio=0.075804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.884920987e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=291643392 dispatch_meta_bytes=341392 combine_payload_bytes=291643392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291984784 ingress_bytes=360849408 link_bytes=360849408 token_bytes=583628176 weight_movement_bytes=69206016 total_nvlink_bytes=652834192 weight_to_token_ratio=0.118579 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357847272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81460 payload_rows_remote=69546 dup_rows=16844 dup_rows_remote=13108 dedup_ratio=0.171346 router_tokens_per_expert=[8, 170, 0, 0, 789, 639, 207, 5, 25, 68, 4377, 0, 903, 1502, 1118, 0, 237, 7, 301, 55, 13, 0, 0, 2419, 349, 582, 6457, 0, 0, 7, 1, 6, 286, 0, 869, 779, 0, 40, 1262, 152, 1238, 1604, 797, 7372, 28, 0, 1222, 4437, 816, 5, 68, 46, 313, 2, 228, 2566, 782, 1432, 183, 1830, 83, 0, 189, 278] payload_rows_per_dst=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] dup_rows_per_dst=[3178, 624, 334, 1484, 3736, 1560, 4256, 1672] combine_rows_per_src=[15202, 6608, 5730, 13320, 11914, 8218, 12586, 7882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78912 payload_rows_remote=64992 dup_rows=19392 dup_rows_remote=17086 dedup_ratio=0.197266 router_tokens_per_expert=[131, 128, 5, 7, 612, 498, 120, 3, 39, 98, 3742, 26, 768, 1830, 539, 2, 186, 0, 324, 46, 3, 1, 3, 3650, 408, 466, 5685, 0, 0, 5, 0, 16, 253, 3, 592, 629, 0, 1, 1136, 27, 3214, 3878, 527, 6609, 126, 0, 1180, 4145, 930, 1, 208, 151, 503, 5, 123, 1541, 1489, 1530, 159, 558, 2, 0, 175, 116] payload_rows_per_dst=[13920, 11862, 10416, 14056, 10986, 4608, 6132, 6932] dup_rows_per_dst=[2306, 2226, 2994, 6860, 2586, 502, 792, 1126] combine_rows_per_src=[13920, 11862, 10416, 14056, 10986, 4608, 6132, 6932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=65954 dup_rows=18000 dup_rows_remote=11466 dedup_ratio=0.183105 router_tokens_per_expert=[14, 451, 0, 0, 483, 87, 88, 2, 20, 31, 3952, 0, 1322, 3294, 1823, 0, 91, 2, 355, 55, 1, 0, 0, 2011, 580, 706, 5179, 0, 6, 34, 0, 19, 281, 0, 259, 545, 0, 1, 1042, 38, 1357, 1486, 178, 7112, 35, 0, 1341, 5726, 641, 11, 143, 202, 138, 1, 262, 2659, 1436, 821, 53, 2148, 8, 0, 390, 232] payload_rows_per_dst=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] dup_rows_per_dst=[2000, 6534, 396, 3408, 2982, 434, 604, 1642] combine_rows_per_src=[14474, 14350, 4634, 12612, 12802, 5388, 7510, 8534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80052 payload_rows_remote=73736 dup_rows=18252 dup_rows_remote=17392 dedup_ratio=0.185669 router_tokens_per_expert=[61, 515, 0, 8, 1514, 707, 77, 75, 150, 101, 3484, 2, 1188, 2082, 873, 3, 231, 2, 977, 73, 7, 0, 0, 2298, 738, 905, 5791, 5, 2, 50, 0, 12, 294, 5, 450, 650, 3, 19, 981, 112, 2021, 1677, 341, 6606, 63, 0, 1891, 4761, 1052, 12, 258, 146, 218, 14, 327, 1279, 1172, 1035, 37, 1372, 13, 8, 238, 166] payload_rows_per_dst=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] dup_rows_per_dst=[4502, 3370, 860, 3316, 2704, 1832, 526, 1142] combine_rows_per_src=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79964 payload_rows_remote=71994 dup_rows=18340 dup_rows_remote=16608 dedup_ratio=0.186564 router_tokens_per_expert=[2829, 0, 0, 518, 52, 1, 3659, 618, 5468, 8098, 105, 2, 0, 276, 18, 183, 169, 10, 1907, 234, 590, 909, 41, 1, 1325, 1374, 50, 6, 80, 0, 41, 590, 0, 2, 109, 811, 3, 210, 4, 1, 305, 485, 279, 605, 1788, 807, 322, 260, 2706, 0, 2, 9, 9, 320, 0, 0, 790, 50, 46, 1, 4, 1760, 3965, 4345] payload_rows_per_dst=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] dup_rows_per_dst=[364, 2, 4410, 4586, 2250, 1732, 2496, 2500] combine_rows_per_src=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78520 payload_rows_remote=63554 dup_rows=19784 dup_rows_remote=12116 dedup_ratio=0.201253 router_tokens_per_expert=[78, 17, 12, 2, 6, 2481, 989, 640, 84, 3, 169, 77, 1, 265, 8, 0, 977, 17, 968, 2759, 6536, 257, 61, 32, 2579, 1, 86, 1060, 1, 4924, 48, 574, 6, 477, 1945, 2, 1, 1840, 119, 0, 88, 15, 1635, 35, 1, 38, 1055, 2, 851, 229, 4844, 557, 4140, 210, 62, 424, 3475, 8, 0, 194, 1129, 0, 58, 0] payload_rows_per_dst=[7198, 13516, 7468, 9006, 7278, 5502, 14966, 13586] dup_rows_per_dst=[1252, 770, 900, 1466, 1502, 236, 7668, 5990] combine_rows_per_src=[7198, 13516, 7468, 9006, 7278, 5502, 14966, 13586] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=291643392 dispatch_meta_bytes=341392 combine_payload_bytes=291643392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291984784 ingress_bytes=360849408 link_bytes=360849408 token_bytes=583628176 weight_movement_bytes=69206016 total_nvlink_bytes=652834192 weight_to_token_ratio=0.118579 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357847272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=284860416 dispatch_meta_bytes=330616 combine_payload_bytes=284860416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285191032 ingress_bytes=319463424 link_bytes=319463424 token_bytes=570051448 weight_movement_bytes=34603008 total_nvlink_bytes=604654456 weight_to_token_ratio=0.060702 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202115146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=328312 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266535544 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532742776 weight_movement_bytes=34603008 total_nvlink_bytes=567345784 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78924 payload_rows_remote=69122 dup_rows=19380 dup_rows_remote=16936 dedup_ratio=0.197144 router_tokens_per_expert=[23, 193, 0, 0, 578, 313, 62, 0, 57, 72, 3251, 0, 1405, 2078, 1221, 0, 153, 0, 354, 31, 1, 0, 0, 2665, 795, 718, 4326, 0, 0, 53, 1, 13, 245, 0, 228, 665, 0, 0, 1849, 9, 2079, 2134, 300, 7292, 35, 0, 1575, 4023, 1594, 12, 263, 104, 343, 4, 574, 1729, 1392, 2135, 59, 1830, 5, 0, 238, 73] payload_rows_per_dst=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] dup_rows_per_dst=[724, 1222, 4334, 1494, 2982, 2444, 3840, 2340] combine_rows_per_src=[5598, 8444, 12674, 10318, 11056, 9802, 11908, 9124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80052 payload_rows_remote=73736 dup_rows=18252 dup_rows_remote=17392 dedup_ratio=0.185669 router_tokens_per_expert=[61, 515, 0, 8, 1514, 707, 77, 75, 150, 101, 3484, 2, 1188, 2082, 873, 3, 231, 2, 977, 73, 7, 0, 0, 2298, 738, 905, 5791, 5, 2, 50, 0, 12, 294, 5, 450, 650, 3, 19, 981, 112, 2021, 1677, 341, 6606, 63, 0, 1891, 4761, 1052, 12, 258, 146, 218, 14, 327, 1279, 1172, 1035, 37, 1372, 13, 8, 238, 166] payload_rows_per_dst=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] dup_rows_per_dst=[4502, 3370, 860, 3316, 2704, 1832, 526, 1142] combine_rows_per_src=[14624, 12396, 6316, 12824, 11846, 9020, 6086, 6940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260317184 dispatch_meta_bytes=302680 combine_payload_bytes=260317184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=260619864 ingress_bytes=364126208 link_bytes=364126208 token_bytes=520937048 weight_movement_bytes=103809024 total_nvlink_bytes=624746072 weight_to_token_ratio=0.199274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370177606e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79868 payload_rows_remote=67836 dup_rows=18436 dup_rows_remote=15734 dedup_ratio=0.187541 router_tokens_per_expert=[3501, 5, 2, 653, 75, 7, 2192, 525, 5172, 7237, 135, 12, 0, 146, 26, 133, 186, 10, 899, 184, 550, 425, 22, 3, 462, 1140, 30, 31, 484, 0, 60, 677, 0, 6, 82, 286, 0, 485, 0, 0, 533, 486, 331, 1201, 3372, 1181, 143, 1297, 4811, 3, 16, 31, 11, 301, 0, 2, 764, 150, 42, 0, 10, 1748, 2939, 3937] payload_rows_per_dst=[8868, 884, 8026, 12554, 14628, 12256, 12032, 10620] dup_rows_per_dst=[668, 20, 1392, 3558, 1564, 4832, 2702, 3700] combine_rows_per_src=[8868, 884, 8026, 12554, 14628, 12256, 12032, 10620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=270147584 dispatch_meta_bytes=309680 combine_payload_bytes=270147584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270457264 ingress_bytes=339353600 link_bytes=339353600 token_bytes=540604848 weight_movement_bytes=69206016 total_nvlink_bytes=609810864 weight_to_token_ratio=0.128016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276960277e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283123712 dispatch_meta_bytes=344232 combine_payload_bytes=283123712 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=283467944 ingress_bytes=490741760 link_bytes=490741760 token_bytes=566591656 weight_movement_bytes=207618048 total_nvlink_bytes=774209704 weight_to_token_ratio=0.366433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846621735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70334 payload_rows_remote=54108 dup_rows=27970 dup_rows_remote=11192 dedup_ratio=0.284526 router_tokens_per_expert=[3461, 13, 0, 846, 25, 3, 3483, 534, 5217, 7637, 82, 6, 0, 89, 4, 113, 241, 12, 1835, 355, 266, 220, 28, 6, 485, 1523, 29, 14, 198, 0, 20, 371, 0, 18, 328, 816, 1, 604, 16, 1, 182, 641, 348, 773, 2188, 839, 336, 525, 4456, 1, 4, 9, 3, 218, 0, 6, 578, 50, 75, 7, 5, 1180, 3969, 3859] payload_rows_per_dst=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] dup_rows_per_dst=[4188, 16778, 2902, 606, 524, 2386, 170, 416] combine_rows_per_src=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=364512 combine_payload_bytes=302022656 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=302387168 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604409824 weight_movement_bytes=69206016 total_nvlink_bytes=673615840 weight_to_token_ratio=0.114502 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78912 payload_rows_remote=64992 dup_rows=19392 dup_rows_remote=17086 dedup_ratio=0.197266 router_tokens_per_expert=[131, 128, 5, 7, 612, 498, 120, 3, 39, 98, 3742, 26, 768, 1830, 539, 2, 186, 0, 324, 46, 3, 1, 3, 3650, 408, 466, 5685, 0, 0, 5, 0, 16, 253, 3, 592, 629, 0, 1, 1136, 27, 3214, 3878, 527, 6609, 126, 0, 1180, 4145, 930, 1, 208, 151, 503, 5, 123, 1541, 1489, 1530, 159, 558, 2, 0, 175, 116] payload_rows_per_dst=[13920, 11862, 10416, 14056, 10986, 4608, 6132, 6932] dup_rows_per_dst=[2306, 2226, 2994, 6860, 2586, 502, 792, 1126] combine_rows_per_src=[13920, 11862, 10416, 14056, 10986, 4608, 6132, 6932] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76456 payload_rows_remote=62980 dup_rows=21848 dup_rows_remote=14710 dedup_ratio=0.222249 router_tokens_per_expert=[2963, 8, 9, 570, 81, 7, 4255, 1123, 4596, 7154, 108, 7, 0, 68, 104, 95, 204, 4, 1465, 488, 386, 227, 51, 0, 494, 1126, 42, 1, 146, 13, 77, 926, 0, 13, 193, 1302, 8, 103, 19, 2, 297, 378, 198, 1550, 2506, 743, 160, 509, 4156, 4, 4, 55, 123, 137, 0, 2, 654, 81, 54, 2, 20, 1832, 3482, 3767] payload_rows_per_dst=[8528, 764, 5264, 11864, 14778, 9144, 12638, 13476] dup_rows_per_dst=[994, 0, 386, 2978, 2810, 2708, 4834, 7138] combine_rows_per_src=[8528, 764, 5264, 11864, 14778, 9144, 12638, 13476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=294887424 dispatch_meta_bytes=354408 combine_payload_bytes=294887424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295241832 ingress_bytes=364093440 link_bytes=364093440 token_bytes=590129256 weight_movement_bytes=69206016 total_nvlink_bytes=659335272 weight_to_token_ratio=0.117273 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370054303e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81938 payload_rows_remote=68874 dup_rows=16366 dup_rows_remote=11076 dedup_ratio=0.166484 router_tokens_per_expert=[3187, 1, 5, 1528, 24, 3, 3342, 1410, 6569, 7421, 115, 26, 0, 146, 14, 234, 335, 23, 1234, 133, 160, 727, 20, 0, 809, 800, 12, 2, 376, 2, 67, 2779, 0, 5, 107, 338, 11, 147, 7, 1, 50, 347, 570, 582, 2001, 507, 60, 401, 2020, 13, 2, 284, 7, 545, 0, 15, 467, 83, 22, 1, 8, 1900, 3612, 3535] payload_rows_per_dst=[10696, 5232, 10106, 13064, 14902, 7858, 10198, 9882] dup_rows_per_dst=[1620, 316, 2228, 5290, 1172, 1178, 2258, 2304] combine_rows_per_src=[10696, 5232, 10106, 13064, 14902, 7858, 10198, 9882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=277856256 dispatch_meta_bytes=334280 combine_payload_bytes=277856256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278190536 ingress_bytes=312459264 link_bytes=312459264 token_bytes=556046792 weight_movement_bytes=34603008 total_nvlink_bytes=590649800 weight_to_token_ratio=0.062230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175759055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81798 payload_rows_remote=66998 dup_rows=16506 dup_rows_remote=14884 dedup_ratio=0.167908 router_tokens_per_expert=[3969, 1, 5, 693, 15, 2, 2939, 335, 5766, 7322, 131, 13, 15, 244, 10, 148, 313, 23, 1605, 120, 448, 222, 215, 16, 628, 675, 27, 21, 31, 3, 72, 280, 0, 24, 197, 324, 4, 335, 5, 0, 180, 324, 117, 686, 2850, 1723, 174, 678, 2922, 10, 5, 20, 17, 334, 0, 4, 1082, 324, 33, 2, 12, 1934, 4123, 4402] payload_rows_per_dst=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] dup_rows_per_dst=[566, 14, 3054, 2436, 1622, 2830, 2322, 3662] combine_rows_per_src=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=221626368 dispatch_meta_bytes=261200 combine_payload_bytes=221626368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221887568 ingress_bytes=325435392 link_bytes=325435392 token_bytes=443513936 weight_movement_bytes=103809024 total_nvlink_bytes=547322960 weight_to_token_ratio=0.234060 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224587180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=328312 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266535544 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532742776 weight_movement_bytes=34603008 total_nvlink_bytes=567345784 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257966080 dispatch_meta_bytes=310760 combine_payload_bytes=257966080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258276840 ingress_bytes=361775104 link_bytes=361775104 token_bytes=516242920 weight_movement_bytes=103809024 total_nvlink_bytes=620051944 weight_to_token_ratio=0.201086 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75556 payload_rows_remote=63936 dup_rows=22748 dup_rows_remote=19276 dedup_ratio=0.231405 router_tokens_per_expert=[50, 242, 0, 2, 628, 143, 404, 4, 15, 106, 3110, 0, 1459, 1934, 1959, 0, 154, 1, 68, 13, 3, 1, 0, 2276, 703, 747, 5712, 0, 1, 8, 7, 23, 234, 0, 693, 516, 0, 1, 1526, 5, 1126, 2212, 547, 7283, 25, 0, 1569, 4579, 1028, 25, 51, 181, 332, 0, 684, 2135, 1030, 835, 104, 2320, 4, 1, 170, 163] payload_rows_per_dst=[2850, 9104, 15298, 3388, 6118, 12894, 11620, 14284] dup_rows_per_dst=[96, 1842, 4300, 224, 502, 6552, 3472, 5760] combine_rows_per_src=[2850, 9104, 15298, 3388, 6118, 12894, 11620, 14284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=282107904 dispatch_meta_bytes=319800 combine_payload_bytes=282107904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282427704 ingress_bytes=316710912 link_bytes=316710912 token_bytes=564535608 weight_movement_bytes=34603008 total_nvlink_bytes=599138616 weight_to_token_ratio=0.061295 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191757665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81798 payload_rows_remote=66998 dup_rows=16506 dup_rows_remote=14884 dedup_ratio=0.167908 router_tokens_per_expert=[3969, 1, 5, 693, 15, 2, 2939, 335, 5766, 7322, 131, 13, 15, 244, 10, 148, 313, 23, 1605, 120, 448, 222, 215, 16, 628, 675, 27, 21, 31, 3, 72, 280, 0, 24, 197, 324, 4, 335, 5, 0, 180, 324, 117, 686, 2850, 1723, 174, 678, 2922, 10, 5, 20, 17, 334, 0, 4, 1082, 324, 33, 2, 12, 1934, 4123, 4402] payload_rows_per_dst=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] dup_rows_per_dst=[566, 14, 3054, 2436, 1622, 2830, 2322, 3662] combine_rows_per_src=[9474, 1108, 11674, 12570, 14800, 10634, 10180, 11358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67510 payload_rows_remote=54226 dup_rows=30794 dup_rows_remote=24642 dedup_ratio=0.313253 router_tokens_per_expert=[2993, 4, 0, 702, 17, 42, 3262, 421, 4604, 7519, 82, 5, 0, 131, 10, 292, 165, 18, 1726, 450, 892, 258, 209, 3, 377, 1675, 4, 8, 79, 1, 15, 443, 0, 4, 95, 770, 3, 235, 37, 2, 274, 219, 250, 1202, 4000, 1521, 221, 233, 2398, 7, 0, 182, 27, 286, 1, 7, 1091, 80, 58, 1, 37, 1110, 4646, 3748] payload_rows_per_dst=[13284, 16274, 11624, 4604, 2250, 9296, 5720, 4458] dup_rows_per_dst=[6152, 18304, 3314, 600, 42, 1990, 96, 296] combine_rows_per_src=[13284, 16274, 11624, 4604, 2250, 9296, 5720, 4458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70334 payload_rows_remote=54108 dup_rows=27970 dup_rows_remote=11192 dedup_ratio=0.284526 router_tokens_per_expert=[3461, 13, 0, 846, 25, 3, 3483, 534, 5217, 7637, 82, 6, 0, 89, 4, 113, 241, 12, 1835, 355, 266, 220, 28, 6, 485, 1523, 29, 14, 198, 0, 20, 371, 0, 18, 328, 816, 1, 604, 16, 1, 182, 641, 348, 773, 2188, 839, 336, 525, 4456, 1, 4, 9, 3, 218, 0, 6, 578, 50, 75, 7, 5, 1180, 3969, 3859] payload_rows_per_dst=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] dup_rows_per_dst=[4188, 16778, 2902, 606, 524, 2386, 170, 416] combine_rows_per_src=[12542, 16226, 10742, 4674, 3044, 9278, 9224, 4604] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76336 payload_rows_remote=66940 dup_rows=21968 dup_rows_remote=20184 dedup_ratio=0.223470 router_tokens_per_expert=[3241, 17, 23, 949, 452, 19, 3820, 349, 5230, 6938, 175, 1, 1, 51, 30, 321, 241, 17, 1314, 353, 342, 159, 14, 3, 1217, 915, 39, 21, 103, 20, 57, 447, 0, 31, 396, 662, 17, 592, 8, 1, 591, 925, 184, 597, 1887, 1859, 387, 676, 2866, 203, 1, 1, 70, 552, 0, 41, 970, 723, 87, 1, 81, 1126, 3591, 3147] payload_rows_per_dst=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] dup_rows_per_dst=[2112, 6970, 1784, 728, 1032, 3736, 2432, 3174] combine_rows_per_src=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78680 payload_rows_remote=67544 dup_rows=19624 dup_rows_remote=15636 dedup_ratio=0.199626 router_tokens_per_expert=[1, 182, 1021, 1615, 506, 1531, 0, 41, 0, 3, 1, 58, 1084, 166, 2618, 1307, 11, 3158, 0, 0, 1, 0, 41, 1, 213, 244, 646, 92, 27, 302, 428, 0, 184, 66, 1330, 2963, 152, 4330, 1, 0, 286, 3, 63, 1659, 13, 0, 1480, 696, 174, 0, 1339, 2507, 1369, 4360, 684, 1, 0, 331, 0, 9, 5211, 4629, 13, 1] payload_rows_per_dst=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] dup_rows_per_dst=[1610, 1598, 2924, 2528, 1362, 3988, 3960, 1654] combine_rows_per_src=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=327528 combine_payload_bytes=274423808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274751336 ingress_bytes=309026816 link_bytes=309026816 token_bytes=549175144 weight_movement_bytes=34603008 total_nvlink_bytes=583778152 weight_to_token_ratio=0.063009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162843030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79964 payload_rows_remote=71994 dup_rows=18340 dup_rows_remote=16608 dedup_ratio=0.186564 router_tokens_per_expert=[2829, 0, 0, 518, 52, 1, 3659, 618, 5468, 8098, 105, 2, 0, 276, 18, 183, 169, 10, 1907, 234, 590, 909, 41, 1, 1325, 1374, 50, 6, 80, 0, 41, 590, 0, 2, 109, 811, 3, 210, 4, 1, 305, 485, 279, 605, 1788, 807, 322, 260, 2706, 0, 2, 9, 9, 320, 0, 0, 790, 50, 46, 1, 4, 1760, 3965, 4345] payload_rows_per_dst=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] dup_rows_per_dst=[364, 2, 4410, 4586, 2250, 1732, 2496, 2500] combine_rows_per_src=[7672, 1166, 12002, 13282, 16226, 7970, 10914, 10732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76336 payload_rows_remote=66940 dup_rows=21968 dup_rows_remote=20184 dedup_ratio=0.223470 router_tokens_per_expert=[3241, 17, 23, 949, 452, 19, 3820, 349, 5230, 6938, 175, 1, 1, 51, 30, 321, 241, 17, 1314, 353, 342, 159, 14, 3, 1217, 915, 39, 21, 103, 20, 57, 447, 0, 31, 396, 662, 17, 592, 8, 1, 591, 925, 184, 597, 1887, 1859, 387, 676, 2866, 203, 1, 1, 70, 552, 0, 41, 970, 723, 87, 1, 81, 1126, 3591, 3147] payload_rows_per_dst=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] dup_rows_per_dst=[2112, 6970, 1784, 728, 1032, 3736, 2432, 3174] combine_rows_per_src=[10576, 14246, 9396, 4910, 6660, 10476, 10088, 9984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261881856 dispatch_meta_bytes=332848 combine_payload_bytes=261881856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262214704 ingress_bytes=296484864 link_bytes=296484864 token_bytes=524096560 weight_movement_bytes=34603008 total_nvlink_bytes=558699568 weight_to_token_ratio=0.066024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115648674e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=222109696 dispatch_meta_bytes=315472 combine_payload_bytes=222109696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222425168 ingress_bytes=291315712 link_bytes=291315712 token_bytes=444534864 weight_movement_bytes=69206016 total_nvlink_bytes=513740880 weight_to_token_ratio=0.155682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096197571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78030 payload_rows_remote=65210 dup_rows=20274 dup_rows_remote=15570 dedup_ratio=0.206238 router_tokens_per_expert=[0, 218, 2361, 2713, 285, 2391, 0, 26, 4, 23, 0, 88, 1542, 241, 3144, 1192, 9, 3967, 0, 0, 51, 7, 23, 1, 57, 264, 485, 130, 45, 86, 588, 0, 305, 313, 688, 1694, 275, 3977, 0, 0, 252, 3, 53, 2308, 31, 3, 1832, 486, 11, 3, 309, 2116, 1229, 4537, 551, 6, 0, 233, 0, 5, 4841, 3086, 18, 46] payload_rows_per_dst=[10982, 10222, 8058, 7838, 6340, 8728, 12820, 13042] dup_rows_per_dst=[3858, 2306, 58, 1644, 1298, 1208, 4704, 5198] combine_rows_per_src=[10982, 10222, 8058, 7838, 6340, 8728, 12820, 13042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=221626368 dispatch_meta_bytes=261200 combine_payload_bytes=221626368 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221887568 ingress_bytes=325435392 link_bytes=325435392 token_bytes=443513936 weight_movement_bytes=103809024 total_nvlink_bytes=547322960 weight_to_token_ratio=0.234060 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224587180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=294887424 dispatch_meta_bytes=354408 combine_payload_bytes=294887424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295241832 ingress_bytes=364093440 link_bytes=364093440 token_bytes=590129256 weight_movement_bytes=69206016 total_nvlink_bytes=659335272 weight_to_token_ratio=0.117273 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370054303e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=274186240 dispatch_meta_bytes=348496 combine_payload_bytes=274186240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274534736 ingress_bytes=308789248 link_bytes=308789248 token_bytes=548720976 weight_movement_bytes=34603008 total_nvlink_bytes=583323984 weight_to_token_ratio=0.063061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161949081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=332720 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276992944 ingress_bytes=311263232 link_bytes=311263232 token_bytes=553653168 weight_movement_bytes=34603008 total_nvlink_bytes=588256176 weight_to_token_ratio=0.062499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171258483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80476 payload_rows_remote=71348 dup_rows=17828 dup_rows_remote=16114 dedup_ratio=0.181356 router_tokens_per_expert=[0, 283, 1080, 3368, 182, 2628, 1, 40, 0, 17, 0, 5, 1190, 264, 2618, 1327, 1, 3488, 0, 0, 2, 4, 30, 4, 80, 192, 457, 53, 45, 50, 677, 0, 180, 112, 998, 627, 28, 4996, 0, 3, 250, 0, 139, 3300, 0, 0, 2081, 579, 269, 25, 223, 2876, 1272, 4113, 518, 2, 0, 340, 0, 1, 4451, 3669, 14, 0] payload_rows_per_dst=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] dup_rows_per_dst=[3326, 1714, 3836, 1888, 2514, 2528, 1828, 194] combine_rows_per_src=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67510 payload_rows_remote=54226 dup_rows=30794 dup_rows_remote=24642 dedup_ratio=0.313253 router_tokens_per_expert=[2993, 4, 0, 702, 17, 42, 3262, 421, 4604, 7519, 82, 5, 0, 131, 10, 292, 165, 18, 1726, 450, 892, 258, 209, 3, 377, 1675, 4, 8, 79, 1, 15, 443, 0, 4, 95, 770, 3, 235, 37, 2, 274, 219, 250, 1202, 4000, 1521, 221, 233, 2398, 7, 0, 182, 27, 286, 1, 7, 1091, 80, 58, 1, 37, 1110, 4646, 3748] payload_rows_per_dst=[13284, 16274, 11624, 4604, 2250, 9296, 5720, 4458] dup_rows_per_dst=[6152, 18304, 3314, 600, 42, 1990, 96, 296] combine_rows_per_src=[13284, 16274, 11624, 4604, 2250, 9296, 5720, 4458] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74584 payload_rows_remote=61072 dup_rows=23720 dup_rows_remote=15176 dedup_ratio=0.241292 router_tokens_per_expert=[0, 160, 840, 2358, 705, 1895, 0, 4, 0, 9, 0, 13, 1408, 236, 2723, 1451, 33, 3386, 0, 0, 7, 13, 88, 1, 158, 310, 426, 310, 40, 99, 505, 0, 240, 117, 799, 1573, 157, 4169, 0, 0, 387, 0, 206, 3365, 2, 0, 1396, 789, 146, 2, 230, 2462, 1382, 4151, 456, 1, 1, 351, 0, 23, 3855, 5707, 3, 4] payload_rows_per_dst=[6488, 9678, 6906, 8180, 6304, 9934, 13582, 13512] dup_rows_per_dst=[720, 2002, 150, 1686, 1046, 2356, 7216, 8544] combine_rows_per_src=[6488, 9678, 6906, 8180, 6304, 9934, 13582, 13512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78150 payload_rows_remote=67544 dup_rows=20154 dup_rows_remote=17198 dedup_ratio=0.205017 router_tokens_per_expert=[2, 411, 1730, 2453, 475, 1734, 0, 2, 1, 32, 0, 105, 1727, 186, 2499, 933, 10, 2497, 0, 0, 7, 0, 58, 0, 65, 685, 411, 203, 10, 93, 610, 0, 597, 690, 800, 1528, 68, 4070, 0, 3, 661, 3, 194, 2708, 11, 0, 3733, 521, 227, 3, 442, 2409, 569, 3629, 459, 3, 0, 135, 0, 26, 3245, 5462, 17, 0] payload_rows_per_dst=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] dup_rows_per_dst=[3086, 1920, 2564, 2936, 2956, 5298, 1350, 44] combine_rows_per_src=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78366 payload_rows_remote=68372 dup_rows=19938 dup_rows_remote=17666 dedup_ratio=0.202820 router_tokens_per_expert=[0, 55, 1160, 2210, 375, 939, 0, 46, 0, 8, 0, 51, 1935, 212, 4507, 2218, 5, 3249, 0, 0, 17, 0, 77, 0, 105, 297, 588, 31, 72, 158, 700, 0, 323, 229, 1675, 2270, 201, 3552, 0, 0, 404, 0, 58, 1178, 55, 0, 1738, 355, 277, 5, 107, 2172, 2294, 4785, 528, 1, 0, 265, 0, 12, 3414, 4182, 32, 25] payload_rows_per_dst=[8166, 12972, 12032, 9994, 12014, 6830, 9056, 7302] dup_rows_per_dst=[1404, 4890, 4234, 2272, 4486, 746, 1712, 194] combine_rows_per_src=[8166, 12972, 12032, 9994, 12014, 6830, 9056, 7302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=323120 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267423280 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534523440 weight_movement_bytes=69206016 total_nvlink_bytes=603729456 weight_to_token_ratio=0.129472 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292241408 dispatch_meta_bytes=349848 combine_payload_bytes=292241408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292591256 ingress_bytes=326844416 link_bytes=326844416 token_bytes=584832664 weight_movement_bytes=34603008 total_nvlink_bytes=619435672 weight_to_token_ratio=0.059167 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229889224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=222109696 dispatch_meta_bytes=315472 combine_payload_bytes=222109696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222425168 ingress_bytes=291315712 link_bytes=291315712 token_bytes=444534864 weight_movement_bytes=69206016 total_nvlink_bytes=513740880 weight_to_token_ratio=0.155682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096197571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79868 payload_rows_remote=67836 dup_rows=18436 dup_rows_remote=15734 dedup_ratio=0.187541 router_tokens_per_expert=[3501, 5, 2, 653, 75, 7, 2192, 525, 5172, 7237, 135, 12, 0, 146, 26, 133, 186, 10, 899, 184, 550, 425, 22, 3, 462, 1140, 30, 31, 484, 0, 60, 677, 0, 6, 82, 286, 0, 485, 0, 0, 533, 486, 331, 1201, 3372, 1181, 143, 1297, 4811, 3, 16, 31, 11, 301, 0, 2, 764, 150, 42, 0, 10, 1748, 2939, 3937] payload_rows_per_dst=[8868, 884, 8026, 12554, 14628, 12256, 12032, 10620] dup_rows_per_dst=[668, 20, 1392, 3558, 1564, 4832, 2702, 3700] combine_rows_per_src=[8868, 884, 8026, 12554, 14628, 12256, 12032, 10620] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250150912 dispatch_meta_bytes=304992 combine_payload_bytes=250150912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250455904 ingress_bytes=319356928 link_bytes=319356928 token_bytes=500606816 weight_movement_bytes=69206016 total_nvlink_bytes=569812832 weight_to_token_ratio=0.138244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201714410e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78150 payload_rows_remote=67544 dup_rows=20154 dup_rows_remote=17198 dedup_ratio=0.205017 router_tokens_per_expert=[2, 411, 1730, 2453, 475, 1734, 0, 2, 1, 32, 0, 105, 1727, 186, 2499, 933, 10, 2497, 0, 0, 7, 0, 58, 0, 65, 685, 411, 203, 10, 93, 610, 0, 597, 690, 800, 1528, 68, 4070, 0, 3, 661, 3, 194, 2708, 11, 0, 3733, 521, 227, 3, 442, 2409, 569, 3629, 459, 3, 0, 135, 0, 26, 3245, 5462, 17, 0] payload_rows_per_dst=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] dup_rows_per_dst=[3086, 1920, 2564, 2936, 2956, 5298, 1350, 44] combine_rows_per_src=[10528, 9046, 9838, 12142, 10606, 12314, 6874, 6802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=280051712 dispatch_meta_bytes=344152 combine_payload_bytes=280051712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280395864 ingress_bytes=314654720 link_bytes=314654720 token_bytes=560447576 weight_movement_bytes=34603008 total_nvlink_bytes=595050584 weight_to_token_ratio=0.061742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184020380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81806 payload_rows_remote=71610 dup_rows=16498 dup_rows_remote=13950 dedup_ratio=0.167826 router_tokens_per_expert=[0, 139, 1599, 2761, 548, 1317, 0, 8, 0, 56, 0, 16, 2067, 85, 2321, 2106, 9, 1699, 0, 0, 5, 1, 147, 1, 234, 141, 427, 316, 27, 129, 597, 0, 125, 150, 1732, 659, 139, 5580, 0, 0, 351, 6, 237, 2077, 1, 8, 2037, 259, 220, 2, 409, 1665, 938, 4207, 428, 48, 0, 242, 0, 5, 5682, 5094, 85, 10] payload_rows_per_dst=[10196, 10372, 9844, 11360, 13118, 8648, 6528, 11740] dup_rows_per_dst=[2548, 2930, 2294, 2570, 3652, 1304, 892, 308] combine_rows_per_src=[10196, 10372, 9844, 11360, 13118, 8648, 6528, 11740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80476 payload_rows_remote=71348 dup_rows=17828 dup_rows_remote=16114 dedup_ratio=0.181356 router_tokens_per_expert=[0, 283, 1080, 3368, 182, 2628, 1, 40, 0, 17, 0, 5, 1190, 264, 2618, 1327, 1, 3488, 0, 0, 2, 4, 30, 4, 80, 192, 457, 53, 45, 50, 677, 0, 180, 112, 998, 627, 28, 4996, 0, 3, 250, 0, 139, 3300, 0, 0, 2081, 579, 269, 25, 223, 2876, 1272, 4113, 518, 2, 0, 340, 0, 1, 4451, 3669, 14, 0] payload_rows_per_dst=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] dup_rows_per_dst=[3326, 1714, 3836, 1888, 2514, 2528, 1828, 194] combine_rows_per_src=[11838, 9128, 11448, 8558, 11374, 10170, 8542, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=338968 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276999192 ingress_bytes=380469248 link_bytes=380469248 token_bytes=553659416 weight_movement_bytes=103809024 total_nvlink_bytes=657468440 weight_to_token_ratio=0.187496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431675150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79224 payload_rows_remote=66292 dup_rows=19080 dup_rows_remote=13864 dedup_ratio=0.194092 router_tokens_per_expert=[387, 221, 190, 13, 0, 241, 302, 710, 0, 7, 5, 4190, 1, 1749, 6538, 4, 123, 9, 243, 625, 3365, 5459, 205, 3, 6, 3559, 485, 256, 10, 0, 596, 22, 0, 0, 2, 525, 0, 3557, 2456, 0, 0, 412, 43, 479, 1540, 3299, 7, 446, 2801, 0, 0, 163, 0, 2, 1313, 144, 162, 299, 441, 354, 30, 0, 0, 1153] payload_rows_per_dst=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] dup_rows_per_dst=[1818, 1364, 894, 1048, 2614, 5216, 2758, 3368] combine_rows_per_src=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=280051712 dispatch_meta_bytes=344152 combine_payload_bytes=280051712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280395864 ingress_bytes=314654720 link_bytes=314654720 token_bytes=560447576 weight_movement_bytes=34603008 total_nvlink_bytes=595050584 weight_to_token_ratio=0.061742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184020380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80172 payload_rows_remote=69008 dup_rows=18132 dup_rows_remote=15280 dedup_ratio=0.184448 router_tokens_per_expert=[13, 166, 1387, 3372, 1015, 1613, 0, 119, 19, 82, 0, 109, 1039, 27, 3051, 998, 1, 2035, 0, 8, 5, 6, 49, 35, 99, 247, 510, 749, 16, 54, 505, 5, 556, 53, 1118, 771, 219, 3885, 0, 0, 441, 0, 57, 3366, 99, 9, 1838, 748, 289, 14, 472, 2341, 863, 4869, 483, 5, 2, 286, 0, 30, 3999, 4852, 148, 5] payload_rows_per_dst=[11882, 9068, 11164, 11130, 10522, 10378, 7370, 8658] dup_rows_per_dst=[3488, 1582, 2852, 2944, 2682, 2738, 1564, 282] combine_rows_per_src=[11882, 9068, 11164, 11130, 10522, 10378, 7370, 8658] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=293314560 dispatch_meta_bytes=342240 combine_payload_bytes=293314560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=293656800 ingress_bytes=397123584 link_bytes=397123584 token_bytes=586971360 weight_movement_bytes=103809024 total_nvlink_bytes=690780384 weight_to_token_ratio=0.176855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494344076e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292241408 dispatch_meta_bytes=349848 combine_payload_bytes=292241408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292591256 ingress_bytes=326844416 link_bytes=326844416 token_bytes=584832664 weight_movement_bytes=34603008 total_nvlink_bytes=619435672 weight_to_token_ratio=0.059167 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229889224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78680 payload_rows_remote=67544 dup_rows=19624 dup_rows_remote=15636 dedup_ratio=0.199626 router_tokens_per_expert=[1, 182, 1021, 1615, 506, 1531, 0, 41, 0, 3, 1, 58, 1084, 166, 2618, 1307, 11, 3158, 0, 0, 1, 0, 41, 1, 213, 244, 646, 92, 27, 302, 428, 0, 184, 66, 1330, 2963, 152, 4330, 1, 0, 286, 3, 63, 1659, 13, 0, 1480, 696, 174, 0, 1339, 2507, 1369, 4360, 684, 1, 0, 331, 0, 9, 5211, 4629, 13, 1] payload_rows_per_dst=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] dup_rows_per_dst=[1610, 1598, 2924, 2528, 1362, 3988, 3960, 1654] combine_rows_per_src=[8184, 8876, 10102, 10634, 8030, 11136, 10306, 11412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=277856256 dispatch_meta_bytes=334280 combine_payload_bytes=277856256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278190536 ingress_bytes=312459264 link_bytes=312459264 token_bytes=556046792 weight_movement_bytes=34603008 total_nvlink_bytes=590649800 weight_to_token_ratio=0.062230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175759055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81340 payload_rows_remote=69714 dup_rows=16964 dup_rows_remote=12658 dedup_ratio=0.172567 router_tokens_per_expert=[364, 428, 241, 107, 0, 143, 362, 249, 4, 18, 148, 3584, 0, 1621, 7353, 2, 39, 2, 219, 216, 2941, 6671, 121, 5, 4, 5968, 260, 371, 28, 2, 1047, 1, 0, 0, 0, 619, 1, 3543, 1979, 0, 0, 356, 34, 234, 1218, 2027, 38, 663, 3174, 2, 2, 255, 0, 7, 807, 135, 108, 176, 350, 298, 67, 0, 5, 535] payload_rows_per_dst=[3216, 15174, 6632, 12840, 9858, 8048, 11626, 13946] dup_rows_per_dst=[572, 3118, 454, 2522, 2426, 1092, 4306, 2474] combine_rows_per_src=[3216, 15174, 6632, 12840, 9858, 8048, 11626, 13946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282656768 dispatch_meta_bytes=337152 combine_payload_bytes=282656768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282993920 ingress_bytes=317259776 link_bytes=317259776 token_bytes=565650688 weight_movement_bytes=34603008 total_nvlink_bytes=600253696 weight_to_token_ratio=0.061174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193822996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=276660224 dispatch_meta_bytes=332720 combine_payload_bytes=276660224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276992944 ingress_bytes=311263232 link_bytes=311263232 token_bytes=553653168 weight_movement_bytes=34603008 total_nvlink_bytes=588256176 weight_to_token_ratio=0.062499 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171258483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81806 payload_rows_remote=71610 dup_rows=16498 dup_rows_remote=13950 dedup_ratio=0.167826 router_tokens_per_expert=[0, 139, 1599, 2761, 548, 1317, 0, 8, 0, 56, 0, 16, 2067, 85, 2321, 2106, 9, 1699, 0, 0, 5, 1, 147, 1, 234, 141, 427, 316, 27, 129, 597, 0, 125, 150, 1732, 659, 139, 5580, 0, 0, 351, 6, 237, 2077, 1, 8, 2037, 259, 220, 2, 409, 1665, 938, 4207, 428, 48, 0, 242, 0, 5, 5682, 5094, 85, 10] payload_rows_per_dst=[10196, 10372, 9844, 11360, 13118, 8648, 6528, 11740] dup_rows_per_dst=[2548, 2930, 2294, 2570, 3652, 1304, 892, 308] combine_rows_per_src=[10196, 10372, 9844, 11360, 13118, 8648, 6528, 11740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80866 payload_rows_remote=70388 dup_rows=17438 dup_rows_remote=16472 dedup_ratio=0.177389 router_tokens_per_expert=[432, 195, 195, 10, 0, 34, 963, 143, 0, 1, 29, 4542, 4, 1146, 6136, 0, 67, 3, 192, 332, 2994, 6053, 135, 26, 9, 4473, 346, 569, 10, 0, 483, 1, 0, 0, 4, 674, 0, 2610, 696, 0, 0, 338, 15, 1201, 1940, 1875, 65, 1691, 3573, 15, 1, 312, 0, 6, 1894, 90, 361, 247, 516, 686, 211, 0, 0, 608] payload_rows_per_dst=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] dup_rows_per_dst=[2814, 966, 5802, 1374, 1022, 2800, 1700, 960] combine_rows_per_src=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78960 payload_rows_remote=65740 dup_rows=19344 dup_rows_remote=16396 dedup_ratio=0.196777 router_tokens_per_expert=[505, 240, 296, 88, 0, 28, 388, 348, 0, 11, 91, 3488, 0, 1073, 6942, 16, 278, 3, 291, 518, 1421, 6217, 422, 8, 19, 6314, 317, 785, 17, 36, 519, 14, 0, 0, 4, 971, 2, 3814, 886, 2, 0, 198, 83, 861, 1448, 1848, 120, 1016, 3071, 3, 0, 549, 0, 14, 1415, 287, 83, 31, 297, 362, 134, 0, 7, 953] payload_rows_per_dst=[3358, 14160, 5462, 9282, 11882, 9262, 12334, 13220] dup_rows_per_dst=[428, 2106, 420, 1484, 4752, 1886, 5320, 2948] combine_rows_per_src=[3358, 14160, 5462, 9282, 11882, 9262, 12334, 13220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=271532032 dispatch_meta_bytes=320624 combine_payload_bytes=271532032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271852656 ingress_bytes=306135040 link_bytes=306135040 token_bytes=543384688 weight_movement_bytes=34603008 total_nvlink_bytes=577987696 weight_to_token_ratio=0.063680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151961510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82104 payload_rows_remote=71034 dup_rows=16200 dup_rows_remote=13946 dedup_ratio=0.164795 router_tokens_per_expert=[332, 199, 162, 20, 0, 35, 216, 98, 0, 2, 69, 3685, 0, 2112, 6303, 1, 69, 2, 194, 319, 2573, 5354, 462, 19, 105, 4990, 820, 520, 3, 2, 687, 0, 0, 0, 4, 1019, 7, 4728, 904, 0, 0, 490, 17, 343, 830, 2306, 99, 828, 3661, 8, 3, 95, 0, 1, 1184, 130, 116, 148, 583, 222, 880, 1, 0, 1192] payload_rows_per_dst=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] dup_rows_per_dst=[1540, 1086, 562, 2882, 2254, 2176, 1376, 4324] combine_rows_per_src=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82294 payload_rows_remote=71340 dup_rows=16010 dup_rows_remote=14362 dedup_ratio=0.162862 router_tokens_per_expert=[444, 245, 127, 270, 0, 59, 122, 252, 0, 3, 4, 4522, 0, 1621, 6787, 15, 123, 69, 393, 563, 3092, 6347, 65, 30, 2, 4640, 553, 637, 23, 2, 404, 40, 0, 0, 0, 1209, 1, 2968, 1218, 0, 0, 374, 17, 478, 1457, 1891, 14, 261, 3059, 10, 0, 78, 0, 1, 1035, 339, 139, 290, 486, 1218, 491, 2, 2, 660] payload_rows_per_dst=[14228, 10664, 9588, 10954, 8788, 8134, 7994, 11944] dup_rows_per_dst=[2384, 1666, 2802, 1648, 2004, 850, 1050, 3606] combine_rows_per_src=[14228, 10664, 9588, 10954, 8788, 8134, 7994, 11944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282656768 dispatch_meta_bytes=337152 combine_payload_bytes=282656768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282993920 ingress_bytes=317259776 link_bytes=317259776 token_bytes=565650688 weight_movement_bytes=34603008 total_nvlink_bytes=600253696 weight_to_token_ratio=0.061174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193822996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=293314560 dispatch_meta_bytes=342240 combine_payload_bytes=293314560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=293656800 ingress_bytes=397123584 link_bytes=397123584 token_bytes=586971360 weight_movement_bytes=103809024 total_nvlink_bytes=690780384 weight_to_token_ratio=0.176855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494344076e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78030 payload_rows_remote=65210 dup_rows=20274 dup_rows_remote=15570 dedup_ratio=0.206238 router_tokens_per_expert=[0, 218, 2361, 2713, 285, 2391, 0, 26, 4, 23, 0, 88, 1542, 241, 3144, 1192, 9, 3967, 0, 0, 51, 7, 23, 1, 57, 264, 485, 130, 45, 86, 588, 0, 305, 313, 688, 1694, 275, 3977, 0, 0, 252, 3, 53, 2308, 31, 3, 1832, 486, 11, 3, 309, 2116, 1229, 4537, 551, 6, 0, 233, 0, 5, 4841, 3086, 18, 46] payload_rows_per_dst=[10982, 10222, 8058, 7838, 6340, 8728, 12820, 13042] dup_rows_per_dst=[3858, 2306, 58, 1644, 1298, 1208, 4704, 5198] combine_rows_per_src=[10982, 10222, 8058, 7838, 6340, 8728, 12820, 13042] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=285548544 dispatch_meta_bytes=329488 combine_payload_bytes=285548544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285878032 ingress_bytes=320151552 link_bytes=320151552 token_bytes=571426576 weight_movement_bytes=34603008 total_nvlink_bytes=606029584 weight_to_token_ratio=0.060555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204704516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288309248 dispatch_meta_bytes=347440 combine_payload_bytes=288309248 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=288656688 ingress_bytes=426721280 link_bytes=426721280 token_bytes=576965936 weight_movement_bytes=138412032 total_nvlink_bytes=715377968 weight_to_token_ratio=0.239896 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605717823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269271040 dispatch_meta_bytes=328544 combine_payload_bytes=269271040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269599584 ingress_bytes=303874048 link_bytes=303874048 token_bytes=538870624 weight_movement_bytes=34603008 total_nvlink_bytes=573473632 weight_to_token_ratio=0.064214 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143453579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82104 payload_rows_remote=71034 dup_rows=16200 dup_rows_remote=13946 dedup_ratio=0.164795 router_tokens_per_expert=[332, 199, 162, 20, 0, 35, 216, 98, 0, 2, 69, 3685, 0, 2112, 6303, 1, 69, 2, 194, 319, 2573, 5354, 462, 19, 105, 4990, 820, 520, 3, 2, 687, 0, 0, 0, 4, 1019, 7, 4728, 904, 0, 0, 490, 17, 343, 830, 2306, 99, 828, 3661, 8, 3, 95, 0, 1, 1184, 130, 116, 148, 583, 222, 880, 1, 0, 1192] payload_rows_per_dst=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] dup_rows_per_dst=[1540, 1086, 562, 2882, 2254, 2176, 1376, 4324] combine_rows_per_src=[13190, 8064, 6714, 11832, 11070, 9778, 8788, 12668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=290955264 dispatch_meta_bytes=339920 combine_payload_bytes=290955264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291295184 ingress_bytes=325558272 link_bytes=325558272 token_bytes=582250448 weight_movement_bytes=34603008 total_nvlink_bytes=616853456 weight_to_token_ratio=0.059430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225049568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81698 payload_rows_remote=68632 dup_rows=16606 dup_rows_remote=11792 dedup_ratio=0.168925 router_tokens_per_expert=[629, 150, 426, 15, 7, 79, 598, 584, 57, 3, 24, 3774, 25, 949, 6331, 0, 267, 3, 278, 556, 1969, 5583, 104, 180, 27, 5173, 288, 355, 6, 33, 372, 7, 1, 2, 0, 188, 47, 3692, 1173, 0, 3, 413, 0, 866, 1489, 1412, 66, 1907, 4149, 0, 6, 156, 0, 53, 1585, 122, 323, 484, 268, 1060, 290, 0, 0, 545] payload_rows_per_dst=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] dup_rows_per_dst=[3572, 674, 4814, 1216, 1356, 2354, 1850, 770] combine_rows_per_src=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79224 payload_rows_remote=66292 dup_rows=19080 dup_rows_remote=13864 dedup_ratio=0.194092 router_tokens_per_expert=[387, 221, 190, 13, 0, 241, 302, 710, 0, 7, 5, 4190, 1, 1749, 6538, 4, 123, 9, 243, 625, 3365, 5459, 205, 3, 6, 3559, 485, 256, 10, 0, 596, 22, 0, 0, 2, 525, 0, 3557, 2456, 0, 0, 412, 43, 479, 1540, 3299, 7, 446, 2801, 0, 0, 163, 0, 2, 1313, 144, 162, 299, 441, 354, 30, 0, 0, 1153] payload_rows_per_dst=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] dup_rows_per_dst=[1818, 1364, 894, 1048, 2614, 5216, 2758, 3368] combine_rows_per_src=[9684, 7870, 8252, 8820, 10466, 12932, 8772, 12428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292208640 dispatch_meta_bytes=342808 combine_payload_bytes=292208640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292551448 ingress_bytes=396017664 link_bytes=396017664 token_bytes=584760088 weight_movement_bytes=103809024 total_nvlink_bytes=688569112 weight_to_token_ratio=0.177524 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490182588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=323120 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267423280 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534523440 weight_movement_bytes=69206016 total_nvlink_bytes=603729456 weight_to_token_ratio=0.129472 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=281116672 dispatch_meta_bytes=321696 combine_payload_bytes=281116672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281438368 ingress_bytes=350322688 link_bytes=350322688 token_bytes=562555040 weight_movement_bytes=69206016 total_nvlink_bytes=631761056 weight_to_token_ratio=0.123021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318236072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=271532032 dispatch_meta_bytes=320624 combine_payload_bytes=271532032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271852656 ingress_bytes=306135040 link_bytes=306135040 token_bytes=543384688 weight_movement_bytes=34603008 total_nvlink_bytes=577987696 weight_to_token_ratio=0.063680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151961510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79324 payload_rows_remote=66984 dup_rows=18980 dup_rows_remote=17288 dedup_ratio=0.193075 router_tokens_per_expert=[266, 207, 168, 3, 0, 274, 221, 128, 0, 3, 52, 4192, 0, 1681, 5749, 34, 100, 19, 390, 537, 2860, 4312, 24, 5, 33, 3051, 795, 1501, 8, 0, 1228, 1, 0, 0, 12, 1334, 216, 2990, 2454, 2, 0, 372, 3, 1283, 1189, 2486, 57, 794, 3563, 2, 5, 327, 0, 20, 677, 434, 272, 438, 606, 640, 62, 0, 0, 1072] payload_rows_per_dst=[12340, 10496, 12726, 8904, 10386, 10614, 8720, 5138] dup_rows_per_dst=[1692, 1428, 6314, 1784, 3630, 1754, 1336, 1042] combine_rows_per_src=[12340, 10496, 12726, 8904, 10386, 10614, 8720, 5138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80866 payload_rows_remote=70388 dup_rows=17438 dup_rows_remote=16472 dedup_ratio=0.177389 router_tokens_per_expert=[432, 195, 195, 10, 0, 34, 963, 143, 0, 1, 29, 4542, 4, 1146, 6136, 0, 67, 3, 192, 332, 2994, 6053, 135, 26, 9, 4473, 346, 569, 10, 0, 483, 1, 0, 0, 4, 674, 0, 2610, 696, 0, 0, 338, 15, 1201, 1940, 1875, 65, 1691, 3573, 15, 1, 312, 0, 6, 1894, 90, 361, 247, 516, 686, 211, 0, 0, 608] payload_rows_per_dst=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] dup_rows_per_dst=[2814, 966, 5802, 1374, 1022, 2800, 1700, 960] combine_rows_per_src=[13402, 10478, 13802, 10408, 6946, 11450, 10082, 4298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78962 payload_rows_remote=69182 dup_rows=19342 dup_rows_remote=16276 dedup_ratio=0.196757 router_tokens_per_expert=[0, 5, 0, 2840, 369, 56, 0, 8, 1121, 1291, 10, 0, 4, 208, 1411, 2041, 102, 4, 126, 660, 3786, 659, 3, 2, 294, 759, 127, 987, 2955, 3413, 5, 1481, 155, 0, 69, 604, 28, 65, 854, 4, 1, 0, 98, 13, 0, 1, 15, 3, 850, 1, 4936, 4157, 912, 0, 519, 1636, 1382, 720, 4792, 0, 2, 0, 1462, 1146] payload_rows_per_dst=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] dup_rows_per_dst=[3596, 3066, 1260, 2650, 1200, 4824, 946, 1800] combine_rows_per_src=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=290955264 dispatch_meta_bytes=339920 combine_payload_bytes=290955264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291295184 ingress_bytes=325558272 link_bytes=325558272 token_bytes=582250448 weight_movement_bytes=34603008 total_nvlink_bytes=616853456 weight_to_token_ratio=0.059430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225049568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84336 payload_rows_remote=72780 dup_rows=13968 dup_rows_remote=13778 dedup_ratio=0.142090 router_tokens_per_expert=[0, 0, 0, 2098, 263, 13, 0, 3, 606, 584, 2, 0, 3, 36, 3396, 2230, 87, 0, 101, 200, 4951, 2446, 0, 3, 14, 2128, 160, 274, 4366, 2583, 0, 1501, 153, 0, 80, 1042, 151, 105, 446, 5, 0, 0, 125, 4, 0, 0, 22, 0, 375, 0, 5722, 3645, 546, 0, 112, 2397, 325, 1160, 3108, 0, 0, 0, 695, 886] payload_rows_per_dst=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] dup_rows_per_dst=[992, 2836, 2354, 2208, 1614, 190, 2372, 1402] combine_rows_per_src=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81698 payload_rows_remote=68632 dup_rows=16606 dup_rows_remote=11792 dedup_ratio=0.168925 router_tokens_per_expert=[629, 150, 426, 15, 7, 79, 598, 584, 57, 3, 24, 3774, 25, 949, 6331, 0, 267, 3, 278, 556, 1969, 5583, 104, 180, 27, 5173, 288, 355, 6, 33, 372, 7, 1, 2, 0, 188, 47, 3692, 1173, 0, 3, 413, 0, 866, 1489, 1412, 66, 1907, 4149, 0, 6, 156, 0, 53, 1585, 122, 323, 484, 268, 1060, 290, 0, 0, 545] payload_rows_per_dst=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] dup_rows_per_dst=[3572, 674, 4814, 1216, 1356, 2354, 1850, 770] combine_rows_per_src=[14066, 8990, 13066, 11306, 8850, 9958, 10292, 5170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274366464 dispatch_meta_bytes=337088 combine_payload_bytes=274366464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274703552 ingress_bytes=308969472 link_bytes=308969472 token_bytes=549070016 weight_movement_bytes=34603008 total_nvlink_bytes=583673024 weight_to_token_ratio=0.063021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162627249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80864 payload_rows_remote=68410 dup_rows=17440 dup_rows_remote=14774 dedup_ratio=0.177409 router_tokens_per_expert=[0, 1, 3, 2806, 286, 7, 1, 11, 276, 1171, 6, 0, 1, 60, 1888, 1393, 37, 2, 119, 439, 3320, 3074, 4, 0, 336, 705, 260, 1024, 3669, 2285, 0, 1585, 97, 1, 103, 1204, 165, 133, 1122, 73, 0, 16, 111, 9, 0, 6, 16, 1, 613, 0, 5355, 4649, 686, 0, 272, 1340, 982, 1608, 4513, 5, 2, 2, 576, 723] payload_rows_per_dst=[8302, 8240, 11646, 6514, 10884, 10828, 12454, 11996] dup_rows_per_dst=[1062, 1350, 2344, 1306, 2250, 200, 2666, 6262] combine_rows_per_src=[8302, 8240, 11646, 6514, 10884, 10828, 12454, 11996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288309248 dispatch_meta_bytes=347440 combine_payload_bytes=288309248 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=288656688 ingress_bytes=426721280 link_bytes=426721280 token_bytes=576965936 weight_movement_bytes=138412032 total_nvlink_bytes=715377968 weight_to_token_ratio=0.239896 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605717823e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=298106880 dispatch_meta_bytes=346232 combine_payload_bytes=298106880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=298453112 ingress_bytes=367312896 link_bytes=367312896 token_bytes=596559992 weight_movement_bytes=69206016 total_nvlink_bytes=665766008 weight_to_token_ratio=0.116008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382168857e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78076 payload_rows_remote=63934 dup_rows=20228 dup_rows_remote=10790 dedup_ratio=0.205770 router_tokens_per_expert=[0, 48, 0, 1825, 225, 28, 7, 14, 70, 852, 1, 5, 1, 61, 3249, 2012, 57, 1, 464, 460, 2850, 2862, 2, 2, 174, 655, 1031, 655, 2813, 3592, 18, 2161, 77, 1, 75, 611, 224, 260, 1234, 23, 0, 3, 119, 22, 1, 0, 52, 0, 1080, 1, 4944, 3166, 386, 0, 67, 2413, 521, 1776, 4889, 9, 5, 4, 700, 294] payload_rows_per_dst=[4130, 10490, 8884, 8150, 9118, 10098, 13064, 14142] dup_rows_per_dst=[164, 2012, 1078, 1238, 1518, 184, 4596, 9438] combine_rows_per_src=[4130, 10490, 8884, 8150, 9118, 10098, 13064, 14142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81436 payload_rows_remote=68536 dup_rows=16868 dup_rows_remote=14142 dedup_ratio=0.171590 router_tokens_per_expert=[0, 15, 0, 2582, 367, 3, 2, 129, 586, 1186, 11, 0, 3, 132, 850, 1091, 177, 15, 388, 154, 4360, 1626, 22, 0, 276, 1503, 787, 1452, 5660, 2142, 2, 488, 289, 0, 96, 469, 311, 129, 845, 14, 1, 0, 178, 2, 0, 2, 8, 0, 465, 1, 4156, 2545, 372, 1, 714, 1316, 1801, 1962, 5722, 2, 15, 17, 804, 906] payload_rows_per_dst=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] dup_rows_per_dst=[4050, 1382, 1952, 2896, 2726, 162, 1464, 2236] combine_rows_per_src=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79388 payload_rows_remote=66998 dup_rows=18916 dup_rows_remote=14242 dedup_ratio=0.192424 router_tokens_per_expert=[1, 178, 0, 1995, 507, 19, 8, 27, 915, 1206, 1, 0, 14, 66, 999, 1485, 60, 16, 76, 154, 4219, 1990, 5, 4, 81, 1748, 177, 570, 3172, 3969, 0, 1987, 248, 0, 94, 2247, 327, 125, 1310, 1, 2, 0, 324, 19, 1, 0, 24, 0, 489, 0, 3826, 4296, 238, 1, 433, 2311, 660, 2132, 2412, 1, 1, 0, 1163, 818] payload_rows_per_dst=[8718, 7568, 11220, 12390, 11446, 8108, 12270, 7668] dup_rows_per_dst=[1576, 1804, 1828, 4674, 3602, 284, 3266, 1882] combine_rows_per_src=[8718, 7568, 11220, 12390, 11446, 8108, 12270, 7668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=281116672 dispatch_meta_bytes=321696 combine_payload_bytes=281116672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281438368 ingress_bytes=350322688 link_bytes=350322688 token_bytes=562555040 weight_movement_bytes=69206016 total_nvlink_bytes=631761056 weight_to_token_ratio=0.123021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318236072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274366464 dispatch_meta_bytes=337088 combine_payload_bytes=274366464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274703552 ingress_bytes=308969472 link_bytes=308969472 token_bytes=549070016 weight_movement_bytes=34603008 total_nvlink_bytes=583673024 weight_to_token_ratio=0.063021 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162627249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81340 payload_rows_remote=69714 dup_rows=16964 dup_rows_remote=12658 dedup_ratio=0.172567 router_tokens_per_expert=[364, 428, 241, 107, 0, 143, 362, 249, 4, 18, 148, 3584, 0, 1621, 7353, 2, 39, 2, 219, 216, 2941, 6671, 121, 5, 4, 5968, 260, 371, 28, 2, 1047, 1, 0, 0, 0, 619, 1, 3543, 1979, 0, 0, 356, 34, 234, 1218, 2027, 38, 663, 3174, 2, 2, 255, 0, 7, 807, 135, 108, 176, 350, 298, 67, 0, 5, 535] payload_rows_per_dst=[3216, 15174, 6632, 12840, 9858, 8048, 11626, 13946] dup_rows_per_dst=[572, 3118, 454, 2522, 2426, 1092, 4306, 2474] combine_rows_per_src=[3216, 15174, 6632, 12840, 9858, 8048, 11626, 13946] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280207360 dispatch_meta_bytes=332736 combine_payload_bytes=280207360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=280540096 ingress_bytes=384016384 link_bytes=384016384 token_bytes=560747456 weight_movement_bytes=103809024 total_nvlink_bytes=664556480 weight_to_token_ratio=0.185126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445022737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81436 payload_rows_remote=68536 dup_rows=16868 dup_rows_remote=14142 dedup_ratio=0.171590 router_tokens_per_expert=[0, 15, 0, 2582, 367, 3, 2, 129, 586, 1186, 11, 0, 3, 132, 850, 1091, 177, 15, 388, 154, 4360, 1626, 22, 0, 276, 1503, 787, 1452, 5660, 2142, 2, 488, 289, 0, 96, 469, 311, 129, 845, 14, 1, 0, 178, 2, 0, 2, 8, 0, 465, 1, 4156, 2545, 372, 1, 714, 1316, 1801, 1962, 5722, 2, 15, 17, 804, 906] payload_rows_per_dst=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] dup_rows_per_dst=[4050, 1382, 1952, 2896, 2726, 162, 1464, 2236] combine_rows_per_src=[13590, 6336, 11532, 10404, 12900, 8532, 9364, 8778] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=261873664 dispatch_meta_bytes=298896 combine_payload_bytes=261873664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262172560 ingress_bytes=331079680 link_bytes=331079680 token_bytes=524046224 weight_movement_bytes=69206016 total_nvlink_bytes=593252240 weight_to_token_ratio=0.132061 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245826182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=324960 combine_payload_bytes=274423808 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274748768 ingress_bytes=412835840 link_bytes=412835840 token_bytes=549172576 weight_movement_bytes=138412032 total_nvlink_bytes=687584608 weight_to_token_ratio=0.252037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553468030e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78540 payload_rows_remote=67172 dup_rows=19764 dup_rows_remote=17508 dedup_ratio=0.201050 router_tokens_per_expert=[0, 45, 0, 2238, 328, 1, 6, 79, 254, 792, 1, 0, 5, 75, 2206, 907, 224, 14, 52, 186, 3985, 2161, 1, 0, 9, 1612, 146, 900, 4029, 4242, 0, 1113, 53, 1, 73, 556, 387, 350, 975, 129, 5, 0, 41, 7, 0, 19, 149, 0, 483, 2, 5560, 3737, 616, 0, 370, 1482, 1304, 1666, 4115, 0, 0, 5, 385, 1071] payload_rows_per_dst=[11368, 12014, 11828, 8652, 7930, 13552, 5596, 7600] dup_rows_per_dst=[2256, 4950, 2538, 1610, 1354, 5484, 310, 1262] combine_rows_per_src=[11368, 12014, 11828, 8652, 7930, 13552, 5596, 7600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280723456 dispatch_meta_bytes=330712 combine_payload_bytes=280723456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281054168 ingress_bytes=315326464 link_bytes=315326464 token_bytes=561777624 weight_movement_bytes=34603008 total_nvlink_bytes=596380632 weight_to_token_ratio=0.061596 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186548098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84054 payload_rows_remote=73736 dup_rows=14250 dup_rows_remote=12174 dedup_ratio=0.144958 router_tokens_per_expert=[0, 102, 8, 131, 136, 1620, 6, 0, 0, 5383, 0, 610, 2, 574, 21, 553, 0, 1041, 3111, 3, 1955, 669, 2, 10, 3, 6457, 3, 131, 6683, 178, 0, 0, 825, 0, 36, 0, 288, 57, 1, 6, 44, 370, 717, 3699, 0, 0, 1357, 10, 0, 0, 337, 500, 0, 0, 450, 2273, 3889, 3421, 50, 0, 0, 1, 1429, 0] payload_rows_per_dst=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] dup_rows_per_dst=[1522, 2508, 750, 246, 1642, 2076, 1616, 3890] combine_rows_per_src=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78962 payload_rows_remote=69182 dup_rows=19342 dup_rows_remote=16276 dedup_ratio=0.196757 router_tokens_per_expert=[0, 5, 0, 2840, 369, 56, 0, 8, 1121, 1291, 10, 0, 4, 208, 1411, 2041, 102, 4, 126, 660, 3786, 659, 3, 2, 294, 759, 127, 987, 2955, 3413, 5, 1481, 155, 0, 69, 604, 28, 65, 854, 4, 1, 0, 98, 13, 0, 1, 15, 3, 850, 1, 4936, 4157, 912, 0, 519, 1636, 1382, 720, 4792, 0, 2, 0, 1462, 1146] payload_rows_per_dst=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] dup_rows_per_dst=[3596, 3066, 1260, 2650, 1200, 4824, 946, 1800] combine_rows_per_src=[12544, 9780, 9424, 10808, 8268, 12900, 7614, 7624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=283369472 dispatch_meta_bytes=341832 combine_payload_bytes=283369472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283711304 ingress_bytes=317972480 link_bytes=317972480 token_bytes=567080776 weight_movement_bytes=34603008 total_nvlink_bytes=601683784 weight_to_token_ratio=0.061020 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196504843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80918 payload_rows_remote=74244 dup_rows=17386 dup_rows_remote=16866 dedup_ratio=0.176860 router_tokens_per_expert=[0, 110, 7, 2608, 287, 60, 2, 424, 905, 1474, 11, 0, 0, 157, 2650, 2602, 166, 13, 158, 237, 2435, 569, 17, 2, 99, 1954, 222, 844, 4224, 1928, 11, 2651, 246, 5, 189, 387, 160, 164, 795, 36, 22, 2, 139, 75, 0, 0, 61, 1, 567, 0, 5401, 1956, 172, 0, 537, 2040, 1003, 1465, 4491, 2, 65, 5, 1258, 1081] payload_rows_per_dst=[12446, 11360, 6674, 11866, 10764, 11122, 9190, 7496] dup_rows_per_dst=[3532, 4238, 520, 3552, 1648, 280, 1354, 2262] combine_rows_per_src=[12446, 11360, 6674, 11866, 10764, 11122, 9190, 7496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=338720 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275475232 ingress_bytes=309739520 link_bytes=309739520 token_bytes=550611744 weight_movement_bytes=34603008 total_nvlink_bytes=585214752 weight_to_token_ratio=0.062845 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165524878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84336 payload_rows_remote=72780 dup_rows=13968 dup_rows_remote=13778 dedup_ratio=0.142090 router_tokens_per_expert=[0, 0, 0, 2098, 263, 13, 0, 3, 606, 584, 2, 0, 3, 36, 3396, 2230, 87, 0, 101, 200, 4951, 2446, 0, 3, 14, 2128, 160, 274, 4366, 2583, 0, 1501, 153, 0, 80, 1042, 151, 105, 446, 5, 0, 0, 125, 4, 0, 0, 22, 0, 375, 0, 5722, 3645, 546, 0, 112, 2397, 325, 1160, 3108, 0, 0, 0, 695, 886] payload_rows_per_dst=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] dup_rows_per_dst=[992, 2836, 2354, 2208, 1614, 190, 2372, 1402] combine_rows_per_src=[9978, 10878, 13222, 9386, 11082, 11556, 11778, 6456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=274423808 dispatch_meta_bytes=324960 combine_payload_bytes=274423808 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274748768 ingress_bytes=412835840 link_bytes=412835840 token_bytes=549172576 weight_movement_bytes=138412032 total_nvlink_bytes=687584608 weight_to_token_ratio=0.252037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553468030e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=285548544 dispatch_meta_bytes=329488 combine_payload_bytes=285548544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285878032 ingress_bytes=320151552 link_bytes=320151552 token_bytes=571426576 weight_movement_bytes=34603008 total_nvlink_bytes=606029584 weight_to_token_ratio=0.060555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204704516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81588 payload_rows_remote=66438 dup_rows=16716 dup_rows_remote=10516 dedup_ratio=0.170044 router_tokens_per_expert=[3, 88, 219, 151, 240, 1335, 30, 1, 4, 5880, 0, 432, 8, 421, 3, 573, 0, 916, 2780, 11, 2049, 972, 0, 9, 1, 6447, 13, 128, 7605, 398, 0, 0, 423, 3, 27, 0, 160, 17, 60, 104, 341, 148, 639, 3335, 1, 0, 282, 7, 0, 0, 360, 457, 0, 5, 246, 3160, 4215, 3251, 105, 1, 0, 0, 1088, 0] payload_rows_per_dst=[10922, 9840, 9052, 1072, 15694, 8406, 15150, 11452] dup_rows_per_dst=[1642, 1628, 2036, 8, 1104, 1100, 6200, 2998] combine_rows_per_src=[10922, 9840, 9052, 1072, 15694, 8406, 15150, 11452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=343640 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302366296 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604388952 weight_movement_bytes=69206016 total_nvlink_bytes=673594968 weight_to_token_ratio=0.114506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=283369472 dispatch_meta_bytes=341832 combine_payload_bytes=283369472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283711304 ingress_bytes=317972480 link_bytes=317972480 token_bytes=567080776 weight_movement_bytes=34603008 total_nvlink_bytes=601683784 weight_to_token_ratio=0.061020 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196504843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77012 payload_rows_remote=65812 dup_rows=21292 dup_rows_remote=18294 dedup_ratio=0.216593 router_tokens_per_expert=[0, 269, 120, 114, 130, 1963, 5, 0, 20, 5250, 6, 605, 27, 525, 67, 455, 0, 1419, 2446, 10, 1450, 505, 6, 132, 8, 4384, 10, 382, 6223, 263, 0, 0, 650, 0, 250, 0, 687, 276, 104, 135, 380, 1171, 761, 3430, 0, 12, 338, 40, 0, 7, 398, 289, 1, 0, 344, 4466, 3966, 3297, 310, 6, 2, 25, 1013, 0] payload_rows_per_dst=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] dup_rows_per_dst=[1998, 136, 3302, 38, 2786, 4190, 5844, 2998] combine_rows_per_src=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84600 payload_rows_remote=73326 dup_rows=13704 dup_rows_remote=13272 dedup_ratio=0.139404 router_tokens_per_expert=[0, 123, 630, 580, 255, 1330, 11, 0, 1, 5791, 0, 848, 25, 454, 5, 1057, 0, 1167, 3121, 6, 2492, 1425, 0, 2, 2, 5316, 1, 15, 6337, 519, 0, 0, 655, 1, 28, 0, 117, 18, 44, 82, 163, 285, 719, 2668, 0, 4, 857, 16, 0, 1, 213, 142, 0, 0, 1197, 2464, 5011, 2049, 50, 1, 0, 0, 854, 0] payload_rows_per_dst=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] dup_rows_per_dst=[3288, 3378, 1658, 432, 1072, 1098, 780, 1998] combine_rows_per_src=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304103424 dispatch_meta_bytes=364440 combine_payload_bytes=304103424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304467864 ingress_bytes=373309440 link_bytes=373309440 token_bytes=608571288 weight_movement_bytes=69206016 total_nvlink_bytes=677777304 weight_to_token_ratio=0.113719 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404733369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=272130048 dispatch_meta_bytes=307816 combine_payload_bytes=272130048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272437864 ingress_bytes=306733056 link_bytes=306733056 token_bytes=544567912 weight_movement_bytes=34603008 total_nvlink_bytes=579170920 weight_to_token_ratio=0.063542 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154211796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=298106880 dispatch_meta_bytes=346232 combine_payload_bytes=298106880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=298453112 ingress_bytes=367312896 link_bytes=367312896 token_bytes=596559992 weight_movement_bytes=69206016 total_nvlink_bytes=665766008 weight_to_token_ratio=0.116008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382168857e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78540 payload_rows_remote=67172 dup_rows=19764 dup_rows_remote=17508 dedup_ratio=0.201050 router_tokens_per_expert=[0, 45, 0, 2238, 328, 1, 6, 79, 254, 792, 1, 0, 5, 75, 2206, 907, 224, 14, 52, 186, 3985, 2161, 1, 0, 9, 1612, 146, 900, 4029, 4242, 0, 1113, 53, 1, 73, 556, 387, 350, 975, 129, 5, 0, 41, 7, 0, 19, 149, 0, 483, 2, 5560, 3737, 616, 0, 370, 1482, 1304, 1666, 4115, 0, 0, 5, 385, 1071] payload_rows_per_dst=[11368, 12014, 11828, 8652, 7930, 13552, 5596, 7600] dup_rows_per_dst=[2256, 4950, 2538, 1610, 1354, 5484, 310, 1262] combine_rows_per_src=[11368, 12014, 11828, 8652, 7930, 13552, 5596, 7600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77012 payload_rows_remote=65812 dup_rows=21292 dup_rows_remote=18294 dedup_ratio=0.216593 router_tokens_per_expert=[0, 269, 120, 114, 130, 1963, 5, 0, 20, 5250, 6, 605, 27, 525, 67, 455, 0, 1419, 2446, 10, 1450, 505, 6, 132, 8, 4384, 10, 382, 6223, 263, 0, 0, 650, 0, 250, 0, 687, 276, 104, 135, 380, 1171, 761, 3430, 0, 12, 338, 40, 0, 7, 398, 289, 1, 0, 344, 4466, 3966, 3297, 310, 6, 2, 25, 1013, 0] payload_rows_per_dst=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] dup_rows_per_dst=[1998, 136, 3302, 38, 2786, 4190, 5844, 2998] combine_rows_per_src=[11136, 3274, 11232, 1288, 13864, 11084, 13934, 11200] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80850 payload_rows_remote=68262 dup_rows=17454 dup_rows_remote=16480 dedup_ratio=0.177551 router_tokens_per_expert=[0, 97, 292, 374, 79, 563, 2, 0, 1, 3328, 6, 259, 57, 411, 27, 163, 0, 1812, 3526, 5, 2127, 1253, 3, 16, 0, 5293, 9, 903, 5788, 291, 0, 0, 392, 27, 94, 0, 152, 30, 194, 104, 241, 1905, 889, 4463, 1, 0, 711, 28, 0, 0, 381, 988, 0, 0, 389, 2795, 2693, 3456, 25, 0, 1, 1, 2503, 4] payload_rows_per_dst=[7398, 7444, 8564, 11884, 12588, 12314, 7882, 12776] dup_rows_per_dst=[802, 1060, 1868, 1108, 974, 4162, 1224, 6256] combine_rows_per_src=[7398, 7444, 8564, 11884, 12588, 12314, 7882, 12776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=269565952 dispatch_meta_bytes=336424 combine_payload_bytes=269565952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269902376 ingress_bytes=373374976 link_bytes=373374976 token_bytes=539468328 weight_movement_bytes=103809024 total_nvlink_bytes=643277352 weight_to_token_ratio=0.192428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404979976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300343296 dispatch_meta_bytes=346392 combine_payload_bytes=300343296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300689688 ingress_bytes=404152320 link_bytes=404152320 token_bytes=601032984 weight_movement_bytes=103809024 total_nvlink_bytes=704842008 weight_to_token_ratio=0.172718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520792643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72866 payload_rows_remote=60736 dup_rows=25438 dup_rows_remote=21226 dedup_ratio=0.258769 router_tokens_per_expert=[3, 80, 320, 289, 36, 2536, 10, 0, 1, 5038, 1, 923, 10, 1794, 1, 221, 0, 1942, 2197, 0, 4206, 449, 0, 2, 2, 3540, 8, 59, 4201, 372, 1, 0, 366, 1, 202, 0, 236, 368, 266, 31, 400, 758, 618, 3995, 2, 0, 353, 26, 3, 0, 736, 321, 6, 0, 495, 5009, 1904, 3490, 17, 0, 0, 8, 1299, 0] payload_rows_per_dst=[12130, 12282, 12642, 14508, 2570, 4130, 11146, 3458] dup_rows_per_dst=[4212, 3696, 4950, 9848, 370, 184, 1994, 184] combine_rows_per_src=[12130, 12282, 12642, 14508, 2570, 4130, 11146, 3458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=338720 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275475232 ingress_bytes=309739520 link_bytes=309739520 token_bytes=550611744 weight_movement_bytes=34603008 total_nvlink_bytes=585214752 weight_to_token_ratio=0.062845 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165524878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=67044 dup_rows=20676 dup_rows_remote=18234 dedup_ratio=0.210327 router_tokens_per_expert=[0, 245, 104, 308, 204, 1739, 4, 0, 0, 4487, 0, 455, 23, 783, 10, 755, 0, 914, 2743, 1, 1574, 732, 0, 36, 0, 6735, 2, 80, 6842, 152, 0, 0, 498, 1, 39, 0, 109, 53, 17, 25, 66, 253, 782, 3678, 0, 1, 1184, 19, 2, 1, 362, 372, 0, 0, 714, 4395, 4376, 2396, 87, 0, 0, 5, 782, 7] payload_rows_per_dst=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] dup_rows_per_dst=[2306, 2442, 2270, 9838, 358, 1452, 1388, 622] combine_rows_per_src=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80864 payload_rows_remote=68410 dup_rows=17440 dup_rows_remote=14774 dedup_ratio=0.177409 router_tokens_per_expert=[0, 1, 3, 2806, 286, 7, 1, 11, 276, 1171, 6, 0, 1, 60, 1888, 1393, 37, 2, 119, 439, 3320, 3074, 4, 0, 336, 705, 260, 1024, 3669, 2285, 0, 1585, 97, 1, 103, 1204, 165, 133, 1122, 73, 0, 16, 111, 9, 0, 6, 16, 1, 613, 0, 5355, 4649, 686, 0, 272, 1340, 982, 1608, 4513, 5, 2, 2, 576, 723] payload_rows_per_dst=[8302, 8240, 11646, 6514, 10884, 10828, 12454, 11996] dup_rows_per_dst=[1062, 1350, 2344, 1306, 2250, 200, 2666, 6262] combine_rows_per_src=[8302, 8240, 11646, 6514, 10884, 10828, 12454, 11996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84600 payload_rows_remote=73326 dup_rows=13704 dup_rows_remote=13272 dedup_ratio=0.139404 router_tokens_per_expert=[0, 123, 630, 580, 255, 1330, 11, 0, 1, 5791, 0, 848, 25, 454, 5, 1057, 0, 1167, 3121, 6, 2492, 1425, 0, 2, 2, 5316, 1, 15, 6337, 519, 0, 0, 655, 1, 28, 0, 117, 18, 44, 82, 163, 285, 719, 2668, 0, 4, 857, 16, 0, 1, 213, 142, 0, 0, 1197, 2464, 5011, 2049, 50, 1, 0, 0, 854, 0] payload_rows_per_dst=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] dup_rows_per_dst=[3288, 3378, 1658, 432, 1072, 1098, 780, 1998] combine_rows_per_src=[12592, 12984, 8526, 11274, 13492, 8326, 7254, 10152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=269565952 dispatch_meta_bytes=336424 combine_payload_bytes=269565952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269902376 ingress_bytes=373374976 link_bytes=373374976 token_bytes=539468328 weight_movement_bytes=103809024 total_nvlink_bytes=643277352 weight_to_token_ratio=0.192428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404979976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=279601152 dispatch_meta_bytes=338968 combine_payload_bytes=279601152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279940120 ingress_bytes=314204160 link_bytes=314204160 token_bytes=559541272 weight_movement_bytes=34603008 total_nvlink_bytes=594144280 weight_to_token_ratio=0.061842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182324959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=69524 dup_rows=12618 dup_rows_remote=8324 dedup_ratio=0.128357 router_tokens_per_expert=[2294, 150, 0, 6113, 0, 4, 177, 284, 0, 45, 110, 7, 0, 0, 162, 2318, 0, 1, 6, 0, 0, 24, 210, 4699, 3, 0, 1, 0, 2325, 0, 0, 0, 20, 1748, 449, 58, 5972, 0, 1481, 4824, 0, 0, 7, 745, 18, 0, 7977, 0, 50, 86, 0, 0, 0, 34, 0, 0, 268, 0, 0, 3228, 14, 1617, 37, 1586] payload_rows_per_dst=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] dup_rows_per_dst=[182, 1168, 128, 2680, 1836, 4294, 122, 2208] combine_rows_per_src=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=248774656 dispatch_meta_bytes=327848 combine_payload_bytes=248774656 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=249102504 ingress_bytes=317980672 link_bytes=317980672 token_bytes=497877160 weight_movement_bytes=69206016 total_nvlink_bytes=567083176 weight_to_token_ratio=0.139002 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196535669e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=67044 dup_rows=20676 dup_rows_remote=18234 dedup_ratio=0.210327 router_tokens_per_expert=[0, 245, 104, 308, 204, 1739, 4, 0, 0, 4487, 0, 455, 23, 783, 10, 755, 0, 914, 2743, 1, 1574, 732, 0, 36, 0, 6735, 2, 80, 6842, 152, 0, 0, 498, 1, 39, 0, 109, 53, 17, 25, 66, 253, 782, 3678, 0, 1, 1184, 19, 2, 1, 362, 372, 0, 0, 714, 4395, 4376, 2396, 87, 0, 0, 5, 782, 7] payload_rows_per_dst=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] dup_rows_per_dst=[2306, 2442, 2270, 9838, 358, 1452, 1388, 622] combine_rows_per_src=[11654, 10584, 9730, 15224, 5474, 8726, 10304, 5932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=341112 combine_payload_bytes=274612224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274953336 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549565560 weight_movement_bytes=69206016 total_nvlink_bytes=618771576 weight_to_token_ratio=0.125929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84054 payload_rows_remote=73736 dup_rows=14250 dup_rows_remote=12174 dedup_ratio=0.144958 router_tokens_per_expert=[0, 102, 8, 131, 136, 1620, 6, 0, 0, 5383, 0, 610, 2, 574, 21, 553, 0, 1041, 3111, 3, 1955, 669, 2, 10, 3, 6457, 3, 131, 6683, 178, 0, 0, 825, 0, 36, 0, 288, 57, 1, 6, 44, 370, 717, 3699, 0, 0, 1357, 10, 0, 0, 337, 500, 0, 0, 450, 2273, 3889, 3421, 50, 0, 0, 1, 1429, 0] payload_rows_per_dst=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] dup_rows_per_dst=[1522, 2508, 750, 246, 1642, 2076, 1616, 3890] combine_rows_per_src=[10262, 11778, 6610, 10730, 14150, 10318, 8072, 12134] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80262 payload_rows_remote=68932 dup_rows=18042 dup_rows_remote=14258 dedup_ratio=0.183533 router_tokens_per_expert=[1, 204, 138, 232, 46, 945, 21, 2, 46, 5214, 0, 476, 62, 1203, 14, 620, 3, 920, 2888, 23, 3993, 2020, 10, 14, 1, 4790, 9, 146, 5071, 455, 0, 2, 288, 13, 8, 36, 476, 76, 41, 74, 286, 358, 1875, 5746, 2, 7, 2086, 9, 1, 3, 460, 302, 3, 0, 307, 1682, 1631, 810, 47, 2, 0, 4, 2903, 47] payload_rows_per_dst=[5706, 12126, 11330, 10390, 11190, 13802, 5214, 10504] dup_rows_per_dst=[734, 3144, 3784, 416, 976, 6936, 302, 1750] combine_rows_per_src=[5706, 12126, 11330, 10390, 11190, 13802, 5214, 10504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284770304 dispatch_meta_bytes=311392 combine_payload_bytes=284770304 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285081696 ingress_bytes=388579328 link_bytes=388579328 token_bytes=569852000 weight_movement_bytes=103809024 total_nvlink_bytes=673661024 weight_to_token_ratio=0.182168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462192728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85668 payload_rows_remote=74242 dup_rows=12636 dup_rows_remote=12310 dedup_ratio=0.128540 router_tokens_per_expert=[1374, 31, 8, 6061, 0, 14, 305, 586, 3, 179, 50, 0, 3, 0, 114, 2453, 0, 6, 56, 0, 0, 99, 35, 5405, 0, 0, 0, 2, 1317, 2, 1, 0, 35, 2151, 1068, 33, 5478, 0, 1268, 4900, 4, 0, 12, 665, 0, 0, 7734, 43, 41, 331, 0, 9, 0, 15, 2, 0, 366, 2, 0, 2883, 11, 1736, 68, 2193] payload_rows_per_dst=[4458, 5422, 10990, 13252, 12888, 15686, 11426, 11546] dup_rows_per_dst=[178, 182, 212, 1514, 3486, 3766, 326, 2972] combine_rows_per_src=[4458, 5422, 10990, 13252, 12888, 15686, 11426, 11546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=280207360 dispatch_meta_bytes=332736 combine_payload_bytes=280207360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=280540096 ingress_bytes=384016384 link_bytes=384016384 token_bytes=560747456 weight_movement_bytes=103809024 total_nvlink_bytes=664556480 weight_to_token_ratio=0.185126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445022737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300343296 dispatch_meta_bytes=346392 combine_payload_bytes=300343296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300689688 ingress_bytes=404152320 link_bytes=404152320 token_bytes=601032984 weight_movement_bytes=103809024 total_nvlink_bytes=704842008 weight_to_token_ratio=0.172718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520792643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85246 payload_rows_remote=73470 dup_rows=13058 dup_rows_remote=9776 dedup_ratio=0.132833 router_tokens_per_expert=[715, 68, 5, 5168, 0, 12, 442, 441, 5, 326, 52, 56, 2, 0, 61, 3326, 1, 1, 57, 2, 0, 10, 10, 5092, 5, 0, 24, 0, 1801, 2, 1, 0, 29, 1852, 105, 884, 6234, 5, 1551, 4878, 0, 0, 0, 487, 3, 0, 7529, 6, 59, 222, 25, 1, 0, 60, 8, 0, 411, 5, 0, 3060, 11, 3072, 100, 870] payload_rows_per_dst=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] dup_rows_per_dst=[160, 404, 56, 1628, 3402, 3770, 356, 3282] combine_rows_per_src=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=72376 dup_rows=12480 dup_rows_remote=9856 dedup_ratio=0.126953 router_tokens_per_expert=[2118, 357, 17, 5723, 0, 33, 199, 291, 8, 141, 52, 18, 0, 0, 34, 1535, 0, 3, 45, 0, 0, 240, 44, 4261, 7, 0, 2, 0, 2304, 0, 0, 0, 57, 1847, 1440, 71, 5198, 0, 2338, 5914, 0, 0, 12, 902, 9, 0, 8028, 0, 67, 151, 0, 0, 0, 89, 10, 0, 488, 5, 0, 2515, 2, 1136, 42, 1399] payload_rows_per_dst=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] dup_rows_per_dst=[374, 2114, 1164, 2624, 2094, 2264, 202, 1644] combine_rows_per_src=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=341112 combine_payload_bytes=274612224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274953336 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549565560 weight_movement_bytes=69206016 total_nvlink_bytes=618771576 weight_to_token_ratio=0.125929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72866 payload_rows_remote=60736 dup_rows=25438 dup_rows_remote=21226 dedup_ratio=0.258769 router_tokens_per_expert=[3, 80, 320, 289, 36, 2536, 10, 0, 1, 5038, 1, 923, 10, 1794, 1, 221, 0, 1942, 2197, 0, 4206, 449, 0, 2, 2, 3540, 8, 59, 4201, 372, 1, 0, 366, 1, 202, 0, 236, 368, 266, 31, 400, 758, 618, 3995, 2, 0, 353, 26, 3, 0, 736, 321, 6, 0, 495, 5009, 1904, 3490, 17, 0, 0, 8, 1299, 0] payload_rows_per_dst=[12130, 12282, 12642, 14508, 2570, 4130, 11146, 3458] dup_rows_per_dst=[4212, 3696, 4950, 9848, 370, 184, 1994, 184] combine_rows_per_src=[12130, 12282, 12642, 14508, 2570, 4130, 11146, 3458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=343640 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302366296 ingress_bytes=371228672 link_bytes=371228672 token_bytes=604388952 weight_movement_bytes=69206016 total_nvlink_bytes=673594968 weight_to_token_ratio=0.114506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396903606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282345472 dispatch_meta_bytes=332760 combine_payload_bytes=282345472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=282678232 ingress_bytes=351551488 link_bytes=351551488 token_bytes=565023704 weight_movement_bytes=69206016 total_nvlink_bytes=634229720 weight_to_token_ratio=0.122483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322859947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=304095232 dispatch_meta_bytes=346208 combine_payload_bytes=304095232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304441440 ingress_bytes=338698240 link_bytes=338698240 token_bytes=608536672 weight_movement_bytes=34603008 total_nvlink_bytes=643139680 weight_to_token_ratio=0.056863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274494210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85246 payload_rows_remote=73470 dup_rows=13058 dup_rows_remote=9776 dedup_ratio=0.132833 router_tokens_per_expert=[715, 68, 5, 5168, 0, 12, 442, 441, 5, 326, 52, 56, 2, 0, 61, 3326, 1, 1, 57, 2, 0, 10, 10, 5092, 5, 0, 24, 0, 1801, 2, 1, 0, 29, 1852, 105, 884, 6234, 5, 1551, 4878, 0, 0, 0, 487, 3, 0, 7529, 6, 59, 222, 25, 1, 0, 60, 8, 0, 411, 5, 0, 3060, 11, 3072, 100, 870] payload_rows_per_dst=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] dup_rows_per_dst=[160, 404, 56, 1628, 3402, 3770, 356, 3282] combine_rows_per_src=[3206, 7252, 10290, 12374, 12104, 15382, 12862, 11776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=248774656 dispatch_meta_bytes=327848 combine_payload_bytes=248774656 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=249102504 ingress_bytes=317980672 link_bytes=317980672 token_bytes=497877160 weight_movement_bytes=69206016 total_nvlink_bytes=567083176 weight_to_token_ratio=0.139002 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196535669e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84936 payload_rows_remote=79378 dup_rows=13368 dup_rows_remote=13108 dedup_ratio=0.135986 router_tokens_per_expert=[899, 113, 29, 6028, 17, 51, 539, 226, 36, 220, 14, 9, 0, 0, 14, 1544, 1, 1, 8, 0, 13, 293, 4, 3959, 2, 4, 7, 3, 2308, 0, 0, 6, 1, 2252, 646, 10, 5865, 0, 1870, 6019, 0, 0, 0, 268, 33, 0, 7801, 3, 26, 312, 1, 0, 0, 149, 0, 0, 336, 1, 2, 2411, 12, 1887, 76, 2823] payload_rows_per_dst=[3616, 13794, 8324, 13750, 5558, 15840, 12272, 11782] dup_rows_per_dst=[132, 1918, 234, 2966, 260, 4110, 434, 3314] combine_rows_per_src=[3616, 13794, 8324, 13750, 5558, 15840, 12272, 11782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77882 payload_rows_remote=65848 dup_rows=20422 dup_rows_remote=19160 dedup_ratio=0.207743 router_tokens_per_expert=[1084, 70, 1, 7055, 2, 8, 275, 692, 11, 215, 42, 3, 0, 0, 51, 1293, 0, 0, 27, 0, 0, 21, 17, 4937, 3, 0, 2, 7, 2696, 0, 0, 0, 35, 883, 722, 93, 7098, 1, 1748, 5033, 0, 0, 7, 274, 2, 0, 8004, 9, 55, 275, 0, 0, 1, 26, 34, 1, 453, 0, 0, 2759, 4, 1791, 32, 1300] payload_rows_per_dst=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] dup_rows_per_dst=[10956, 1262, 5032, 14, 748, 0, 540, 1870] combine_rows_per_src=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81588 payload_rows_remote=66438 dup_rows=16716 dup_rows_remote=10516 dedup_ratio=0.170044 router_tokens_per_expert=[3, 88, 219, 151, 240, 1335, 30, 1, 4, 5880, 0, 432, 8, 421, 3, 573, 0, 916, 2780, 11, 2049, 972, 0, 9, 1, 6447, 13, 128, 7605, 398, 0, 0, 423, 3, 27, 0, 160, 17, 60, 104, 341, 148, 639, 3335, 1, 0, 282, 7, 0, 0, 360, 457, 0, 5, 246, 3160, 4215, 3251, 105, 1, 0, 0, 1088, 0] payload_rows_per_dst=[10922, 9840, 9052, 1072, 15694, 8406, 15150, 11452] dup_rows_per_dst=[1642, 1628, 2036, 8, 1104, 1100, 6200, 2998] combine_rows_per_src=[10922, 9840, 9052, 1072, 15694, 8406, 15150, 11452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=72376 dup_rows=12480 dup_rows_remote=9856 dedup_ratio=0.126953 router_tokens_per_expert=[2118, 357, 17, 5723, 0, 33, 199, 291, 8, 141, 52, 18, 0, 0, 34, 1535, 0, 3, 45, 0, 0, 240, 44, 4261, 7, 0, 2, 0, 2304, 0, 0, 0, 57, 1847, 1440, 71, 5198, 0, 2338, 5914, 0, 0, 12, 902, 9, 0, 8028, 0, 67, 151, 0, 0, 0, 89, 10, 0, 488, 5, 0, 2515, 2, 1136, 42, 1399] payload_rows_per_dst=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] dup_rows_per_dst=[374, 2114, 1164, 2624, 2094, 2264, 202, 1644] combine_rows_per_src=[5656, 13290, 9968, 13448, 8662, 14442, 10828, 9530] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300933120 dispatch_meta_bytes=332984 combine_payload_bytes=300933120 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301266104 ingress_bytes=335536128 link_bytes=335536128 token_bytes=602199224 weight_movement_bytes=34603008 total_nvlink_bytes=636802232 weight_to_token_ratio=0.057461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=369944 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=325502232 ingress_bytes=463544320 link_bytes=463544320 token_bytes=650634520 weight_movement_bytes=138412032 total_nvlink_bytes=789046552 weight_to_token_ratio=0.212734 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.744279958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=69524 dup_rows=12618 dup_rows_remote=8324 dedup_ratio=0.128357 router_tokens_per_expert=[2294, 150, 0, 6113, 0, 4, 177, 284, 0, 45, 110, 7, 0, 0, 162, 2318, 0, 1, 6, 0, 0, 24, 210, 4699, 3, 0, 1, 0, 2325, 0, 0, 0, 20, 1748, 449, 58, 5972, 0, 1481, 4824, 0, 0, 7, 745, 18, 0, 7977, 0, 50, 86, 0, 0, 0, 34, 0, 0, 268, 0, 0, 3228, 14, 1617, 37, 1586] payload_rows_per_dst=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] dup_rows_per_dst=[182, 1168, 128, 2680, 1836, 4294, 122, 2208] combine_rows_per_src=[5636, 8130, 9752, 14204, 8348, 16162, 12162, 11292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81912 payload_rows_remote=65968 dup_rows=16392 dup_rows_remote=10702 dedup_ratio=0.166748 router_tokens_per_expert=[925, 293, 3, 5757, 51, 21, 861, 466, 3, 499, 18, 92, 38, 0, 16, 1328, 2, 13, 48, 1, 18, 27, 40, 2782, 1, 3, 301, 0, 3235, 1, 3, 6, 30, 2086, 276, 25, 6598, 5, 3148, 6123, 1, 9, 0, 150, 72, 53, 7886, 0, 48, 722, 0, 1, 0, 236, 0, 20, 491, 10, 5, 1418, 5, 1847, 92, 943] payload_rows_per_dst=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] dup_rows_per_dst=[3042, 2214, 5690, 664, 2226, 20, 1226, 1310] combine_rows_per_src=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=340032 combine_payload_bytes=269713408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270053440 ingress_bytes=304316416 link_bytes=304316416 token_bytes=539766848 weight_movement_bytes=34603008 total_nvlink_bytes=574369856 weight_to_token_ratio=0.064107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145118174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=272130048 dispatch_meta_bytes=307816 combine_payload_bytes=272130048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272437864 ingress_bytes=306733056 link_bytes=306733056 token_bytes=544567912 weight_movement_bytes=34603008 total_nvlink_bytes=579170920 weight_to_token_ratio=0.063542 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154211796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296452096 dispatch_meta_bytes=328928 combine_payload_bytes=296452096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296781024 ingress_bytes=331055104 link_bytes=331055104 token_bytes=593233120 weight_movement_bytes=34603008 total_nvlink_bytes=627836128 weight_to_token_ratio=0.058330 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245733704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70176 payload_rows_remote=53848 dup_rows=28128 dup_rows_remote=11276 dedup_ratio=0.286133 router_tokens_per_expert=[2124, 47, 0, 6172, 1, 3, 315, 276, 0, 301, 108, 29, 9, 0, 79, 1189, 0, 1, 17, 14, 0, 107, 31, 3602, 0, 0, 4, 0, 2867, 0, 0, 0, 11, 1691, 248, 57, 5942, 0, 4032, 5835, 0, 0, 0, 710, 5, 2, 7652, 0, 109, 153, 0, 19, 0, 9, 3, 3, 534, 3, 15, 2381, 2, 1559, 70, 811] payload_rows_per_dst=[16328, 13178, 7476, 5742, 12996, 1432, 6938, 6086] dup_rows_per_dst=[16852, 1922, 68, 0, 8566, 2, 146, 572] combine_rows_per_src=[16328, 13178, 7476, 5742, 12996, 1432, 6938, 6086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84374 payload_rows_remote=74606 dup_rows=13930 dup_rows_remote=11932 dedup_ratio=0.141703 router_tokens_per_expert=[0, 8, 218, 1232, 4, 6, 6649, 1, 258, 146, 0, 0, 3671, 0, 1595, 1, 7, 7389, 1386, 0, 0, 1, 1, 41, 94, 0, 0, 1, 0, 20, 176, 10, 983, 643, 0, 3, 0, 34, 1018, 0, 25, 21, 3783, 0, 0, 2345, 2983, 0, 8, 7068, 0, 1812, 628, 0, 4014, 6, 0, 5, 846, 0, 10, 0, 0, 2] payload_rows_per_dst=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] dup_rows_per_dst=[1038, 1060, 2594, 328, 2492, 1998, 2352, 2068] combine_rows_per_src=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284770304 dispatch_meta_bytes=311392 combine_payload_bytes=284770304 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285081696 ingress_bytes=388579328 link_bytes=388579328 token_bytes=569852000 weight_movement_bytes=103809024 total_nvlink_bytes=673661024 weight_to_token_ratio=0.182168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462192728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270204928 dispatch_meta_bytes=306680 combine_payload_bytes=270204928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270511608 ingress_bytes=304807936 link_bytes=304807936 token_bytes=540716536 weight_movement_bytes=34603008 total_nvlink_bytes=575319544 weight_to_token_ratio=0.063995 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146967724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=369944 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=325502232 ingress_bytes=463544320 link_bytes=463544320 token_bytes=650634520 weight_movement_bytes=138412032 total_nvlink_bytes=789046552 weight_to_token_ratio=0.212734 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.744279958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=220561408 dispatch_meta_bytes=260496 combine_payload_bytes=220561408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=220821904 ingress_bytes=289767424 link_bytes=289767424 token_bytes=441383312 weight_movement_bytes=69206016 total_nvlink_bytes=510589328 weight_to_token_ratio=0.156793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090371488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77882 payload_rows_remote=65848 dup_rows=20422 dup_rows_remote=19160 dedup_ratio=0.207743 router_tokens_per_expert=[1084, 70, 1, 7055, 2, 8, 275, 692, 11, 215, 42, 3, 0, 0, 51, 1293, 0, 0, 27, 0, 0, 21, 17, 4937, 3, 0, 2, 7, 2696, 0, 0, 0, 35, 883, 722, 93, 7098, 1, 1748, 5033, 0, 0, 7, 274, 2, 0, 8004, 9, 55, 275, 0, 0, 1, 26, 34, 1, 453, 0, 0, 2759, 4, 1791, 32, 1300] payload_rows_per_dst=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] dup_rows_per_dst=[10956, 1262, 5032, 14, 748, 0, 540, 1870] combine_rows_per_src=[15422, 12034, 12976, 5402, 6216, 584, 14440, 10808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305586176 dispatch_meta_bytes=346152 combine_payload_bytes=305586176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305932328 ingress_bytes=409395200 link_bytes=409395200 token_bytes=611518504 weight_movement_bytes=103809024 total_nvlink_bytes=715327528 weight_to_token_ratio=0.169756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540521179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81912 payload_rows_remote=65968 dup_rows=16392 dup_rows_remote=10702 dedup_ratio=0.166748 router_tokens_per_expert=[925, 293, 3, 5757, 51, 21, 861, 466, 3, 499, 18, 92, 38, 0, 16, 1328, 2, 13, 48, 1, 18, 27, 40, 2782, 1, 3, 301, 0, 3235, 1, 3, 6, 30, 2086, 276, 25, 6598, 5, 3148, 6123, 1, 9, 0, 150, 72, 53, 7886, 0, 48, 722, 0, 1, 0, 236, 0, 20, 491, 10, 5, 1418, 5, 1847, 92, 943] payload_rows_per_dst=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] dup_rows_per_dst=[3042, 2214, 5690, 664, 2226, 20, 1226, 1310] combine_rows_per_src=[12106, 14020, 15944, 8042, 8914, 550, 14024, 8312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80288 payload_rows_remote=66362 dup_rows=18016 dup_rows_remote=14198 dedup_ratio=0.183268 router_tokens_per_expert=[0, 0, 183, 828, 7, 13, 7645, 5, 114, 67, 0, 0, 1658, 0, 1576, 3, 58, 7135, 938, 0, 0, 0, 0, 104, 35, 0, 0, 0, 0, 58, 180, 159, 1386, 515, 0, 0, 1, 75, 806, 0, 48, 8, 2605, 0, 0, 2172, 3411, 2, 18, 7049, 1, 2519, 41, 0, 6280, 13, 0, 3, 1419, 3, 4, 4, 3, 0] payload_rows_per_dst=[2024, 6474, 2174, 14326, 15802, 14598, 13926, 10964] dup_rows_per_dst=[48, 362, 26, 636, 5054, 4132, 3818, 3940] combine_rows_per_src=[2024, 6474, 2174, 14326, 15802, 14598, 13926, 10964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73674 payload_rows_remote=62536 dup_rows=24630 dup_rows_remote=20736 dedup_ratio=0.250549 router_tokens_per_expert=[0, 78, 114, 1930, 12, 115, 7473, 42, 247, 175, 1, 11, 3440, 0, 750, 49, 120, 7549, 757, 4, 0, 1, 11, 152, 356, 2, 0, 0, 3, 79, 16, 23, 939, 1781, 5, 3, 1, 36, 1181, 72, 41, 17, 2855, 0, 0, 1231, 3395, 0, 56, 6690, 0, 1857, 49, 6, 4145, 16, 1, 3, 1229, 1, 6, 1, 0, 25] payload_rows_per_dst=[4426, 8692, 2078, 3208, 15520, 13690, 14922, 11138] dup_rows_per_dst=[156, 654, 12, 212, 7462, 2170, 10070, 3894] combine_rows_per_src=[4426, 8692, 2078, 3208, 15520, 13690, 14922, 11138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89228 payload_rows_remote=76054 dup_rows=9076 dup_rows_remote=8794 dedup_ratio=0.092326 router_tokens_per_expert=[0, 1, 170, 941, 2, 11, 6649, 6, 205, 152, 0, 0, 2923, 0, 1508, 3, 4, 7242, 1011, 0, 0, 4, 1, 248, 181, 1, 0, 1, 0, 22, 5, 14, 1638, 1303, 3, 0, 0, 13, 2358, 1, 71, 4, 3539, 0, 0, 1117, 4049, 2, 100, 6504, 1, 740, 45, 1, 5272, 8, 0, 3, 1061, 2, 4, 0, 0, 8] payload_rows_per_dst=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] dup_rows_per_dst=[1624, 856, 2062, 282, 1720, 602, 1068, 862] combine_rows_per_src=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=269713408 dispatch_meta_bytes=340032 combine_payload_bytes=269713408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270053440 ingress_bytes=304316416 link_bytes=304316416 token_bytes=539766848 weight_movement_bytes=34603008 total_nvlink_bytes=574369856 weight_to_token_ratio=0.064107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145118174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=220561408 dispatch_meta_bytes=260496 combine_payload_bytes=220561408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=220821904 ingress_bytes=289767424 link_bytes=289767424 token_bytes=441383312 weight_movement_bytes=69206016 total_nvlink_bytes=510589328 weight_to_token_ratio=0.156793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090371488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270204928 dispatch_meta_bytes=306680 combine_payload_bytes=270204928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270511608 ingress_bytes=304807936 link_bytes=304807936 token_bytes=540716536 weight_movement_bytes=34603008 total_nvlink_bytes=575319544 weight_to_token_ratio=0.063995 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146967724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=322240 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=272140992 ingress_bytes=375627776 link_bytes=375627776 token_bytes=543959744 weight_movement_bytes=103809024 total_nvlink_bytes=647768768 weight_to_token_ratio=0.190840 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413457081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88878 payload_rows_remote=81306 dup_rows=9426 dup_rows_remote=8132 dedup_ratio=0.095886 router_tokens_per_expert=[0, 38, 30, 1799, 0, 19, 5178, 8, 61, 73, 3, 0, 2164, 0, 1099, 3, 13, 6872, 505, 0, 0, 0, 18, 60, 40, 8, 0, 0, 0, 39, 11, 17, 2096, 705, 14, 13, 1, 12, 1589, 3, 34, 46, 5243, 0, 0, 1988, 4913, 3, 29, 6288, 1, 1101, 11, 0, 6002, 26, 0, 4, 962, 0, 9, 1, 0, 0] payload_rows_per_dst=[12250, 6396, 14164, 12646, 7572, 12330, 12690, 10830] dup_rows_per_dst=[1894, 410, 772, 160, 1294, 2298, 1650, 948] combine_rows_per_src=[12250, 6396, 14164, 12646, 7572, 12330, 12690, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84550 payload_rows_remote=69778 dup_rows=13754 dup_rows_remote=5730 dedup_ratio=0.139913 router_tokens_per_expert=[0, 54, 69, 1017, 6, 42, 7509, 3, 592, 452, 2, 0, 3907, 0, 1507, 2, 10, 7496, 860, 0, 0, 1, 1, 169, 34, 0, 0, 0, 0, 10, 14, 0, 1310, 637, 3, 18, 0, 55, 1077, 0, 12, 6, 2585, 0, 1, 1013, 4339, 0, 14, 6538, 1, 1250, 24, 1, 5482, 3, 0, 15, 1008, 0, 0, 1, 0, 2] payload_rows_per_dst=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] dup_rows_per_dst=[2038, 8024, 1788, 104, 624, 318, 224, 634] combine_rows_per_src=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256147456 dispatch_meta_bytes=333088 combine_payload_bytes=256147456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=256480544 ingress_bytes=325353472 link_bytes=325353472 token_bytes=512628000 weight_movement_bytes=69206016 total_nvlink_bytes=581834016 weight_to_token_ratio=0.135002 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.022500 gemm_active_groups=11 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224278922e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311517184 dispatch_meta_bytes=339392 combine_payload_bytes=311517184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311856576 ingress_bytes=346120192 link_bytes=346120192 token_bytes=623373760 weight_movement_bytes=34603008 total_nvlink_bytes=657976768 weight_to_token_ratio=0.055509 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302422418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83444 payload_rows_remote=68766 dup_rows=14860 dup_rows_remote=12482 dedup_ratio=0.151164 router_tokens_per_expert=[3, 1, 174, 1429, 2, 57, 6861, 1, 95, 72, 6, 0, 4580, 0, 816, 14, 2, 6004, 730, 0, 2, 1, 17, 135, 173, 7, 0, 1, 3, 28, 2, 17, 1497, 492, 0, 13, 0, 1, 2064, 0, 43, 102, 3092, 0, 0, 1235, 4815, 1, 14, 5515, 0, 1174, 7, 8, 6618, 5, 0, 1, 1170, 0, 24, 9, 0, 19] payload_rows_per_dst=[14678, 15640, 12850, 11258, 7364, 8296, 2414, 10944] dup_rows_per_dst=[2378, 8762, 932, 234, 770, 650, 2, 1132] combine_rows_per_src=[14678, 15640, 12850, 11258, 7364, 8296, 2414, 10944] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84122 payload_rows_remote=69390 dup_rows=14182 dup_rows_remote=9476 dedup_ratio=0.144267 router_tokens_per_expert=[36, 0, 6767, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 1293, 105, 2550, 840, 193, 6, 287, 2392, 0, 145, 16, 0, 0, 0, 630, 1, 2438, 34, 0, 0, 0, 0, 886, 17, 5, 2535, 5, 0, 0, 2824, 0, 0, 6890, 0, 2219, 0, 966, 3860, 246, 1357, 260, 0, 0, 343, 40, 7011, 7, 3, 0, 0] payload_rows_per_dst=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] dup_rows_per_dst=[84, 606, 2962, 790, 2928, 4706, 1464, 642] combine_rows_per_src=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89228 payload_rows_remote=76054 dup_rows=9076 dup_rows_remote=8794 dedup_ratio=0.092326 router_tokens_per_expert=[0, 1, 170, 941, 2, 11, 6649, 6, 205, 152, 0, 0, 2923, 0, 1508, 3, 4, 7242, 1011, 0, 0, 4, 1, 248, 181, 1, 0, 1, 0, 22, 5, 14, 1638, 1303, 3, 0, 0, 13, 2358, 1, 71, 4, 3539, 0, 0, 1117, 4049, 2, 100, 6504, 1, 740, 45, 1, 5272, 8, 0, 3, 1061, 2, 4, 0, 0, 8] payload_rows_per_dst=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] dup_rows_per_dst=[1624, 856, 2062, 282, 1720, 602, 1068, 862] combine_rows_per_src=[13936, 8726, 14958, 13174, 8912, 8864, 11266, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85668 payload_rows_remote=74242 dup_rows=12636 dup_rows_remote=12310 dedup_ratio=0.128540 router_tokens_per_expert=[1374, 31, 8, 6061, 0, 14, 305, 586, 3, 179, 50, 0, 3, 0, 114, 2453, 0, 6, 56, 0, 0, 99, 35, 5405, 0, 0, 0, 2, 1317, 2, 1, 0, 35, 2151, 1068, 33, 5478, 0, 1268, 4900, 4, 0, 12, 665, 0, 0, 7734, 43, 41, 331, 0, 9, 0, 15, 2, 0, 366, 2, 0, 2883, 11, 1736, 68, 2193] payload_rows_per_dst=[4458, 5422, 10990, 13252, 12888, 15686, 11426, 11546] dup_rows_per_dst=[178, 182, 212, 1514, 3486, 3766, 326, 2972] combine_rows_per_src=[4458, 5422, 10990, 13252, 12888, 15686, 11426, 11546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333029376 dispatch_meta_bytes=357752 combine_payload_bytes=333029376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333387128 ingress_bytes=367632384 link_bytes=367632384 token_bytes=666416504 weight_movement_bytes=34603008 total_nvlink_bytes=701019512 weight_to_token_ratio=0.051924 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383371064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=281665536 dispatch_meta_bytes=324992 combine_payload_bytes=281665536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281990528 ingress_bytes=350871552 link_bytes=350871552 token_bytes=563656064 weight_movement_bytes=69206016 total_nvlink_bytes=632862080 weight_to_token_ratio=0.122781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=284221440 dispatch_meta_bytes=315464 combine_payload_bytes=284221440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284536904 ingress_bytes=353427456 link_bytes=353427456 token_bytes=568758344 weight_movement_bytes=69206016 total_nvlink_bytes=637964360 weight_to_token_ratio=0.121679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329919064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84550 payload_rows_remote=69778 dup_rows=13754 dup_rows_remote=5730 dedup_ratio=0.139913 router_tokens_per_expert=[0, 54, 69, 1017, 6, 42, 7509, 3, 592, 452, 2, 0, 3907, 0, 1507, 2, 10, 7496, 860, 0, 0, 1, 1, 169, 34, 0, 0, 0, 0, 10, 14, 0, 1310, 637, 3, 18, 0, 55, 1077, 0, 12, 6, 2585, 0, 1, 1013, 4339, 0, 14, 6538, 1, 1250, 24, 1, 5482, 3, 0, 15, 1008, 0, 0, 1, 0, 2] payload_rows_per_dst=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] dup_rows_per_dst=[2038, 8024, 1788, 104, 624, 318, 224, 634] combine_rows_per_src=[15362, 14772, 15286, 13088, 5576, 6916, 3454, 10096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84374 payload_rows_remote=74606 dup_rows=13930 dup_rows_remote=11932 dedup_ratio=0.141703 router_tokens_per_expert=[0, 8, 218, 1232, 4, 6, 6649, 1, 258, 146, 0, 0, 3671, 0, 1595, 1, 7, 7389, 1386, 0, 0, 1, 1, 41, 94, 0, 0, 1, 0, 20, 176, 10, 983, 643, 0, 3, 0, 34, 1018, 0, 25, 21, 3783, 0, 0, 2345, 2983, 0, 8, 7068, 0, 1812, 628, 0, 4014, 6, 0, 5, 846, 0, 10, 0, 0, 2] payload_rows_per_dst=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] dup_rows_per_dst=[1038, 1060, 2594, 328, 2492, 1998, 2352, 2068] combine_rows_per_src=[8292, 10282, 13048, 14410, 9142, 9768, 10584, 8848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87112 payload_rows_remote=73522 dup_rows=11192 dup_rows_remote=9222 dedup_ratio=0.113851 router_tokens_per_expert=[56, 91, 66, 2212, 8, 38, 6597, 12, 799, 286, 26, 0, 1359, 0, 951, 176, 0, 6381, 1066, 8, 0, 7, 45, 273, 112, 35, 0, 32, 0, 25, 34, 34, 1663, 713, 1, 4, 36, 30, 2053, 0, 61, 52, 5013, 0, 1, 701, 4513, 9, 73, 6847, 0, 1467, 293, 18, 2939, 142, 7, 7, 1661, 3, 2, 75, 3, 36] payload_rows_per_dst=[14580, 6702, 13590, 13806, 7648, 11034, 8576, 11176] dup_rows_per_dst=[3580, 492, 1970, 432, 1352, 640, 1288, 1438] combine_rows_per_src=[14580, 6702, 13590, 13806, 7648, 11034, 8576, 11176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83410 payload_rows_remote=73066 dup_rows=14894 dup_rows_remote=12702 dedup_ratio=0.151510 router_tokens_per_expert=[49, 0, 6103, 0, 3, 0, 4, 0, 1, 4, 2, 0, 0, 2544, 971, 79, 3243, 535, 57, 3, 220, 2795, 0, 1286, 0, 0, 0, 0, 177, 4, 1767, 64, 2, 0, 3, 9, 856, 40, 7, 1689, 384, 2, 0, 2059, 1, 0, 6567, 1, 1724, 0, 1965, 4342, 122, 1793, 664, 0, 0, 235, 62, 6664, 2, 47, 1, 0] payload_rows_per_dst=[12220, 6752, 11572, 3848, 11252, 13894, 10344, 13528] dup_rows_per_dst=[98, 450, 4706, 176, 2644, 4134, 2192, 494] combine_rows_per_src=[12220, 6752, 11572, 3848, 11252, 13894, 10344, 13528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75890 payload_rows_remote=61440 dup_rows=22414 dup_rows_remote=17408 dedup_ratio=0.228007 router_tokens_per_expert=[5, 0, 5765, 16, 49, 0, 4, 0, 0, 0, 9, 0, 0, 2444, 2712, 162, 4790, 786, 358, 11, 749, 2385, 5, 516, 2, 0, 0, 52, 1025, 75, 1350, 69, 20, 0, 1, 6, 1083, 9, 9, 855, 339, 11, 0, 2978, 4, 0, 5652, 3, 1041, 0, 917, 4264, 72, 936, 287, 9, 1, 325, 61, 6881, 1, 31, 2, 15] payload_rows_per_dst=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] dup_rows_per_dst=[72, 1342, 6078, 1124, 426, 4578, 3788, 5006] combine_rows_per_src=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85156 payload_rows_remote=70822 dup_rows=13148 dup_rows_remote=11070 dedup_ratio=0.133748 router_tokens_per_expert=[120, 0, 6912, 0, 9, 0, 4, 2, 0, 3, 0, 0, 2, 2773, 1340, 8, 2849, 978, 84, 20, 235, 2225, 0, 355, 36, 0, 0, 6, 26, 26, 1047, 137, 3, 0, 0, 0, 2298, 14, 3, 846, 5, 26, 0, 641, 0, 0, 6928, 3, 2096, 8, 2013, 4838, 920, 1669, 184, 0, 5, 330, 49, 7061, 4, 7, 0, 4] payload_rows_per_dst=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] dup_rows_per_dst=[246, 2692, 998, 2078, 3714, 0, 2774, 646] combine_rows_per_src=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12860363600 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6434067280 ingress_bytes=8052637696 link_bytes=8052637696 total_nvlink_bytes=14486704976 weight_to_token_ratio=0.126462 prefetch_share_of_ingress=0.094536 dedup_ratio=0.195833 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.317453363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285810688 dispatch_meta_bytes=302032 combine_payload_bytes=285810688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286112720 ingress_bytes=320413696 link_bytes=320413696 token_bytes=571923408 weight_movement_bytes=34603008 total_nvlink_bytes=606526416 weight_to_token_ratio=0.060503 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205690943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311517184 dispatch_meta_bytes=339392 combine_payload_bytes=311517184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311856576 ingress_bytes=346120192 link_bytes=346120192 token_bytes=623373760 weight_movement_bytes=34603008 total_nvlink_bytes=657976768 weight_to_token_ratio=0.055509 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302422418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=304095232 dispatch_meta_bytes=346208 combine_payload_bytes=304095232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304441440 ingress_bytes=338698240 link_bytes=338698240 token_bytes=608536672 weight_movement_bytes=34603008 total_nvlink_bytes=643139680 weight_to_token_ratio=0.056863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274494210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83444 payload_rows_remote=68766 dup_rows=14860 dup_rows_remote=12482 dedup_ratio=0.151164 router_tokens_per_expert=[3, 1, 174, 1429, 2, 57, 6861, 1, 95, 72, 6, 0, 4580, 0, 816, 14, 2, 6004, 730, 0, 2, 1, 17, 135, 173, 7, 0, 1, 3, 28, 2, 17, 1497, 492, 0, 13, 0, 1, 2064, 0, 43, 102, 3092, 0, 0, 1235, 4815, 1, 14, 5515, 0, 1174, 7, 8, 6618, 5, 0, 1, 1170, 0, 24, 9, 0, 19] payload_rows_per_dst=[14678, 15640, 12850, 11258, 7364, 8296, 2414, 10944] dup_rows_per_dst=[2378, 8762, 932, 234, 770, 650, 2, 1132] combine_rows_per_src=[14678, 15640, 12850, 11258, 7364, 8296, 2414, 10944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305586176 dispatch_meta_bytes=346152 combine_payload_bytes=305586176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305932328 ingress_bytes=409395200 link_bytes=409395200 token_bytes=611518504 weight_movement_bytes=103809024 total_nvlink_bytes=715327528 weight_to_token_ratio=0.169756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540521179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=301146112 dispatch_meta_bytes=330976 combine_payload_bytes=301146112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301477088 ingress_bytes=404955136 link_bytes=404955136 token_bytes=602623200 weight_movement_bytes=103809024 total_nvlink_bytes=706432224 weight_to_token_ratio=0.172262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523813575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75890 payload_rows_remote=61440 dup_rows=22414 dup_rows_remote=17408 dedup_ratio=0.228007 router_tokens_per_expert=[5, 0, 5765, 16, 49, 0, 4, 0, 0, 0, 9, 0, 0, 2444, 2712, 162, 4790, 786, 358, 11, 749, 2385, 5, 516, 2, 0, 0, 52, 1025, 75, 1350, 69, 20, 0, 1, 6, 1083, 9, 9, 855, 339, 11, 0, 2978, 4, 0, 5652, 3, 1041, 0, 917, 4264, 72, 936, 287, 9, 1, 325, 61, 6881, 1, 31, 2, 15] payload_rows_per_dst=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] dup_rows_per_dst=[72, 1342, 6078, 1124, 426, 4578, 3788, 5006] combine_rows_per_src=[5676, 9312, 13122, 4022, 5234, 13396, 10678, 14450] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84826 payload_rows_remote=74368 dup_rows=13478 dup_rows_remote=11670 dedup_ratio=0.137105 router_tokens_per_expert=[156, 0, 6711, 0, 7, 0, 0, 11, 0, 9, 93, 0, 5, 2854, 2709, 238, 2045, 567, 497, 86, 305, 3284, 0, 922, 432, 0, 0, 23, 305, 1, 2077, 134, 0, 0, 0, 27, 1262, 35, 4, 606, 164, 17, 2, 940, 76, 0, 7043, 0, 1639, 0, 908, 4199, 337, 1022, 546, 6, 3, 33, 90, 6618, 59, 42, 3, 0] payload_rows_per_dst=[13502, 11478, 9704, 5706, 10458, 12998, 7618, 13362] dup_rows_per_dst=[268, 3806, 2240, 1814, 1808, 1910, 1298, 334] combine_rows_per_src=[13502, 11478, 9704, 5706, 10458, 12998, 7618, 13362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=343072 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299621408 ingress_bytes=403087360 link_bytes=403087360 token_bytes=598899744 weight_movement_bytes=103809024 total_nvlink_bytes=702708768 weight_to_token_ratio=0.173333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516785284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=251658240 dispatch_meta_bytes=315392 combine_payload_bytes=251658240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251973632 ingress_bytes=320864256 link_bytes=320864256 token_bytes=503631872 weight_movement_bytes=69206016 total_nvlink_bytes=572837888 weight_to_token_ratio=0.137414 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207386364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82022 payload_rows_remote=70466 dup_rows=16282 dup_rows_remote=15994 dedup_ratio=0.165629 router_tokens_per_expert=[189, 0, 5723, 0, 5, 0, 0, 5, 1, 2, 3, 0, 101, 1137, 2809, 20, 3707, 345, 396, 0, 184, 1391, 3, 1283, 7, 0, 0, 0, 1023, 5, 1850, 37, 3, 0, 1, 21, 1246, 11, 0, 997, 38, 2, 0, 774, 7, 0, 6628, 0, 4020, 1, 797, 5402, 152, 943, 648, 4, 3, 51, 179, 6979, 3, 13, 3, 0] payload_rows_per_dst=[11556, 12340, 8736, 14170, 12440, 14198, 8086, 496] dup_rows_per_dst=[288, 3220, 2056, 4930, 2922, 1402, 1456, 8] combine_rows_per_src=[11556, 12340, 8736, 14170, 12440, 14198, 8086, 496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290086912 dispatch_meta_bytes=327568 combine_payload_bytes=290086912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290414480 ingress_bytes=324689920 link_bytes=324689920 token_bytes=580501392 weight_movement_bytes=34603008 total_nvlink_bytes=615104400 weight_to_token_ratio=0.059609 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221782029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=281665536 dispatch_meta_bytes=324992 combine_payload_bytes=281665536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281990528 ingress_bytes=350871552 link_bytes=350871552 token_bytes=563656064 weight_movement_bytes=69206016 total_nvlink_bytes=632862080 weight_to_token_ratio=0.122781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82608 payload_rows_remote=70584 dup_rows=15696 dup_rows_remote=12120 dedup_ratio=0.159668 router_tokens_per_expert=[6, 0, 6135, 0, 13, 0, 0, 0, 0, 25, 0, 0, 4, 2285, 1098, 137, 4251, 459, 300, 10, 278, 3733, 0, 776, 45, 0, 2, 23, 964, 4, 2348, 36, 0, 1, 23, 0, 806, 28, 4, 2663, 72, 4, 0, 1469, 0, 0, 5952, 1, 1108, 1, 1292, 3439, 244, 1255, 889, 4, 0, 139, 52, 6752, 10, 8, 3, 1] payload_rows_per_dst=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] dup_rows_per_dst=[30, 3576, 1770, 4868, 2726, 1026, 1510, 190] combine_rows_per_src=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=251658240 dispatch_meta_bytes=315392 combine_payload_bytes=251658240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=251973632 ingress_bytes=320864256 link_bytes=320864256 token_bytes=503631872 weight_movement_bytes=69206016 total_nvlink_bytes=572837888 weight_to_token_ratio=0.137414 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207386364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85156 payload_rows_remote=70822 dup_rows=13148 dup_rows_remote=11070 dedup_ratio=0.133748 router_tokens_per_expert=[120, 0, 6912, 0, 9, 0, 4, 2, 0, 3, 0, 0, 2, 2773, 1340, 8, 2849, 978, 84, 20, 235, 2225, 0, 355, 36, 0, 0, 6, 26, 26, 1047, 137, 3, 0, 0, 0, 2298, 14, 3, 846, 5, 26, 0, 641, 0, 0, 6928, 3, 2096, 8, 2013, 4838, 920, 1669, 184, 0, 5, 330, 49, 7061, 4, 7, 0, 4] payload_rows_per_dst=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] dup_rows_per_dst=[246, 2692, 998, 2078, 3714, 0, 2774, 646] combine_rows_per_src=[13848, 11258, 6796, 14334, 12290, 1350, 11006, 14274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80288 payload_rows_remote=66362 dup_rows=18016 dup_rows_remote=14198 dedup_ratio=0.183268 router_tokens_per_expert=[0, 0, 183, 828, 7, 13, 7645, 5, 114, 67, 0, 0, 1658, 0, 1576, 3, 58, 7135, 938, 0, 0, 0, 0, 104, 35, 0, 0, 0, 0, 58, 180, 159, 1386, 515, 0, 0, 1, 75, 806, 0, 48, 8, 2605, 0, 0, 2172, 3411, 2, 18, 7049, 1, 2519, 41, 0, 6280, 13, 0, 3, 1419, 3, 4, 4, 3, 0] payload_rows_per_dst=[2024, 6474, 2174, 14326, 15802, 14598, 13926, 10964] dup_rows_per_dst=[48, 362, 26, 636, 5054, 4132, 3818, 3940] combine_rows_per_src=[2024, 6474, 2174, 14326, 15802, 14598, 13926, 10964] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=304611328 dispatch_meta_bytes=344152 combine_payload_bytes=304611328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304955480 ingress_bytes=339214336 link_bytes=339214336 token_bytes=609566808 weight_movement_bytes=34603008 total_nvlink_bytes=644169816 weight_to_token_ratio=0.056767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276436237e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84122 payload_rows_remote=69390 dup_rows=14182 dup_rows_remote=9476 dedup_ratio=0.144267 router_tokens_per_expert=[36, 0, 6767, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1936, 1293, 105, 2550, 840, 193, 6, 287, 2392, 0, 145, 16, 0, 0, 0, 630, 1, 2438, 34, 0, 0, 0, 0, 886, 17, 5, 2535, 5, 0, 0, 2824, 0, 0, 6890, 0, 2219, 0, 966, 3860, 246, 1357, 260, 0, 0, 343, 40, 7011, 7, 3, 0, 0] payload_rows_per_dst=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] dup_rows_per_dst=[84, 606, 2962, 790, 2928, 4706, 1464, 642] combine_rows_per_src=[13540, 6062, 9864, 5448, 11678, 14732, 8632, 14166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289112064 dispatch_meta_bytes=330816 combine_payload_bytes=289112064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289442880 ingress_bytes=323715072 link_bytes=323715072 token_bytes=578554944 weight_movement_bytes=34603008 total_nvlink_bytes=613157952 weight_to_token_ratio=0.059809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218113755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11972773824 prefetch_bytes=830472192 reduce_bytes=311427072 weight_movement_bytes=1141899264 egress_bytes=5990131648 ingress_bytes=7124541440 link_bytes=7124541440 total_nvlink_bytes=13114673088 weight_to_token_ratio=0.095375 prefetch_share_of_ingress=0.116565 dedup_ratio=0.230778 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=9 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.165611994e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12415861280 prefetch_bytes=761266176 reduce_bytes=830472192 weight_movement_bytes=1591738368 egress_bytes=6211732000 ingress_bytes=7795867648 link_bytes=7795867648 total_nvlink_bytes=14007599648 weight_to_token_ratio=0.128202 prefetch_share_of_ingress=0.097650 dedup_ratio=0.211088 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=24 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.275444450e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=288628736 dispatch_meta_bytes=345840 combine_payload_bytes=288628736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288974576 ingress_bytes=357834752 link_bytes=357834752 token_bytes=577603312 weight_movement_bytes=69206016 total_nvlink_bytes=646809328 weight_to_token_ratio=0.119816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346503364e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13026883584 prefetch_bytes=622854144 reduce_bytes=1211105280 weight_movement_bytes=1833959424 egress_bytes=6517340160 ingress_bytes=8343502848 link_bytes=8343502848 total_nvlink_bytes=14860843008 weight_to_token_ratio=0.140783 prefetch_share_of_ingress=0.074651 dedup_ratio=0.185480 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=35 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.365040413e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290086912 dispatch_meta_bytes=327568 combine_payload_bytes=290086912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290414480 ingress_bytes=324689920 link_bytes=324689920 token_bytes=580501392 weight_movement_bytes=34603008 total_nvlink_bytes=615104400 weight_to_token_ratio=0.059609 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221782029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81012 payload_rows_remote=73798 dup_rows=17292 dup_rows_remote=16148 dedup_ratio=0.175903 router_tokens_per_expert=[23, 4, 7155, 0, 87, 0, 0, 30, 0, 1, 0, 0, 12, 2386, 1153, 230, 2911, 1250, 159, 5, 322, 2246, 0, 197, 254, 0, 0, 9, 532, 14, 2842, 19, 5, 0, 17, 50, 848, 46, 12, 377, 12, 2, 15, 1232, 6, 0, 6888, 0, 1124, 7, 2191, 1936, 2363, 3456, 491, 59, 16, 292, 195, 5643, 3, 7, 3, 15] payload_rows_per_dst=[14368, 10540, 7214, 14960, 6092, 2518, 13538, 11782] dup_rows_per_dst=[230, 2846, 1144, 6156, 490, 16, 5844, 566] combine_rows_per_src=[14368, 10540, 7214, 14960, 6092, 2518, 13538, 11782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82608 payload_rows_remote=70584 dup_rows=15696 dup_rows_remote=12120 dedup_ratio=0.159668 router_tokens_per_expert=[6, 0, 6135, 0, 13, 0, 0, 0, 0, 25, 0, 0, 4, 2285, 1098, 137, 4251, 459, 300, 10, 278, 3733, 0, 776, 45, 0, 2, 23, 964, 4, 2348, 36, 0, 1, 23, 0, 806, 28, 4, 2663, 72, 4, 0, 1469, 0, 0, 5952, 1, 1108, 1, 1292, 3439, 244, 1255, 889, 4, 0, 139, 52, 6752, 10, 8, 3, 1] payload_rows_per_dst=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] dup_rows_per_dst=[30, 3576, 1770, 4868, 2726, 1026, 1510, 190] combine_rows_per_src=[12278, 12024, 9342, 13880, 11202, 7446, 8076, 8360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=322240 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=272140992 ingress_bytes=375627776 link_bytes=375627776 token_bytes=543959744 weight_movement_bytes=103809024 total_nvlink_bytes=647768768 weight_to_token_ratio=0.190840 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413457081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13077949840 prefetch_bytes=830472192 reduce_bytes=449839104 weight_movement_bytes=1280311296 egress_bytes=6542904720 ingress_bytes=7815356416 link_bytes=7815356416 total_nvlink_bytes=14358261136 weight_to_token_ratio=0.097898 prefetch_share_of_ingress=0.106262 dedup_ratio=0.189238 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=13 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.278632913e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12131284608 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6069376640 ingress_bytes=7480631296 link_bytes=7480631296 total_nvlink_bytes=13550007936 weight_to_token_ratio=0.116947 prefetch_share_of_ingress=0.092514 dedup_ratio=0.219138 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223870145e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12594937392 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6301285936 ingress_bytes=7746977792 link_bytes=7746977792 total_nvlink_bytes=14048263728 weight_to_token_ratio=0.115390 prefetch_share_of_ingress=0.080400 dedup_ratio=0.204756 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267445815e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82022 payload_rows_remote=70466 dup_rows=16282 dup_rows_remote=15994 dedup_ratio=0.165629 router_tokens_per_expert=[189, 0, 5723, 0, 5, 0, 0, 5, 1, 2, 3, 0, 101, 1137, 2809, 20, 3707, 345, 396, 0, 184, 1391, 3, 1283, 7, 0, 0, 0, 1023, 5, 1850, 37, 3, 0, 1, 21, 1246, 11, 0, 997, 38, 2, 0, 774, 7, 0, 6628, 0, 4020, 1, 797, 5402, 152, 943, 648, 4, 3, 51, 179, 6979, 3, 13, 3, 0] payload_rows_per_dst=[11556, 12340, 8736, 14170, 12440, 14198, 8086, 496] dup_rows_per_dst=[288, 3220, 2056, 4930, 2922, 1402, 1456, 8] combine_rows_per_src=[11556, 12340, 8736, 14170, 12440, 14198, 8086, 496] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=284221440 dispatch_meta_bytes=315464 combine_payload_bytes=284221440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284536904 ingress_bytes=353427456 link_bytes=353427456 token_bytes=568758344 weight_movement_bytes=69206016 total_nvlink_bytes=637964360 weight_to_token_ratio=0.121679 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329919064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13026883584 prefetch_bytes=622854144 reduce_bytes=1211105280 weight_movement_bytes=1833959424 egress_bytes=6517340160 ingress_bytes=8343502848 link_bytes=8343502848 total_nvlink_bytes=14860843008 weight_to_token_ratio=0.140783 prefetch_share_of_ingress=0.074651 dedup_ratio=0.185480 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=35 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.365040413e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=302276608 dispatch_meta_bytes=359784 combine_payload_bytes=302276608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302636392 ingress_bytes=371482624 link_bytes=371482624 token_bytes=604913000 weight_movement_bytes=69206016 total_nvlink_bytes=674119016 weight_to_token_ratio=0.114407 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397859207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289112064 dispatch_meta_bytes=330816 combine_payload_bytes=289112064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289442880 ingress_bytes=323715072 link_bytes=323715072 token_bytes=578554944 weight_movement_bytes=34603008 total_nvlink_bytes=613157952 weight_to_token_ratio=0.059809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218113755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=288628736 dispatch_meta_bytes=345840 combine_payload_bytes=288628736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288974576 ingress_bytes=357834752 link_bytes=357834752 token_bytes=577603312 weight_movement_bytes=69206016 total_nvlink_bytes=646809328 weight_to_token_ratio=0.119816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346503364e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12860363600 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6434067280 ingress_bytes=8052637696 link_bytes=8052637696 total_nvlink_bytes=14486704976 weight_to_token_ratio=0.126462 prefetch_share_of_ingress=0.094536 dedup_ratio=0.195833 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.317453363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83410 payload_rows_remote=73066 dup_rows=14894 dup_rows_remote=12702 dedup_ratio=0.151510 router_tokens_per_expert=[49, 0, 6103, 0, 3, 0, 4, 0, 1, 4, 2, 0, 0, 2544, 971, 79, 3243, 535, 57, 3, 220, 2795, 0, 1286, 0, 0, 0, 0, 177, 4, 1767, 64, 2, 0, 3, 9, 856, 40, 7, 1689, 384, 2, 0, 2059, 1, 0, 6567, 1, 1724, 0, 1965, 4342, 122, 1793, 664, 0, 0, 235, 62, 6664, 2, 47, 1, 0] payload_rows_per_dst=[12220, 6752, 11572, 3848, 11252, 13894, 10344, 13528] dup_rows_per_dst=[98, 450, 4706, 176, 2644, 4134, 2192, 494] combine_rows_per_src=[12220, 6752, 11572, 3848, 11252, 13894, 10344, 13528] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12981233208 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6494537272 ingress_bytes=7974625280 link_bytes=7974625280 total_nvlink_bytes=14469162552 weight_to_token_ratio=0.114622 prefetch_share_of_ingress=0.099800 dedup_ratio=0.196294 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.304690127e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12594937392 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6301285936 ingress_bytes=7746977792 link_bytes=7746977792 total_nvlink_bytes=14048263728 weight_to_token_ratio=0.115390 prefetch_share_of_ingress=0.080400 dedup_ratio=0.204756 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267445815e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none - [2026-08-26 16:55:23] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 794.5 | throughput per GPU (TFLOP/s/GPU): 160.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.956084E+00 | loss scale: 1.0 | grad norm: 42.533 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75274 payload_rows_remote=65854 dup_rows=23030 dup_rows_remote=20502 dedup_ratio=0.234273 router_tokens_per_expert=[2760, 669, 420, 727, 379, 1296, 974, 98, 2583, 707, 1317, 494, 18, 1144, 275, 581, 988, 1206, 480, 774, 291, 356, 522, 586, 798, 779, 166, 353, 944, 821, 445, 1095, 1306, 340, 1049, 1826, 610, 350, 894, 368, 168, 344, 410, 274, 1652, 1954, 785, 165, 301, 583, 613, 812, 1032, 1323, 592, 257, 400, 827, 1484, 516, 397, 369, 1435, 670] payload_rows_per_dst=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] dup_rows_per_dst=[2172, 2798, 4320, 3182, 3668, 2528, 1966, 2396] combine_rows_per_src=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74908 payload_rows_remote=64270 dup_rows=23396 dup_rows_remote=19560 dedup_ratio=0.237996 router_tokens_per_expert=[2756, 594, 463, 601, 423, 1455, 970, 83, 2803, 692, 1254, 462, 38, 1144, 240, 604, 888, 1175, 432, 901, 318, 352, 512, 634, 801, 824, 147, 419, 914, 802, 391, 977, 1157, 347, 976, 1810, 569, 408, 995, 321, 148, 322, 398, 197, 1716, 1983, 784, 171, 284, 645, 604, 792, 907, 1544, 705, 212, 440, 781, 1544, 520, 420, 464, 1455, 464] payload_rows_per_dst=[10968, 10638, 8148, 7986, 9342, 9258, 8922, 9646] dup_rows_per_dst=[3722, 3836, 2276, 2564, 3824, 2180, 2464, 2530] combine_rows_per_src=[10968, 10638, 8148, 7986, 9342, 9258, 8922, 9646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12131284608 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6069376640 ingress_bytes=7480631296 link_bytes=7480631296 total_nvlink_bytes=13550007936 weight_to_token_ratio=0.116947 prefetch_share_of_ingress=0.092514 dedup_ratio=0.219138 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223870145e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=343072 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299621408 ingress_bytes=403087360 link_bytes=403087360 token_bytes=598899744 weight_movement_bytes=103809024 total_nvlink_bytes=702708768 weight_to_token_ratio=0.173333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.516785284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12415861280 prefetch_bytes=761266176 reduce_bytes=830472192 weight_movement_bytes=1591738368 egress_bytes=6211732000 ingress_bytes=7795867648 link_bytes=7795867648 total_nvlink_bytes=14007599648 weight_to_token_ratio=0.128202 prefetch_share_of_ingress=0.097650 dedup_ratio=0.211088 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=24 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.275444450e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none + [2026-08-26 18:07:30] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 793.3 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.956084E+00 | loss scale: 1.0 | grad norm: 42.533 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74816 payload_rows_remote=64918 dup_rows=23488 dup_rows_remote=20596 dedup_ratio=0.238932 router_tokens_per_expert=[2653, 588, 408, 758, 454, 1290, 938, 87, 2844, 641, 1278, 527, 23, 1112, 252, 584, 1015, 1176, 488, 762, 288, 355, 495, 541, 825, 914, 118, 385, 926, 771, 402, 1115, 1211, 300, 971, 1718, 626, 407, 888, 359, 211, 358, 379, 186, 1679, 1900, 741, 158, 312, 627, 561, 847, 1085, 1350, 722, 249, 376, 774, 1605, 593, 357, 465, 1527, 597] payload_rows_per_dst=[6914, 7798, 7908, 12186, 7414, 10822, 11876, 9898] dup_rows_per_dst=[2132, 1036, 2332, 4414, 1908, 3838, 4936, 2892] combine_rows_per_src=[6914, 7798, 7908, 12186, 7414, 10822, 11876, 9898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75138 payload_rows_remote=67086 dup_rows=23166 dup_rows_remote=20772 dedup_ratio=0.235657 router_tokens_per_expert=[2779, 671, 380, 712, 527, 1204, 1112, 100, 2432, 601, 1220, 530, 28, 1143, 260, 672, 1051, 1088, 502, 898, 302, 335, 496, 551, 907, 811, 130, 440, 871, 859, 420, 1017, 1143, 351, 1053, 1571, 630, 385, 978, 442, 202, 360, 373, 225, 1648, 1880, 723, 179, 349, 562, 666, 756, 1013, 1267, 606, 216, 365, 913, 1522, 503, 495, 547, 1484, 696] payload_rows_per_dst=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] dup_rows_per_dst=[3790, 3406, 2394, 2792, 3704, 2234, 1960, 2886] combine_rows_per_src=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74908 payload_rows_remote=64270 dup_rows=23396 dup_rows_remote=19560 dedup_ratio=0.237996 router_tokens_per_expert=[2756, 594, 463, 601, 423, 1455, 970, 83, 2803, 692, 1254, 462, 38, 1144, 240, 604, 888, 1175, 432, 901, 318, 352, 512, 634, 801, 824, 147, 419, 914, 802, 391, 977, 1157, 347, 976, 1810, 569, 408, 995, 321, 148, 322, 398, 197, 1716, 1983, 784, 171, 284, 645, 604, 792, 907, 1544, 705, 212, 440, 781, 1544, 520, 420, 464, 1455, 464] payload_rows_per_dst=[10968, 10638, 8148, 7986, 9342, 9258, 8922, 9646] dup_rows_per_dst=[3722, 3836, 2276, 2564, 3824, 2180, 2464, 2530] combine_rows_per_src=[10968, 10638, 8148, 7986, 9342, 9258, 8922, 9646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75262 payload_rows_remote=66936 dup_rows=23042 dup_rows_remote=20396 dedup_ratio=0.234395 router_tokens_per_expert=[2567, 569, 402, 778, 376, 1363, 967, 106, 2601, 735, 1254, 522, 21, 1070, 277, 487, 932, 1224, 460, 923, 369, 359, 486, 608, 801, 793, 147, 400, 972, 769, 430, 1174, 1283, 358, 990, 1657, 693, 380, 954, 371, 183, 365, 369, 249, 1599, 2024, 695, 179, 263, 619, 642, 819, 996, 1583, 642, 358, 414, 779, 1505, 496, 434, 447, 1386, 478] payload_rows_per_dst=[10776, 10382, 8352, 8326, 9518, 9050, 9392, 9466] dup_rows_per_dst=[3244, 3552, 2606, 2646, 3854, 2276, 2452, 2412] combine_rows_per_src=[10776, 10382, 8352, 8326, 9518, 9050, 9392, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75872 payload_rows_remote=65950 dup_rows=22432 dup_rows_remote=19452 dedup_ratio=0.228190 router_tokens_per_expert=[2682, 722, 503, 716, 404, 1196, 968, 127, 2552, 693, 1209, 490, 30, 1118, 286, 639, 949, 1177, 431, 793, 329, 346, 536, 550, 833, 898, 140, 397, 1022, 738, 375, 929, 1289, 392, 923, 1668, 578, 356, 1060, 404, 174, 430, 407, 257, 1729, 1821, 750, 199, 294, 549, 573, 874, 1038, 1528, 688, 271, 385, 733, 1513, 568, 403, 417, 1510, 593] payload_rows_per_dst=[7076, 7862, 10672, 11752, 7872, 10928, 9922, 9788] dup_rows_per_dst=[2196, 1068, 3642, 4016, 2132, 3942, 2980, 2456] combine_rows_per_src=[7076, 7862, 10672, 11752, 7872, 10928, 9922, 9788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75274 payload_rows_remote=65854 dup_rows=23030 dup_rows_remote=20502 dedup_ratio=0.234273 router_tokens_per_expert=[2760, 669, 420, 727, 379, 1296, 974, 98, 2583, 707, 1317, 494, 18, 1144, 275, 581, 988, 1206, 480, 774, 291, 356, 522, 586, 798, 779, 166, 353, 944, 821, 445, 1095, 1306, 340, 1049, 1826, 610, 350, 894, 368, 168, 344, 410, 274, 1652, 1954, 785, 165, 301, 583, 613, 812, 1032, 1323, 592, 257, 400, 827, 1484, 516, 397, 369, 1435, 670] payload_rows_per_dst=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] dup_rows_per_dst=[2172, 2798, 4320, 3182, 3668, 2528, 1966, 2396] combine_rows_per_src=[6954, 9640, 11606, 9420, 9374, 9420, 9060, 9800] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74808 payload_rows_remote=65360 dup_rows=23496 dup_rows_remote=19504 dedup_ratio=0.239014 router_tokens_per_expert=[2715, 640, 488, 746, 384, 1314, 949, 108, 2759, 669, 1343, 493, 27, 1166, 265, 577, 946, 1143, 433, 824, 279, 341, 470, 535, 766, 780, 142, 380, 967, 862, 364, 873, 1245, 360, 985, 1741, 637, 369, 993, 390, 185, 390, 380, 244, 1623, 1988, 704, 206, 321, 634, 601, 816, 1013, 1482, 702, 284, 380, 699, 1635, 482, 396, 403, 1444, 672] payload_rows_per_dst=[7068, 10794, 11366, 7668, 9448, 9258, 9374, 9832] dup_rows_per_dst=[2190, 3804, 4006, 2600, 3992, 2182, 2332, 2390] combine_rows_per_src=[7068, 10794, 11366, 7668, 9448, 9258, 9374, 9832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269737984 dispatch_meta_bytes=345424 combine_payload_bytes=269737984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270083408 ingress_bytes=304340992 link_bytes=304340992 token_bytes=539821392 weight_movement_bytes=34603008 total_nvlink_bytes=574424400 weight_to_token_ratio=0.064101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145210651e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263249920 dispatch_meta_bytes=335320 combine_payload_bytes=263249920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263585240 ingress_bytes=367058944 link_bytes=367058944 token_bytes=526835160 weight_movement_bytes=103809024 total_nvlink_bytes=630644184 weight_to_token_ratio=0.197043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381213256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342056 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266246184 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532150312 weight_movement_bytes=69206016 total_nvlink_bytes=601356328 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=349328 combine_payload_bytes=274169856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274519184 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548689040 weight_movement_bytes=34603008 total_nvlink_bytes=583292048 weight_to_token_ratio=0.063065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75138 payload_rows_remote=67086 dup_rows=23166 dup_rows_remote=20772 dedup_ratio=0.235657 router_tokens_per_expert=[2779, 671, 380, 712, 527, 1204, 1112, 100, 2432, 601, 1220, 530, 28, 1143, 260, 672, 1051, 1088, 502, 898, 302, 335, 496, 551, 907, 811, 130, 440, 871, 859, 420, 1017, 1143, 351, 1053, 1571, 630, 385, 978, 442, 202, 360, 373, 225, 1648, 1880, 723, 179, 349, 562, 666, 756, 1013, 1267, 606, 216, 365, 913, 1522, 503, 495, 547, 1484, 696] payload_rows_per_dst=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] dup_rows_per_dst=[3790, 3406, 2394, 2792, 3704, 2234, 1960, 2886] combine_rows_per_src=[11180, 10366, 8052, 8118, 9402, 8946, 8910, 10164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274784256 dispatch_meta_bytes=351432 combine_payload_bytes=274784256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275135688 ingress_bytes=309387264 link_bytes=309387264 token_bytes=549919944 weight_movement_bytes=34603008 total_nvlink_bytes=584522952 weight_to_token_ratio=0.062924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263249920 dispatch_meta_bytes=335320 combine_payload_bytes=263249920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263585240 ingress_bytes=367058944 link_bytes=367058944 token_bytes=526835160 weight_movement_bytes=103809024 total_nvlink_bytes=630644184 weight_to_token_ratio=0.197043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381213256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270131200 dispatch_meta_bytes=341608 combine_payload_bytes=270131200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270472808 ingress_bytes=304734208 link_bytes=304734208 token_bytes=540604008 weight_movement_bytes=34603008 total_nvlink_bytes=575207016 weight_to_token_ratio=0.064008 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146690292e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=349328 combine_payload_bytes=274169856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274519184 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548689040 weight_movement_bytes=34603008 total_nvlink_bytes=583292048 weight_to_token_ratio=0.063065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269737984 dispatch_meta_bytes=345424 combine_payload_bytes=269737984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270083408 ingress_bytes=304340992 link_bytes=304340992 token_bytes=539821392 weight_movement_bytes=34603008 total_nvlink_bytes=574424400 weight_to_token_ratio=0.064101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145210651e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=62728 dup_rows=23670 dup_rows_remote=17654 dedup_ratio=0.240784 router_tokens_per_expert=[2641, 653, 429, 788, 389, 1311, 978, 84, 2484, 683, 1298, 474, 20, 1176, 303, 582, 1074, 1205, 514, 834, 258, 358, 490, 579, 756, 837, 141, 397, 947, 765, 427, 1047, 1260, 324, 1194, 1429, 546, 370, 889, 414, 181, 381, 406, 230, 1718, 1842, 729, 234, 311, 610, 643, 958, 1031, 1411, 623, 292, 414, 702, 1497, 544, 412, 487, 1489, 659] payload_rows_per_dst=[11906, 9162, 8242, 8100, 9248, 9208, 9354, 9414] dup_rows_per_dst=[6016, 2162, 2382, 2534, 3604, 2234, 2404, 2334] combine_rows_per_src=[11906, 9162, 8242, 8100, 9248, 9208, 9354, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75872 payload_rows_remote=65950 dup_rows=22432 dup_rows_remote=19452 dedup_ratio=0.228190 router_tokens_per_expert=[2682, 722, 503, 716, 404, 1196, 968, 127, 2552, 693, 1209, 490, 30, 1118, 286, 639, 949, 1177, 431, 793, 329, 346, 536, 550, 833, 898, 140, 397, 1022, 738, 375, 929, 1289, 392, 923, 1668, 578, 356, 1060, 404, 174, 430, 407, 257, 1729, 1821, 750, 199, 294, 549, 573, 874, 1038, 1528, 688, 271, 385, 733, 1513, 568, 403, 417, 1510, 593] payload_rows_per_dst=[7076, 7862, 10672, 11752, 7872, 10928, 9922, 9788] dup_rows_per_dst=[2196, 1068, 3642, 4016, 2132, 3942, 2980, 2456] combine_rows_per_src=[7076, 7862, 10672, 11752, 7872, 10928, 9922, 9788] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267714560 dispatch_meta_bytes=339456 combine_payload_bytes=267714560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268054016 ingress_bytes=336920576 link_bytes=336920576 token_bytes=535768576 weight_movement_bytes=69206016 total_nvlink_bytes=604974592 weight_to_token_ratio=0.129171 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267805003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78558 payload_rows_remote=66206 dup_rows=19746 dup_rows_remote=14730 dedup_ratio=0.200867 router_tokens_per_expert=[128, 397, 272, 187, 1833, 304, 2681, 168, 40, 336, 1007, 388, 726, 749, 121, 2881, 584, 1084, 1226, 1358, 510, 1588, 834, 417, 221, 203, 993, 4555, 51, 975, 77, 834, 179, 14, 21, 243, 524, 19, 250, 603, 1565, 648, 0, 485, 167, 366, 7, 891, 5635, 203, 1862, 21, 313, 319, 922, 785, 930, 1154, 613, 543, 1028, 260, 446, 408] payload_rows_per_dst=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] dup_rows_per_dst=[2438, 2760, 1286, 560, 1814, 5016, 2436, 3436] combine_rows_per_src=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274784256 dispatch_meta_bytes=351432 combine_payload_bytes=274784256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275135688 ingress_bytes=309387264 link_bytes=309387264 token_bytes=549919944 weight_movement_bytes=34603008 total_nvlink_bytes=584522952 weight_to_token_ratio=0.062924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79338 payload_rows_remote=69456 dup_rows=18966 dup_rows_remote=16558 dedup_ratio=0.192932 router_tokens_per_expert=[109, 343, 154, 139, 1808, 304, 2918, 212, 23, 330, 930, 332, 703, 699, 89, 3039, 539, 1033, 1223, 1313, 437, 1688, 752, 437, 190, 226, 1114, 4830, 38, 1146, 74, 741, 122, 9, 9, 227, 448, 10, 219, 463, 1559, 605, 0, 457, 148, 334, 7, 1155, 5949, 143, 2012, 14, 319, 278, 934, 665, 976, 1092, 582, 564, 969, 181, 393, 396] payload_rows_per_dst=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] dup_rows_per_dst=[2230, 2408, 4006, 4234, 1688, 1470, 1310, 1620] combine_rows_per_src=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=64238 dup_rows=23736 dup_rows_remote=20520 dedup_ratio=0.241455 router_tokens_per_expert=[166, 372, 155, 141, 1924, 317, 3084, 210, 25, 380, 1239, 443, 731, 568, 135, 2773, 587, 995, 1222, 1146, 437, 1551, 863, 438, 219, 209, 931, 4672, 44, 973, 57, 877, 161, 9, 12, 244, 507, 27, 251, 481, 1637, 654, 0, 415, 156, 355, 3, 879, 5690, 195, 1922, 27, 233, 341, 1075, 772, 889, 1021, 616, 510, 1008, 235, 518, 425] payload_rows_per_dst=[9950, 8514, 8022, 7310, 3320, 12678, 14444, 10330] dup_rows_per_dst=[2788, 2308, 1166, 972, 64, 4864, 8358, 3216] combine_rows_per_src=[9950, 8514, 8022, 7310, 3320, 12678, 14444, 10330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79138 payload_rows_remote=67140 dup_rows=19166 dup_rows_remote=15520 dedup_ratio=0.194967 router_tokens_per_expert=[170, 420, 228, 147, 1863, 417, 2693, 176, 33, 349, 897, 423, 754, 634, 166, 2891, 512, 1011, 1233, 1408, 508, 1610, 788, 387, 202, 210, 973, 4589, 36, 937, 76, 799, 134, 14, 18, 291, 509, 14, 211, 617, 1438, 611, 0, 495, 143, 385, 3, 1250, 5586, 173, 1921, 17, 241, 316, 982, 807, 935, 1076, 665, 571, 941, 228, 560, 460] payload_rows_per_dst=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] dup_rows_per_dst=[2392, 2594, 3934, 3646, 1854, 1416, 1408, 1922] combine_rows_per_src=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=68500 dup_rows=19206 dup_rows_remote=15456 dedup_ratio=0.195374 router_tokens_per_expert=[213, 347, 214, 157, 1915, 288, 2820, 209, 34, 400, 1084, 418, 725, 742, 161, 2777, 663, 934, 1193, 1299, 532, 1448, 689, 416, 284, 273, 1065, 4309, 42, 1030, 101, 709, 176, 17, 13, 246, 546, 15, 256, 616, 1460, 729, 1, 475, 148, 398, 7, 925, 5580, 202, 1834, 14, 243, 342, 1034, 750, 1025, 1234, 649, 633, 897, 298, 490, 408] payload_rows_per_dst=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] dup_rows_per_dst=[2530, 2940, 3750, 3650, 1774, 1118, 1262, 2182] combine_rows_per_src=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=256933888 dispatch_meta_bytes=321528 combine_payload_bytes=256933888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=257255416 ingress_bytes=360742912 link_bytes=360742912 token_bytes=514189304 weight_movement_bytes=103809024 total_nvlink_bytes=617998328 weight_to_token_ratio=0.201889 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357446536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75560 payload_rows_remote=62636 dup_rows=22744 dup_rows_remote=15728 dedup_ratio=0.231364 router_tokens_per_expert=[176, 416, 194, 146, 1681, 288, 2723, 209, 44, 439, 1092, 406, 792, 680, 152, 3035, 571, 1175, 1143, 1192, 520, 1647, 864, 373, 229, 251, 1139, 4523, 48, 975, 60, 812, 165, 12, 21, 250, 549, 24, 228, 532, 1481, 638, 0, 437, 172, 390, 8, 935, 5475, 195, 1743, 15, 257, 285, 1148, 825, 998, 1060, 619, 565, 972, 220, 485, 453] payload_rows_per_dst=[9448, 10222, 8036, 6406, 5404, 12394, 12924, 10726] dup_rows_per_dst=[2218, 3058, 1256, 622, 488, 4774, 7016, 3312] combine_rows_per_src=[9448, 10222, 8036, 6406, 5404, 12394, 12924, 10726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271179776 dispatch_meta_bytes=323744 combine_payload_bytes=271179776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271503520 ingress_bytes=305782784 link_bytes=305782784 token_bytes=542683296 weight_movement_bytes=34603008 total_nvlink_bytes=577286304 weight_to_token_ratio=0.063763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150635999e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79138 payload_rows_remote=67140 dup_rows=19166 dup_rows_remote=15520 dedup_ratio=0.194967 router_tokens_per_expert=[170, 420, 228, 147, 1863, 417, 2693, 176, 33, 349, 897, 423, 754, 634, 166, 2891, 512, 1011, 1233, 1408, 508, 1610, 788, 387, 202, 210, 973, 4589, 36, 937, 76, 799, 134, 14, 18, 291, 509, 14, 211, 617, 1438, 611, 0, 495, 143, 385, 3, 1250, 5586, 173, 1921, 17, 241, 316, 982, 807, 935, 1076, 665, 571, 941, 228, 560, 460] payload_rows_per_dst=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] dup_rows_per_dst=[2392, 2594, 3934, 3646, 1854, 1416, 1408, 1922] combine_rows_per_src=[9836, 9700, 10980, 11998, 12934, 7234, 7506, 8950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78558 payload_rows_remote=66206 dup_rows=19746 dup_rows_remote=14730 dedup_ratio=0.200867 router_tokens_per_expert=[128, 397, 272, 187, 1833, 304, 2681, 168, 40, 336, 1007, 388, 726, 749, 121, 2881, 584, 1084, 1226, 1358, 510, 1588, 834, 417, 221, 203, 993, 4555, 51, 975, 77, 834, 179, 14, 21, 243, 524, 19, 250, 603, 1565, 648, 0, 485, 167, 366, 7, 891, 5635, 203, 1862, 21, 313, 319, 922, 785, 930, 1154, 613, 543, 1028, 260, 446, 408] payload_rows_per_dst=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] dup_rows_per_dst=[2438, 2760, 1286, 560, 1814, 5016, 2436, 3436] combine_rows_per_src=[9502, 9736, 8024, 6148, 13162, 12352, 9130, 10504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270131200 dispatch_meta_bytes=341608 combine_payload_bytes=270131200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270472808 ingress_bytes=304734208 link_bytes=304734208 token_bytes=540604008 weight_movement_bytes=34603008 total_nvlink_bytes=575207016 weight_to_token_ratio=0.064008 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146690292e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79338 payload_rows_remote=69456 dup_rows=18966 dup_rows_remote=16558 dedup_ratio=0.192932 router_tokens_per_expert=[109, 343, 154, 139, 1808, 304, 2918, 212, 23, 330, 930, 332, 703, 699, 89, 3039, 539, 1033, 1223, 1313, 437, 1688, 752, 437, 190, 226, 1114, 4830, 38, 1146, 74, 741, 122, 9, 9, 227, 448, 10, 219, 463, 1559, 605, 0, 457, 148, 334, 7, 1155, 5949, 143, 2012, 14, 319, 278, 934, 665, 976, 1092, 582, 564, 969, 181, 393, 396] payload_rows_per_dst=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] dup_rows_per_dst=[2230, 2408, 4006, 4234, 1688, 1470, 1310, 1620] combine_rows_per_src=[9744, 9882, 10838, 12484, 13224, 7060, 7420, 8686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263118848 dispatch_meta_bytes=339032 combine_payload_bytes=263118848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263457880 ingress_bytes=297721856 link_bytes=297721856 token_bytes=526576728 weight_movement_bytes=34603008 total_nvlink_bytes=561179736 weight_to_token_ratio=0.065713 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120303376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280576000 dispatch_meta_bytes=335824 combine_payload_bytes=280576000 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280911824 ingress_bytes=418988032 link_bytes=418988032 token_bytes=561487824 weight_movement_bytes=138412032 total_nvlink_bytes=699899856 weight_to_token_ratio=0.246509 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576618233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275005440 dispatch_meta_bytes=330640 combine_payload_bytes=275005440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275336080 ingress_bytes=344211456 link_bytes=344211456 token_bytes=550341520 weight_movement_bytes=69206016 total_nvlink_bytes=619547536 weight_to_token_ratio=0.125751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295239998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78822 payload_rows_remote=65832 dup_rows=19482 dup_rows_remote=17888 dedup_ratio=0.198181 router_tokens_per_expert=[122, 346, 155, 163, 1870, 385, 2870, 153, 19, 282, 900, 367, 753, 630, 141, 3020, 610, 1023, 1419, 1277, 434, 1626, 790, 306, 213, 240, 1036, 4703, 25, 969, 49, 812, 122, 18, 17, 248, 411, 13, 221, 506, 1540, 704, 0, 494, 178, 336, 10, 1186, 5736, 164, 1891, 9, 270, 273, 954, 774, 924, 1118, 566, 536, 961, 217, 562, 485] payload_rows_per_dst=[9710, 9732, 9570, 9974, 12990, 9274, 7458, 10114] dup_rows_per_dst=[2418, 2492, 2752, 2666, 1594, 3076, 1346, 3138] combine_rows_per_src=[9710, 9732, 9570, 9974, 12990, 9274, 7458, 10114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263118848 dispatch_meta_bytes=339032 combine_payload_bytes=263118848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263457880 ingress_bytes=297721856 link_bytes=297721856 token_bytes=526576728 weight_movement_bytes=34603008 total_nvlink_bytes=561179736 weight_to_token_ratio=0.065713 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120303376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271179776 dispatch_meta_bytes=323744 combine_payload_bytes=271179776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271503520 ingress_bytes=305782784 link_bytes=305782784 token_bytes=542683296 weight_movement_bytes=34603008 total_nvlink_bytes=577286304 weight_to_token_ratio=0.063763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150635999e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284491776 dispatch_meta_bytes=344056 combine_payload_bytes=284491776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284835832 ingress_bytes=353697792 link_bytes=353697792 token_bytes=569327608 weight_movement_bytes=69206016 total_nvlink_bytes=638533624 weight_to_token_ratio=0.121557 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330936316e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275005440 dispatch_meta_bytes=330640 combine_payload_bytes=275005440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275336080 ingress_bytes=344211456 link_bytes=344211456 token_bytes=550341520 weight_movement_bytes=69206016 total_nvlink_bytes=619547536 weight_to_token_ratio=0.125751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295239998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=256557056 dispatch_meta_bytes=313456 combine_payload_bytes=256557056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256870512 ingress_bytes=325763072 link_bytes=325763072 token_bytes=513427568 weight_movement_bytes=69206016 total_nvlink_bytes=582633584 weight_to_token_ratio=0.134792 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225820214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269647872 dispatch_meta_bytes=334880 combine_payload_bytes=269647872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269982752 ingress_bytes=304250880 link_bytes=304250880 token_bytes=539630624 weight_movement_bytes=34603008 total_nvlink_bytes=574233632 weight_to_token_ratio=0.064124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144871567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=68500 dup_rows=19206 dup_rows_remote=15456 dedup_ratio=0.195374 router_tokens_per_expert=[213, 347, 214, 157, 1915, 288, 2820, 209, 34, 400, 1084, 418, 725, 742, 161, 2777, 663, 934, 1193, 1299, 532, 1448, 689, 416, 284, 273, 1065, 4309, 42, 1030, 101, 709, 176, 17, 13, 246, 546, 15, 256, 616, 1460, 729, 1, 475, 148, 398, 7, 925, 5580, 202, 1834, 14, 243, 342, 1034, 750, 1025, 1234, 649, 633, 897, 298, 490, 408] payload_rows_per_dst=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] dup_rows_per_dst=[2530, 2940, 3750, 3650, 1774, 1118, 1262, 2182] combine_rows_per_src=[9796, 9742, 10598, 11976, 13156, 7168, 7576, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78612 payload_rows_remote=66490 dup_rows=19692 dup_rows_remote=15304 dedup_ratio=0.200317 router_tokens_per_expert=[1, 259, 14, 199, 33, 35, 1694, 4695, 1, 802, 341, 1388, 1992, 119, 155, 92, 2948, 1315, 1002, 1098, 82, 3, 4697, 307, 391, 1367, 4581, 1054, 602, 345, 182, 24, 272, 456, 1416, 112, 4223, 410, 571, 178, 199, 299, 0, 104, 1147, 2095, 128, 60, 309, 1634, 284, 43, 47, 71, 447, 4, 1112, 70, 72, 57, 8, 592, 34, 880] payload_rows_per_dst=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] dup_rows_per_dst=[2342, 1612, 3294, 1076, 732, 4388, 3316, 2932] combine_rows_per_src=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78968 payload_rows_remote=68448 dup_rows=19336 dup_rows_remote=16552 dedup_ratio=0.196696 router_tokens_per_expert=[126, 440, 223, 151, 1991, 308, 2764, 158, 35, 343, 963, 345, 810, 507, 149, 2956, 522, 1035, 1188, 1451, 573, 1499, 812, 454, 248, 234, 874, 4532, 39, 1089, 76, 737, 182, 12, 21, 266, 539, 18, 223, 630, 1520, 628, 0, 484, 129, 361, 8, 1075, 5528, 182, 1877, 23, 328, 265, 1025, 655, 958, 1081, 570, 549, 1050, 243, 583, 507] payload_rows_per_dst=[10520, 9590, 10354, 12096, 13036, 7038, 7302, 9032] dup_rows_per_dst=[2784, 2626, 3732, 3562, 1802, 1372, 1408, 2050] combine_rows_per_src=[10520, 9590, 10354, 12096, 13036, 7038, 7302, 9032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280576000 dispatch_meta_bytes=335824 combine_payload_bytes=280576000 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280911824 ingress_bytes=418988032 link_bytes=418988032 token_bytes=561487824 weight_movement_bytes=138412032 total_nvlink_bytes=699899856 weight_to_token_ratio=0.246509 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576618233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272343040 dispatch_meta_bytes=327176 combine_payload_bytes=272343040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272670216 ingress_bytes=306946048 link_bytes=306946048 token_bytes=545013256 weight_movement_bytes=34603008 total_nvlink_bytes=579616264 weight_to_token_ratio=0.063490 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155013267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76232 payload_rows_remote=69776 dup_rows=22072 dup_rows_remote=20956 dedup_ratio=0.224528 router_tokens_per_expert=[1, 376, 8, 157, 47, 32, 1575, 4403, 0, 741, 351, 1449, 2110, 168, 287, 159, 3103, 1271, 961, 1141, 118, 4, 4123, 291, 454, 1425, 4283, 1043, 701, 351, 190, 32, 279, 427, 1309, 151, 4143, 488, 673, 190, 304, 331, 3, 142, 1250, 2112, 106, 44, 429, 1279, 208, 41, 54, 103, 552, 8, 1269, 94, 34, 64, 9, 626, 56, 1019] payload_rows_per_dst=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] dup_rows_per_dst=[2374, 5856, 1116, 1812, 3866, 1214, 3060, 2774] combine_rows_per_src=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76924 payload_rows_remote=65024 dup_rows=21380 dup_rows_remote=18408 dedup_ratio=0.217489 router_tokens_per_expert=[2, 302, 11, 211, 24, 34, 1768, 4817, 0, 718, 325, 1279, 1922, 115, 188, 100, 3052, 1385, 949, 1114, 77, 1, 4394, 322, 468, 1479, 4684, 1006, 676, 338, 149, 12, 262, 390, 1272, 105, 4210, 428, 625, 169, 235, 236, 2, 127, 1270, 2247, 77, 63, 377, 1558, 238, 19, 46, 61, 459, 2, 1090, 42, 49, 48, 3, 585, 29, 906] payload_rows_per_dst=[4522, 7906, 6932, 12738, 9384, 12382, 11160, 11900] dup_rows_per_dst=[182, 1388, 1206, 5152, 2780, 4552, 3148, 2972] combine_rows_per_src=[4522, 7906, 6932, 12738, 9384, 12382, 11160, 11900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78706 payload_rows_remote=72106 dup_rows=19598 dup_rows_remote=18632 dedup_ratio=0.199361 router_tokens_per_expert=[3, 340, 3, 176, 30, 57, 1771, 4772, 0, 777, 373, 1356, 1910, 123, 185, 136, 3348, 1286, 858, 1139, 78, 4, 4391, 350, 437, 1396, 4589, 941, 550, 269, 174, 16, 269, 396, 1357, 136, 4440, 422, 588, 173, 190, 309, 0, 97, 1199, 2050, 110, 78, 330, 1559, 259, 27, 65, 78, 463, 3, 1076, 54, 42, 64, 7, 545, 47, 881] payload_rows_per_dst=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] dup_rows_per_dst=[2590, 2120, 3130, 966, 3724, 1016, 3070, 2982] combine_rows_per_src=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77452 payload_rows_remote=66230 dup_rows=20852 dup_rows_remote=17834 dedup_ratio=0.212118 router_tokens_per_expert=[3, 342, 13, 210, 34, 18, 1955, 4568, 1, 862, 305, 1440, 1443, 152, 191, 134, 3130, 1342, 929, 947, 130, 3, 4421, 299, 528, 1361, 4345, 1060, 645, 350, 169, 33, 283, 465, 1509, 144, 4110, 403, 577, 190, 217, 395, 0, 122, 1289, 2394, 101, 93, 316, 1603, 265, 64, 57, 78, 307, 9, 1011, 86, 57, 66, 3, 617, 49, 909] payload_rows_per_dst=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] dup_rows_per_dst=[1146, 1548, 3284, 3492, 780, 4724, 3018, 2860] combine_rows_per_src=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=66030 dup_rows=21336 dup_rows_remote=15516 dedup_ratio=0.217041 router_tokens_per_expert=[2, 222, 12, 189, 25, 24, 2048, 5208, 1, 846, 415, 1512, 2112, 114, 161, 94, 3124, 1051, 923, 893, 83, 7, 4524, 316, 347, 1247, 4766, 1045, 644, 301, 130, 17, 267, 434, 1476, 108, 4243, 402, 487, 153, 181, 237, 0, 69, 1353, 1869, 82, 27, 223, 1693, 261, 19, 44, 90, 397, 5, 1093, 50, 24, 37, 2, 652, 24, 747] payload_rows_per_dst=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] dup_rows_per_dst=[2768, 5820, 2694, 4540, 3434, 938, 926, 216] combine_rows_per_src=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=340000 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280703008 ingress_bytes=314966016 link_bytes=314966016 token_bytes=561066016 weight_movement_bytes=34603008 total_nvlink_bytes=595669024 weight_to_token_ratio=0.061674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185191761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75560 payload_rows_remote=62636 dup_rows=22744 dup_rows_remote=15728 dedup_ratio=0.231364 router_tokens_per_expert=[176, 416, 194, 146, 1681, 288, 2723, 209, 44, 439, 1092, 406, 792, 680, 152, 3035, 571, 1175, 1143, 1192, 520, 1647, 864, 373, 229, 251, 1139, 4523, 48, 975, 60, 812, 165, 12, 21, 250, 549, 24, 228, 532, 1481, 638, 0, 437, 172, 390, 8, 935, 5475, 195, 1743, 15, 257, 285, 1148, 825, 998, 1060, 619, 565, 972, 220, 485, 453] payload_rows_per_dst=[9448, 10222, 8036, 6406, 5404, 12394, 12924, 10726] dup_rows_per_dst=[2218, 3058, 1256, 622, 488, 4774, 7016, 3312] combine_rows_per_src=[9448, 10222, 8036, 6406, 5404, 12394, 12924, 10726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78706 payload_rows_remote=72106 dup_rows=19598 dup_rows_remote=18632 dedup_ratio=0.199361 router_tokens_per_expert=[3, 340, 3, 176, 30, 57, 1771, 4772, 0, 777, 373, 1356, 1910, 123, 185, 136, 3348, 1286, 858, 1139, 78, 4, 4391, 350, 437, 1396, 4589, 941, 550, 269, 174, 16, 269, 396, 1357, 136, 4440, 422, 588, 173, 190, 309, 0, 97, 1199, 2050, 110, 78, 330, 1559, 259, 27, 65, 78, 463, 3, 1076, 54, 42, 64, 7, 545, 47, 881] payload_rows_per_dst=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] dup_rows_per_dst=[2590, 2120, 3130, 966, 3724, 1016, 3070, 2982] combine_rows_per_src=[11714, 8570, 10026, 6600, 11838, 7050, 11280, 11628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=340000 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280703008 ingress_bytes=314966016 link_bytes=314966016 token_bytes=561066016 weight_movement_bytes=34603008 total_nvlink_bytes=595669024 weight_to_token_ratio=0.061674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185191761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76968 payload_rows_remote=66030 dup_rows=21336 dup_rows_remote=15516 dedup_ratio=0.217041 router_tokens_per_expert=[2, 222, 12, 189, 25, 24, 2048, 5208, 1, 846, 415, 1512, 2112, 114, 161, 94, 3124, 1051, 923, 893, 83, 7, 4524, 316, 347, 1247, 4766, 1045, 644, 301, 130, 17, 267, 434, 1476, 108, 4243, 402, 487, 153, 181, 237, 0, 69, 1353, 1869, 82, 27, 223, 1693, 261, 19, 44, 90, 397, 5, 1093, 50, 24, 37, 2, 652, 24, 747] payload_rows_per_dst=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] dup_rows_per_dst=[2768, 5820, 2694, 4540, 3434, 938, 926, 216] combine_rows_per_src=[12692, 10938, 10926, 12454, 11706, 6698, 6512, 5042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285802496 dispatch_meta_bytes=362928 combine_payload_bytes=285802496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286165424 ingress_bytes=424214528 link_bytes=424214528 token_bytes=571967920 weight_movement_bytes=138412032 total_nvlink_bytes=710379952 weight_to_token_ratio=0.241993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596285117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=333728 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266672032 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533010336 weight_movement_bytes=34603008 total_nvlink_bytes=567613344 weight_to_token_ratio=0.064920 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78612 payload_rows_remote=66490 dup_rows=19692 dup_rows_remote=15304 dedup_ratio=0.200317 router_tokens_per_expert=[1, 259, 14, 199, 33, 35, 1694, 4695, 1, 802, 341, 1388, 1992, 119, 155, 92, 2948, 1315, 1002, 1098, 82, 3, 4697, 307, 391, 1367, 4581, 1054, 602, 345, 182, 24, 272, 456, 1416, 112, 4223, 410, 571, 178, 199, 299, 0, 104, 1147, 2095, 128, 60, 309, 1634, 284, 43, 47, 71, 447, 4, 1112, 70, 72, 57, 8, 592, 34, 880] payload_rows_per_dst=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] dup_rows_per_dst=[2342, 1612, 3294, 1076, 732, 4388, 3316, 2932] combine_rows_per_src=[11518, 8168, 10216, 6854, 6098, 12122, 11756, 11880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=256557056 dispatch_meta_bytes=313456 combine_payload_bytes=256557056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=256870512 ingress_bytes=325763072 link_bytes=325763072 token_bytes=513427568 weight_movement_bytes=69206016 total_nvlink_bytes=582633584 weight_to_token_ratio=0.134792 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225820214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=333728 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266672032 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533010336 weight_movement_bytes=34603008 total_nvlink_bytes=567613344 weight_to_token_ratio=0.064920 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79222 payload_rows_remote=70906 dup_rows=19082 dup_rows_remote=17178 dedup_ratio=0.194112 router_tokens_per_expert=[2, 227, 4, 215, 34, 28, 1678, 4865, 1, 760, 459, 1288, 2149, 126, 173, 113, 3014, 1326, 1052, 865, 85, 3, 4692, 398, 334, 1365, 4823, 1036, 598, 303, 115, 18, 238, 519, 1429, 98, 4295, 482, 612, 148, 192, 264, 0, 88, 1397, 1975, 58, 65, 223, 1536, 231, 20, 41, 57, 339, 4, 1055, 34, 35, 47, 3, 537, 28, 953] payload_rows_per_dst=[11656, 8510, 10302, 6596, 8316, 10370, 11400, 12072] dup_rows_per_dst=[2450, 1628, 3184, 942, 1904, 3130, 2886, 2958] combine_rows_per_src=[11656, 8510, 10302, 6596, 8316, 10370, 11400, 12072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=295346176 dispatch_meta_bytes=362952 combine_payload_bytes=295346176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295709128 ingress_bytes=364552192 link_bytes=364552192 token_bytes=591055304 weight_movement_bytes=69206016 total_nvlink_bytes=660261320 weight_to_token_ratio=0.117089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371780550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=336256 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271614336 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542892416 weight_movement_bytes=34603008 total_nvlink_bytes=577495424 weight_to_token_ratio=0.063738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270458880 dispatch_meta_bytes=326184 combine_payload_bytes=270458880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270785064 ingress_bytes=305061888 link_bytes=305061888 token_bytes=541243944 weight_movement_bytes=34603008 total_nvlink_bytes=575846952 weight_to_token_ratio=0.063932 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147923325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80684 payload_rows_remote=72484 dup_rows=17620 dup_rows_remote=14676 dedup_ratio=0.179240 router_tokens_per_expert=[845, 68, 1463, 33, 709, 269, 604, 502, 141, 121, 102, 76, 317, 19, 586, 290, 5486, 305, 189, 4, 275, 116, 0, 3907, 250, 0, 1918, 2661, 852, 1179, 6022, 56, 768, 1135, 23, 42, 1180, 301, 1028, 200, 427, 1729, 52, 1735, 61, 365, 5, 1198, 1, 154, 2289, 1129, 81, 746, 23, 981, 321, 0, 817, 88, 228, 2301, 194, 185] payload_rows_per_dst=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] dup_rows_per_dst=[1088, 2552, 1036, 830, 4628, 2944, 1742, 2800] combine_rows_per_src=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272343040 dispatch_meta_bytes=327176 combine_payload_bytes=272343040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272670216 ingress_bytes=306946048 link_bytes=306946048 token_bytes=545013256 weight_movement_bytes=34603008 total_nvlink_bytes=579616264 weight_to_token_ratio=0.063490 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155013267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290430976 dispatch_meta_bytes=352336 combine_payload_bytes=290430976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290783312 ingress_bytes=359636992 link_bytes=359636992 token_bytes=581214288 weight_movement_bytes=69206016 total_nvlink_bytes=650420304 weight_to_token_ratio=0.119071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353285048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76232 payload_rows_remote=69776 dup_rows=22072 dup_rows_remote=20956 dedup_ratio=0.224528 router_tokens_per_expert=[1, 376, 8, 157, 47, 32, 1575, 4403, 0, 741, 351, 1449, 2110, 168, 287, 159, 3103, 1271, 961, 1141, 118, 4, 4123, 291, 454, 1425, 4283, 1043, 701, 351, 190, 32, 279, 427, 1309, 151, 4143, 488, 673, 190, 304, 331, 3, 142, 1250, 2112, 106, 44, 429, 1279, 208, 41, 54, 103, 552, 8, 1269, 94, 34, 64, 9, 626, 56, 1019] payload_rows_per_dst=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] dup_rows_per_dst=[2374, 5856, 1116, 1812, 3866, 1214, 3060, 2774] combine_rows_per_src=[10824, 10880, 6456, 7610, 11454, 7370, 10534, 11104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76236 payload_rows_remote=64506 dup_rows=22068 dup_rows_remote=19540 dedup_ratio=0.224487 router_tokens_per_expert=[5, 381, 14, 68, 36, 44, 1914, 4667, 2, 853, 383, 1455, 1825, 122, 204, 123, 2959, 1132, 988, 1023, 86, 5, 4006, 275, 464, 1436, 4496, 984, 622, 319, 167, 28, 248, 405, 1459, 119, 4637, 367, 568, 160, 254, 324, 1, 165, 1819, 2035, 118, 100, 313, 1373, 219, 39, 70, 80, 409, 5, 1149, 67, 75, 59, 4, 567, 41, 817] payload_rows_per_dst=[11730, 10662, 11620, 12106, 11978, 8088, 4744, 5308] dup_rows_per_dst=[2528, 5190, 3410, 4926, 3948, 1544, 272, 250] combine_rows_per_src=[11730, 10662, 11620, 12106, 11978, 8088, 4744, 5308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=296894464 dispatch_meta_bytes=348640 combine_payload_bytes=296894464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297243104 ingress_bytes=331497472 link_bytes=331497472 token_bytes=594137568 weight_movement_bytes=34603008 total_nvlink_bytes=628740576 weight_to_token_ratio=0.058241 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247398299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285802496 dispatch_meta_bytes=362928 combine_payload_bytes=285802496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286165424 ingress_bytes=424214528 link_bytes=424214528 token_bytes=571967920 weight_movement_bytes=138412032 total_nvlink_bytes=710379952 weight_to_token_ratio=0.241993 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596285117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78264 payload_rows_remote=70690 dup_rows=20040 dup_rows_remote=19262 dedup_ratio=0.203857 router_tokens_per_expert=[868, 49, 1292, 14, 613, 215, 822, 373, 116, 116, 94, 62, 353, 38, 568, 196, 5157, 348, 272, 2, 372, 105, 1, 4120, 216, 0, 2049, 2757, 868, 1006, 6132, 37, 735, 982, 16, 60, 1428, 378, 1024, 210, 331, 1582, 60, 1997, 156, 389, 1, 1126, 2, 132, 2208, 908, 61, 718, 30, 968, 239, 0, 1125, 66, 260, 2290, 190, 249] payload_rows_per_dst=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] dup_rows_per_dst=[910, 2396, 6922, 778, 1472, 2886, 1644, 3032] combine_rows_per_src=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77962 payload_rows_remote=68888 dup_rows=20342 dup_rows_remote=18514 dedup_ratio=0.206930 router_tokens_per_expert=[815, 71, 1322, 35, 680, 283, 453, 502, 273, 127, 127, 130, 317, 24, 669, 214, 5138, 320, 220, 8, 257, 157, 0, 3659, 290, 2, 2327, 2812, 852, 1132, 5940, 87, 709, 1015, 17, 47, 1440, 184, 979, 230, 370, 1922, 83, 1827, 43, 326, 6, 1067, 4, 128, 2278, 1289, 72, 598, 33, 1042, 295, 0, 807, 82, 248, 2303, 197, 268] payload_rows_per_dst=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] dup_rows_per_dst=[4894, 3004, 1828, 2052, 1340, 3068, 1762, 2394] combine_rows_per_src=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73590 payload_rows_remote=62404 dup_rows=24714 dup_rows_remote=21750 dedup_ratio=0.251404 router_tokens_per_expert=[860, 67, 1355, 22, 661, 257, 426, 516, 236, 115, 70, 55, 333, 21, 489, 240, 5671, 348, 199, 7, 290, 122, 1, 3877, 204, 0, 2113, 2517, 915, 1170, 5967, 70, 549, 881, 8, 43, 1182, 223, 1123, 190, 448, 1717, 52, 1849, 71, 472, 5, 1033, 0, 156, 2490, 1055, 54, 535, 35, 1229, 265, 0, 1100, 113, 222, 2393, 215, 250] payload_rows_per_dst=[7476, 3026, 1836, 13248, 11976, 10680, 14162, 11186] dup_rows_per_dst=[852, 92, 98, 3404, 4176, 4840, 8288, 2964] combine_rows_per_src=[7476, 3026, 1836, 13248, 11976, 10680, 14162, 11186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66156 dup_rows=22084 dup_rows_remote=18826 dedup_ratio=0.224650 router_tokens_per_expert=[830, 51, 1273, 19, 490, 215, 710, 459, 225, 103, 77, 57, 330, 20, 445, 87, 5314, 384, 265, 1, 417, 139, 0, 3869, 208, 2, 2064, 2640, 1061, 1373, 6055, 61, 735, 1035, 28, 38, 1220, 230, 1043, 242, 380, 1710, 62, 2162, 168, 471, 3, 906, 1, 160, 2228, 810, 81, 692, 32, 1239, 269, 0, 1042, 76, 349, 2073, 240, 183] payload_rows_per_dst=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] dup_rows_per_dst=[828, 130, 1154, 5158, 4666, 4084, 3258, 2806] combine_rows_per_src=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77452 payload_rows_remote=66230 dup_rows=20852 dup_rows_remote=17834 dedup_ratio=0.212118 router_tokens_per_expert=[3, 342, 13, 210, 34, 18, 1955, 4568, 1, 862, 305, 1440, 1443, 152, 191, 134, 3130, 1342, 929, 947, 130, 3, 4421, 299, 528, 1361, 4345, 1060, 645, 350, 169, 33, 283, 465, 1509, 144, 4110, 403, 577, 190, 217, 395, 0, 122, 1289, 2394, 101, 93, 316, 1603, 265, 64, 57, 78, 307, 9, 1011, 86, 57, 66, 3, 617, 49, 909] payload_rows_per_dst=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] dup_rows_per_dst=[1146, 1548, 3284, 3492, 780, 4724, 3018, 2860] combine_rows_per_src=[7268, 7508, 10508, 10440, 6362, 12718, 11222, 11426] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78264 payload_rows_remote=70690 dup_rows=20040 dup_rows_remote=19262 dedup_ratio=0.203857 router_tokens_per_expert=[868, 49, 1292, 14, 613, 215, 822, 373, 116, 116, 94, 62, 353, 38, 568, 196, 5157, 348, 272, 2, 372, 105, 1, 4120, 216, 0, 2049, 2757, 868, 1006, 6132, 37, 735, 982, 16, 60, 1428, 378, 1024, 210, 331, 1582, 60, 1997, 156, 389, 1, 1126, 2, 132, 2208, 908, 61, 718, 30, 968, 239, 0, 1125, 66, 260, 2290, 190, 249] payload_rows_per_dst=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] dup_rows_per_dst=[910, 2396, 6922, 778, 1472, 2886, 1644, 3032] combine_rows_per_src=[7582, 12954, 13832, 7574, 8194, 8398, 8410, 11320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77242 payload_rows_remote=64642 dup_rows=21062 dup_rows_remote=18226 dedup_ratio=0.214254 router_tokens_per_expert=[784, 58, 1680, 15, 593, 210, 574, 493, 217, 108, 86, 47, 307, 8, 732, 184, 5628, 327, 184, 3, 380, 91, 0, 4062, 237, 0, 2041, 2742, 1009, 1061, 6029, 64, 605, 996, 29, 39, 1374, 299, 998, 199, 331, 1631, 37, 2038, 102, 409, 0, 1106, 1, 135, 2156, 769, 40, 697, 29, 1311, 219, 0, 888, 87, 141, 2096, 187, 249] payload_rows_per_dst=[13688, 12600, 9134, 9940, 7844, 8368, 8720, 6948] dup_rows_per_dst=[6382, 2836, 960, 4368, 1234, 2940, 1556, 786] combine_rows_per_src=[13688, 12600, 9134, 9940, 7844, 8368, 8720, 6948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264216576 dispatch_meta_bytes=336184 combine_payload_bytes=264216576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264552760 ingress_bytes=333422592 link_bytes=333422592 token_bytes=528769336 weight_movement_bytes=69206016 total_nvlink_bytes=597975352 weight_to_token_ratio=0.130881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254642371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289546240 dispatch_meta_bytes=359808 combine_payload_bytes=289546240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=289906048 ingress_bytes=427958272 link_bytes=427958272 token_bytes=579452288 weight_movement_bytes=138412032 total_nvlink_bytes=717864320 weight_to_token_ratio=0.238867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.610372524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=349608 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282514856 ingress_bytes=420577280 link_bytes=420577280 token_bytes=564680104 weight_movement_bytes=138412032 total_nvlink_bytes=703092136 weight_to_token_ratio=0.245116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.582598445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=255606784 dispatch_meta_bytes=336616 combine_payload_bytes=255606784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255943400 ingress_bytes=290209792 link_bytes=290209792 token_bytes=511550184 weight_movement_bytes=34603008 total_nvlink_bytes=546153192 weight_to_token_ratio=0.067643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092036083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78324 payload_rows_remote=69360 dup_rows=19980 dup_rows_remote=17884 dedup_ratio=0.203247 router_tokens_per_expert=[797, 54, 1449, 26, 572, 205, 662, 472, 90, 84, 76, 50, 337, 17, 497, 204, 5699, 342, 228, 3, 556, 103, 0, 3904, 267, 0, 1945, 2669, 796, 946, 6152, 82, 790, 1151, 23, 54, 1456, 250, 992, 218, 321, 1635, 44, 1931, 125, 448, 0, 1244, 2, 120, 2232, 871, 50, 631, 29, 1238, 235, 0, 1028, 89, 247, 1974, 162, 278] payload_rows_per_dst=[7490, 12798, 13818, 7328, 8964, 8464, 8786, 10676] dup_rows_per_dst=[984, 2216, 6660, 744, 2096, 3032, 1560, 2688] combine_rows_per_src=[7490, 12798, 13818, 7328, 8964, 8464, 8786, 10676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80684 payload_rows_remote=72484 dup_rows=17620 dup_rows_remote=14676 dedup_ratio=0.179240 router_tokens_per_expert=[845, 68, 1463, 33, 709, 269, 604, 502, 141, 121, 102, 76, 317, 19, 586, 290, 5486, 305, 189, 4, 275, 116, 0, 3907, 250, 0, 1918, 2661, 852, 1179, 6022, 56, 768, 1135, 23, 42, 1180, 301, 1028, 200, 427, 1729, 52, 1735, 61, 365, 5, 1198, 1, 154, 2289, 1129, 81, 746, 23, 981, 321, 0, 817, 88, 228, 2301, 194, 185] payload_rows_per_dst=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] dup_rows_per_dst=[1088, 2552, 1036, 830, 4628, 2944, 1742, 2800] combine_rows_per_src=[7898, 12796, 11714, 7680, 12540, 8200, 9066, 10790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=336256 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271614336 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542892416 weight_movement_bytes=34603008 total_nvlink_bytes=577495424 weight_to_token_ratio=0.063738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270974976 dispatch_meta_bytes=339928 combine_payload_bytes=270974976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271314904 ingress_bytes=305577984 link_bytes=305577984 token_bytes=542289880 weight_movement_bytes=34603008 total_nvlink_bytes=576892888 weight_to_token_ratio=0.063809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149865353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=67142 dup_rows=17818 dup_rows_remote=12466 dedup_ratio=0.181254 router_tokens_per_expert=[3, 269, 1554, 0, 20, 415, 26, 389, 1, 465, 476, 201, 0, 0, 4, 90, 0, 1115, 0, 7288, 562, 172, 363, 906, 85, 1, 345, 4593, 129, 5546, 45, 0, 0, 4, 0, 7561, 2355, 3, 66, 590, 11, 7, 10, 1122, 322, 0, 2489, 68, 0, 55, 1415, 23, 5, 330, 1245, 1261, 8, 720, 1, 5, 4338, 62, 9, 4] payload_rows_per_dst=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] dup_rows_per_dst=[310, 1404, 5614, 748, 432, 5352, 3272, 686] combine_rows_per_src=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78324 payload_rows_remote=69360 dup_rows=19980 dup_rows_remote=17884 dedup_ratio=0.203247 router_tokens_per_expert=[797, 54, 1449, 26, 572, 205, 662, 472, 90, 84, 76, 50, 337, 17, 497, 204, 5699, 342, 228, 3, 556, 103, 0, 3904, 267, 0, 1945, 2669, 796, 946, 6152, 82, 790, 1151, 23, 54, 1456, 250, 992, 218, 321, 1635, 44, 1931, 125, 448, 0, 1244, 2, 120, 2232, 871, 50, 631, 29, 1238, 235, 0, 1028, 89, 247, 1974, 162, 278] payload_rows_per_dst=[7490, 12798, 13818, 7328, 8964, 8464, 8786, 10676] dup_rows_per_dst=[984, 2216, 6660, 744, 2096, 3032, 1560, 2688] combine_rows_per_src=[7490, 12798, 13818, 7328, 8964, 8464, 8786, 10676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=289546240 dispatch_meta_bytes=359808 combine_payload_bytes=289546240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=289906048 ingress_bytes=427958272 link_bytes=427958272 token_bytes=579452288 weight_movement_bytes=138412032 total_nvlink_bytes=717864320 weight_to_token_ratio=0.238867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.610372524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=331472 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265105104 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529878736 weight_movement_bytes=34603008 total_nvlink_bytes=564481744 weight_to_token_ratio=0.065304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77962 payload_rows_remote=68888 dup_rows=20342 dup_rows_remote=18514 dedup_ratio=0.206930 router_tokens_per_expert=[815, 71, 1322, 35, 680, 283, 453, 502, 273, 127, 127, 130, 317, 24, 669, 214, 5138, 320, 220, 8, 257, 157, 0, 3659, 290, 2, 2327, 2812, 852, 1132, 5940, 87, 709, 1015, 17, 47, 1440, 184, 979, 230, 370, 1922, 83, 1827, 43, 326, 6, 1067, 4, 128, 2278, 1289, 72, 598, 33, 1042, 295, 0, 807, 82, 248, 2303, 197, 268] payload_rows_per_dst=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] dup_rows_per_dst=[4894, 3004, 1828, 2052, 1340, 3068, 1762, 2394] combine_rows_per_src=[12044, 12638, 9074, 8844, 7902, 8220, 9126, 10114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=296894464 dispatch_meta_bytes=348640 combine_payload_bytes=296894464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297243104 ingress_bytes=331497472 link_bytes=331497472 token_bytes=594137568 weight_movement_bytes=34603008 total_nvlink_bytes=628740576 weight_to_token_ratio=0.058241 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247398299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=284098560 dispatch_meta_bytes=348976 combine_payload_bytes=284098560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284447536 ingress_bytes=318701568 link_bytes=318701568 token_bytes=568546096 weight_movement_bytes=34603008 total_nvlink_bytes=603149104 weight_to_token_ratio=0.060862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.199248343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275013632 dispatch_meta_bytes=318432 combine_payload_bytes=275013632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275332064 ingress_bytes=309616640 link_bytes=309616640 token_bytes=550345696 weight_movement_bytes=34603008 total_nvlink_bytes=584948704 weight_to_token_ratio=0.062875 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165062490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=349608 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=282514856 ingress_bytes=420577280 link_bytes=420577280 token_bytes=564680104 weight_movement_bytes=138412032 total_nvlink_bytes=703092136 weight_to_token_ratio=0.245116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.582598445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77948 payload_rows_remote=64704 dup_rows=20356 dup_rows_remote=14678 dedup_ratio=0.207072 router_tokens_per_expert=[827, 95, 1430, 20, 533, 226, 495, 430, 193, 128, 105, 61, 351, 13, 598, 199, 5405, 247, 230, 14, 320, 139, 0, 3553, 267, 1, 2200, 2800, 896, 1128, 6053, 90, 659, 1007, 23, 69, 1168, 351, 1044, 208, 343, 1828, 62, 2127, 139, 375, 5, 1186, 4, 141, 2284, 804, 74, 700, 30, 1043, 268, 0, 911, 129, 317, 2379, 244, 183] payload_rows_per_dst=[13244, 12802, 9586, 9410, 7724, 8754, 8642, 7786] dup_rows_per_dst=[5678, 2600, 1602, 3172, 1334, 3376, 1518, 1076] combine_rows_per_src=[13244, 12802, 9586, 9410, 7724, 8754, 8642, 7786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=72056 dup_rows=16082 dup_rows_remote=15248 dedup_ratio=0.163595 router_tokens_per_expert=[3, 232, 1787, 0, 20, 447, 18, 451, 0, 509, 441, 223, 1, 0, 13, 57, 0, 1179, 0, 7335, 571, 183, 306, 797, 63, 0, 399, 4841, 90, 5560, 105, 2, 0, 7, 0, 7453, 2221, 6, 57, 573, 7, 4, 18, 1098, 390, 0, 2757, 75, 0, 46, 989, 57, 10, 327, 1333, 1207, 7, 704, 0, 13, 4064, 76, 7, 13] payload_rows_per_dst=[13414, 12082, 7610, 10166, 13766, 9482, 6562, 9140] dup_rows_per_dst=[4142, 1526, 1492, 834, 4024, 2060, 1376, 628] combine_rows_per_src=[13414, 12082, 7610, 10166, 13766, 9482, 6562, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=76372 dup_rows=16018 dup_rows_remote=15500 dedup_ratio=0.162944 router_tokens_per_expert=[5, 230, 1727, 2, 21, 353, 20, 468, 5, 576, 526, 362, 0, 0, 18, 63, 0, 1046, 0, 7228, 474, 127, 437, 1132, 75, 0, 316, 4870, 95, 4901, 50, 1, 0, 2, 0, 7281, 2310, 13, 87, 787, 15, 10, 27, 1392, 267, 0, 2929, 70, 1, 94, 834, 51, 8, 370, 1356, 1213, 18, 718, 2, 16, 4028, 101, 8, 16] payload_rows_per_dst=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] dup_rows_per_dst=[4776, 1458, 518, 730, 5344, 932, 1532, 728] combine_rows_per_src=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73338 payload_rows_remote=57498 dup_rows=24966 dup_rows_remote=15814 dedup_ratio=0.253967 router_tokens_per_expert=[0, 420, 1432, 1, 11, 446, 15, 403, 0, 444, 537, 226, 0, 0, 4, 88, 1, 1085, 0, 7543, 574, 285, 470, 1010, 94, 1, 425, 4599, 80, 5572, 44, 1, 0, 3, 0, 7422, 2393, 6, 67, 583, 14, 7, 34, 1138, 287, 0, 2610, 45, 1, 44, 767, 44, 8, 341, 1400, 1174, 18, 560, 2, 8, 4295, 57, 4, 9] payload_rows_per_dst=[5246, 2490, 6192, 14484, 5866, 7596, 15624, 15840] dup_rows_per_dst=[210, 108, 658, 6886, 500, 674, 6778, 9152] combine_rows_per_src=[5246, 2490, 6192, 14484, 5866, 7596, 15624, 15840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79360 payload_rows_remote=63620 dup_rows=18944 dup_rows_remote=11772 dedup_ratio=0.192708 router_tokens_per_expert=[8, 311, 1190, 3, 24, 469, 23, 490, 4, 392, 577, 215, 2, 0, 25, 75, 0, 1272, 0, 7184, 523, 251, 416, 938, 59, 2, 442, 4680, 99, 5802, 70, 2, 2, 1, 0, 7548, 2219, 8, 73, 463, 18, 5, 29, 1202, 390, 0, 2703, 84, 0, 48, 1174, 52, 7, 333, 1124, 1170, 22, 619, 0, 34, 4159, 83, 21, 13] payload_rows_per_dst=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] dup_rows_per_dst=[274, 1400, 4416, 1522, 270, 742, 7172, 3148] combine_rows_per_src=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77948 payload_rows_remote=64704 dup_rows=20356 dup_rows_remote=14678 dedup_ratio=0.207072 router_tokens_per_expert=[827, 95, 1430, 20, 533, 226, 495, 430, 193, 128, 105, 61, 351, 13, 598, 199, 5405, 247, 230, 14, 320, 139, 0, 3553, 267, 1, 2200, 2800, 896, 1128, 6053, 90, 659, 1007, 23, 69, 1168, 351, 1044, 208, 343, 1828, 62, 2127, 139, 375, 5, 1186, 4, 141, 2284, 804, 74, 700, 30, 1043, 268, 0, 911, 129, 317, 2379, 244, 183] payload_rows_per_dst=[13244, 12802, 9586, 9410, 7724, 8754, 8642, 7786] dup_rows_per_dst=[5678, 2600, 1602, 3172, 1334, 3376, 1518, 1076] combine_rows_per_src=[13244, 12802, 9586, 9410, 7724, 8754, 8642, 7786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76220 payload_rows_remote=66156 dup_rows=22084 dup_rows_remote=18826 dedup_ratio=0.224650 router_tokens_per_expert=[830, 51, 1273, 19, 490, 215, 710, 459, 225, 103, 77, 57, 330, 20, 445, 87, 5314, 384, 265, 1, 417, 139, 0, 3869, 208, 2, 2064, 2640, 1061, 1373, 6055, 61, 735, 1035, 28, 38, 1220, 230, 1043, 242, 380, 1710, 62, 2162, 168, 471, 3, 906, 1, 160, 2228, 810, 81, 692, 32, 1239, 269, 0, 1042, 76, 349, 2073, 240, 183] payload_rows_per_dst=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] dup_rows_per_dst=[828, 130, 1154, 5158, 4666, 4084, 3258, 2806] combine_rows_per_src=[7266, 2864, 9050, 13774, 12214, 10050, 10064, 10938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=317528 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265345112 ingress_bytes=299630592 link_bytes=299630592 token_bytes=530372696 weight_movement_bytes=34603008 total_nvlink_bytes=564975704 weight_to_token_ratio=0.065243 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127485795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=72056 dup_rows=16082 dup_rows_remote=15248 dedup_ratio=0.163595 router_tokens_per_expert=[3, 232, 1787, 0, 20, 447, 18, 451, 0, 509, 441, 223, 1, 0, 13, 57, 0, 1179, 0, 7335, 571, 183, 306, 797, 63, 0, 399, 4841, 90, 5560, 105, 2, 0, 7, 0, 7453, 2221, 6, 57, 573, 7, 4, 18, 1098, 390, 0, 2757, 75, 0, 46, 989, 57, 10, 327, 1333, 1207, 7, 704, 0, 13, 4064, 76, 7, 13] payload_rows_per_dst=[13414, 12082, 7610, 10166, 13766, 9482, 6562, 9140] dup_rows_per_dst=[4142, 1526, 1492, 834, 4024, 2060, 1376, 628] combine_rows_per_src=[13414, 12082, 7610, 10166, 13766, 9482, 6562, 9140] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83878 payload_rows_remote=70830 dup_rows=14426 dup_rows_remote=12962 dedup_ratio=0.146749 router_tokens_per_expert=[1, 192, 1514, 0, 12, 347, 11, 346, 1, 421, 474, 186, 0, 0, 10, 98, 0, 1048, 0, 7298, 611, 170, 416, 854, 77, 0, 321, 4736, 77, 6066, 54, 0, 0, 2, 0, 7490, 2257, 3, 50, 581, 9, 1, 17, 1145, 264, 0, 2819, 47, 0, 56, 1253, 32, 4, 358, 1185, 1265, 11, 660, 0, 7, 4203, 81, 3, 8] payload_rows_per_dst=[15412, 13048, 5682, 9910, 15794, 7876, 6850, 9306] dup_rows_per_dst=[4030, 1464, 516, 620, 4972, 728, 1456, 640] combine_rows_per_src=[15412, 13048, 5682, 9910, 15794, 7876, 6850, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=295141376 dispatch_meta_bytes=349216 combine_payload_bytes=295141376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295490592 ingress_bytes=364347392 link_bytes=364347392 token_bytes=590631968 weight_movement_bytes=69206016 total_nvlink_bytes=659837984 weight_to_token_ratio=0.117173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371009904e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80282 payload_rows_remote=74716 dup_rows=18022 dup_rows_remote=17666 dedup_ratio=0.183329 router_tokens_per_expert=[6, 199, 1664, 1, 9, 369, 10, 381, 0, 392, 499, 176, 0, 0, 12, 75, 0, 1189, 0, 7261, 614, 194, 346, 808, 94, 0, 330, 4977, 70, 5863, 54, 0, 0, 2, 0, 7592, 2328, 7, 66, 558, 4, 3, 21, 1161, 267, 0, 2808, 42, 0, 42, 1161, 29, 9, 305, 1143, 1123, 9, 628, 0, 9, 4155, 66, 3, 18] payload_rows_per_dst=[4970, 12668, 15242, 10364, 5566, 15826, 6478, 9168] dup_rows_per_dst=[308, 1366, 5582, 686, 356, 7970, 1146, 608] combine_rows_per_src=[4970, 12668, 15242, 10364, 5566, 15826, 6478, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=367488 combine_payload_bytes=312819712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=313187200 ingress_bytes=416628736 link_bytes=416628736 token_bytes=626006912 weight_movement_bytes=103809024 total_nvlink_bytes=729815936 weight_to_token_ratio=0.165827 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=235511808 dispatch_meta_bytes=293248 combine_payload_bytes=235511808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235805056 ingress_bytes=270114816 link_bytes=270114816 token_bytes=471316864 weight_movement_bytes=34603008 total_nvlink_bytes=505919872 weight_to_token_ratio=0.073418 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016420307e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260587520 dispatch_meta_bytes=301568 combine_payload_bytes=260587520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260889088 ingress_bytes=295190528 link_bytes=295190528 token_bytes=521476608 weight_movement_bytes=34603008 total_nvlink_bytes=556079616 weight_to_token_ratio=0.066356 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110778192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=317528 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265345112 ingress_bytes=299630592 link_bytes=299630592 token_bytes=530372696 weight_movement_bytes=34603008 total_nvlink_bytes=564975704 weight_to_token_ratio=0.065243 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127485795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270974976 dispatch_meta_bytes=339928 combine_payload_bytes=270974976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271314904 ingress_bytes=305577984 link_bytes=305577984 token_bytes=542289880 weight_movement_bytes=34603008 total_nvlink_bytes=576892888 weight_to_token_ratio=0.063809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149865353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=67142 dup_rows=17818 dup_rows_remote=12466 dedup_ratio=0.181254 router_tokens_per_expert=[3, 269, 1554, 0, 20, 415, 26, 389, 1, 465, 476, 201, 0, 0, 4, 90, 0, 1115, 0, 7288, 562, 172, 363, 906, 85, 1, 345, 4593, 129, 5546, 45, 0, 0, 4, 0, 7561, 2355, 3, 66, 590, 11, 7, 10, 1122, 322, 0, 2489, 68, 0, 55, 1415, 23, 5, 330, 1245, 1261, 8, 720, 1, 5, 4338, 62, 9, 4] payload_rows_per_dst=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] dup_rows_per_dst=[310, 1404, 5614, 748, 432, 5352, 3272, 686] combine_rows_per_src=[5042, 12162, 15198, 9648, 5604, 13344, 9880, 9608] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84984 payload_rows_remote=70776 dup_rows=13320 dup_rows_remote=10394 dedup_ratio=0.135498 router_tokens_per_expert=[7, 707, 2011, 0, 444, 24, 33, 417, 124, 0, 10, 1361, 0, 3, 2478, 1577, 5939, 7814, 12, 4, 4734, 2064, 2075, 0, 0, 42, 363, 0, 0, 0, 35, 0, 11, 0, 0, 1, 0, 1, 2773, 178, 6698, 6, 268, 0, 0, 10, 0, 1585, 14, 14, 248, 3203, 8, 26, 0, 1076, 1, 18, 0, 8, 0, 615, 0, 112] payload_rows_per_dst=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] dup_rows_per_dst=[714, 2124, 10, 856, 3138, 2926, 2450, 1102] combine_rows_per_src=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=76372 dup_rows=16018 dup_rows_remote=15500 dedup_ratio=0.162944 router_tokens_per_expert=[5, 230, 1727, 2, 21, 353, 20, 468, 5, 576, 526, 362, 0, 0, 18, 63, 0, 1046, 0, 7228, 474, 127, 437, 1132, 75, 0, 316, 4870, 95, 4901, 50, 1, 0, 2, 0, 7281, 2310, 13, 87, 787, 15, 10, 27, 1392, 267, 0, 2929, 70, 1, 94, 834, 51, 8, 370, 1356, 1213, 18, 718, 2, 16, 4028, 101, 8, 16] payload_rows_per_dst=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] dup_rows_per_dst=[4776, 1458, 518, 730, 5344, 932, 1532, 728] combine_rows_per_src=[15332, 11444, 5914, 10084, 15616, 8488, 6322, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80282 payload_rows_remote=74716 dup_rows=18022 dup_rows_remote=17666 dedup_ratio=0.183329 router_tokens_per_expert=[6, 199, 1664, 1, 9, 369, 10, 381, 0, 392, 499, 176, 0, 0, 12, 75, 0, 1189, 0, 7261, 614, 194, 346, 808, 94, 0, 330, 4977, 70, 5863, 54, 0, 0, 2, 0, 7592, 2328, 7, 66, 558, 4, 3, 21, 1161, 267, 0, 2808, 42, 0, 42, 1161, 29, 9, 305, 1143, 1123, 9, 628, 0, 9, 4155, 66, 3, 18] payload_rows_per_dst=[4970, 12668, 15242, 10364, 5566, 15826, 6478, 9168] dup_rows_per_dst=[308, 1366, 5582, 686, 356, 7970, 1146, 608] combine_rows_per_src=[4970, 12668, 15242, 10364, 5566, 15826, 6478, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=295141376 dispatch_meta_bytes=349216 combine_payload_bytes=295141376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295490592 ingress_bytes=364347392 link_bytes=364347392 token_bytes=590631968 weight_movement_bytes=69206016 total_nvlink_bytes=659837984 weight_to_token_ratio=0.117173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371009904e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290119680 dispatch_meta_bytes=335168 combine_payload_bytes=290119680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290454848 ingress_bytes=324722688 link_bytes=324722688 token_bytes=580574528 weight_movement_bytes=34603008 total_nvlink_bytes=615177536 weight_to_token_ratio=0.059601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221905333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275013632 dispatch_meta_bytes=318432 combine_payload_bytes=275013632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275332064 ingress_bytes=309616640 link_bytes=309616640 token_bytes=550345696 weight_movement_bytes=34603008 total_nvlink_bytes=584948704 weight_to_token_ratio=0.062875 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165062490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=306036736 dispatch_meta_bytes=369528 combine_payload_bytes=306036736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=306406264 ingress_bytes=444448768 link_bytes=444448768 token_bytes=612443000 weight_movement_bytes=138412032 total_nvlink_bytes=750855032 weight_to_token_ratio=0.226000 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.672424933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289898496 dispatch_meta_bytes=324680 combine_payload_bytes=289898496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=290223176 ingress_bytes=359104512 link_bytes=359104512 token_bytes=580121672 weight_movement_bytes=69206016 total_nvlink_bytes=649327688 weight_to_token_ratio=0.119296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351281368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82560 payload_rows_remote=67272 dup_rows=15744 dup_rows_remote=10956 dedup_ratio=0.160156 router_tokens_per_expert=[12, 308, 1649, 7, 22, 487, 19, 394, 3, 602, 554, 316, 3, 0, 23, 55, 0, 1040, 0, 7140, 481, 124, 521, 925, 60, 0, 493, 4726, 106, 5673, 54, 0, 0, 5, 0, 7373, 1892, 11, 56, 572, 33, 3, 45, 1209, 347, 0, 2855, 65, 0, 57, 841, 37, 10, 391, 1463, 1259, 16, 652, 0, 17, 4014, 117, 7, 8] payload_rows_per_dst=[15288, 12566, 8922, 10090, 11854, 8272, 6590, 8978] dup_rows_per_dst=[4788, 1892, 2146, 788, 3078, 842, 1526, 684] combine_rows_per_src=[15288, 12566, 8922, 10090, 11854, 8272, 6590, 8978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=367488 combine_payload_bytes=312819712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=313187200 ingress_bytes=416628736 link_bytes=416628736 token_bytes=626006912 weight_movement_bytes=103809024 total_nvlink_bytes=729815936 weight_to_token_ratio=0.165827 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=69334 dup_rows=13086 dup_rows_remote=12318 dedup_ratio=0.133118 router_tokens_per_expert=[0, 654, 2125, 0, 438, 31, 45, 277, 122, 0, 12, 1312, 0, 1, 2373, 1133, 6031, 7939, 19, 3, 4360, 2280, 2566, 0, 1, 33, 321, 0, 0, 0, 32, 0, 10, 0, 0, 2, 0, 1, 2613, 315, 6810, 4, 212, 1, 0, 14, 1, 1710, 15, 9, 237, 3616, 6, 37, 0, 696, 6, 26, 1, 29, 0, 549, 0, 124] payload_rows_per_dst=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] dup_rows_per_dst=[1166, 1380, 20, 768, 3140, 2810, 2740, 1062] combine_rows_per_src=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83994 payload_rows_remote=73466 dup_rows=14310 dup_rows_remote=12036 dedup_ratio=0.145569 router_tokens_per_expert=[5, 593, 1623, 0, 402, 26, 13, 362, 117, 0, 13, 1470, 1, 10, 3040, 1476, 5589, 7832, 17, 4, 4951, 1992, 2180, 0, 0, 27, 366, 0, 0, 1, 55, 0, 14, 0, 0, 1, 0, 1, 3021, 257, 6623, 7, 274, 1, 0, 17, 0, 1839, 7, 9, 342, 3069, 3, 28, 0, 763, 2, 40, 0, 11, 1, 517, 1, 139] payload_rows_per_dst=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] dup_rows_per_dst=[466, 2580, 12, 876, 3746, 3398, 2274, 958] combine_rows_per_src=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82078 payload_rows_remote=72584 dup_rows=16226 dup_rows_remote=14904 dedup_ratio=0.165059 router_tokens_per_expert=[11, 680, 1943, 0, 560, 26, 45, 432, 82, 0, 8, 1454, 1, 2, 2187, 1669, 5636, 7768, 5, 6, 4431, 2103, 2090, 0, 0, 46, 336, 0, 0, 1, 39, 0, 6, 0, 0, 1, 0, 3, 3084, 322, 6955, 6, 229, 2, 1, 5, 0, 1583, 16, 15, 253, 3256, 2, 65, 0, 926, 2, 27, 0, 23, 0, 652, 0, 157] payload_rows_per_dst=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] dup_rows_per_dst=[6270, 3942, 1322, 672, 400, 92, 2338, 1190] combine_rows_per_src=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85082 payload_rows_remote=72288 dup_rows=13222 dup_rows_remote=12056 dedup_ratio=0.134501 router_tokens_per_expert=[1, 683, 1748, 0, 489, 26, 20, 455, 44, 0, 4, 1201, 0, 4, 2305, 1244, 6193, 7890, 12, 5, 4837, 2171, 2096, 0, 0, 23, 328, 0, 0, 1, 21, 0, 8, 0, 2, 1, 0, 2, 3096, 182, 6831, 11, 258, 0, 0, 13, 0, 1734, 12, 5, 245, 3236, 6, 34, 0, 888, 0, 31, 0, 13, 0, 619, 0, 124] payload_rows_per_dst=[6266, 8184, 4358, 15784, 12508, 14458, 10730, 12794] dup_rows_per_dst=[578, 1420, 18, 742, 3748, 3236, 2314, 1166] combine_rows_per_src=[6266, 8184, 4358, 15784, 12508, 14458, 10730, 12794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82560 payload_rows_remote=67272 dup_rows=15744 dup_rows_remote=10956 dedup_ratio=0.160156 router_tokens_per_expert=[12, 308, 1649, 7, 22, 487, 19, 394, 3, 602, 554, 316, 3, 0, 23, 55, 0, 1040, 0, 7140, 481, 124, 521, 925, 60, 0, 493, 4726, 106, 5673, 54, 0, 0, 5, 0, 7373, 1892, 11, 56, 572, 33, 3, 45, 1209, 347, 0, 2855, 65, 0, 57, 841, 37, 10, 391, 1463, 1259, 16, 652, 0, 17, 4014, 117, 7, 8] payload_rows_per_dst=[15288, 12566, 8922, 10090, 11854, 8272, 6590, 8978] dup_rows_per_dst=[4788, 1892, 2146, 788, 3078, 842, 1526, 684] combine_rows_per_src=[15288, 12566, 8922, 10090, 11854, 8272, 6590, 8978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79360 payload_rows_remote=63620 dup_rows=18944 dup_rows_remote=11772 dedup_ratio=0.192708 router_tokens_per_expert=[8, 311, 1190, 3, 24, 469, 23, 490, 4, 392, 577, 215, 2, 0, 25, 75, 0, 1272, 0, 7184, 523, 251, 416, 938, 59, 2, 442, 4680, 99, 5802, 70, 2, 2, 1, 0, 7548, 2219, 8, 73, 463, 18, 5, 29, 1202, 390, 0, 2703, 84, 0, 48, 1174, 52, 7, 333, 1124, 1170, 22, 619, 0, 34, 4159, 83, 21, 13] payload_rows_per_dst=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] dup_rows_per_dst=[274, 1400, 4416, 1522, 270, 742, 7172, 3148] combine_rows_per_src=[4762, 11226, 12602, 10744, 5262, 8120, 15740, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275546112 dispatch_meta_bytes=312912 combine_payload_bytes=275546112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275859024 ingress_bytes=310149120 link_bytes=310149120 token_bytes=551405136 weight_movement_bytes=34603008 total_nvlink_bytes=586008144 weight_to_token_ratio=0.062754 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167066170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283992064 dispatch_meta_bytes=326608 combine_payload_bytes=283992064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284318672 ingress_bytes=318595072 link_bytes=318595072 token_bytes=568310736 weight_movement_bytes=34603008 total_nvlink_bytes=602913744 weight_to_token_ratio=0.060887 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198847607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=69334 dup_rows=13086 dup_rows_remote=12318 dedup_ratio=0.133118 router_tokens_per_expert=[0, 654, 2125, 0, 438, 31, 45, 277, 122, 0, 12, 1312, 0, 1, 2373, 1133, 6031, 7939, 19, 3, 4360, 2280, 2566, 0, 1, 33, 321, 0, 0, 0, 32, 0, 10, 0, 0, 2, 0, 1, 2613, 315, 6810, 4, 212, 1, 0, 14, 1, 1710, 15, 9, 237, 3616, 6, 37, 0, 696, 6, 26, 1, 29, 0, 549, 0, 124] payload_rows_per_dst=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] dup_rows_per_dst=[1166, 1380, 20, 768, 3140, 2810, 2740, 1062] combine_rows_per_src=[7226, 8526, 4654, 15884, 11462, 13372, 11624, 12470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73420 payload_rows_remote=62318 dup_rows=24884 dup_rows_remote=21408 dedup_ratio=0.253133 router_tokens_per_expert=[0, 582, 1948, 0, 439, 23, 11, 314, 71, 0, 4, 1193, 0, 3, 2444, 1291, 5884, 7852, 22, 4, 5172, 2283, 2355, 0, 0, 29, 264, 1, 0, 0, 34, 0, 2, 0, 1, 2, 0, 0, 2991, 179, 7066, 4, 164, 1, 0, 3, 0, 1498, 4, 11, 267, 3449, 4, 15, 0, 646, 0, 16, 0, 6, 0, 490, 0, 110] payload_rows_per_dst=[15130, 11102, 16114, 656, 6304, 3266, 8710, 12138] dup_rows_per_dst=[5636, 3476, 13828, 0, 46, 74, 950, 874] combine_rows_per_src=[15130, 11102, 16114, 656, 6304, 3266, 8710, 12138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=300916736 dispatch_meta_bytes=342008 combine_payload_bytes=300916736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301258744 ingress_bytes=335519744 link_bytes=335519744 token_bytes=602175480 weight_movement_bytes=34603008 total_nvlink_bytes=636778488 weight_to_token_ratio=0.057463 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=297304064 dispatch_meta_bytes=349952 combine_payload_bytes=297304064 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=297654016 ingress_bytes=504922112 link_bytes=504922112 token_bytes=594958080 weight_movement_bytes=207618048 total_nvlink_bytes=802576128 weight_to_token_ratio=0.348962 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.899981258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=296091648 dispatch_meta_bytes=337376 combine_payload_bytes=296091648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296429024 ingress_bytes=330694656 link_bytes=330694656 token_bytes=592520672 weight_movement_bytes=34603008 total_nvlink_bytes=627123680 weight_to_token_ratio=0.058400 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244377367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85184 payload_rows_remote=73142 dup_rows=13120 dup_rows_remote=10040 dedup_ratio=0.133464 router_tokens_per_expert=[0, 535, 2088, 0, 409, 27, 27, 272, 63, 0, 2, 1344, 0, 2, 2704, 1280, 5806, 7970, 16, 6, 4922, 2178, 2378, 0, 0, 24, 268, 0, 0, 1, 26, 0, 3, 0, 0, 2, 0, 0, 2419, 215, 7159, 4, 207, 1, 0, 16, 0, 1801, 6, 7, 250, 3275, 3, 13, 0, 733, 1, 29, 0, 11, 0, 543, 0, 106] payload_rows_per_dst=[6118, 8918, 4386, 15942, 12042, 14902, 10862, 12014] dup_rows_per_dst=[598, 1872, 14, 636, 3080, 3474, 2468, 978] combine_rows_per_src=[6118, 8918, 4386, 15942, 12042, 14902, 10862, 12014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=71524 dup_rows=12480 dup_rows_remote=8226 dedup_ratio=0.126953 router_tokens_per_expert=[1951, 0, 7110, 11, 0, 0, 2, 4948, 0, 0, 728, 1996, 6570, 6228, 401, 14, 0, 0, 0, 227, 0, 3, 2335, 673, 0, 76, 145, 1935, 0, 0, 0, 0, 3, 1, 4940, 1, 0, 0, 0, 0, 3, 1838, 0, 30, 7, 0, 1066, 105, 16, 0, 5160, 533, 29, 31, 0, 0, 27, 3, 0, 0, 1, 0, 5, 0] payload_rows_per_dst=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] dup_rows_per_dst=[20, 454, 3666, 3590, 4, 4254, 428, 64] combine_rows_per_src=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=275546112 dispatch_meta_bytes=312912 combine_payload_bytes=275546112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275859024 ingress_bytes=310149120 link_bytes=310149120 token_bytes=551405136 weight_movement_bytes=34603008 total_nvlink_bytes=586008144 weight_to_token_ratio=0.062754 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167066170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260587520 dispatch_meta_bytes=301568 combine_payload_bytes=260587520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260889088 ingress_bytes=295190528 link_bytes=295190528 token_bytes=521476608 weight_movement_bytes=34603008 total_nvlink_bytes=556079616 weight_to_token_ratio=0.066356 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110778192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84984 payload_rows_remote=70776 dup_rows=13320 dup_rows_remote=10394 dedup_ratio=0.135498 router_tokens_per_expert=[7, 707, 2011, 0, 444, 24, 33, 417, 124, 0, 10, 1361, 0, 3, 2478, 1577, 5939, 7814, 12, 4, 4734, 2064, 2075, 0, 0, 42, 363, 0, 0, 0, 35, 0, 11, 0, 0, 1, 0, 1, 2773, 178, 6698, 6, 268, 0, 0, 10, 0, 1585, 14, 14, 248, 3203, 8, 26, 0, 1076, 1, 18, 0, 8, 0, 615, 0, 112] payload_rows_per_dst=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] dup_rows_per_dst=[714, 2124, 10, 856, 3138, 2926, 2450, 1102] combine_rows_per_src=[6572, 8982, 4150, 15652, 12258, 14208, 10878, 12284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82078 payload_rows_remote=72584 dup_rows=16226 dup_rows_remote=14904 dedup_ratio=0.165059 router_tokens_per_expert=[11, 680, 1943, 0, 560, 26, 45, 432, 82, 0, 8, 1454, 1, 2, 2187, 1669, 5636, 7768, 5, 6, 4431, 2103, 2090, 0, 0, 46, 336, 0, 0, 1, 39, 0, 6, 0, 0, 1, 0, 3, 3084, 322, 6955, 6, 229, 2, 1, 5, 0, 1583, 16, 15, 253, 3256, 2, 65, 0, 926, 2, 27, 0, 23, 0, 652, 0, 157] payload_rows_per_dst=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] dup_rows_per_dst=[6270, 3942, 1322, 672, 400, 92, 2338, 1190] combine_rows_per_src=[15034, 11062, 9494, 13294, 6922, 3560, 10908, 11804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85184 payload_rows_remote=73142 dup_rows=13120 dup_rows_remote=10040 dedup_ratio=0.133464 router_tokens_per_expert=[0, 535, 2088, 0, 409, 27, 27, 272, 63, 0, 2, 1344, 0, 2, 2704, 1280, 5806, 7970, 16, 6, 4922, 2178, 2378, 0, 0, 24, 268, 0, 0, 1, 26, 0, 3, 0, 0, 2, 0, 0, 2419, 215, 7159, 4, 207, 1, 0, 16, 0, 1801, 6, 7, 250, 3275, 3, 13, 0, 733, 1, 29, 0, 11, 0, 543, 0, 106] payload_rows_per_dst=[6118, 8918, 4386, 15942, 12042, 14902, 10862, 12014] dup_rows_per_dst=[598, 1872, 14, 636, 3080, 3474, 2468, 978] combine_rows_per_src=[6118, 8918, 4386, 15942, 12042, 14902, 10862, 12014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283992064 dispatch_meta_bytes=326608 combine_payload_bytes=283992064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284318672 ingress_bytes=318595072 link_bytes=318595072 token_bytes=568310736 weight_movement_bytes=34603008 total_nvlink_bytes=602913744 weight_to_token_ratio=0.060887 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198847607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=255254528 dispatch_meta_bytes=334904 combine_payload_bytes=255254528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255589432 ingress_bytes=289857536 link_bytes=289857536 token_bytes=510843960 weight_movement_bytes=34603008 total_nvlink_bytes=545446968 weight_to_token_ratio=0.067737 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090710573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289898496 dispatch_meta_bytes=324680 combine_payload_bytes=289898496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=290223176 ingress_bytes=359104512 link_bytes=359104512 token_bytes=580121672 weight_movement_bytes=69206016 total_nvlink_bytes=649327688 weight_to_token_ratio=0.119296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.351281368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299589632 dispatch_meta_bytes=332728 combine_payload_bytes=299589632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299922360 ingress_bytes=334192640 link_bytes=334192640 token_bytes=599511992 weight_movement_bytes=34603008 total_nvlink_bytes=634115000 weight_to_token_ratio=0.057719 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257540000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=319000 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293281304 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586243608 weight_movement_bytes=34603008 total_nvlink_bytes=620846616 weight_to_token_ratio=0.059025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=297304064 dispatch_meta_bytes=349952 combine_payload_bytes=297304064 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=297654016 ingress_bytes=504922112 link_bytes=504922112 token_bytes=594958080 weight_movement_bytes=207618048 total_nvlink_bytes=802576128 weight_to_token_ratio=0.348962 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.899981258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86206 payload_rows_remote=79846 dup_rows=12098 dup_rows_remote=11394 dedup_ratio=0.123067 router_tokens_per_expert=[2122, 0, 6830, 10, 0, 0, 0, 4842, 0, 0, 758, 2151, 6548, 5941, 440, 13, 0, 0, 1, 264, 0, 14, 2274, 979, 0, 96, 161, 1803, 0, 0, 1, 0, 6, 33, 4797, 6, 0, 0, 0, 0, 1, 1961, 0, 67, 13, 0, 1140, 118, 21, 0, 5065, 464, 111, 36, 0, 0, 59, 4, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] dup_rows_per_dst=[2536, 4716, 704, 3354, 22, 302, 356, 108] combine_rows_per_src=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80074 payload_rows_remote=66284 dup_rows=18230 dup_rows_remote=18192 dedup_ratio=0.185445 router_tokens_per_expert=[2057, 0, 7164, 6, 0, 0, 0, 4699, 0, 0, 812, 2359, 6880, 6035, 383, 10, 0, 0, 0, 197, 0, 11, 2496, 787, 0, 98, 154, 1743, 0, 0, 2, 0, 0, 7, 4056, 3, 1, 0, 0, 0, 3, 1983, 0, 54, 5, 1, 1179, 124, 21, 0, 5263, 492, 8, 25, 0, 0, 18, 8, 0, 0, 5, 0, 2, 1] payload_rows_per_dst=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] dup_rows_per_dst=[12, 508, 3816, 20, 7544, 3318, 2974, 38] combine_rows_per_src=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67268 payload_rows_remote=52278 dup_rows=31036 dup_rows_remote=25078 dedup_ratio=0.315715 router_tokens_per_expert=[6, 636, 2072, 0, 485, 37, 19, 341, 131, 0, 16, 1334, 0, 2, 2404, 1459, 5598, 7857, 31, 6, 4444, 2185, 2222, 0, 0, 24, 361, 0, 0, 0, 39, 4, 8, 0, 4, 3, 0, 6, 3010, 352, 6878, 9, 265, 2, 0, 5, 2, 1919, 14, 9, 274, 3228, 4, 33, 0, 703, 1, 54, 5, 17, 0, 494, 1, 139] payload_rows_per_dst=[14990, 11242, 16274, 856, 6432, 4264, 7880, 5330] dup_rows_per_dst=[5958, 3820, 19636, 0, 334, 140, 650, 498] combine_rows_per_src=[14990, 11242, 16274, 856, 6432, 4264, 7880, 5330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83994 payload_rows_remote=73466 dup_rows=14310 dup_rows_remote=12036 dedup_ratio=0.145569 router_tokens_per_expert=[5, 593, 1623, 0, 402, 26, 13, 362, 117, 0, 13, 1470, 1, 10, 3040, 1476, 5589, 7832, 17, 4, 4951, 1992, 2180, 0, 0, 27, 366, 0, 0, 1, 55, 0, 14, 0, 0, 1, 0, 1, 3021, 257, 6623, 7, 274, 1, 0, 17, 0, 1839, 7, 9, 342, 3069, 3, 28, 0, 763, 2, 40, 0, 11, 1, 517, 1, 139] payload_rows_per_dst=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] dup_rows_per_dst=[466, 2580, 12, 876, 3746, 3398, 2274, 958] combine_rows_per_src=[5582, 9674, 4014, 15686, 12744, 14124, 10528, 11642] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86766 payload_rows_remote=71698 dup_rows=11538 dup_rows_remote=8710 dedup_ratio=0.117371 router_tokens_per_expert=[1928, 1, 7244, 7, 1, 0, 0, 5156, 0, 0, 868, 2161, 6232, 6095, 216, 8, 0, 0, 2, 167, 0, 2, 2444, 675, 0, 70, 128, 1500, 0, 0, 6, 0, 3, 3, 5285, 1, 0, 0, 0, 0, 7, 1570, 0, 42, 3, 0, 1214, 112, 28, 0, 5185, 665, 20, 24, 1, 0, 43, 1, 0, 0, 0, 0, 31, 3] payload_rows_per_dst=[9850, 12650, 7794, 15068, 10580, 7022, 11238, 12564] dup_rows_per_dst=[1822, 3988, 1300, 2828, 4, 932, 608, 56] combine_rows_per_src=[9850, 12650, 7794, 15068, 10580, 7022, 11238, 12564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83520 payload_rows_remote=72482 dup_rows=14784 dup_rows_remote=14342 dedup_ratio=0.150391 router_tokens_per_expert=[1953, 0, 7047, 11, 0, 0, 1, 4711, 0, 2, 768, 1959, 6759, 5943, 419, 12, 0, 0, 0, 362, 0, 0, 2636, 765, 0, 75, 142, 1645, 0, 1, 0, 0, 7, 4, 4880, 2, 0, 1, 0, 0, 1, 1794, 0, 65, 29, 0, 1289, 96, 17, 0, 5102, 579, 13, 29, 0, 0, 26, 3, 1, 0, 3, 0, 0, 0] payload_rows_per_dst=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] dup_rows_per_dst=[14, 410, 4052, 1582, 3938, 4288, 442, 58] combine_rows_per_src=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80074 payload_rows_remote=66284 dup_rows=18230 dup_rows_remote=18192 dedup_ratio=0.185445 router_tokens_per_expert=[2057, 0, 7164, 6, 0, 0, 0, 4699, 0, 0, 812, 2359, 6880, 6035, 383, 10, 0, 0, 0, 197, 0, 11, 2496, 787, 0, 98, 154, 1743, 0, 0, 2, 0, 0, 7, 4056, 3, 1, 0, 0, 0, 3, 1983, 0, 54, 5, 1, 1179, 124, 21, 0, 5263, 492, 8, 25, 0, 0, 18, 8, 0, 0, 5, 0, 2, 1] payload_rows_per_dst=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] dup_rows_per_dst=[12, 508, 3816, 20, 7544, 3318, 2974, 38] combine_rows_per_src=[4114, 6620, 12564, 3974, 14918, 11604, 12490, 13790] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271499264 dispatch_meta_bytes=337904 combine_payload_bytes=271499264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271837168 ingress_bytes=306102272 link_bytes=306102272 token_bytes=543336432 weight_movement_bytes=34603008 total_nvlink_bytes=577939440 weight_to_token_ratio=0.063686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151838206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327049216 dispatch_meta_bytes=364960 combine_payload_bytes=327049216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327414176 ingress_bytes=361652224 link_bytes=361652224 token_bytes=654463392 weight_movement_bytes=34603008 total_nvlink_bytes=689066400 weight_to_token_ratio=0.052872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360868204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83452 payload_rows_remote=70010 dup_rows=14852 dup_rows_remote=11212 dedup_ratio=0.151082 router_tokens_per_expert=[1932, 0, 7136, 6, 0, 0, 0, 4997, 0, 0, 699, 1834, 6400, 5669, 334, 5, 0, 0, 0, 124, 0, 2, 2887, 567, 0, 58, 148, 1614, 0, 0, 1, 0, 2, 0, 5140, 1, 0, 0, 0, 0, 1, 1986, 0, 40, 5, 0, 1091, 119, 29, 0, 5658, 595, 30, 12, 0, 0, 27, 1, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] dup_rows_per_dst=[8746, 3640, 556, 1002, 2, 300, 552, 54] combine_rows_per_src=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=214130688 dispatch_meta_bytes=309424 combine_payload_bytes=214130688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214440112 ingress_bytes=248733696 link_bytes=248733696 token_bytes=428570800 weight_movement_bytes=34603008 total_nvlink_bytes=463173808 weight_to_token_ratio=0.080740 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.359648733e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293675008 dispatch_meta_bytes=321632 combine_payload_bytes=293675008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293996640 ingress_bytes=328278016 link_bytes=328278016 token_bytes=587671648 weight_movement_bytes=34603008 total_nvlink_bytes=622274656 weight_to_token_ratio=0.058882 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235283746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83452 payload_rows_remote=70010 dup_rows=14852 dup_rows_remote=11212 dedup_ratio=0.151082 router_tokens_per_expert=[1932, 0, 7136, 6, 0, 0, 0, 4997, 0, 0, 699, 1834, 6400, 5669, 334, 5, 0, 0, 0, 124, 0, 2, 2887, 567, 0, 58, 148, 1614, 0, 0, 1, 0, 2, 0, 5140, 1, 0, 0, 0, 0, 1, 1986, 0, 40, 5, 0, 1091, 119, 29, 0, 5658, 595, 30, 12, 0, 0, 27, 1, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] dup_rows_per_dst=[8746, 3640, 556, 1002, 2, 300, 552, 54] combine_rows_per_src=[14648, 13442, 6604, 7388, 10284, 6184, 12096, 12806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=296886272 dispatch_meta_bytes=347296 combine_payload_bytes=296886272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297233568 ingress_bytes=331489280 link_bytes=331489280 token_bytes=594119840 weight_movement_bytes=34603008 total_nvlink_bytes=628722848 weight_to_token_ratio=0.058242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247367474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271499264 dispatch_meta_bytes=337904 combine_payload_bytes=271499264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271837168 ingress_bytes=306102272 link_bytes=306102272 token_bytes=543336432 weight_movement_bytes=34603008 total_nvlink_bytes=577939440 weight_to_token_ratio=0.063686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151838206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=300916736 dispatch_meta_bytes=342008 combine_payload_bytes=300916736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301258744 ingress_bytes=335519744 link_bytes=335519744 token_bytes=602175480 weight_movement_bytes=34603008 total_nvlink_bytes=636778488 weight_to_token_ratio=0.057463 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85824 payload_rows_remote=71524 dup_rows=12480 dup_rows_remote=8226 dedup_ratio=0.126953 router_tokens_per_expert=[1951, 0, 7110, 11, 0, 0, 2, 4948, 0, 0, 728, 1996, 6570, 6228, 401, 14, 0, 0, 0, 227, 0, 3, 2335, 673, 0, 76, 145, 1935, 0, 0, 0, 0, 3, 1, 4940, 1, 0, 0, 0, 0, 3, 1838, 0, 30, 7, 0, 1066, 105, 16, 0, 5160, 533, 29, 31, 0, 0, 27, 3, 0, 0, 1, 0, 5, 0] payload_rows_per_dst=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] dup_rows_per_dst=[20, 454, 3666, 3590, 4, 4254, 428, 64] combine_rows_per_src=[3908, 5824, 12706, 14942, 9886, 14300, 11110, 13148] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85740 payload_rows_remote=75800 dup_rows=12564 dup_rows_remote=12558 dedup_ratio=0.127808 router_tokens_per_expert=[1751, 0, 7142, 8, 0, 0, 0, 5415, 0, 0, 649, 2029, 6393, 6136, 364, 6, 0, 0, 0, 127, 0, 0, 3139, 622, 0, 62, 118, 1234, 0, 1, 5, 0, 5, 0, 4965, 3, 0, 0, 0, 0, 11, 2122, 0, 48, 3, 0, 959, 113, 12, 1, 5169, 501, 1, 7, 0, 0, 24, 1, 0, 0, 0, 0, 6, 0] payload_rows_per_dst=[3502, 5680, 13776, 14860, 9940, 14192, 10984, 12806] dup_rows_per_dst=[16, 416, 4830, 2264, 6, 4592, 398, 42] combine_rows_per_src=[3502, 5680, 13776, 14860, 9940, 14192, 10984, 12806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86806 payload_rows_remote=70422 dup_rows=11498 dup_rows_remote=5708 dedup_ratio=0.116964 router_tokens_per_expert=[892, 383, 0, 60, 0, 0, 0, 0, 0, 57, 7831, 0, 0, 2510, 7376, 8, 0, 0, 0, 0, 1084, 42, 0, 5, 18, 222, 11, 548, 0, 8191, 2, 49, 3467, 0, 1137, 0, 5350, 0, 38, 0, 411, 2336, 0, 0, 2, 147, 0, 0, 214, 0, 1, 13, 41, 0, 5680, 0, 2, 659, 0, 0, 0, 0, 336, 29] payload_rows_per_dst=[12296, 5128, 15102, 1686, 8770, 16384, 11584, 15856] dup_rows_per_dst=[1074, 22, 1912, 14, 514, 5790, 314, 1858] combine_rows_per_src=[12296, 5128, 15102, 1686, 8770, 16384, 11584, 15856] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293675008 dispatch_meta_bytes=321632 combine_payload_bytes=293675008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293996640 ingress_bytes=328278016 link_bytes=328278016 token_bytes=587671648 weight_movement_bytes=34603008 total_nvlink_bytes=622274656 weight_to_token_ratio=0.058882 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235283746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86206 payload_rows_remote=79846 dup_rows=12098 dup_rows_remote=11394 dedup_ratio=0.123067 router_tokens_per_expert=[2122, 0, 6830, 10, 0, 0, 0, 4842, 0, 0, 758, 2151, 6548, 5941, 440, 13, 0, 0, 1, 264, 0, 14, 2274, 979, 0, 96, 161, 1803, 0, 0, 1, 0, 6, 33, 4797, 6, 0, 0, 0, 0, 1, 1961, 0, 67, 13, 0, 1140, 118, 21, 0, 5065, 464, 111, 36, 0, 0, 59, 4, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] dup_rows_per_dst=[2536, 4716, 704, 3354, 22, 302, 356, 108] combine_rows_per_src=[11412, 13890, 6360, 14428, 9662, 6298, 11038, 13118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=286760960 dispatch_meta_bytes=324888 combine_payload_bytes=286760960 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=287085848 ingress_bytes=425172992 link_bytes=425172992 token_bytes=573846808 weight_movement_bytes=138412032 total_nvlink_bytes=712258840 weight_to_token_ratio=0.241200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599891740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=319000 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293281304 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586243608 weight_movement_bytes=34603008 total_nvlink_bytes=620846616 weight_to_token_ratio=0.059025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=310476800 dispatch_meta_bytes=353432 combine_payload_bytes=310476800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310830232 ingress_bytes=345079808 link_bytes=345079808 token_bytes=621307032 weight_movement_bytes=34603008 total_nvlink_bytes=655910040 weight_to_token_ratio=0.055694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298507536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288448512 dispatch_meta_bytes=304520 combine_payload_bytes=288448512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288753032 ingress_bytes=323051520 link_bytes=323051520 token_bytes=577201544 weight_movement_bytes=34603008 total_nvlink_bytes=611804552 weight_to_token_ratio=0.059950 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215616862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327049216 dispatch_meta_bytes=364960 combine_payload_bytes=327049216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327414176 ingress_bytes=361652224 link_bytes=361652224 token_bytes=654463392 weight_movement_bytes=34603008 total_nvlink_bytes=689066400 weight_to_token_ratio=0.052872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360868204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82980 payload_rows_remote=67276 dup_rows=15324 dup_rows_remote=13678 dedup_ratio=0.155884 router_tokens_per_expert=[851, 399, 0, 79, 1, 0, 0, 0, 4, 60, 7755, 0, 1, 2478, 7573, 27, 0, 0, 0, 0, 1222, 30, 0, 19, 13, 230, 1, 425, 0, 8188, 0, 16, 3190, 0, 981, 0, 5613, 0, 29, 0, 384, 2319, 0, 3, 2, 183, 0, 0, 218, 0, 1, 14, 51, 0, 5872, 0, 4, 526, 0, 0, 0, 0, 356, 34] payload_rows_per_dst=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] dup_rows_per_dst=[40, 24, 1142, 638, 5110, 5774, 950, 1646] combine_rows_per_src=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86336 payload_rows_remote=73536 dup_rows=11968 dup_rows_remote=10442 dedup_ratio=0.121745 router_tokens_per_expert=[867, 460, 0, 48, 0, 0, 0, 0, 4, 97, 7726, 2, 0, 2746, 7361, 14, 0, 0, 0, 0, 1046, 37, 0, 4, 58, 225, 5, 607, 0, 8187, 1, 42, 3083, 1, 1136, 0, 5164, 0, 30, 0, 397, 2607, 0, 1, 3, 198, 0, 0, 193, 0, 3, 7, 73, 9, 5667, 0, 5, 613, 0, 0, 1, 0, 392, 32] payload_rows_per_dst=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] dup_rows_per_dst=[1016, 882, 1526, 1872, 5862, 60, 298, 452] combine_rows_per_src=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73558 payload_rows_remote=57338 dup_rows=24746 dup_rows_remote=7178 dedup_ratio=0.251729 router_tokens_per_expert=[1696, 0, 7128, 13, 0, 0, 0, 4914, 0, 1, 774, 2338, 6903, 5873, 324, 18, 0, 0, 0, 101, 2, 11, 2453, 773, 0, 67, 120, 1447, 0, 0, 0, 0, 5, 22, 4547, 4, 1, 0, 0, 2, 11, 1671, 0, 78, 6, 0, 1403, 117, 31, 0, 5650, 565, 9, 21, 0, 0, 40, 0, 1, 0, 7, 0, 5, 0] payload_rows_per_dst=[16220, 14598, 6110, 3250, 9144, 6104, 12038, 6094] dup_rows_per_dst=[17568, 5558, 570, 18, 18, 468, 514, 32] combine_rows_per_src=[16220, 14598, 6110, 3250, 9144, 6104, 12038, 6094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83520 payload_rows_remote=72482 dup_rows=14784 dup_rows_remote=14342 dedup_ratio=0.150391 router_tokens_per_expert=[1953, 0, 7047, 11, 0, 0, 1, 4711, 0, 2, 768, 1959, 6759, 5943, 419, 12, 0, 0, 0, 362, 0, 0, 2636, 765, 0, 75, 142, 1645, 0, 1, 0, 0, 7, 4, 4880, 2, 0, 1, 0, 0, 1, 1794, 0, 65, 29, 0, 1289, 96, 17, 0, 5102, 579, 13, 29, 0, 0, 26, 3, 1, 0, 3, 0, 0, 0] payload_rows_per_dst=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] dup_rows_per_dst=[14, 410, 4052, 1582, 3938, 4288, 442, 58] combine_rows_per_src=[3916, 5910, 12896, 9736, 12352, 14146, 11038, 13526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89780 payload_rows_remote=73404 dup_rows=8524 dup_rows_remote=6088 dedup_ratio=0.086711 router_tokens_per_expert=[792, 407, 0, 86, 0, 0, 1, 3, 0, 60, 7736, 0, 2, 2612, 7473, 17, 0, 0, 2, 0, 1174, 53, 0, 4, 27, 237, 10, 888, 3, 8180, 0, 61, 3037, 0, 1297, 0, 5503, 1, 20, 0, 358, 1574, 2, 1, 7, 161, 0, 0, 247, 1, 0, 16, 73, 14, 5971, 0, 4, 669, 0, 1, 0, 1, 349, 17] payload_rows_per_dst=[12440, 5366, 15274, 16376, 8218, 4162, 12208, 15736] dup_rows_per_dst=[1144, 16, 2138, 2436, 492, 44, 436, 1818] combine_rows_per_src=[12440, 5366, 15274, 16376, 8218, 4162, 12208, 15736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86180 payload_rows_remote=74262 dup_rows=12124 dup_rows_remote=11708 dedup_ratio=0.123332 router_tokens_per_expert=[974, 478, 0, 124, 0, 0, 0, 0, 2, 73, 7705, 0, 0, 2643, 7492, 39, 0, 0, 0, 0, 1192, 38, 0, 12, 21, 344, 9, 555, 0, 8185, 1, 58, 2738, 0, 1215, 0, 4961, 0, 36, 0, 424, 2335, 0, 5, 3, 197, 0, 0, 273, 0, 2, 6, 71, 5, 5810, 0, 5, 606, 0, 0, 0, 0, 492, 23] payload_rows_per_dst=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] dup_rows_per_dst=[1184, 40, 2168, 34, 428, 5914, 416, 1940] combine_rows_per_src=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82980 payload_rows_remote=67276 dup_rows=15324 dup_rows_remote=13678 dedup_ratio=0.155884 router_tokens_per_expert=[851, 399, 0, 79, 1, 0, 0, 0, 4, 60, 7755, 0, 1, 2478, 7573, 27, 0, 0, 0, 0, 1222, 30, 0, 19, 13, 230, 1, 425, 0, 8188, 0, 16, 3190, 0, 981, 0, 5613, 0, 29, 0, 384, 2319, 0, 3, 2, 183, 0, 0, 218, 0, 1, 14, 51, 0, 5872, 0, 4, 526, 0, 0, 0, 0, 356, 34] payload_rows_per_dst=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] dup_rows_per_dst=[40, 24, 1142, 638, 5110, 5774, 950, 1646] combine_rows_per_src=[2778, 4012, 8380, 8898, 14358, 16384, 12466, 15704] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275562496 dispatch_meta_bytes=323816 combine_payload_bytes=275562496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275886312 ingress_bytes=310165504 link_bytes=310165504 token_bytes=551448808 weight_movement_bytes=34603008 total_nvlink_bytes=586051816 weight_to_token_ratio=0.062749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167127821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301203456 dispatch_meta_bytes=335912 combine_payload_bytes=301203456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301539368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=602742824 weight_movement_bytes=34603008 total_nvlink_bytes=637345832 weight_to_token_ratio=0.057409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77760 payload_rows_remote=61700 dup_rows=20544 dup_rows_remote=8270 dedup_ratio=0.208984 router_tokens_per_expert=[955, 411, 0, 58, 0, 0, 0, 0, 0, 45, 7871, 0, 0, 2258, 7501, 15, 0, 0, 0, 0, 1083, 26, 0, 0, 25, 210, 3, 843, 0, 8192, 0, 11, 3807, 0, 1036, 0, 5713, 0, 15, 0, 372, 1578, 0, 0, 8, 196, 0, 0, 145, 0, 0, 15, 62, 0, 5726, 0, 0, 663, 0, 0, 0, 0, 277, 32] payload_rows_per_dst=[12934, 16060, 2214, 16384, 12378, 4266, 11610, 1914] dup_rows_per_dst=[1340, 12274, 4, 2184, 4384, 42, 286, 30] combine_rows_per_src=[12934, 16060, 2214, 16384, 12378, 4266, 11610, 1914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86806 payload_rows_remote=70422 dup_rows=11498 dup_rows_remote=5708 dedup_ratio=0.116964 router_tokens_per_expert=[892, 383, 0, 60, 0, 0, 0, 0, 0, 57, 7831, 0, 0, 2510, 7376, 8, 0, 0, 0, 0, 1084, 42, 0, 5, 18, 222, 11, 548, 0, 8191, 2, 49, 3467, 0, 1137, 0, 5350, 0, 38, 0, 411, 2336, 0, 0, 2, 147, 0, 0, 214, 0, 1, 13, 41, 0, 5680, 0, 2, 659, 0, 0, 0, 0, 336, 29] payload_rows_per_dst=[12296, 5128, 15102, 1686, 8770, 16384, 11584, 15856] dup_rows_per_dst=[1074, 22, 1912, 14, 514, 5790, 314, 1858] combine_rows_per_src=[12296, 5128, 15102, 1686, 8770, 16384, 11584, 15856] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=234856448 dispatch_meta_bytes=258064 combine_payload_bytes=234856448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=235114512 ingress_bytes=373268480 link_bytes=373268480 token_bytes=469970960 weight_movement_bytes=138412032 total_nvlink_bytes=608382992 weight_to_token_ratio=0.294512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404579240e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=296886272 dispatch_meta_bytes=347296 combine_payload_bytes=296886272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297233568 ingress_bytes=331489280 link_bytes=331489280 token_bytes=594119840 weight_movement_bytes=34603008 total_nvlink_bytes=628722848 weight_to_token_ratio=0.058242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247367474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300662784 dispatch_meta_bytes=317968 combine_payload_bytes=300662784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=300980752 ingress_bytes=369868800 link_bytes=369868800 token_bytes=601643536 weight_movement_bytes=69206016 total_nvlink_bytes=670849552 weight_to_token_ratio=0.115028 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=304177152 dispatch_meta_bytes=343880 combine_payload_bytes=304177152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521032 ingress_bytes=338780160 link_bytes=338780160 token_bytes=608698184 weight_movement_bytes=34603008 total_nvlink_bytes=643301192 weight_to_token_ratio=0.056848 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274802468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77760 payload_rows_remote=61700 dup_rows=20544 dup_rows_remote=8270 dedup_ratio=0.208984 router_tokens_per_expert=[955, 411, 0, 58, 0, 0, 0, 0, 0, 45, 7871, 0, 0, 2258, 7501, 15, 0, 0, 0, 0, 1083, 26, 0, 0, 25, 210, 3, 843, 0, 8192, 0, 11, 3807, 0, 1036, 0, 5713, 0, 15, 0, 372, 1578, 0, 0, 8, 196, 0, 0, 145, 0, 0, 15, 62, 0, 5726, 0, 0, 663, 0, 0, 0, 0, 277, 32] payload_rows_per_dst=[12934, 16060, 2214, 16384, 12378, 4266, 11610, 1914] dup_rows_per_dst=[1340, 12274, 4, 2184, 4384, 42, 286, 30] combine_rows_per_src=[12934, 16060, 2214, 16384, 12378, 4266, 11610, 1914] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89336 payload_rows_remote=80844 dup_rows=8968 dup_rows_remote=8512 dedup_ratio=0.091227 router_tokens_per_expert=[840, 363, 0, 59, 0, 0, 0, 1, 1, 59, 7804, 0, 0, 2714, 7458, 38, 0, 1, 0, 0, 1320, 20, 0, 2, 17, 241, 4, 740, 2, 8191, 5, 15, 3552, 1, 915, 0, 5788, 0, 6, 0, 382, 1572, 0, 0, 4, 117, 0, 0, 167, 0, 0, 10, 76, 3, 5732, 0, 0, 641, 0, 0, 0, 0, 265, 26] payload_rows_per_dst=[12906, 5604, 15310, 9786, 8492, 9798, 11684, 15756] dup_rows_per_dst=[1196, 20, 2292, 1160, 456, 1836, 292, 1716] combine_rows_per_src=[12906, 5604, 15310, 9786, 8492, 9798, 11684, 15756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275562496 dispatch_meta_bytes=323816 combine_payload_bytes=275562496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275886312 ingress_bytes=310165504 link_bytes=310165504 token_bytes=551448808 weight_movement_bytes=34603008 total_nvlink_bytes=586051816 weight_to_token_ratio=0.062749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167127821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81382 payload_rows_remote=68742 dup_rows=16922 dup_rows_remote=14402 dedup_ratio=0.172139 router_tokens_per_expert=[3241, 709, 0, 182, 4, 0, 382, 0, 0, 0, 0, 2768, 0, 0, 3, 135, 0, 47, 0, 0, 351, 98, 0, 0, 0, 0, 794, 0, 0, 1572, 3603, 62, 0, 822, 0, 0, 0, 0, 0, 0, 0, 1150, 9, 0, 7721, 0, 0, 4880, 1, 0, 0, 0, 2742, 6, 736, 0, 62, 1304, 0, 102, 0, 4, 7949, 7713] payload_rows_per_dst=[8138, 7564, 12198, 10608, 8464, 12640, 12888, 8882] dup_rows_per_dst=[898, 1062, 818, 2138, 646, 2520, 3842, 4998] combine_rows_per_src=[8138, 7564, 12198, 10608, 8464, 12640, 12888, 8882] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86336 payload_rows_remote=73536 dup_rows=11968 dup_rows_remote=10442 dedup_ratio=0.121745 router_tokens_per_expert=[867, 460, 0, 48, 0, 0, 0, 0, 4, 97, 7726, 2, 0, 2746, 7361, 14, 0, 0, 0, 0, 1046, 37, 0, 4, 58, 225, 5, 607, 0, 8187, 1, 42, 3083, 1, 1136, 0, 5164, 0, 30, 0, 397, 2607, 0, 1, 3, 198, 0, 0, 193, 0, 3, 7, 73, 9, 5667, 0, 5, 613, 0, 0, 1, 0, 392, 32] payload_rows_per_dst=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] dup_rows_per_dst=[1016, 882, 1526, 1872, 5862, 60, 298, 452] combine_rows_per_src=[12062, 7414, 12800, 16378, 13668, 6352, 11606, 6056] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=252723200 dispatch_meta_bytes=279880 combine_payload_bytes=252723200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=253003080 ingress_bytes=425738240 link_bytes=425738240 token_bytes=505726280 weight_movement_bytes=173015040 total_nvlink_bytes=678741320 weight_to_token_ratio=0.342112 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.602018722e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288448512 dispatch_meta_bytes=304520 combine_payload_bytes=288448512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288753032 ingress_bytes=323051520 link_bytes=323051520 token_bytes=577201544 weight_movement_bytes=34603008 total_nvlink_bytes=611804552 weight_to_token_ratio=0.059950 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215616862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=331137024 dispatch_meta_bytes=357424 combine_payload_bytes=331137024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331494448 ingress_bytes=400343040 link_bytes=400343040 token_bytes=662631472 weight_movement_bytes=69206016 total_nvlink_bytes=731837488 weight_to_token_ratio=0.104441 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506458629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281567232 dispatch_meta_bytes=332576 combine_payload_bytes=281567232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281899808 ingress_bytes=385376256 link_bytes=385376256 token_bytes=563467040 weight_movement_bytes=103809024 total_nvlink_bytes=667276064 weight_to_token_ratio=0.184233 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450139826e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301203456 dispatch_meta_bytes=335912 combine_payload_bytes=301203456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301539368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=602742824 weight_movement_bytes=34603008 total_nvlink_bytes=637345832 weight_to_token_ratio=0.057409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85746 payload_rows_remote=69904 dup_rows=12558 dup_rows_remote=11448 dedup_ratio=0.127747 router_tokens_per_expert=[3290, 561, 0, 197, 9, 0, 422, 0, 0, 0, 0, 3095, 0, 1, 1, 123, 0, 23, 2, 0, 428, 106, 0, 1, 0, 0, 801, 0, 0, 1758, 3280, 54, 0, 869, 0, 0, 0, 1, 0, 0, 0, 1254, 9, 0, 7487, 0, 0, 4778, 2, 0, 0, 1, 2860, 19, 687, 0, 118, 1488, 0, 101, 0, 0, 7916, 7410] payload_rows_per_dst=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] dup_rows_per_dst=[774, 5986, 1110, 1862, 1360, 1266, 136, 64] combine_rows_per_src=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72332 payload_rows_remote=55950 dup_rows=25972 dup_rows_remote=8056 dedup_ratio=0.264201 router_tokens_per_expert=[3112, 757, 0, 170, 6, 0, 447, 0, 0, 0, 0, 3228, 0, 0, 3, 96, 0, 23, 0, 0, 303, 99, 0, 2, 0, 0, 759, 0, 0, 1642, 3613, 71, 1, 979, 0, 1, 3, 0, 0, 0, 0, 1164, 10, 5, 7588, 0, 0, 4925, 1, 0, 0, 0, 2299, 3, 693, 0, 50, 1265, 0, 101, 0, 2, 8052, 7679] payload_rows_per_dst=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] dup_rows_per_dst=[866, 40, 4, 1838, 4, 2028, 3276, 17916] combine_rows_per_src=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=61702 dup_rows=24020 dup_rows_remote=22598 dedup_ratio=0.244344 router_tokens_per_expert=[986, 479, 0, 76, 0, 0, 0, 1, 1, 78, 7708, 0, 0, 2420, 7666, 111, 1, 0, 1, 0, 1307, 47, 0, 15, 50, 333, 14, 641, 0, 8166, 5, 49, 2770, 0, 1026, 0, 5460, 0, 16, 0, 400, 1902, 0, 0, 33, 222, 0, 0, 232, 0, 2, 15, 94, 3, 5772, 0, 0, 598, 0, 0, 0, 0, 395, 57] payload_rows_per_dst=[12582, 16376, 2732, 16356, 7306, 4996, 11852, 2084] dup_rows_per_dst=[1422, 19592, 10, 2160, 318, 118, 384, 16] combine_rows_per_src=[12582, 16376, 2732, 16356, 7306, 4996, 11852, 2084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86180 payload_rows_remote=74262 dup_rows=12124 dup_rows_remote=11708 dedup_ratio=0.123332 router_tokens_per_expert=[974, 478, 0, 124, 0, 0, 0, 0, 2, 73, 7705, 0, 0, 2643, 7492, 39, 0, 0, 0, 0, 1192, 38, 0, 12, 21, 344, 9, 555, 0, 8185, 1, 58, 2738, 0, 1215, 0, 4961, 0, 36, 0, 424, 2335, 0, 5, 3, 197, 0, 0, 273, 0, 2, 6, 71, 5, 5810, 0, 5, 606, 0, 0, 0, 0, 492, 23] payload_rows_per_dst=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] dup_rows_per_dst=[1184, 40, 2168, 34, 428, 5914, 416, 1940] combine_rows_per_src=[11890, 5474, 15300, 1942, 7550, 16384, 11918, 15722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85206 payload_rows_remote=74896 dup_rows=13098 dup_rows_remote=11204 dedup_ratio=0.133240 router_tokens_per_expert=[2835, 662, 0, 205, 20, 0, 463, 0, 0, 0, 0, 2897, 0, 1, 4, 125, 1, 33, 0, 0, 275, 119, 1, 0, 0, 0, 823, 0, 2, 1695, 3475, 107, 0, 757, 1, 0, 0, 0, 0, 0, 0, 1235, 20, 5, 7712, 0, 0, 4855, 0, 1, 0, 0, 3183, 13, 531, 0, 37, 1355, 0, 125, 0, 15, 7907, 7657] payload_rows_per_dst=[7532, 15756, 15826, 10310, 15666, 7126, 10054, 2936] dup_rows_per_dst=[838, 5612, 846, 1894, 1274, 714, 1792, 128] combine_rows_per_src=[7532, 15756, 15826, 10310, 15666, 7126, 10054, 2936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71522 payload_rows_remote=58956 dup_rows=26782 dup_rows_remote=22950 dedup_ratio=0.272441 router_tokens_per_expert=[2762, 789, 0, 192, 12, 1, 535, 0, 0, 0, 0, 2854, 0, 0, 4, 111, 0, 17, 0, 0, 403, 86, 0, 0, 0, 0, 776, 0, 0, 1756, 3656, 59, 0, 845, 0, 0, 0, 1, 0, 0, 0, 1408, 10, 1, 7646, 0, 0, 4705, 4, 0, 1, 1, 2832, 26, 629, 1, 118, 1300, 0, 152, 0, 0, 7930, 7529] payload_rows_per_dst=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] dup_rows_per_dst=[848, 34, 0, 1902, 0, 2482, 3832, 17684] combine_rows_per_src=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72332 payload_rows_remote=55950 dup_rows=25972 dup_rows_remote=8056 dedup_ratio=0.264201 router_tokens_per_expert=[3112, 757, 0, 170, 6, 0, 447, 0, 0, 0, 0, 3228, 0, 0, 3, 96, 0, 23, 0, 0, 303, 99, 0, 2, 0, 0, 759, 0, 0, 1642, 3613, 71, 1, 979, 0, 1, 3, 0, 0, 0, 0, 1164, 10, 5, 7588, 0, 0, 4925, 1, 0, 0, 0, 2299, 3, 693, 0, 50, 1265, 0, 101, 0, 2, 8052, 7679] payload_rows_per_dst=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] dup_rows_per_dst=[866, 40, 4, 1838, 4, 2028, 3276, 17916] combine_rows_per_src=[8118, 6614, 850, 10332, 1964, 15506, 12566, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286326784 dispatch_meta_bytes=325408 combine_payload_bytes=286326784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286652192 ingress_bytes=320929792 link_bytes=320929792 token_bytes=572978976 weight_movement_bytes=34603008 total_nvlink_bytes=607581984 weight_to_token_ratio=0.060391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207632970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=229171200 dispatch_meta_bytes=256024 combine_payload_bytes=229171200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=229427224 ingress_bytes=402186240 link_bytes=402186240 token_bytes=458598424 weight_movement_bytes=173015040 total_nvlink_bytes=631613464 weight_to_token_ratio=0.377269 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513394442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=304177152 dispatch_meta_bytes=343880 combine_payload_bytes=304177152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521032 ingress_bytes=338780160 link_bytes=338780160 token_bytes=608698184 weight_movement_bytes=34603008 total_nvlink_bytes=643301192 weight_to_token_ratio=0.056848 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274802468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85518 payload_rows_remote=78878 dup_rows=12786 dup_rows_remote=12328 dedup_ratio=0.130066 router_tokens_per_expert=[2806, 673, 0, 208, 2, 0, 437, 0, 0, 0, 0, 2921, 0, 0, 0, 114, 0, 42, 0, 0, 44, 106, 0, 0, 0, 0, 732, 0, 0, 1580, 3906, 46, 0, 946, 0, 0, 0, 0, 0, 0, 0, 1204, 6, 0, 7717, 0, 0, 5287, 0, 0, 0, 0, 2483, 4, 615, 0, 62, 1358, 0, 112, 0, 6, 8001, 7734] payload_rows_per_dst=[15588, 6640, 16004, 10620, 15778, 11712, 6138, 3038] dup_rows_per_dst=[7104, 458, 382, 1908, 1548, 1282, 66, 38] combine_rows_per_src=[15588, 6640, 16004, 10620, 15778, 11712, 6138, 3038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84244 payload_rows_remote=68378 dup_rows=14060 dup_rows_remote=12372 dedup_ratio=0.143026 router_tokens_per_expert=[2709, 740, 0, 196, 5, 1, 436, 0, 0, 0, 0, 2792, 0, 0, 7, 92, 0, 50, 0, 0, 133, 132, 0, 0, 0, 0, 777, 0, 0, 1561, 3626, 51, 0, 956, 0, 1, 0, 0, 0, 0, 0, 1283, 7, 1, 7820, 0, 0, 5005, 1, 0, 0, 0, 2733, 9, 638, 0, 60, 1533, 0, 93, 0, 5, 7929, 7770] payload_rows_per_dst=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] dup_rows_per_dst=[790, 5488, 626, 1742, 1688, 0, 3662, 64] combine_rows_per_src=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81382 payload_rows_remote=68742 dup_rows=16922 dup_rows_remote=14402 dedup_ratio=0.172139 router_tokens_per_expert=[3241, 709, 0, 182, 4, 0, 382, 0, 0, 0, 0, 2768, 0, 0, 3, 135, 0, 47, 0, 0, 351, 98, 0, 0, 0, 0, 794, 0, 0, 1572, 3603, 62, 0, 822, 0, 0, 0, 0, 0, 0, 0, 1150, 9, 0, 7721, 0, 0, 4880, 1, 0, 0, 0, 2742, 6, 736, 0, 62, 1304, 0, 102, 0, 4, 7949, 7713] payload_rows_per_dst=[8138, 7564, 12198, 10608, 8464, 12640, 12888, 8882] dup_rows_per_dst=[898, 1062, 818, 2138, 646, 2520, 3842, 4998] combine_rows_per_src=[8138, 7564, 12198, 10608, 8464, 12640, 12888, 8882] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=252731392 dispatch_meta_bytes=337200 combine_payload_bytes=252731392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253068592 ingress_bytes=287334400 link_bytes=287334400 token_bytes=505799984 weight_movement_bytes=34603008 total_nvlink_bytes=540402992 weight_to_token_ratio=0.068412 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=38144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.081216215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306774016 dispatch_meta_bytes=344400 combine_payload_bytes=306774016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307118416 ingress_bytes=341377024 link_bytes=341377024 token_bytes=613892432 weight_movement_bytes=34603008 total_nvlink_bytes=648495440 weight_to_token_ratio=0.056367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284574258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84244 payload_rows_remote=68378 dup_rows=14060 dup_rows_remote=12372 dedup_ratio=0.143026 router_tokens_per_expert=[2709, 740, 0, 196, 5, 1, 436, 0, 0, 0, 0, 2792, 0, 0, 7, 92, 0, 50, 0, 0, 133, 132, 0, 0, 0, 0, 777, 0, 0, 1561, 3626, 51, 0, 956, 0, 1, 0, 0, 0, 0, 0, 1283, 7, 1, 7820, 0, 0, 5005, 1, 0, 0, 0, 2733, 9, 638, 0, 60, 1533, 0, 93, 0, 5, 7929, 7770] payload_rows_per_dst=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] dup_rows_per_dst=[790, 5488, 626, 1742, 1688, 0, 3662, 64] combine_rows_per_src=[7384, 15834, 15862, 10288, 15866, 2582, 13110, 3318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85518 payload_rows_remote=78878 dup_rows=12786 dup_rows_remote=12328 dedup_ratio=0.130066 router_tokens_per_expert=[2806, 673, 0, 208, 2, 0, 437, 0, 0, 0, 0, 2921, 0, 0, 0, 114, 0, 42, 0, 0, 44, 106, 0, 0, 0, 0, 732, 0, 0, 1580, 3906, 46, 0, 946, 0, 0, 0, 0, 0, 0, 0, 1204, 6, 0, 7717, 0, 0, 5287, 0, 0, 0, 0, 2483, 4, 615, 0, 62, 1358, 0, 112, 0, 6, 8001, 7734] payload_rows_per_dst=[15588, 6640, 16004, 10620, 15778, 11712, 6138, 3038] dup_rows_per_dst=[7104, 458, 382, 1908, 1548, 1282, 66, 38] combine_rows_per_src=[15588, 6640, 16004, 10620, 15778, 11712, 6138, 3038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=241483776 dispatch_meta_bytes=327624 combine_payload_bytes=241483776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241811400 ingress_bytes=276086784 link_bytes=276086784 token_bytes=483295176 weight_movement_bytes=34603008 total_nvlink_bytes=517898184 weight_to_token_ratio=0.071598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.038892341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88486 payload_rows_remote=77514 dup_rows=9818 dup_rows_remote=7814 dedup_ratio=0.099874 router_tokens_per_expert=[1407, 2520, 5990, 756, 0, 5044, 58, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 4797, 0, 0, 0, 380, 3, 257, 1091, 2803, 8192, 0, 169, 0, 0, 0, 0, 0, 3, 0, 6, 0, 1621, 4541, 0, 0, 326, 0, 0, 0, 0, 5137, 0, 884, 0, 0, 0, 0, 0, 29, 1331, 0, 1715, 0, 0, 0] payload_rows_per_dst=[8288, 14130, 10348, 8928, 11980, 10972, 11288, 12552] dup_rows_per_dst=[1194, 182, 6, 1974, 18, 2004, 754, 3686] combine_rows_per_src=[8288, 14130, 10348, 8928, 11980, 10972, 11288, 12552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=229171200 dispatch_meta_bytes=256024 combine_payload_bytes=229171200 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=229427224 ingress_bytes=402186240 link_bytes=402186240 token_bytes=458598424 weight_movement_bytes=173015040 total_nvlink_bytes=631613464 weight_to_token_ratio=0.377269 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513394442e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85746 payload_rows_remote=69904 dup_rows=12558 dup_rows_remote=11448 dedup_ratio=0.127747 router_tokens_per_expert=[3290, 561, 0, 197, 9, 0, 422, 0, 0, 0, 0, 3095, 0, 1, 1, 123, 0, 23, 2, 0, 428, 106, 0, 1, 0, 0, 801, 0, 0, 1758, 3280, 54, 0, 869, 0, 0, 0, 1, 0, 0, 0, 1254, 9, 0, 7487, 0, 0, 4778, 2, 0, 0, 1, 2860, 19, 687, 0, 118, 1488, 0, 101, 0, 0, 7916, 7410] payload_rows_per_dst=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] dup_rows_per_dst=[774, 5986, 1110, 1862, 1360, 1266, 136, 64] combine_rows_per_src=[8184, 15274, 15842, 9924, 15354, 10816, 7002, 3350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280076288 dispatch_meta_bytes=323000 combine_payload_bytes=280076288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280399288 ingress_bytes=314679296 link_bytes=314679296 token_bytes=560475576 weight_movement_bytes=34603008 total_nvlink_bytes=595078584 weight_to_token_ratio=0.061739 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184112857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323084288 dispatch_meta_bytes=364824 combine_payload_bytes=323084288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323449112 ingress_bytes=357687296 link_bytes=357687296 token_bytes=646533400 weight_movement_bytes=34603008 total_nvlink_bytes=681136408 weight_to_token_ratio=0.053521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345948499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317497344 dispatch_meta_bytes=341312 combine_payload_bytes=317497344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317838656 ingress_bytes=421306368 link_bytes=421306368 token_bytes=635336000 weight_movement_bytes=103809024 total_nvlink_bytes=739145024 weight_to_token_ratio=0.163392 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.585341945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=286326784 dispatch_meta_bytes=325408 combine_payload_bytes=286326784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286652192 ingress_bytes=320929792 link_bytes=320929792 token_bytes=572978976 weight_movement_bytes=34603008 total_nvlink_bytes=607581984 weight_to_token_ratio=0.060391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207632970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85302 payload_rows_remote=69238 dup_rows=13002 dup_rows_remote=5558 dedup_ratio=0.132263 router_tokens_per_expert=[2815, 697, 0, 217, 24, 0, 591, 0, 0, 1, 0, 2878, 0, 0, 6, 134, 0, 28, 2, 0, 448, 107, 0, 0, 0, 0, 868, 0, 0, 1640, 3569, 96, 1, 810, 1, 0, 0, 2, 0, 0, 0, 894, 14, 2, 7686, 0, 0, 5081, 1, 0, 0, 2, 3136, 6, 542, 0, 114, 1343, 0, 104, 0, 5, 7877, 7410] payload_rows_per_dst=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] dup_rows_per_dst=[7444, 38, 1150, 1996, 1364, 912, 60, 38] combine_rows_per_src=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=280076288 dispatch_meta_bytes=323000 combine_payload_bytes=280076288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280399288 ingress_bytes=314679296 link_bytes=314679296 token_bytes=560475576 weight_movement_bytes=34603008 total_nvlink_bytes=595078584 weight_to_token_ratio=0.061739 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184112857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281567232 dispatch_meta_bytes=332576 combine_payload_bytes=281567232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281899808 ingress_bytes=385376256 link_bytes=385376256 token_bytes=563467040 weight_movement_bytes=103809024 total_nvlink_bytes=667276064 weight_to_token_ratio=0.184233 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450139826e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87100 payload_rows_remote=76206 dup_rows=11204 dup_rows_remote=9522 dedup_ratio=0.113973 router_tokens_per_expert=[1042, 2552, 6210, 863, 0, 5628, 24, 0, 0, 0, 0, 0, 1, 117, 0, 0, 0, 0, 0, 4564, 0, 0, 0, 340, 7, 254, 1183, 2842, 8192, 0, 221, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1617, 4352, 0, 0, 262, 0, 0, 0, 0, 4647, 0, 953, 0, 0, 0, 0, 0, 19, 1302, 0, 1951, 0, 0, 4] payload_rows_per_dst=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] dup_rows_per_dst=[4602, 236, 1682, 594, 10, 2020, 588, 1472] combine_rows_per_src=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79798 payload_rows_remote=68018 dup_rows=18506 dup_rows_remote=14966 dedup_ratio=0.188253 router_tokens_per_expert=[1186, 2342, 5793, 807, 0, 4540, 28, 0, 0, 0, 0, 0, 1, 201, 0, 0, 0, 1, 0, 4840, 0, 0, 0, 342, 3, 218, 1206, 2976, 8191, 0, 174, 1, 0, 0, 0, 0, 8, 0, 4, 0, 1681, 4631, 1, 0, 376, 0, 0, 0, 0, 5512, 0, 969, 0, 0, 0, 0, 0, 17, 1020, 1, 2081, 0, 0, 1] payload_rows_per_dst=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] dup_rows_per_dst=[470, 0, 0, 9154, 1818, 2296, 1228, 3540] combine_rows_per_src=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71522 payload_rows_remote=58956 dup_rows=26782 dup_rows_remote=22950 dedup_ratio=0.272441 router_tokens_per_expert=[2762, 789, 0, 192, 12, 1, 535, 0, 0, 0, 0, 2854, 0, 0, 4, 111, 0, 17, 0, 0, 403, 86, 0, 0, 0, 0, 776, 0, 0, 1756, 3656, 59, 0, 845, 0, 0, 0, 1, 0, 0, 0, 1408, 10, 1, 7646, 0, 0, 4705, 4, 0, 1, 1, 2832, 26, 629, 1, 118, 1300, 0, 152, 0, 0, 7930, 7529] payload_rows_per_dst=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] dup_rows_per_dst=[848, 34, 0, 1902, 0, 2482, 3832, 17684] combine_rows_per_src=[7734, 5904, 1012, 10592, 1692, 15648, 12566, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89092 payload_rows_remote=80464 dup_rows=9212 dup_rows_remote=8050 dedup_ratio=0.093709 router_tokens_per_expert=[1425, 2403, 5965, 839, 0, 4862, 62, 0, 0, 0, 0, 1, 0, 99, 0, 2, 0, 0, 0, 4646, 0, 0, 0, 328, 2, 268, 1188, 3278, 8188, 0, 159, 0, 1, 0, 0, 0, 7, 0, 3, 1, 1813, 3678, 0, 0, 351, 2, 1, 2, 1, 5808, 0, 966, 0, 0, 1, 0, 1, 19, 1056, 0, 1718, 0, 0, 8] payload_rows_per_dst=[8088, 16380, 9936, 8628, 11940, 9836, 12166, 12118] dup_rows_per_dst=[1370, 200, 12, 1162, 14, 1858, 1386, 3210] combine_rows_per_src=[8088, 16380, 9936, 8628, 11940, 9836, 12166, 12118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81532 payload_rows_remote=69852 dup_rows=16772 dup_rows_remote=15756 dedup_ratio=0.170614 router_tokens_per_expert=[965, 2373, 5824, 843, 0, 5462, 39, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 0, 4948, 0, 0, 0, 372, 1, 230, 1221, 2815, 8192, 0, 138, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1464, 4716, 0, 0, 301, 0, 0, 0, 4, 5291, 0, 1053, 0, 0, 0, 0, 0, 10, 1160, 0, 1701, 0, 0, 5] payload_rows_per_dst=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] dup_rows_per_dst=[1166, 0, 10, 8810, 2, 2046, 1016, 3722] combine_rows_per_src=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85302 payload_rows_remote=69238 dup_rows=13002 dup_rows_remote=5558 dedup_ratio=0.132263 router_tokens_per_expert=[2815, 697, 0, 217, 24, 0, 591, 0, 0, 1, 0, 2878, 0, 0, 6, 134, 0, 28, 2, 0, 448, 107, 0, 0, 0, 0, 868, 0, 0, 1640, 3569, 96, 1, 810, 1, 0, 0, 2, 0, 0, 0, 894, 14, 2, 7686, 0, 0, 5081, 1, 0, 0, 2, 3136, 6, 542, 0, 114, 1343, 0, 104, 0, 5, 7877, 7410] payload_rows_per_dst=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] dup_rows_per_dst=[7444, 38, 1150, 1996, 1364, 912, 60, 38] combine_rows_per_src=[16064, 6000, 15774, 10350, 15636, 11070, 7314, 3094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312139776 dispatch_meta_bytes=342912 combine_payload_bytes=312139776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312482688 ingress_bytes=346742784 link_bytes=346742784 token_bytes=624622464 weight_movement_bytes=34603008 total_nvlink_bytes=659225472 weight_to_token_ratio=0.055398 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304765181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79798 payload_rows_remote=68018 dup_rows=18506 dup_rows_remote=14966 dedup_ratio=0.188253 router_tokens_per_expert=[1186, 2342, 5793, 807, 0, 4540, 28, 0, 0, 0, 0, 0, 1, 201, 0, 0, 0, 1, 0, 4840, 0, 0, 0, 342, 3, 218, 1206, 2976, 8191, 0, 174, 1, 0, 0, 0, 0, 8, 0, 4, 0, 1681, 4631, 1, 0, 376, 0, 0, 0, 0, 5512, 0, 969, 0, 0, 0, 0, 0, 17, 1020, 1, 2081, 0, 0, 1] payload_rows_per_dst=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] dup_rows_per_dst=[470, 0, 0, 9154, 1818, 2296, 1228, 3540] combine_rows_per_src=[4948, 404, 10366, 16384, 13100, 11082, 11734, 11780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283598848 dispatch_meta_bytes=299184 combine_payload_bytes=283598848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283898032 ingress_bytes=318201856 link_bytes=318201856 token_bytes=567496880 weight_movement_bytes=34603008 total_nvlink_bytes=602099888 weight_to_token_ratio=0.060975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197367967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=278601728 dispatch_meta_bytes=331936 combine_payload_bytes=278601728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278933664 ingress_bytes=313204736 link_bytes=313204736 token_bytes=557535392 weight_movement_bytes=34603008 total_nvlink_bytes=592138400 weight_to_token_ratio=0.062064 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178564207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=241483776 dispatch_meta_bytes=327624 combine_payload_bytes=241483776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241811400 ingress_bytes=276086784 link_bytes=276086784 token_bytes=483295176 weight_movement_bytes=34603008 total_nvlink_bytes=517898184 weight_to_token_ratio=0.071598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.038892341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85212 payload_rows_remote=68828 dup_rows=13092 dup_rows_remote=12730 dedup_ratio=0.133179 router_tokens_per_expert=[1380, 2351, 6066, 590, 0, 4608, 21, 0, 0, 0, 0, 0, 1, 180, 0, 0, 0, 0, 0, 5259, 0, 0, 0, 345, 2, 200, 1085, 3198, 8192, 0, 188, 0, 0, 0, 0, 0, 6, 0, 1, 0, 1855, 4720, 0, 0, 306, 0, 0, 0, 0, 4951, 0, 707, 0, 0, 0, 0, 0, 27, 1236, 0, 1677, 0, 0, 0] payload_rows_per_dst=[12950, 16384, 13004, 2932, 12134, 11408, 10844, 5556] dup_rows_per_dst=[4950, 362, 4600, 18, 12, 2354, 472, 324] combine_rows_per_src=[12950, 16384, 13004, 2932, 12134, 11408, 10844, 5556] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89812 payload_rows_remote=77568 dup_rows=8492 dup_rows_remote=8470 dedup_ratio=0.086385 router_tokens_per_expert=[1234, 2437, 6120, 767, 0, 4440, 52, 0, 1, 0, 0, 0, 0, 346, 0, 0, 0, 0, 0, 4944, 0, 0, 0, 362, 2, 247, 1250, 3069, 8191, 0, 151, 1, 0, 0, 0, 0, 4, 0, 9, 0, 1778, 4968, 0, 0, 288, 0, 0, 0, 0, 5536, 0, 716, 0, 0, 0, 0, 0, 20, 795, 1, 1422, 0, 0, 1] payload_rows_per_dst=[7760, 16382, 10596, 8416, 12244, 11506, 11960, 10948] dup_rows_per_dst=[1220, 694, 16, 1024, 22, 2562, 544, 2410] combine_rows_per_src=[7760, 16382, 10596, 8416, 12244, 11506, 11960, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88486 payload_rows_remote=77514 dup_rows=9818 dup_rows_remote=7814 dedup_ratio=0.099874 router_tokens_per_expert=[1407, 2520, 5990, 756, 0, 5044, 58, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 4797, 0, 0, 0, 380, 3, 257, 1091, 2803, 8192, 0, 169, 0, 0, 0, 0, 0, 3, 0, 6, 0, 1621, 4541, 0, 0, 326, 0, 0, 0, 0, 5137, 0, 884, 0, 0, 0, 0, 0, 29, 1331, 0, 1715, 0, 0, 0] payload_rows_per_dst=[8288, 14130, 10348, 8928, 11980, 10972, 11288, 12552] dup_rows_per_dst=[1194, 182, 6, 1974, 18, 2004, 754, 3686] combine_rows_per_src=[8288, 14130, 10348, 8928, 11980, 10972, 11288, 12552] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=329580544 dispatch_meta_bytes=354056 combine_payload_bytes=329580544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329934600 ingress_bytes=433389568 link_bytes=433389568 token_bytes=659515144 weight_movement_bytes=103809024 total_nvlink_bytes=763324168 weight_to_token_ratio=0.157402 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630810054e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286113792 dispatch_meta_bytes=342432 combine_payload_bytes=286113792 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=286456224 ingress_bytes=320716800 link_bytes=320716800 token_bytes=572570016 weight_movement_bytes=34603008 total_nvlink_bytes=607173024 weight_to_token_ratio=0.060435 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206831499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85212 payload_rows_remote=68828 dup_rows=13092 dup_rows_remote=12730 dedup_ratio=0.133179 router_tokens_per_expert=[1380, 2351, 6066, 590, 0, 4608, 21, 0, 0, 0, 0, 0, 1, 180, 0, 0, 0, 0, 0, 5259, 0, 0, 0, 345, 2, 200, 1085, 3198, 8192, 0, 188, 0, 0, 0, 0, 0, 6, 0, 1, 0, 1855, 4720, 0, 0, 306, 0, 0, 0, 0, 4951, 0, 707, 0, 0, 0, 0, 0, 27, 1236, 0, 1677, 0, 0, 0] payload_rows_per_dst=[12950, 16384, 13004, 2932, 12134, 11408, 10844, 5556] dup_rows_per_dst=[4950, 362, 4600, 18, 12, 2354, 472, 324] combine_rows_per_src=[12950, 16384, 13004, 2932, 12134, 11408, 10844, 5556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91278 payload_rows_remote=86522 dup_rows=7026 dup_rows_remote=6798 dedup_ratio=0.071472 router_tokens_per_expert=[0, 0, 0, 7939, 35, 3194, 0, 0, 337, 0, 6810, 0, 0, 0, 0, 166, 0, 1340, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 162, 0, 5832, 0, 0, 0, 0, 751, 0, 0, 0, 0, 722, 334, 8192, 1243, 0, 0, 6811, 0, 4157, 0, 1, 0, 0, 0, 325, 0, 0, 311, 284, 136, 32, 0, 0] payload_rows_per_dst=[6448, 12872, 16078, 11792, 16070, 4756, 9174, 14088] dup_rows_per_dst=[10, 394, 2544, 208, 1430, 228, 1152, 1060] combine_rows_per_src=[6448, 12872, 16078, 11792, 16070, 4756, 9174, 14088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=278601728 dispatch_meta_bytes=331936 combine_payload_bytes=278601728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278933664 ingress_bytes=313204736 link_bytes=313204736 token_bytes=557535392 weight_movement_bytes=34603008 total_nvlink_bytes=592138400 weight_to_token_ratio=0.062064 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178564207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=283598848 dispatch_meta_bytes=299184 combine_payload_bytes=283598848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283898032 ingress_bytes=318201856 link_bytes=318201856 token_bytes=567496880 weight_movement_bytes=34603008 total_nvlink_bytes=602099888 weight_to_token_ratio=0.060975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197367967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317718528 dispatch_meta_bytes=344152 combine_payload_bytes=317718528 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=318062680 ingress_bytes=386924544 link_bytes=386924544 token_bytes=635781208 weight_movement_bytes=69206016 total_nvlink_bytes=704987224 weight_to_token_ratio=0.108852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455965909e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87100 payload_rows_remote=76206 dup_rows=11204 dup_rows_remote=9522 dedup_ratio=0.113973 router_tokens_per_expert=[1042, 2552, 6210, 863, 0, 5628, 24, 0, 0, 0, 0, 0, 1, 117, 0, 0, 0, 0, 0, 4564, 0, 0, 0, 340, 7, 254, 1183, 2842, 8192, 0, 221, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1617, 4352, 0, 0, 262, 0, 0, 0, 0, 4647, 0, 953, 0, 0, 0, 0, 0, 19, 1302, 0, 1951, 0, 0, 4] payload_rows_per_dst=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] dup_rows_per_dst=[4602, 236, 1682, 594, 10, 2020, 588, 1472] combine_rows_per_src=[12336, 16384, 10894, 5652, 12420, 10442, 10612, 8360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281919488 dispatch_meta_bytes=326232 combine_payload_bytes=281919488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282245720 ingress_bytes=316522496 link_bytes=316522496 token_bytes=564165208 weight_movement_bytes=34603008 total_nvlink_bytes=598768216 weight_to_token_ratio=0.061335 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191048670e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=354394112 dispatch_meta_bytes=373280 combine_payload_bytes=354394112 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=354767392 ingress_bytes=492806144 link_bytes=492806144 token_bytes=709161504 weight_movement_bytes=138412032 total_nvlink_bytes=847573536 weight_to_token_ratio=0.195177 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.854389846e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83870 payload_rows_remote=69954 dup_rows=14434 dup_rows_remote=6662 dedup_ratio=0.146830 router_tokens_per_expert=[1453, 2469, 5451, 741, 0, 5228, 57, 0, 3, 0, 0, 0, 2, 61, 3, 0, 0, 0, 0, 4779, 0, 0, 0, 361, 10, 255, 1211, 2962, 8192, 0, 133, 0, 0, 0, 0, 0, 6, 2, 11, 0, 1761, 4635, 0, 0, 349, 0, 0, 0, 0, 5260, 0, 921, 0, 0, 0, 0, 0, 5, 1129, 0, 1701, 0, 0, 1] payload_rows_per_dst=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] dup_rows_per_dst=[7772, 138, 3808, 748, 16, 828, 794, 330] combine_rows_per_src=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312139776 dispatch_meta_bytes=342912 combine_payload_bytes=312139776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312482688 ingress_bytes=346742784 link_bytes=346742784 token_bytes=624622464 weight_movement_bytes=34603008 total_nvlink_bytes=659225472 weight_to_token_ratio=0.055398 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304765181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91112 payload_rows_remote=80676 dup_rows=7192 dup_rows_remote=7038 dedup_ratio=0.073161 router_tokens_per_expert=[0, 0, 4, 7928, 1, 3137, 0, 2, 391, 0, 6798, 0, 0, 1, 0, 177, 0, 1275, 46, 1, 2, 0, 0, 2, 0, 1, 0, 0, 5, 137, 1, 5151, 1, 0, 1, 0, 957, 2, 1, 0, 0, 666, 475, 8187, 1484, 0, 0, 6623, 0, 4342, 0, 0, 0, 0, 0, 387, 0, 0, 453, 282, 161, 70, 0, 0] payload_rows_per_dst=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] dup_rows_per_dst=[4, 530, 2438, 154, 1918, 176, 606, 1366] combine_rows_per_src=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81764 payload_rows_remote=66466 dup_rows=16540 dup_rows_remote=9410 dedup_ratio=0.168254 router_tokens_per_expert=[0, 0, 0, 7979, 59, 2948, 0, 0, 449, 0, 6860, 0, 0, 0, 0, 206, 0, 1452, 37, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6, 154, 0, 5353, 0, 0, 1, 1, 1161, 0, 0, 0, 0, 835, 458, 8192, 1183, 0, 0, 6421, 0, 4002, 0, 20, 0, 0, 0, 332, 0, 0, 451, 324, 194, 70, 0, 0] payload_rows_per_dst=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] dup_rows_per_dst=[14, 0, 2830, 190, 2, 4952, 7130, 1422] combine_rows_per_src=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317718528 dispatch_meta_bytes=344152 combine_payload_bytes=317718528 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=318062680 ingress_bytes=386924544 link_bytes=386924544 token_bytes=635781208 weight_movement_bytes=69206016 total_nvlink_bytes=704987224 weight_to_token_ratio=0.108852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455965909e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=317497344 dispatch_meta_bytes=341312 combine_payload_bytes=317497344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317838656 ingress_bytes=421306368 link_bytes=421306368 token_bytes=635336000 weight_movement_bytes=103809024 total_nvlink_bytes=739145024 weight_to_token_ratio=0.163392 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.585341945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88202 payload_rows_remote=81094 dup_rows=10102 dup_rows_remote=9240 dedup_ratio=0.102763 router_tokens_per_expert=[0, 0, 0, 7937, 80, 3007, 0, 0, 345, 0, 6777, 0, 0, 0, 0, 278, 0, 1667, 47, 5, 0, 0, 0, 0, 0, 1, 0, 0, 8, 152, 0, 4820, 0, 0, 0, 0, 1043, 0, 0, 0, 1, 857, 424, 8192, 1314, 0, 0, 6789, 0, 3943, 0, 0, 0, 0, 0, 244, 0, 0, 646, 324, 208, 43, 0, 0] payload_rows_per_dst=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] dup_rows_per_dst=[4210, 572, 862, 148, 2086, 158, 338, 1728] combine_rows_per_src=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78128 payload_rows_remote=64030 dup_rows=20176 dup_rows_remote=18994 dedup_ratio=0.205241 router_tokens_per_expert=[0, 0, 0, 8030, 47, 3240, 0, 0, 362, 0, 6837, 1, 0, 0, 0, 139, 0, 1385, 15, 2, 0, 1, 0, 0, 0, 0, 0, 0, 1, 176, 0, 5695, 0, 0, 1, 0, 829, 0, 0, 1, 0, 851, 437, 8181, 1335, 0, 0, 6777, 0, 3647, 0, 10, 0, 0, 0, 289, 0, 0, 369, 272, 197, 25, 0, 0] payload_rows_per_dst=[6568, 1004, 8538, 14590, 1662, 16366, 15302, 14098] dup_rows_per_dst=[6, 0, 962, 6520, 0, 5242, 6264, 1182] combine_rows_per_src=[6568, 1004, 8538, 14590, 1662, 16366, 15302, 14098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90598 payload_rows_remote=74216 dup_rows=7706 dup_rows_remote=5558 dedup_ratio=0.078389 router_tokens_per_expert=[0, 0, 0, 7988, 20, 3317, 0, 0, 358, 0, 6667, 0, 0, 0, 0, 144, 0, 1706, 16, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 164, 0, 5284, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 599, 385, 8191, 1403, 0, 0, 6868, 0, 3865, 0, 0, 0, 0, 0, 329, 0, 0, 414, 185, 142, 26, 0, 0] payload_rows_per_dst=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] dup_rows_per_dst=[6, 392, 3258, 188, 2148, 100, 526, 1088] combine_rows_per_src=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286531584 dispatch_meta_bytes=306464 combine_payload_bytes=286531584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286838048 ingress_bytes=424943616 link_bytes=424943616 token_bytes=573369632 weight_movement_bytes=138412032 total_nvlink_bytes=711781664 weight_to_token_ratio=0.241401 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599028616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=330448896 dispatch_meta_bytes=350856 combine_payload_bytes=330448896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330799752 ingress_bytes=365051904 link_bytes=365051904 token_bytes=661248648 weight_movement_bytes=34603008 total_nvlink_bytes=695851656 weight_to_token_ratio=0.052330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373660926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=303504 combine_payload_bytes=272244736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272548240 ingress_bytes=306847744 link_bytes=306847744 token_bytes=544792976 weight_movement_bytes=34603008 total_nvlink_bytes=579395984 weight_to_token_ratio=0.063516 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154643357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81532 payload_rows_remote=69852 dup_rows=16772 dup_rows_remote=15756 dedup_ratio=0.170614 router_tokens_per_expert=[965, 2373, 5824, 843, 0, 5462, 39, 0, 0, 0, 0, 0, 1, 21, 0, 0, 0, 0, 0, 4948, 0, 0, 0, 372, 1, 230, 1221, 2815, 8192, 0, 138, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1464, 4716, 0, 0, 301, 0, 0, 0, 4, 5291, 0, 1053, 0, 0, 0, 0, 0, 10, 1160, 0, 1701, 0, 0, 5] payload_rows_per_dst=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] dup_rows_per_dst=[1166, 0, 10, 8810, 2, 2046, 1016, 3722] combine_rows_per_src=[7274, 44, 10630, 16384, 11650, 10916, 11680, 12954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=332161024 dispatch_meta_bytes=361336 combine_payload_bytes=332161024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332522360 ingress_bytes=366764032 link_bytes=366764032 token_bytes=664683384 weight_movement_bytes=34603008 total_nvlink_bytes=699286392 weight_to_token_ratio=0.052059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380103525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72496 dup_rows=12006 dup_rows_remote=11518 dedup_ratio=0.122131 router_tokens_per_expert=[0, 0, 0, 7924, 7, 3164, 0, 0, 363, 0, 6599, 0, 0, 0, 0, 183, 0, 1485, 11, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 99, 0, 5914, 0, 0, 0, 0, 767, 0, 0, 0, 0, 675, 393, 8192, 1272, 0, 0, 6889, 0, 4039, 0, 3, 0, 0, 0, 356, 0, 0, 489, 198, 82, 43, 0, 0] payload_rows_per_dst=[16026, 13802, 2994, 11898, 16384, 13002, 8250, 3942] dup_rows_per_dst=[6164, 488, 0, 136, 1534, 2934, 546, 204] combine_rows_per_src=[16026, 13802, 2994, 11898, 16384, 13002, 8250, 3942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82030 payload_rows_remote=67810 dup_rows=16274 dup_rows_remote=16142 dedup_ratio=0.165548 router_tokens_per_expert=[0, 8092, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 7104, 0, 0, 0, 12, 216, 0, 1478, 0, 2174, 0, 7, 0, 1012, 362, 2, 2, 7109, 0, 0, 441, 0, 0, 0, 223, 0, 7031, 0, 0, 145, 0, 0, 0, 0, 0, 0, 19, 0, 2559, 0, 260, 0, 0, 0, 0, 8179, 1148, 0, 0, 0] payload_rows_per_dst=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] dup_rows_per_dst=[44, 384, 588, 12, 596, 132, 5052, 9466] combine_rows_per_src=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91112 payload_rows_remote=80676 dup_rows=7192 dup_rows_remote=7038 dedup_ratio=0.073161 router_tokens_per_expert=[0, 0, 4, 7928, 1, 3137, 0, 2, 391, 0, 6798, 0, 0, 1, 0, 177, 0, 1275, 46, 1, 2, 0, 0, 2, 0, 1, 0, 0, 5, 137, 1, 5151, 1, 0, 1, 0, 957, 2, 1, 0, 0, 666, 475, 8187, 1484, 0, 0, 6623, 0, 4342, 0, 0, 0, 0, 0, 387, 0, 0, 453, 282, 161, 70, 0, 0] payload_rows_per_dst=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] dup_rows_per_dst=[4, 530, 2438, 154, 1918, 176, 606, 1366] combine_rows_per_src=[6284, 14204, 16070, 10436, 16380, 5074, 8852, 13812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83870 payload_rows_remote=69954 dup_rows=14434 dup_rows_remote=6662 dedup_ratio=0.146830 router_tokens_per_expert=[1453, 2469, 5451, 741, 0, 5228, 57, 0, 3, 0, 0, 0, 2, 61, 3, 0, 0, 0, 0, 4779, 0, 0, 0, 361, 10, 255, 1211, 2962, 8192, 0, 133, 0, 0, 0, 0, 0, 6, 2, 11, 0, 1761, 4635, 0, 0, 349, 0, 0, 0, 0, 5260, 0, 921, 0, 0, 0, 0, 0, 5, 1129, 0, 1701, 0, 0, 1] payload_rows_per_dst=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] dup_rows_per_dst=[7772, 138, 3808, 748, 16, 828, 794, 330] combine_rows_per_src=[13916, 16384, 12396, 7496, 10924, 6174, 11238, 5342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=262266880 dispatch_meta_bytes=332096 combine_payload_bytes=262266880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262598976 ingress_bytes=296869888 link_bytes=296869888 token_bytes=524865856 weight_movement_bytes=34603008 total_nvlink_bytes=559468864 weight_to_token_ratio=0.065927 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117097489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286113792 dispatch_meta_bytes=342432 combine_payload_bytes=286113792 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=286456224 ingress_bytes=320716800 link_bytes=320716800 token_bytes=572570016 weight_movement_bytes=34603008 total_nvlink_bytes=607173024 weight_to_token_ratio=0.060435 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206831499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72496 dup_rows=12006 dup_rows_remote=11518 dedup_ratio=0.122131 router_tokens_per_expert=[0, 0, 0, 7924, 7, 3164, 0, 0, 363, 0, 6599, 0, 0, 0, 0, 183, 0, 1485, 11, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 99, 0, 5914, 0, 0, 0, 0, 767, 0, 0, 0, 0, 675, 393, 8192, 1272, 0, 0, 6889, 0, 4039, 0, 3, 0, 0, 0, 356, 0, 0, 489, 198, 82, 43, 0, 0] payload_rows_per_dst=[16026, 13802, 2994, 11898, 16384, 13002, 8250, 3942] dup_rows_per_dst=[6164, 488, 0, 136, 1534, 2934, 546, 204] combine_rows_per_src=[16026, 13802, 2994, 11898, 16384, 13002, 8250, 3942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90598 payload_rows_remote=74216 dup_rows=7706 dup_rows_remote=5558 dedup_ratio=0.078389 router_tokens_per_expert=[0, 0, 0, 7988, 20, 3317, 0, 0, 358, 0, 6667, 0, 0, 0, 0, 144, 0, 1706, 16, 5, 0, 0, 0, 0, 0, 0, 0, 0, 2, 164, 0, 5284, 0, 0, 0, 0, 1074, 0, 0, 0, 0, 599, 385, 8191, 1403, 0, 0, 6868, 0, 3865, 0, 0, 0, 0, 0, 329, 0, 0, 414, 185, 142, 26, 0, 0] payload_rows_per_dst=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] dup_rows_per_dst=[6, 392, 3258, 188, 2148, 100, 526, 1088] combine_rows_per_src=[6668, 13946, 16172, 10712, 16382, 4674, 7862, 14182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91278 payload_rows_remote=86522 dup_rows=7026 dup_rows_remote=6798 dedup_ratio=0.071472 router_tokens_per_expert=[0, 0, 0, 7939, 35, 3194, 0, 0, 337, 0, 6810, 0, 0, 0, 0, 166, 0, 1340, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 162, 0, 5832, 0, 0, 0, 0, 751, 0, 0, 0, 0, 722, 334, 8192, 1243, 0, 0, 6811, 0, 4157, 0, 1, 0, 0, 0, 325, 0, 0, 311, 284, 136, 32, 0, 0] payload_rows_per_dst=[6448, 12872, 16078, 11792, 16070, 4756, 9174, 14088] dup_rows_per_dst=[10, 394, 2544, 208, 1430, 228, 1152, 1060] combine_rows_per_src=[6448, 12872, 16078, 11792, 16070, 4756, 9174, 14088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=330448896 dispatch_meta_bytes=350856 combine_payload_bytes=330448896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=330799752 ingress_bytes=365051904 link_bytes=365051904 token_bytes=661248648 weight_movement_bytes=34603008 total_nvlink_bytes=695851656 weight_to_token_ratio=0.052330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373660926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303988736 dispatch_meta_bytes=319096 combine_payload_bytes=303988736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304307832 ingress_bytes=338591744 link_bytes=338591744 token_bytes=608296568 weight_movement_bytes=34603008 total_nvlink_bytes=642899576 weight_to_token_ratio=0.056885 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274093474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88202 payload_rows_remote=81094 dup_rows=10102 dup_rows_remote=9240 dedup_ratio=0.102763 router_tokens_per_expert=[0, 0, 0, 7937, 80, 3007, 0, 0, 345, 0, 6777, 0, 0, 0, 0, 278, 0, 1667, 47, 5, 0, 0, 0, 0, 0, 1, 0, 0, 8, 152, 0, 4820, 0, 0, 0, 0, 1043, 0, 0, 0, 1, 857, 424, 8192, 1314, 0, 0, 6789, 0, 3943, 0, 0, 0, 0, 0, 244, 0, 0, 646, 324, 208, 43, 0, 0] payload_rows_per_dst=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] dup_rows_per_dst=[4210, 572, 862, 148, 2086, 158, 338, 1728] combine_rows_per_src=[13306, 14228, 7108, 9814, 16384, 5034, 8036, 14292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=286531584 dispatch_meta_bytes=306464 combine_payload_bytes=286531584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=286838048 ingress_bytes=424943616 link_bytes=424943616 token_bytes=573369632 weight_movement_bytes=138412032 total_nvlink_bytes=711781664 weight_to_token_ratio=0.241401 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599028616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296943616 dispatch_meta_bytes=336056 combine_payload_bytes=296943616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297279672 ingress_bytes=366149632 link_bytes=366149632 token_bytes=594223288 weight_movement_bytes=69206016 total_nvlink_bytes=663429304 weight_to_token_ratio=0.116465 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377791588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=335808 combine_payload_bytes=277749760 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=278085568 ingress_bytes=381558784 link_bytes=381558784 token_bytes=555835328 weight_movement_bytes=103809024 total_nvlink_bytes=659644352 weight_to_token_ratio=0.186762 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435774986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73112 payload_rows_remote=56996 dup_rows=25192 dup_rows_remote=14718 dedup_ratio=0.256266 router_tokens_per_expert=[0, 0, 0, 7943, 28, 3169, 0, 0, 364, 0, 6588, 0, 0, 0, 0, 227, 0, 1140, 32, 0, 0, 0, 0, 0, 0, 5, 0, 1, 3, 191, 0, 5137, 1, 0, 9, 0, 859, 2, 0, 1, 3, 819, 645, 8185, 1553, 0, 0, 6592, 0, 4106, 0, 55, 0, 0, 0, 323, 0, 0, 625, 300, 203, 43, 0, 0] payload_rows_per_dst=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] dup_rows_per_dst=[10474, 9186, 4, 192, 220, 4442, 606, 68] combine_rows_per_src=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=332161024 dispatch_meta_bytes=361336 combine_payload_bytes=332161024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332522360 ingress_bytes=366764032 link_bytes=366764032 token_bytes=664683384 weight_movement_bytes=34603008 total_nvlink_bytes=699286392 weight_to_token_ratio=0.052059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380103525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91174 payload_rows_remote=83784 dup_rows=7130 dup_rows_remote=6760 dedup_ratio=0.072530 router_tokens_per_expert=[0, 8041, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 6799, 0, 0, 0, 3, 207, 0, 1920, 0, 1750, 0, 5, 0, 1175, 449, 3, 4, 7208, 0, 0, 288, 1, 0, 0, 237, 0, 6658, 0, 0, 225, 0, 0, 0, 0, 0, 0, 53, 0, 2886, 0, 432, 0, 0, 0, 0, 8176, 1252, 0, 0, 0] payload_rows_per_dst=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] dup_rows_per_dst=[98, 2196, 370, 3096, 1052, 138, 180, 0] combine_rows_per_src=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303988736 dispatch_meta_bytes=319096 combine_payload_bytes=303988736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304307832 ingress_bytes=338591744 link_bytes=338591744 token_bytes=608296568 weight_movement_bytes=34603008 total_nvlink_bytes=642899576 weight_to_token_ratio=0.056885 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274093474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=354394112 dispatch_meta_bytes=373280 combine_payload_bytes=354394112 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=354767392 ingress_bytes=492806144 link_bytes=492806144 token_bytes=709161504 weight_movement_bytes=138412032 total_nvlink_bytes=847573536 weight_to_token_ratio=0.195177 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.854389846e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69064 payload_rows_remote=52680 dup_rows=29240 dup_rows_remote=13150 dedup_ratio=0.297445 router_tokens_per_expert=[0, 8059, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 7016, 0, 0, 0, 2, 284, 0, 1334, 0, 2077, 0, 4, 0, 1197, 355, 4, 3, 7202, 0, 0, 299, 0, 0, 0, 245, 1, 7188, 0, 0, 114, 0, 0, 0, 0, 0, 0, 53, 0, 2621, 0, 410, 0, 0, 0, 0, 8184, 1035, 0, 0, 2] payload_rows_per_dst=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] dup_rows_per_dst=[0, 0, 7366, 16, 0, 150, 5618, 16090] combine_rows_per_src=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=343179264 dispatch_meta_bytes=362176 combine_payload_bytes=343179264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343541440 ingress_bytes=377782272 link_bytes=377782272 token_bytes=686720704 weight_movement_bytes=34603008 total_nvlink_bytes=721323712 weight_to_token_ratio=0.050389 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421564276e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81764 payload_rows_remote=66466 dup_rows=16540 dup_rows_remote=9410 dedup_ratio=0.168254 router_tokens_per_expert=[0, 0, 0, 7979, 59, 2948, 0, 0, 449, 0, 6860, 0, 0, 0, 0, 206, 0, 1452, 37, 0, 0, 0, 0, 0, 1, 3, 0, 0, 6, 154, 0, 5353, 0, 0, 1, 1, 1161, 0, 0, 0, 0, 835, 458, 8192, 1183, 0, 0, 6421, 0, 4002, 0, 20, 0, 0, 0, 332, 0, 0, 451, 324, 194, 70, 0, 0] payload_rows_per_dst=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] dup_rows_per_dst=[14, 0, 2830, 190, 2, 4952, 7130, 1422] combine_rows_per_src=[6000, 1310, 16106, 10844, 2324, 16384, 15298, 13498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90482 payload_rows_remote=76212 dup_rows=7822 dup_rows_remote=5230 dedup_ratio=0.079569 router_tokens_per_expert=[0, 8064, 32, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1644, 1, 7142, 1, 3, 0, 8, 228, 0, 1866, 0, 2220, 0, 8, 0, 1212, 324, 2, 10, 6875, 0, 0, 486, 0, 0, 0, 190, 0, 6747, 0, 0, 167, 0, 0, 0, 0, 3, 6, 16, 0, 2265, 2, 287, 0, 0, 0, 0, 8183, 1159, 0, 0, 0] payload_rows_per_dst=[16138, 14796, 7836, 14270, 16368, 13670, 5086, 2318] dup_rows_per_dst=[56, 2780, 814, 2592, 1350, 158, 72, 0] combine_rows_per_src=[16138, 14796, 7836, 14270, 16368, 13670, 5086, 2318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68212 payload_rows_remote=53470 dup_rows=30092 dup_rows_remote=25080 dedup_ratio=0.306112 router_tokens_per_expert=[0, 8081, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1455, 0, 6598, 0, 1, 0, 8, 176, 0, 2158, 2, 2357, 0, 5, 0, 1236, 473, 3, 8, 7067, 0, 0, 172, 0, 0, 0, 250, 1, 6764, 0, 0, 117, 0, 0, 0, 0, 0, 0, 26, 0, 2479, 0, 305, 0, 0, 0, 0, 8177, 1211, 0, 0, 0] payload_rows_per_dst=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] dup_rows_per_dst=[0, 0, 9362, 26, 0, 102, 5012, 15590] combine_rows_per_src=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73112 payload_rows_remote=56996 dup_rows=25192 dup_rows_remote=14718 dedup_ratio=0.256266 router_tokens_per_expert=[0, 0, 0, 7943, 28, 3169, 0, 0, 364, 0, 6588, 0, 0, 0, 0, 227, 0, 1140, 32, 0, 0, 0, 0, 0, 0, 5, 0, 1, 3, 191, 0, 5137, 1, 0, 9, 0, 859, 2, 0, 1, 3, 819, 645, 8185, 1553, 0, 0, 6592, 0, 4106, 0, 55, 0, 0, 0, 323, 0, 0, 625, 300, 203, 43, 0, 0] payload_rows_per_dst=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] dup_rows_per_dst=[10474, 9186, 4, 192, 220, 4442, 606, 68] combine_rows_per_src=[16116, 15376, 2340, 10482, 3380, 14782, 8362, 2274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=215777280 dispatch_meta_bytes=263320 combine_payload_bytes=215777280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=216040600 ingress_bytes=284983296 link_bytes=284983296 token_bytes=431817880 weight_movement_bytes=69206016 total_nvlink_bytes=501023896 weight_to_token_ratio=0.160267 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072369200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89274 payload_rows_remote=74832 dup_rows=9030 dup_rows_remote=6268 dedup_ratio=0.091858 router_tokens_per_expert=[0, 8066, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 6909, 0, 0, 0, 4, 247, 0, 1898, 0, 1840, 0, 11, 0, 1135, 434, 0, 0, 7002, 0, 0, 333, 0, 0, 0, 153, 0, 7335, 0, 0, 159, 0, 0, 0, 0, 0, 0, 21, 0, 2531, 0, 183, 0, 0, 0, 0, 8188, 1002, 0, 0, 0] payload_rows_per_dst=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] dup_rows_per_dst=[16, 2762, 492, 4466, 972, 252, 70, 0] combine_rows_per_src=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272244736 dispatch_meta_bytes=303504 combine_payload_bytes=272244736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272548240 ingress_bytes=306847744 link_bytes=306847744 token_bytes=544792976 weight_movement_bytes=34603008 total_nvlink_bytes=579395984 weight_to_token_ratio=0.063516 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154643357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82030 payload_rows_remote=67810 dup_rows=16274 dup_rows_remote=16142 dedup_ratio=0.165548 router_tokens_per_expert=[0, 8092, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1554, 0, 7104, 0, 0, 0, 12, 216, 0, 1478, 0, 2174, 0, 7, 0, 1012, 362, 2, 2, 7109, 0, 0, 441, 0, 0, 0, 223, 0, 7031, 0, 0, 145, 0, 0, 0, 0, 0, 0, 19, 0, 2559, 0, 260, 0, 0, 0, 0, 8179, 1148, 0, 0, 0] payload_rows_per_dst=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] dup_rows_per_dst=[44, 384, 588, 12, 596, 132, 5052, 9466] combine_rows_per_src=[15818, 5622, 7540, 2758, 8112, 14220, 14842, 13118] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89530 payload_rows_remote=73180 dup_rows=8774 dup_rows_remote=7508 dedup_ratio=0.089254 router_tokens_per_expert=[0, 8086, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1585, 0, 7013, 0, 0, 0, 0, 311, 0, 1824, 0, 2003, 0, 13, 0, 1278, 554, 2, 5, 6844, 0, 0, 457, 0, 0, 0, 177, 0, 7136, 0, 0, 144, 0, 0, 0, 0, 0, 0, 7, 0, 2301, 0, 189, 0, 0, 0, 0, 8174, 1015, 0, 0, 0] payload_rows_per_dst=[16172, 14528, 7642, 7060, 16350, 14302, 11446, 2030] dup_rows_per_dst=[68, 2668, 634, 868, 1266, 258, 3012, 0] combine_rows_per_src=[16172, 14528, 7642, 7060, 16350, 14302, 11446, 2030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69064 payload_rows_remote=52680 dup_rows=29240 dup_rows_remote=13150 dedup_ratio=0.297445 router_tokens_per_expert=[0, 8059, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1439, 0, 7016, 0, 0, 0, 2, 284, 0, 1334, 0, 2077, 0, 4, 0, 1197, 355, 4, 3, 7202, 0, 0, 299, 0, 0, 0, 245, 1, 7188, 0, 0, 114, 0, 0, 0, 0, 0, 0, 53, 0, 2621, 0, 410, 0, 0, 0, 0, 8184, 1035, 0, 0, 2] payload_rows_per_dst=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] dup_rows_per_dst=[0, 0, 7366, 16, 0, 150, 5618, 16090] combine_rows_per_src=[48, 2878, 16146, 3110, 1090, 14454, 14954, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233455616 dispatch_meta_bytes=286856 combine_payload_bytes=233455616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=233742472 ingress_bytes=337264640 link_bytes=337264640 token_bytes=467198088 weight_movement_bytes=103809024 total_nvlink_bytes=571007112 weight_to_token_ratio=0.222195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269099688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=312164352 dispatch_meta_bytes=325768 combine_payload_bytes=312164352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312490120 ingress_bytes=381370368 link_bytes=381370368 token_bytes=624654472 weight_movement_bytes=69206016 total_nvlink_bytes=693860488 weight_to_token_ratio=0.110791 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435065992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219013120 dispatch_meta_bytes=314200 combine_payload_bytes=219013120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219327320 ingress_bytes=253616128 link_bytes=253616128 token_bytes=438340440 weight_movement_bytes=34603008 total_nvlink_bytes=472943448 weight_to_token_ratio=0.078941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.543370719e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89274 payload_rows_remote=74832 dup_rows=9030 dup_rows_remote=6268 dedup_ratio=0.091858 router_tokens_per_expert=[0, 8066, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 6909, 0, 0, 0, 4, 247, 0, 1898, 0, 1840, 0, 11, 0, 1135, 434, 0, 0, 7002, 0, 0, 333, 0, 0, 0, 153, 0, 7335, 0, 0, 159, 0, 0, 0, 0, 0, 0, 21, 0, 2531, 0, 183, 0, 0, 0, 0, 8188, 1002, 0, 0, 0] payload_rows_per_dst=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] dup_rows_per_dst=[16, 2762, 492, 4466, 972, 252, 70, 0] combine_rows_per_src=[16132, 14442, 7486, 14554, 16376, 12880, 5400, 2004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90260 payload_rows_remote=87108 dup_rows=8044 dup_rows_remote=7288 dedup_ratio=0.081828 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 0, 2602, 0, 3507, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1824, 384, 0, 0, 0, 17, 0, 0, 3865, 1, 6718, 8192, 0, 0, 997, 957, 0, 0, 0, 0, 0, 0, 0, 7790, 0, 0, 0, 265, 0, 511, 0, 0, 791, 0] payload_rows_per_dst=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] dup_rows_per_dst=[0, 760, 0, 3388, 6, 756, 0, 3134] combine_rows_per_src=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299745280 dispatch_meta_bytes=322752 combine_payload_bytes=299745280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300068032 ingress_bytes=334348288 link_bytes=334348288 token_bytes=599813312 weight_movement_bytes=34603008 total_nvlink_bytes=634416320 weight_to_token_ratio=0.057690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=215777280 dispatch_meta_bytes=263320 combine_payload_bytes=215777280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=216040600 ingress_bytes=284983296 link_bytes=284983296 token_bytes=431817880 weight_movement_bytes=69206016 total_nvlink_bytes=501023896 weight_to_token_ratio=0.160267 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072369200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91174 payload_rows_remote=83784 dup_rows=7130 dup_rows_remote=6760 dedup_ratio=0.072530 router_tokens_per_expert=[0, 8041, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1329, 0, 6799, 0, 0, 0, 3, 207, 0, 1920, 0, 1750, 0, 5, 0, 1175, 449, 3, 4, 7208, 0, 0, 288, 1, 0, 0, 237, 0, 6658, 0, 0, 225, 0, 0, 0, 0, 0, 0, 53, 0, 2886, 0, 432, 0, 0, 0, 0, 8176, 1252, 0, 0, 0] payload_rows_per_dst=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] dup_rows_per_dst=[98, 2196, 370, 3096, 1052, 138, 180, 0] combine_rows_per_src=[16086, 14060, 7390, 14592, 16352, 13628, 6562, 2504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=233455616 dispatch_meta_bytes=286856 combine_payload_bytes=233455616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=233742472 ingress_bytes=337264640 link_bytes=337264640 token_bytes=467198088 weight_movement_bytes=103809024 total_nvlink_bytes=571007112 weight_to_token_ratio=0.222195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269099688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306511872 dispatch_meta_bytes=324400 combine_payload_bytes=306511872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306836272 ingress_bytes=375717888 link_bytes=375717888 token_bytes=613348144 weight_movement_bytes=69206016 total_nvlink_bytes=682554160 weight_to_token_ratio=0.112833 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=356794368 dispatch_meta_bytes=377584 combine_payload_bytes=356794368 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=357171952 ingress_bytes=495206400 link_bytes=495206400 token_bytes=713966320 weight_movement_bytes=138412032 total_nvlink_bytes=852378352 weight_to_token_ratio=0.193864 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.863421816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79232 payload_rows_remote=63046 dup_rows=19072 dup_rows_remote=17674 dedup_ratio=0.194010 router_tokens_per_expert=[0, 8084, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 6648, 0, 0, 0, 3, 174, 0, 1833, 0, 2207, 0, 7, 0, 1194, 631, 0, 3, 7308, 0, 2, 355, 0, 0, 0, 245, 0, 6560, 0, 0, 150, 0, 0, 0, 0, 0, 1, 42, 0, 2800, 0, 285, 0, 0, 0, 1, 8185, 1208, 0, 0, 0] payload_rows_per_dst=[16186, 14148, 7900, 15778, 16370, 300, 6134, 2416] dup_rows_per_dst=[1398, 4130, 534, 11682, 1204, 0, 122, 2] combine_rows_per_src=[16186, 14148, 7900, 15778, 16370, 300, 6134, 2416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=343179264 dispatch_meta_bytes=362176 combine_payload_bytes=343179264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343541440 ingress_bytes=377782272 link_bytes=377782272 token_bytes=686720704 weight_movement_bytes=34603008 total_nvlink_bytes=721323712 weight_to_token_ratio=0.050389 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421564276e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86010 payload_rows_remote=69626 dup_rows=12294 dup_rows_remote=5550 dedup_ratio=0.125061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 0, 0, 1, 0, 2739, 0, 3372, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1921, 546, 0, 0, 0, 4, 0, 0, 3946, 2, 6343, 8192, 0, 0, 916, 899, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 274, 0, 509, 0, 0, 708, 0] payload_rows_per_dst=[16384, 10042, 16384, 14120, 7902, 3052, 15596, 2530] dup_rows_per_dst=[0, 1010, 6744, 3508, 2, 578, 0, 452] combine_rows_per_src=[16384, 10042, 16384, 14120, 7902, 3052, 15596, 2530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=335808 combine_payload_bytes=277749760 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=278085568 ingress_bytes=381558784 link_bytes=381558784 token_bytes=555835328 weight_movement_bytes=103809024 total_nvlink_bytes=659644352 weight_to_token_ratio=0.186762 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.435774986e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=299745280 dispatch_meta_bytes=322752 combine_payload_bytes=299745280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300068032 ingress_bytes=334348288 link_bytes=334348288 token_bytes=599813312 weight_movement_bytes=34603008 total_nvlink_bytes=634416320 weight_to_token_ratio=0.057690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70686 payload_rows_remote=54302 dup_rows=27618 dup_rows_remote=25606 dedup_ratio=0.280945 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2084, 0, 0, 1, 0, 2281, 0, 3568, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1899, 350, 0, 0, 0, 15, 0, 0, 4768, 2, 7091, 8191, 0, 0, 894, 1007, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 200, 0, 265, 0, 0, 541, 0] payload_rows_per_dst=[0, 8312, 2, 4440, 9562, 16384, 15602, 16384] dup_rows_per_dst=[0, 420, 0, 60, 8, 25118, 0, 2012] combine_rows_per_src=[0, 8312, 2, 4440, 9562, 16384, 15602, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285188096 dispatch_meta_bytes=300704 combine_payload_bytes=285188096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285488800 ingress_bytes=388997120 link_bytes=388997120 token_bytes=570676896 weight_movement_bytes=103809024 total_nvlink_bytes=674485920 weight_to_token_ratio=0.181905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463764846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68212 payload_rows_remote=53470 dup_rows=30092 dup_rows_remote=25080 dedup_ratio=0.306112 router_tokens_per_expert=[0, 8081, 21, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1455, 0, 6598, 0, 1, 0, 8, 176, 0, 2158, 2, 2357, 0, 5, 0, 1236, 473, 3, 8, 7067, 0, 0, 172, 0, 0, 0, 250, 1, 6764, 0, 0, 117, 0, 0, 0, 0, 0, 0, 26, 0, 2479, 0, 305, 0, 0, 0, 0, 8177, 1211, 0, 0, 0] payload_rows_per_dst=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] dup_rows_per_dst=[0, 0, 9362, 26, 0, 102, 5012, 15590] combine_rows_per_src=[44, 2910, 16204, 3424, 846, 13660, 14742, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89512 payload_rows_remote=75312 dup_rows=8792 dup_rows_remote=5354 dedup_ratio=0.089437 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 2167, 0, 0, 0, 0, 2347, 0, 3934, 8189, 1, 2, 0, 0, 0, 0, 0, 1, 3, 0, 0, 1, 2038, 359, 0, 0, 0, 4, 0, 0, 4079, 0, 6417, 8186, 0, 0, 1211, 988, 3, 0, 0, 0, 0, 0, 2, 7850, 0, 4, 0, 265, 2, 461, 2, 0, 633, 1] payload_rows_per_dst=[16376, 8536, 9392, 14200, 8164, 3672, 15708, 13464] dup_rows_per_dst=[0, 492, 1860, 3438, 2, 732, 4, 2264] combine_rows_per_src=[16376, 8536, 9392, 14200, 8164, 3672, 15708, 13464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78140 payload_rows_remote=62506 dup_rows=20164 dup_rows_remote=20158 dedup_ratio=0.205119 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 2716, 0, 3025, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1682, 403, 0, 0, 0, 4, 0, 0, 5029, 0, 6684, 8189, 0, 0, 1107, 1263, 0, 0, 0, 3, 0, 0, 0, 7817, 0, 0, 0, 246, 0, 175, 0, 0, 734, 0] payload_rows_per_dst=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] dup_rows_per_dst=[0, 600, 0, 1328, 0, 15922, 6, 2308] combine_rows_per_src=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79232 payload_rows_remote=63046 dup_rows=19072 dup_rows_remote=17674 dedup_ratio=0.194010 router_tokens_per_expert=[0, 8084, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1202, 0, 6648, 0, 0, 0, 3, 174, 0, 1833, 0, 2207, 0, 7, 0, 1194, 631, 0, 3, 7308, 0, 2, 355, 0, 0, 0, 245, 0, 6560, 0, 0, 150, 0, 0, 0, 0, 0, 1, 42, 0, 2800, 0, 285, 0, 0, 0, 1, 8185, 1208, 0, 0, 0] payload_rows_per_dst=[16186, 14148, 7900, 15778, 16370, 300, 6134, 2416] dup_rows_per_dst=[1398, 4130, 534, 11682, 1204, 0, 122, 2] combine_rows_per_src=[16186, 14148, 7900, 15778, 16370, 300, 6134, 2416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=222420992 dispatch_meta_bytes=319632 combine_payload_bytes=222420992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=222740624 ingress_bytes=257024000 link_bytes=257024000 token_bytes=445161616 weight_movement_bytes=34603008 total_nvlink_bytes=479764624 weight_to_token_ratio=0.077731 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.671606199e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=258236416 dispatch_meta_bytes=322880 combine_payload_bytes=258236416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258559296 ingress_bytes=327442432 link_bytes=327442432 token_bytes=516795712 weight_movement_bytes=69206016 total_nvlink_bytes=586001728 weight_to_token_ratio=0.133914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232139510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=219013120 dispatch_meta_bytes=314200 combine_payload_bytes=219013120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219327320 ingress_bytes=253616128 link_bytes=253616128 token_bytes=438340440 weight_movement_bytes=34603008 total_nvlink_bytes=472943448 weight_to_token_ratio=0.078941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.543370719e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77820 payload_rows_remote=64438 dup_rows=20484 dup_rows_remote=15208 dedup_ratio=0.208374 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 0, 0, 2485, 0, 3443, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2165, 301, 0, 0, 0, 10, 0, 0, 4630, 0, 6779, 8192, 0, 0, 1064, 923, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 159, 0, 142, 0, 0, 675, 0] payload_rows_per_dst=[16384, 13382, 16384, 7966, 14002, 3230, 4538, 1934] dup_rows_per_dst=[0, 5276, 6886, 1130, 6430, 744, 0, 18] combine_rows_per_src=[16384, 13382, 16384, 7966, 14002, 3230, 4538, 1934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90260 payload_rows_remote=87108 dup_rows=8044 dup_rows_remote=7288 dedup_ratio=0.081828 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 0, 0, 0, 2602, 0, 3507, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1824, 384, 0, 0, 0, 17, 0, 0, 3865, 1, 6718, 8192, 0, 0, 997, 957, 0, 0, 0, 0, 0, 0, 0, 7790, 0, 0, 0, 265, 0, 511, 0, 0, 791, 0] payload_rows_per_dst=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] dup_rows_per_dst=[0, 760, 0, 3388, 6, 756, 0, 3134] combine_rows_per_src=[16384, 9514, 7014, 14472, 7760, 3152, 15580, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91528 payload_rows_remote=80816 dup_rows=6776 dup_rows_remote=6766 dedup_ratio=0.068929 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1903, 0, 0, 1, 0, 2530, 0, 2933, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1818, 238, 0, 0, 0, 16, 0, 0, 5344, 1, 6936, 8191, 0, 0, 1153, 992, 0, 0, 0, 0, 0, 0, 0, 7818, 0, 0, 0, 276, 0, 227, 0, 0, 582, 0] payload_rows_per_dst=[16382, 8364, 5866, 14704, 10712, 3480, 15636, 16384] dup_rows_per_dst=[0, 504, 0, 3282, 10, 810, 0, 2170] combine_rows_per_src=[16382, 8364, 5866, 14704, 10712, 3480, 15636, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70686 payload_rows_remote=54302 dup_rows=27618 dup_rows_remote=25606 dedup_ratio=0.280945 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2084, 0, 0, 1, 0, 2281, 0, 3568, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1899, 350, 0, 0, 0, 15, 0, 0, 4768, 2, 7091, 8191, 0, 0, 894, 1007, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 200, 0, 265, 0, 0, 541, 0] payload_rows_per_dst=[0, 8312, 2, 4440, 9562, 16384, 15602, 16384] dup_rows_per_dst=[0, 420, 0, 60, 8, 25118, 0, 2012] combine_rows_per_src=[0, 8312, 2, 4440, 9562, 16384, 15602, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=308477952 dispatch_meta_bytes=322664 combine_payload_bytes=308477952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308800616 ingress_bytes=343080960 link_bytes=343080960 token_bytes=617278568 weight_movement_bytes=34603008 total_nvlink_bytes=651881576 weight_to_token_ratio=0.056057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290986032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=258236416 dispatch_meta_bytes=322880 combine_payload_bytes=258236416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=258559296 ingress_bytes=327442432 link_bytes=327442432 token_bytes=516795712 weight_movement_bytes=69206016 total_nvlink_bytes=586001728 weight_to_token_ratio=0.133914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232139510e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91966 payload_rows_remote=80542 dup_rows=6338 dup_rows_remote=4500 dedup_ratio=0.064473 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 2, 35, 0, 3004, 3539, 51, 0, 0, 447, 0, 0, 5746, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 5818, 70, 0, 0, 7765, 0, 0, 0, 0, 1235, 0, 0] payload_rows_per_dst=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] dup_rows_per_dst=[0, 3230, 1838, 606, 0, 0, 664, 0] combine_rows_per_src=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263938048 dispatch_meta_bytes=318584 combine_payload_bytes=263938048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264256632 ingress_bytes=298541056 link_bytes=298541056 token_bytes=528194680 weight_movement_bytes=34603008 total_nvlink_bytes=562797688 weight_to_token_ratio=0.065512 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123385959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=356794368 dispatch_meta_bytes=377584 combine_payload_bytes=356794368 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=357171952 ingress_bytes=495206400 link_bytes=495206400 token_bytes=713966320 weight_movement_bytes=138412032 total_nvlink_bytes=852378352 weight_to_token_ratio=0.193864 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.863421816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331022336 dispatch_meta_bytes=350328 combine_payload_bytes=331022336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331372664 ingress_bytes=365625344 link_bytes=365625344 token_bytes=662395000 weight_movement_bytes=34603008 total_nvlink_bytes=696998008 weight_to_token_ratio=0.052239 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375818734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71520 payload_rows_remote=55136 dup_rows=26784 dup_rows_remote=6654 dedup_ratio=0.272461 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 1, 23, 0, 2862, 3422, 82, 0, 1, 284, 0, 0, 5147, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 7426, 0, 0, 1, 0, 0, 0, 0, 2094, 0, 5756, 39, 0, 0, 7861, 0, 0, 0, 0, 2243, 0, 0] payload_rows_per_dst=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] dup_rows_per_dst=[0, 0, 628, 388, 0, 0, 5638, 20130] combine_rows_per_src=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78140 payload_rows_remote=62506 dup_rows=20164 dup_rows_remote=20158 dedup_ratio=0.205119 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1881, 0, 0, 0, 0, 2716, 0, 3025, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 1682, 403, 0, 0, 0, 4, 0, 0, 5029, 0, 6684, 8189, 0, 0, 1107, 1263, 0, 0, 0, 3, 0, 0, 0, 7817, 0, 0, 0, 246, 0, 175, 0, 0, 734, 0] payload_rows_per_dst=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] dup_rows_per_dst=[0, 600, 0, 1328, 0, 15922, 6, 2308] combine_rows_per_src=[14, 8594, 3086, 7988, 10066, 16374, 15634, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=329900032 dispatch_meta_bytes=340168 combine_payload_bytes=329900032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330240200 ingress_bytes=399106048 link_bytes=399106048 token_bytes=660140232 weight_movement_bytes=69206016 total_nvlink_bytes=729346248 weight_to_token_ratio=0.104835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501803928e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=83908 dup_rows=3934 dup_rows_remote=3604 dedup_ratio=0.040019 router_tokens_per_expert=[0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4271, 3, 0, 0, 38, 3, 2833, 3706, 36, 0, 1, 294, 0, 1, 5092, 1, 7, 1, 0, 0, 0, 8147, 1, 0, 3, 0, 0, 0, 0, 7421, 0, 0, 0, 0, 0, 0, 0, 1920, 1, 5583, 38, 0, 0, 7765, 0, 0, 0, 1, 1976, 0, 1] payload_rows_per_dst=[15536, 8554, 11562, 10462, 16296, 14842, 13164, 3954] dup_rows_per_dst=[0, 4, 1672, 330, 6, 0, 1920, 2] combine_rows_per_src=[15536, 8554, 11562, 10462, 16296, 14842, 13164, 3954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=225837056 dispatch_meta_bytes=247160 combine_payload_bytes=225837056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=226084216 ingress_bytes=295043072 link_bytes=295043072 token_bytes=451921272 weight_movement_bytes=69206016 total_nvlink_bytes=521127288 weight_to_token_ratio=0.153137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110223327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=256024576 dispatch_meta_bytes=330656 combine_payload_bytes=256024576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=256355232 ingress_bytes=325230592 link_bytes=325230592 token_bytes=512379808 weight_movement_bytes=69206016 total_nvlink_bytes=581585824 weight_to_token_ratio=0.135068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223816534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77820 payload_rows_remote=64438 dup_rows=20484 dup_rows_remote=15208 dedup_ratio=0.208374 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2146, 0, 0, 0, 0, 2485, 0, 3443, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2165, 301, 0, 0, 0, 10, 0, 0, 4630, 0, 6779, 8192, 0, 0, 1064, 923, 0, 0, 0, 0, 0, 0, 0, 7845, 0, 0, 0, 159, 0, 142, 0, 0, 675, 0] payload_rows_per_dst=[16384, 13382, 16384, 7966, 14002, 3230, 4538, 1934] dup_rows_per_dst=[0, 5276, 6886, 1130, 6430, 744, 0, 18] combine_rows_per_src=[16384, 13382, 16384, 7966, 14002, 3230, 4538, 1934] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73360 payload_rows_remote=56980 dup_rows=24944 dup_rows_remote=24944 dedup_ratio=0.253743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 0, 0, 0, 0, 2433, 0, 3225, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1877, 259, 0, 1, 0, 13, 0, 0, 4930, 0, 6780, 8190, 0, 0, 1167, 983, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 286, 0, 224, 0, 0, 731, 0] payload_rows_per_dst=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] dup_rows_per_dst=[0, 7054, 8302, 62, 8644, 816, 0, 66] combine_rows_per_src=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85124 payload_rows_remote=70334 dup_rows=13180 dup_rows_remote=7656 dedup_ratio=0.134074 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4489, 0, 0, 0, 39, 0, 2970, 3328, 2, 0, 0, 260, 0, 0, 5466, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 0, 1915, 0, 5668, 0, 0, 0, 7741, 0, 0, 0, 0, 1727, 0, 0] payload_rows_per_dst=[15482, 14790, 11098, 11042, 16382, 9046, 3830, 3454] dup_rows_per_dst=[0, 5524, 1580, 410, 5666, 0, 0, 0] combine_rows_per_src=[15482, 14790, 11098, 11042, 16382, 9046, 3830, 3454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92086 payload_rows_remote=77206 dup_rows=6218 dup_rows_remote=6218 dedup_ratio=0.063253 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 0, 0, 35, 0, 2813, 3670, 37, 0, 0, 295, 0, 0, 5491, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 7440, 0, 0, 0, 0, 0, 0, 0, 1842, 0, 5967, 20, 0, 0, 7771, 0, 0, 0, 0, 1557, 0, 0] payload_rows_per_dst=[15542, 8106, 11476, 11170, 5220, 14880, 13744, 11948] dup_rows_per_dst=[0, 0, 1634, 402, 0, 0, 1914, 2268] combine_rows_per_src=[15542, 8106, 11476, 11170, 5220, 14880, 13744, 11948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331022336 dispatch_meta_bytes=350328 combine_payload_bytes=331022336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331372664 ingress_bytes=365625344 link_bytes=365625344 token_bytes=662395000 weight_movement_bytes=34603008 total_nvlink_bytes=696998008 weight_to_token_ratio=0.052239 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375818734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=222420992 dispatch_meta_bytes=319632 combine_payload_bytes=222420992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=222740624 ingress_bytes=257024000 link_bytes=257024000 token_bytes=445161616 weight_movement_bytes=34603008 total_nvlink_bytes=479764624 weight_to_token_ratio=0.077731 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.671606199e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=343687168 dispatch_meta_bytes=350048 combine_payload_bytes=343687168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=344037216 ingress_bytes=378290176 link_bytes=378290176 token_bytes=687724384 weight_movement_bytes=34603008 total_nvlink_bytes=722327392 weight_to_token_ratio=0.050315 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.423475477e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94612 payload_rows_remote=78252 dup_rows=3692 dup_rows_remote=3692 dedup_ratio=0.037557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4647, 0, 0, 1, 28, 0, 2972, 3208, 49, 0, 0, 234, 0, 0, 5069, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 1, 0, 0, 0, 0, 1810, 0, 5844, 11, 0, 0, 7713, 0, 0, 0, 0, 1949, 0, 0] payload_rows_per_dst=[15426, 9294, 10914, 10314, 16360, 14870, 13536, 3898] dup_rows_per_dst=[2, 0, 1602, 292, 0, 2, 1794, 0] combine_rows_per_src=[15426, 9294, 10914, 10314, 16360, 14870, 13536, 3898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86010 payload_rows_remote=69626 dup_rows=12294 dup_rows_remote=5550 dedup_ratio=0.125061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 0, 0, 1, 0, 2739, 0, 3372, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1921, 546, 0, 0, 0, 4, 0, 0, 3946, 2, 6343, 8192, 0, 0, 916, 899, 0, 0, 0, 0, 0, 0, 0, 7798, 0, 0, 0, 274, 0, 509, 0, 0, 708, 0] payload_rows_per_dst=[16384, 10042, 16384, 14120, 7902, 3052, 15596, 2530] dup_rows_per_dst=[0, 1010, 6744, 3508, 2, 578, 0, 452] combine_rows_per_src=[16384, 10042, 16384, 14120, 7902, 3052, 15596, 2530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263938048 dispatch_meta_bytes=318584 combine_payload_bytes=263938048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264256632 ingress_bytes=298541056 link_bytes=298541056 token_bytes=528194680 weight_movement_bytes=34603008 total_nvlink_bytes=562797688 weight_to_token_ratio=0.065512 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123385959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233390080 dispatch_meta_bytes=327696 combine_payload_bytes=233390080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233717776 ingress_bytes=267993088 link_bytes=267993088 token_bytes=467107856 weight_movement_bytes=34603008 total_nvlink_bytes=501710864 weight_to_token_ratio=0.074079 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.008436415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288088064 dispatch_meta_bytes=311960 combine_payload_bytes=288088064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288400024 ingress_bytes=322691072 link_bytes=322691072 token_bytes=576488088 weight_movement_bytes=34603008 total_nvlink_bytes=611091096 weight_to_token_ratio=0.060024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214260525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89928 payload_rows_remote=74818 dup_rows=8376 dup_rows_remote=6388 dedup_ratio=0.085205 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 527, 5, 0, 0, 0, 3693, 0, 0, 0, 7902, 0, 37, 0, 0, 0, 0, 7736, 0, 0, 7325, 0, 0, 0, 0, 0, 0, 0, 837, 91, 0, 0, 4650, 0, 0, 2602, 6936, 178, 0, 0, 160, 0, 0, 0, 0, 0, 5012, 0, 517, 0, 0, 0, 0, 0, 611, 0, 0, 8] payload_rows_per_dst=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] dup_rows_per_dst=[1424, 2796, 1988, 0, 896, 284, 620, 368] combine_rows_per_src=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316235776 dispatch_meta_bytes=333696 combine_payload_bytes=316235776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316569472 ingress_bytes=420044800 link_bytes=420044800 token_bytes=632805248 weight_movement_bytes=103809024 total_nvlink_bytes=736614272 weight_to_token_ratio=0.164046 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580594766e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73360 payload_rows_remote=56980 dup_rows=24944 dup_rows_remote=24944 dedup_ratio=0.253743 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2036, 0, 0, 0, 0, 2433, 0, 3225, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1877, 259, 0, 1, 0, 13, 0, 0, 4930, 0, 6780, 8190, 0, 0, 1167, 983, 0, 0, 0, 0, 0, 0, 0, 7821, 0, 0, 0, 286, 0, 224, 0, 0, 731, 0] payload_rows_per_dst=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] dup_rows_per_dst=[0, 7054, 8302, 62, 8644, 816, 0, 66] combine_rows_per_src=[16380, 15444, 16384, 4218, 15034, 3484, 0, 2416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=83908 dup_rows=3934 dup_rows_remote=3604 dedup_ratio=0.040019 router_tokens_per_expert=[0, 2, 0, 1, 0, 0, 0, 0, 1, 0, 4, 0, 0, 4271, 3, 0, 0, 38, 3, 2833, 3706, 36, 0, 1, 294, 0, 1, 5092, 1, 7, 1, 0, 0, 0, 8147, 1, 0, 3, 0, 0, 0, 0, 7421, 0, 0, 0, 0, 0, 0, 0, 1920, 1, 5583, 38, 0, 0, 7765, 0, 0, 0, 1, 1976, 0, 1] payload_rows_per_dst=[15536, 8554, 11562, 10462, 16296, 14842, 13164, 3954] dup_rows_per_dst=[0, 4, 1672, 330, 6, 0, 1920, 2] combine_rows_per_src=[15536, 8554, 11562, 10462, 16296, 14842, 13164, 3954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285188096 dispatch_meta_bytes=300704 combine_payload_bytes=285188096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285488800 ingress_bytes=388997120 link_bytes=388997120 token_bytes=570676896 weight_movement_bytes=103809024 total_nvlink_bytes=674485920 weight_to_token_ratio=0.181905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463764846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320520192 dispatch_meta_bytes=327776 combine_payload_bytes=320520192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320847968 ingress_bytes=389726208 link_bytes=389726208 token_bytes=641368160 weight_movement_bytes=69206016 total_nvlink_bytes=710574176 weight_to_token_ratio=0.107904 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.466508345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80022 payload_rows_remote=63892 dup_rows=18282 dup_rows_remote=16732 dedup_ratio=0.185974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 236, 515, 10, 0, 0, 0, 3972, 0, 0, 0, 7989, 0, 2, 0, 0, 0, 0, 7706, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 522, 135, 0, 0, 4451, 0, 0, 3339, 6575, 142, 0, 0, 133, 0, 0, 0, 0, 0, 4713, 0, 628, 0, 0, 0, 0, 0, 826, 0, 0, 25] payload_rows_per_dst=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] dup_rows_per_dst=[0, 0, 4, 0, 7026, 2054, 7648, 1550] combine_rows_per_src=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80486 payload_rows_remote=66192 dup_rows=17818 dup_rows_remote=15658 dedup_ratio=0.181254 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4368, 0, 0, 0, 34, 1, 2294, 3283, 71, 0, 0, 367, 0, 0, 5106, 0, 1, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 7446, 0, 0, 2, 0, 0, 1, 0, 1904, 0, 6256, 66, 0, 0, 7838, 0, 0, 1, 0, 1993, 0, 0] payload_rows_per_dst=[5262, 8736, 10374, 10592, 0, 14896, 14294, 16332] dup_rows_per_dst=[0, 0, 992, 356, 0, 0, 2160, 14310] combine_rows_per_src=[5262, 8736, 10374, 10592, 0, 14896, 14294, 16332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=324824 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306779352 ingress_bytes=410263552 link_bytes=410263552 token_bytes=613233880 weight_movement_bytes=103809024 total_nvlink_bytes=717042904 weight_to_token_ratio=0.169281 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543788717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92576 payload_rows_remote=77858 dup_rows=5728 dup_rows_remote=5728 dedup_ratio=0.058268 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 290, 760, 7, 0, 0, 3, 3901, 0, 0, 2, 7925, 0, 32, 2, 0, 0, 0, 7626, 0, 0, 7359, 0, 0, 0, 0, 0, 0, 0, 733, 109, 1, 0, 4503, 0, 0, 2339, 6874, 152, 1, 0, 89, 0, 0, 0, 0, 0, 4840, 0, 780, 0, 0, 4, 0, 0, 798, 0, 2, 19] payload_rows_per_dst=[15668, 7820, 10216, 14718, 10046, 7648, 10412, 16048] dup_rows_per_dst=[1686, 6, 46, 0, 646, 1068, 828, 1448] combine_rows_per_src=[15668, 7820, 10216, 14718, 10046, 7648, 10412, 16048] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94612 payload_rows_remote=78252 dup_rows=3692 dup_rows_remote=3692 dedup_ratio=0.037557 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4647, 0, 0, 1, 28, 0, 2972, 3208, 49, 0, 0, 234, 0, 0, 5069, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 7435, 0, 0, 1, 0, 0, 0, 0, 1810, 0, 5844, 11, 0, 0, 7713, 0, 0, 0, 0, 1949, 0, 0] payload_rows_per_dst=[15426, 9294, 10914, 10314, 16360, 14870, 13536, 3898] dup_rows_per_dst=[2, 0, 1602, 292, 0, 2, 1794, 0] combine_rows_per_src=[15426, 9294, 10914, 10314, 16360, 14870, 13536, 3898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71520 payload_rows_remote=55136 dup_rows=26784 dup_rows_remote=6654 dedup_ratio=0.272461 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 1, 23, 0, 2862, 3422, 82, 0, 1, 284, 0, 0, 5147, 0, 2, 0, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 7426, 0, 0, 1, 0, 0, 0, 0, 2094, 0, 5756, 39, 0, 0, 7861, 0, 0, 0, 0, 2243, 0, 0] payload_rows_per_dst=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] dup_rows_per_dst=[0, 0, 628, 388, 0, 0, 5638, 20130] combine_rows_per_src=[0, 7510, 7492, 10478, 0, 14854, 14802, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=233390080 dispatch_meta_bytes=327696 combine_payload_bytes=233390080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233717776 ingress_bytes=267993088 link_bytes=267993088 token_bytes=467107856 weight_movement_bytes=34603008 total_nvlink_bytes=501710864 weight_to_token_ratio=0.074079 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.008436415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=343687168 dispatch_meta_bytes=350048 combine_payload_bytes=343687168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=344037216 ingress_bytes=378290176 link_bytes=378290176 token_bytes=687724384 weight_movement_bytes=34603008 total_nvlink_bytes=722327392 weight_to_token_ratio=0.050315 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.423475477e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85124 payload_rows_remote=70334 dup_rows=13180 dup_rows_remote=7656 dedup_ratio=0.134074 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4489, 0, 0, 0, 39, 0, 2970, 3328, 2, 0, 0, 260, 0, 0, 5466, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7357, 0, 0, 0, 0, 0, 0, 0, 1915, 0, 5668, 0, 0, 0, 7741, 0, 0, 0, 0, 1727, 0, 0] payload_rows_per_dst=[15482, 14790, 11098, 11042, 16382, 9046, 3830, 3454] dup_rows_per_dst=[0, 5524, 1580, 410, 5666, 0, 0, 0] combine_rows_per_src=[15482, 14790, 11098, 11042, 16382, 9046, 3830, 3454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261701632 dispatch_meta_bytes=322496 combine_payload_bytes=261701632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262024128 ingress_bytes=296304640 link_bytes=296304640 token_bytes=523725760 weight_movement_bytes=34603008 total_nvlink_bytes=558328768 weight_to_token_ratio=0.066071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114970506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79744 payload_rows_remote=64220 dup_rows=18560 dup_rows_remote=18560 dedup_ratio=0.188802 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4019, 0, 0, 5, 61, 0, 3116, 3631, 49, 0, 0, 311, 0, 0, 5266, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 3, 0, 0, 0, 0, 7342, 0, 0, 0, 0, 0, 0, 0, 2016, 0, 5835, 10, 0, 0, 7762, 0, 0, 0, 0, 1577, 0, 0] payload_rows_per_dst=[15524, 14734, 11906, 14086, 16308, 0, 4032, 3154] dup_rows_per_dst=[0, 4982, 2112, 5596, 5850, 0, 20, 0] combine_rows_per_src=[15524, 14734, 11906, 14086, 16308, 0, 4032, 3154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88942 payload_rows_remote=73540 dup_rows=9362 dup_rows_remote=3032 dedup_ratio=0.095235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 302, 578, 8, 0, 0, 0, 4075, 0, 0, 0, 7931, 0, 7, 0, 0, 0, 0, 7813, 0, 0, 7350, 0, 0, 0, 0, 0, 0, 0, 582, 85, 0, 0, 4638, 0, 0, 2875, 6783, 140, 0, 0, 151, 0, 0, 0, 0, 0, 4499, 0, 649, 0, 0, 0, 0, 0, 644, 0, 0, 42] payload_rows_per_dst=[15920, 15402, 15862, 14700, 10026, 6050, 9610, 1372] dup_rows_per_dst=[1466, 6330, 14, 0, 584, 282, 686, 0] combine_rows_per_src=[15920, 15402, 15862, 14700, 10026, 6050, 9610, 1372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=271122432 dispatch_meta_bytes=327400 combine_payload_bytes=271122432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271449832 ingress_bytes=340328448 link_bytes=340328448 token_bytes=542572264 weight_movement_bytes=69206016 total_nvlink_bytes=611778280 weight_to_token_ratio=0.127552 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280628551e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=318906368 dispatch_meta_bytes=334344 combine_payload_bytes=318906368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=319240712 ingress_bytes=388112384 link_bytes=388112384 token_bytes=638147080 weight_movement_bytes=69206016 total_nvlink_bytes=707353096 weight_to_token_ratio=0.108448 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460435655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90438 payload_rows_remote=74862 dup_rows=7866 dup_rows_remote=3168 dedup_ratio=0.080017 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 289, 525, 9, 0, 0, 0, 3934, 0, 0, 0, 7990, 0, 13, 0, 0, 0, 0, 7502, 0, 0, 7488, 0, 0, 0, 0, 0, 0, 0, 743, 134, 0, 0, 4164, 0, 0, 3005, 6822, 178, 0, 0, 132, 0, 0, 0, 0, 0, 4839, 0, 693, 0, 0, 0, 0, 0, 680, 0, 0, 12] payload_rows_per_dst=[7614, 7884, 8478, 14976, 9514, 15576, 10284, 16112] dup_rows_per_dst=[552, 2, 14, 0, 568, 4698, 780, 1252] combine_rows_per_src=[7614, 7884, 8478, 14976, 9514, 15576, 10284, 16112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320520192 dispatch_meta_bytes=327776 combine_payload_bytes=320520192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320847968 ingress_bytes=389726208 link_bytes=389726208 token_bytes=641368160 weight_movement_bytes=69206016 total_nvlink_bytes=710574176 weight_to_token_ratio=0.107904 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.466508345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=225837056 dispatch_meta_bytes=247160 combine_payload_bytes=225837056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=226084216 ingress_bytes=295043072 link_bytes=295043072 token_bytes=451921272 weight_movement_bytes=69206016 total_nvlink_bytes=521127288 weight_to_token_ratio=0.153137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110223327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89794 payload_rows_remote=79928 dup_rows=8510 dup_rows_remote=8116 dedup_ratio=0.086568 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 235, 663, 6, 0, 0, 0, 4532, 0, 0, 0, 7974, 0, 6, 0, 0, 0, 0, 7684, 0, 0, 7248, 0, 0, 0, 1, 1, 0, 0, 431, 109, 0, 0, 4590, 0, 0, 2976, 6631, 130, 0, 0, 102, 0, 0, 0, 0, 0, 4312, 0, 616, 0, 0, 0, 0, 0, 865, 0, 0, 40] payload_rows_per_dst=[15700, 9074, 12, 14498, 9866, 15310, 9242, 16092] dup_rows_per_dst=[1464, 2, 0, 2, 394, 4368, 614, 1666] combine_rows_per_src=[15700, 9074, 12, 14498, 9866, 15310, 9242, 16092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91966 payload_rows_remote=80542 dup_rows=6338 dup_rows_remote=4500 dedup_ratio=0.064473 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3971, 0, 0, 2, 35, 0, 3004, 3539, 51, 0, 0, 447, 0, 0, 5746, 0, 0, 0, 0, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 7457, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 5818, 70, 0, 0, 7765, 0, 0, 0, 0, 1235, 0, 0] payload_rows_per_dst=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] dup_rows_per_dst=[0, 3230, 1838, 606, 0, 0, 664, 0] combine_rows_per_src=[15530, 12840, 11424, 11780, 16216, 14914, 6792, 2470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288088064 dispatch_meta_bytes=311960 combine_payload_bytes=288088064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288400024 ingress_bytes=322691072 link_bytes=322691072 token_bytes=576488088 weight_movement_bytes=34603008 total_nvlink_bytes=611091096 weight_to_token_ratio=0.060024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214260525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=263045120 dispatch_meta_bytes=331120 combine_payload_bytes=263045120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263376240 ingress_bytes=297648128 link_bytes=297648128 token_bytes=526421360 weight_movement_bytes=34603008 total_nvlink_bytes=561024368 weight_to_token_ratio=0.065733 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120025943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301219840 dispatch_meta_bytes=306288 combine_payload_bytes=301219840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301526128 ingress_bytes=335822848 link_bytes=335822848 token_bytes=602745968 weight_movement_bytes=34603008 total_nvlink_bytes=637348976 weight_to_token_ratio=0.057409 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306634752 dispatch_meta_bytes=312120 combine_payload_bytes=306634752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=306946872 ingress_bytes=445046784 link_bytes=445046784 token_bytes=613581624 weight_movement_bytes=138412032 total_nvlink_bytes=751993656 weight_to_token_ratio=0.225580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674675219e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79744 payload_rows_remote=64220 dup_rows=18560 dup_rows_remote=18560 dedup_ratio=0.188802 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 4019, 0, 0, 5, 61, 0, 3116, 3631, 49, 0, 0, 311, 0, 0, 5266, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 3, 0, 0, 0, 0, 7342, 0, 0, 0, 0, 0, 0, 0, 2016, 0, 5835, 10, 0, 0, 7762, 0, 0, 0, 0, 1577, 0, 0] payload_rows_per_dst=[15524, 14734, 11906, 14086, 16308, 0, 4032, 3154] dup_rows_per_dst=[0, 4982, 2112, 5596, 5850, 0, 20, 0] combine_rows_per_src=[15524, 14734, 11906, 14086, 16308, 0, 4032, 3154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92576 payload_rows_remote=77858 dup_rows=5728 dup_rows_remote=5728 dedup_ratio=0.058268 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 290, 760, 7, 0, 0, 3, 3901, 0, 0, 2, 7925, 0, 32, 2, 0, 0, 0, 7626, 0, 0, 7359, 0, 0, 0, 0, 0, 0, 0, 733, 109, 1, 0, 4503, 0, 0, 2339, 6874, 152, 1, 0, 89, 0, 0, 0, 0, 0, 4840, 0, 780, 0, 0, 4, 0, 0, 798, 0, 2, 19] payload_rows_per_dst=[15668, 7820, 10216, 14718, 10046, 7648, 10412, 16048] dup_rows_per_dst=[1686, 6, 46, 0, 646, 1068, 828, 1448] combine_rows_per_src=[15668, 7820, 10216, 14718, 10046, 7648, 10412, 16048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=329900032 dispatch_meta_bytes=340168 combine_payload_bytes=329900032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330240200 ingress_bytes=399106048 link_bytes=399106048 token_bytes=660140232 weight_movement_bytes=69206016 total_nvlink_bytes=729346248 weight_to_token_ratio=0.104835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501803928e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97484 payload_rows_remote=83840 dup_rows=820 dup_rows_remote=126 dedup_ratio=0.008341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 8192, 0, 5821, 0, 458, 0, 0, 0, 13, 0, 0, 0, 0, 177, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5361, 0, 0, 4676, 0, 0, 6698, 0, 0, 0, 0, 0, 0, 7648, 0, 0, 0, 2, 3257, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] dup_rows_per_dst=[0, 0, 694, 122, 0, 0, 4, 0] combine_rows_per_src=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=327385088 dispatch_meta_bytes=352176 combine_payload_bytes=327385088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327737264 ingress_bytes=361988096 link_bytes=361988096 token_bytes=655122352 weight_movement_bytes=34603008 total_nvlink_bytes=689725360 weight_to_token_ratio=0.052819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362132063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86606 payload_rows_remote=72122 dup_rows=11698 dup_rows_remote=8700 dedup_ratio=0.118998 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5771, 0, 0, 0, 8190, 0, 5968, 0, 477, 0, 0, 0, 17, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5488, 0, 0, 5755, 0, 0, 6783, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 1, 2769, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] dup_rows_per_dst=[0, 0, 80, 42, 0, 5556, 3022, 2998] combine_rows_per_src=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87782 payload_rows_remote=76692 dup_rows=10522 dup_rows_remote=9438 dedup_ratio=0.107035 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 297, 552, 13, 0, 0, 0, 3427, 0, 0, 0, 7959, 0, 2, 0, 0, 0, 0, 7585, 0, 1, 7326, 0, 0, 0, 0, 0, 0, 0, 743, 123, 0, 0, 4122, 0, 0, 2873, 6848, 130, 0, 0, 97, 1, 0, 0, 0, 0, 5029, 0, 1058, 0, 0, 0, 0, 0, 945, 0, 0, 20] payload_rows_per_dst=[1694, 6872, 15174, 9738, 11564, 15540, 11090, 16110] dup_rows_per_dst=[6, 8, 0, 2, 3326, 4358, 1084, 1738] combine_rows_per_src=[1694, 6872, 15174, 9738, 11564, 15540, 11090, 16110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95962 payload_rows_remote=82046 dup_rows=2342 dup_rows_remote=2250 dedup_ratio=0.023824 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 0, 0, 5, 6870, 0, 0, 0, 8188, 0, 5996, 0, 395, 0, 0, 0, 27, 0, 0, 0, 0, 127, 0, 0, 0, 5, 2, 2, 0, 0, 0, 2, 0, 5290, 0, 0, 5704, 0, 0, 6234, 0, 0, 2, 0, 0, 0, 7713, 3, 0, 0, 12, 2571, 0, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[16378, 5350, 12688, 13916, 10584, 11412, 15440, 10194] dup_rows_per_dst=[2, 6, 624, 92, 4, 0, 16, 1598] combine_rows_per_src=[16378, 5350, 12688, 13916, 10584, 11412, 15440, 10194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343408640 dispatch_meta_bytes=335864 combine_payload_bytes=343408640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343744504 ingress_bytes=378011648 link_bytes=378011648 token_bytes=687153144 weight_movement_bytes=34603008 total_nvlink_bytes=721756152 weight_to_token_ratio=0.050357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422427399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89794 payload_rows_remote=79928 dup_rows=8510 dup_rows_remote=8116 dedup_ratio=0.086568 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 235, 663, 6, 0, 0, 0, 4532, 0, 0, 0, 7974, 0, 6, 0, 0, 0, 0, 7684, 0, 0, 7248, 0, 0, 0, 1, 1, 0, 0, 431, 109, 0, 0, 4590, 0, 0, 2976, 6631, 130, 0, 0, 102, 0, 0, 0, 0, 0, 4312, 0, 616, 0, 0, 0, 0, 0, 865, 0, 0, 40] payload_rows_per_dst=[15700, 9074, 12, 14498, 9866, 15310, 9242, 16092] dup_rows_per_dst=[1464, 2, 0, 2, 394, 4368, 614, 1666] combine_rows_per_src=[15700, 9074, 12, 14498, 9866, 15310, 9242, 16092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=263045120 dispatch_meta_bytes=331120 combine_payload_bytes=263045120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263376240 ingress_bytes=297648128 link_bytes=297648128 token_bytes=526421360 weight_movement_bytes=34603008 total_nvlink_bytes=561024368 weight_to_token_ratio=0.065733 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120025943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=318906368 dispatch_meta_bytes=334344 combine_payload_bytes=318906368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=319240712 ingress_bytes=388112384 link_bytes=388112384 token_bytes=638147080 weight_movement_bytes=69206016 total_nvlink_bytes=707353096 weight_to_token_ratio=0.108448 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460435655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80022 payload_rows_remote=63892 dup_rows=18282 dup_rows_remote=16732 dedup_ratio=0.185974 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 236, 515, 10, 0, 0, 0, 3972, 0, 0, 0, 7989, 0, 2, 0, 0, 0, 0, 7706, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 0, 522, 135, 0, 0, 4451, 0, 0, 3339, 6575, 142, 0, 0, 133, 0, 0, 0, 0, 0, 4713, 0, 628, 0, 0, 0, 0, 0, 826, 0, 0, 25] payload_rows_per_dst=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] dup_rows_per_dst=[0, 0, 4, 0, 7026, 2054, 7648, 1550] combine_rows_per_src=[1502, 7964, 15412, 0, 14680, 9816, 14518, 16130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88942 payload_rows_remote=73540 dup_rows=9362 dup_rows_remote=3032 dedup_ratio=0.095235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 302, 578, 8, 0, 0, 0, 4075, 0, 0, 0, 7931, 0, 7, 0, 0, 0, 0, 7813, 0, 0, 7350, 0, 0, 0, 0, 0, 0, 0, 582, 85, 0, 0, 4638, 0, 0, 2875, 6783, 140, 0, 0, 151, 0, 0, 0, 0, 0, 4499, 0, 649, 0, 0, 0, 0, 0, 644, 0, 0, 42] payload_rows_per_dst=[15920, 15402, 15862, 14700, 10026, 6050, 9610, 1372] dup_rows_per_dst=[1466, 6330, 14, 0, 584, 282, 686, 0] combine_rows_per_src=[15920, 15402, 15862, 14700, 10026, 6050, 9610, 1372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=295411712 dispatch_meta_bytes=323288 combine_payload_bytes=295411712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295735000 ingress_bytes=330014720 link_bytes=330014720 token_bytes=591146712 weight_movement_bytes=34603008 total_nvlink_bytes=625749720 weight_to_token_ratio=0.058535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241818823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90310 payload_rows_remote=78066 dup_rows=7994 dup_rows_remote=7994 dedup_ratio=0.081319 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6979, 0, 0, 0, 8192, 0, 6122, 0, 360, 0, 0, 0, 7, 0, 0, 0, 0, 91, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 5678, 0, 0, 6322, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 2, 2605, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12244, 12866, 14082, 15272, 11356, 2896, 5210] dup_rows_per_dst=[0, 0, 512, 60, 7420, 0, 2, 0] combine_rows_per_src=[16384, 12244, 12866, 14082, 15272, 11356, 2896, 5210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88142 payload_rows_remote=72242 dup_rows=10162 dup_rows_remote=8206 dedup_ratio=0.103373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 434, 688, 12, 0, 0, 0, 4370, 0, 0, 0, 7967, 0, 20, 0, 0, 0, 0, 7806, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 471, 110, 0, 0, 3873, 0, 0, 2436, 6849, 275, 0, 0, 117, 0, 0, 0, 0, 0, 4693, 0, 532, 0, 0, 0, 0, 0, 971, 0, 0, 18] payload_rows_per_dst=[15900, 15606, 15934, 15020, 8564, 5318, 9822, 1978] dup_rows_per_dst=[1956, 6856, 40, 0, 344, 338, 628, 0] combine_rows_per_src=[15900, 15606, 15934, 15020, 8564, 5318, 9822, 1978] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=314130432 dispatch_meta_bytes=344520 combine_payload_bytes=314130432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314474952 ingress_bytes=348733440 link_bytes=348733440 token_bytes=628605384 weight_movement_bytes=34603008 total_nvlink_bytes=663208392 weight_to_token_ratio=0.055047 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312255859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=336060416 dispatch_meta_bytes=337184 combine_payload_bytes=336060416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336397600 ingress_bytes=370663424 link_bytes=370663424 token_bytes=672458016 weight_movement_bytes=34603008 total_nvlink_bytes=707061024 weight_to_token_ratio=0.051457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394776624e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94712 payload_rows_remote=83754 dup_rows=3592 dup_rows_remote=3592 dedup_ratio=0.036540 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6835, 0, 0, 0, 8192, 0, 5660, 0, 399, 0, 0, 0, 12, 0, 0, 0, 0, 134, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5354, 0, 0, 5479, 0, 0, 6607, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 2, 2850, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 13352, 13842, 10708, 10958, 15254, 14214] dup_rows_per_dst=[0, 0, 684, 98, 0, 0, 4, 2806] combine_rows_per_src=[16384, 0, 13352, 13842, 10708, 10958, 15254, 14214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=327385088 dispatch_meta_bytes=352176 combine_payload_bytes=327385088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327737264 ingress_bytes=361988096 link_bytes=361988096 token_bytes=655122352 weight_movement_bytes=34603008 total_nvlink_bytes=689725360 weight_to_token_ratio=0.052819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362132063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94672 payload_rows_remote=84880 dup_rows=3632 dup_rows_remote=3632 dedup_ratio=0.036947 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 8192, 0, 5972, 0, 412, 0, 0, 0, 6, 0, 0, 0, 0, 130, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 4896, 0, 0, 5851, 0, 0, 6335, 0, 0, 0, 0, 0, 0, 7642, 0, 0, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 12862, 14200, 9792, 11702, 15284, 14448] dup_rows_per_dst=[0, 0, 644, 130, 0, 0, 0, 2858] combine_rows_per_src=[16384, 0, 12862, 14200, 9792, 11702, 15284, 14448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=261701632 dispatch_meta_bytes=322496 combine_payload_bytes=261701632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262024128 ingress_bytes=296304640 link_bytes=296304640 token_bytes=523725760 weight_movement_bytes=34603008 total_nvlink_bytes=558328768 weight_to_token_ratio=0.066071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114970506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89928 payload_rows_remote=74818 dup_rows=8376 dup_rows_remote=6388 dedup_ratio=0.085205 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 527, 5, 0, 0, 0, 3693, 0, 0, 0, 7902, 0, 37, 0, 0, 0, 0, 7736, 0, 0, 7325, 0, 0, 0, 0, 0, 0, 0, 837, 91, 0, 0, 4650, 0, 0, 2602, 6936, 178, 0, 0, 160, 0, 0, 0, 0, 0, 5012, 0, 517, 0, 0, 0, 0, 0, 611, 0, 0, 8] payload_rows_per_dst=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] dup_rows_per_dst=[1424, 2796, 1988, 0, 896, 284, 620, 368] combine_rows_per_src=[15752, 11682, 15110, 14650, 10260, 5596, 10438, 6440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301219840 dispatch_meta_bytes=306288 combine_payload_bytes=301219840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301526128 ingress_bytes=335822848 link_bytes=335822848 token_bytes=602745968 weight_movement_bytes=34603008 total_nvlink_bytes=637348976 weight_to_token_ratio=0.057409 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=319758336 dispatch_meta_bytes=344240 combine_payload_bytes=319758336 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=320102576 ingress_bytes=458170368 link_bytes=458170368 token_bytes=639860912 weight_movement_bytes=138412032 total_nvlink_bytes=778272944 weight_to_token_ratio=0.216316 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724058209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=321792 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296225024 ingress_bytes=330506240 link_bytes=330506240 token_bytes=592128256 weight_movement_bytes=34603008 total_nvlink_bytes=626731264 weight_to_token_ratio=0.058438 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243668373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343056384 dispatch_meta_bytes=349384 combine_payload_bytes=343056384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=343405768 ingress_bytes=412262400 link_bytes=412262400 token_bytes=686462152 weight_movement_bytes=69206016 total_nvlink_bytes=755668168 weight_to_token_ratio=0.100815 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.551310221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95962 payload_rows_remote=82046 dup_rows=2342 dup_rows_remote=2250 dedup_ratio=0.023824 router_tokens_per_expert=[1, 1, 0, 0, 0, 0, 0, 0, 5, 6870, 0, 0, 0, 8188, 0, 5996, 0, 395, 0, 0, 0, 27, 0, 0, 0, 0, 127, 0, 0, 0, 5, 2, 2, 0, 0, 0, 2, 0, 5290, 0, 0, 5704, 0, 0, 6234, 0, 0, 2, 0, 0, 0, 7713, 3, 0, 0, 12, 2571, 0, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[16378, 5350, 12688, 13916, 10584, 11412, 15440, 10194] dup_rows_per_dst=[2, 6, 624, 92, 4, 0, 16, 1598] combine_rows_per_src=[16378, 5350, 12688, 13916, 10584, 11412, 15440, 10194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88142 payload_rows_remote=72242 dup_rows=10162 dup_rows_remote=8206 dedup_ratio=0.103373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 434, 688, 12, 0, 0, 0, 4370, 0, 0, 0, 7967, 0, 20, 0, 0, 0, 0, 7806, 0, 0, 7510, 0, 0, 0, 0, 0, 0, 0, 471, 110, 0, 0, 3873, 0, 0, 2436, 6849, 275, 0, 0, 117, 0, 0, 0, 0, 0, 4693, 0, 532, 0, 0, 0, 0, 0, 971, 0, 0, 18] payload_rows_per_dst=[15900, 15606, 15934, 15020, 8564, 5318, 9822, 1978] dup_rows_per_dst=[1956, 6856, 40, 0, 344, 338, 628, 0] combine_rows_per_src=[15900, 15606, 15934, 15020, 8564, 5318, 9822, 1978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=324824 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306779352 ingress_bytes=410263552 link_bytes=410263552 token_bytes=613233880 weight_movement_bytes=103809024 total_nvlink_bytes=717042904 weight_to_token_ratio=0.169281 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543788717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95638 payload_rows_remote=82918 dup_rows=2666 dup_rows_remote=2666 dedup_ratio=0.027120 router_tokens_per_expert=[0, 0, 3765, 0, 0, 0, 1891, 11, 0, 7741, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 59, 0, 0, 0, 609, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5704, 6380, 7556, 6360] payload_rows_per_dst=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] dup_rows_per_dst=[1324, 268, 0, 354, 0, 720, 0, 0] combine_rows_per_src=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=347668480 dispatch_meta_bytes=354048 combine_payload_bytes=347668480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348022528 ingress_bytes=382271488 link_bytes=382271488 token_bytes=695691008 weight_movement_bytes=34603008 total_nvlink_bytes=730294016 weight_to_token_ratio=0.049739 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.438456834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51400 payload_rows_remote=35016 dup_rows=46904 dup_rows_remote=1078 dedup_ratio=0.477132 router_tokens_per_expert=[0, 0, 3407, 0, 0, 0, 1726, 10, 0, 7748, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 101, 1, 0, 0, 755, 33, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5415, 6695, 7674, 6550] payload_rows_per_dst=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] dup_rows_per_dst=[870, 104, 0, 98, 0, 6, 0, 45826] combine_rows_per_src=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96136 payload_rows_remote=83174 dup_rows=2168 dup_rows_remote=1722 dedup_ratio=0.022054 router_tokens_per_expert=[0, 0, 3368, 0, 0, 0, 1519, 11, 0, 7636, 1, 0, 0, 1, 0, 127, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 723, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 2, 64, 0, 0, 4, 493, 19, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5972, 6166, 7777, 7074] payload_rows_per_dst=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] dup_rows_per_dst=[940, 246, 0, 446, 0, 536, 0, 0] combine_rows_per_src=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94576 payload_rows_remote=79180 dup_rows=3728 dup_rows_remote=3726 dedup_ratio=0.037923 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6803, 0, 0, 0, 8192, 0, 5535, 0, 351, 0, 0, 1, 50, 0, 0, 0, 0, 175, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 5044, 0, 0, 5659, 0, 0, 6571, 0, 0, 0, 0, 0, 0, 7695, 0, 0, 0, 4, 3065, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 16384, 13354, 13802, 10088, 11318, 15396, 14234] dup_rows_per_dst=[0, 0, 592, 168, 0, 0, 2, 2966] combine_rows_per_src=[0, 16384, 13354, 13802, 10088, 11318, 15396, 14234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=336060416 dispatch_meta_bytes=337184 combine_payload_bytes=336060416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=336397600 ingress_bytes=370663424 link_bytes=370663424 token_bytes=672458016 weight_movement_bytes=34603008 total_nvlink_bytes=707061024 weight_to_token_ratio=0.051457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394776624e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94672 payload_rows_remote=84880 dup_rows=3632 dup_rows_remote=3632 dedup_ratio=0.036947 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7027, 0, 0, 0, 8192, 0, 5972, 0, 412, 0, 0, 0, 6, 0, 0, 0, 0, 130, 0, 0, 0, 5, 3, 0, 0, 0, 0, 0, 0, 4896, 0, 0, 5851, 0, 0, 6335, 0, 0, 0, 0, 0, 0, 7642, 0, 0, 0, 0, 2681, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 12862, 14200, 9792, 11702, 15284, 14448] dup_rows_per_dst=[0, 0, 644, 130, 0, 0, 0, 2858] combine_rows_per_src=[16384, 0, 12862, 14200, 9792, 11702, 15284, 14448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=339632128 dispatch_meta_bytes=342336 combine_payload_bytes=339632128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=339974464 ingress_bytes=374235136 link_bytes=374235136 token_bytes=679606592 weight_movement_bytes=34603008 total_nvlink_bytes=714209600 weight_to_token_ratio=0.050916 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=321792 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296225024 ingress_bytes=330506240 link_bytes=330506240 token_bytes=592128256 weight_movement_bytes=34603008 total_nvlink_bytes=626731264 weight_to_token_ratio=0.058438 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.243668373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86606 payload_rows_remote=72122 dup_rows=11698 dup_rows_remote=8700 dedup_ratio=0.118998 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5771, 0, 0, 0, 8190, 0, 5968, 0, 477, 0, 0, 0, 17, 0, 0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5488, 0, 0, 5755, 0, 0, 6783, 0, 0, 0, 0, 0, 0, 7749, 0, 0, 0, 1, 2769, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] dup_rows_per_dst=[0, 0, 80, 42, 0, 5556, 3022, 2998] combine_rows_per_src=[0, 16380, 2338, 11860, 10976, 14880, 15688, 14484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90310 payload_rows_remote=78066 dup_rows=7994 dup_rows_remote=7994 dedup_ratio=0.081319 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6979, 0, 0, 0, 8192, 0, 6122, 0, 360, 0, 0, 0, 7, 0, 0, 0, 0, 91, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5122, 0, 0, 5678, 0, 0, 6322, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 2, 2605, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12244, 12866, 14082, 15272, 11356, 2896, 5210] dup_rows_per_dst=[0, 0, 512, 60, 7420, 0, 2, 0] combine_rows_per_src=[16384, 12244, 12866, 14082, 15272, 11356, 2896, 5210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=82446 dup_rows=2148 dup_rows_remote=1394 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 3366, 0, 0, 0, 1393, 3, 0, 7722, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 38, 0, 0, 0, 690, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5823, 6467, 7736, 6810] payload_rows_per_dst=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] dup_rows_per_dst=[710, 254, 0, 430, 0, 754, 0, 0] combine_rows_per_src=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=143425536 dispatch_meta_bytes=144376 combine_payload_bytes=143425536 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=143569912 ingress_bytes=316440576 link_bytes=316440576 token_bytes=286995448 weight_movement_bytes=173015040 total_nvlink_bytes=460010488 weight_to_token_ratio=0.602849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190740412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95900 payload_rows_remote=80838 dup_rows=2404 dup_rows_remote=2182 dedup_ratio=0.024455 router_tokens_per_expert=[0, 0, 3587, 0, 0, 0, 1391, 0, 0, 7692, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 59, 0, 0, 0, 540, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 6356, 7586, 6799] payload_rows_per_dst=[9186, 15062, 13598, 13198, 16376, 13308, 15172, 0] dup_rows_per_dst=[1096, 222, 0, 458, 0, 628, 0, 0] combine_rows_per_src=[9186, 15062, 13598, 13198, 16376, 13308, 15172, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=324321280 dispatch_meta_bytes=331624 combine_payload_bytes=324321280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324652904 ingress_bytes=428130304 link_bytes=428130304 token_bytes=648974184 weight_movement_bytes=103809024 total_nvlink_bytes=752783208 weight_to_token_ratio=0.159959 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611019867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=347668480 dispatch_meta_bytes=354048 combine_payload_bytes=347668480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348022528 ingress_bytes=382271488 link_bytes=382271488 token_bytes=695691008 weight_movement_bytes=34603008 total_nvlink_bytes=730294016 weight_to_token_ratio=0.049739 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.438456834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=295411712 dispatch_meta_bytes=323288 combine_payload_bytes=295411712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295735000 ingress_bytes=330014720 link_bytes=330014720 token_bytes=591146712 weight_movement_bytes=34603008 total_nvlink_bytes=625749720 weight_to_token_ratio=0.058535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241818823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=319758336 dispatch_meta_bytes=344240 combine_payload_bytes=319758336 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=320102576 ingress_bytes=458170368 link_bytes=458170368 token_bytes=639860912 weight_movement_bytes=138412032 total_nvlink_bytes=778272944 weight_to_token_ratio=0.216316 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724058209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97484 payload_rows_remote=83840 dup_rows=820 dup_rows_remote=126 dedup_ratio=0.008341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 6846, 0, 0, 0, 8192, 0, 5821, 0, 458, 0, 0, 0, 13, 0, 0, 0, 0, 177, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 5361, 0, 0, 4676, 0, 0, 6698, 0, 0, 0, 0, 0, 0, 7648, 0, 0, 0, 2, 3257, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] dup_rows_per_dst=[0, 0, 694, 122, 0, 0, 4, 0] combine_rows_per_src=[16384, 11642, 13644, 13930, 10722, 9352, 15296, 6514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=337698816 dispatch_meta_bytes=335360 combine_payload_bytes=337698816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338034176 ingress_bytes=372301824 link_bytes=372301824 token_bytes=675732992 weight_movement_bytes=34603008 total_nvlink_bytes=710336000 weight_to_token_ratio=0.051208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.400941791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96136 payload_rows_remote=83174 dup_rows=2168 dup_rows_remote=1722 dedup_ratio=0.022054 router_tokens_per_expert=[0, 0, 3368, 0, 0, 0, 1519, 11, 0, 7636, 1, 0, 0, 1, 0, 127, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1, 723, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 2, 64, 0, 0, 4, 493, 19, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 5972, 6166, 7777, 7074] payload_rows_per_dst=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] dup_rows_per_dst=[940, 246, 0, 446, 0, 536, 0, 0] combine_rows_per_src=[8856, 15284, 14150, 12962, 16362, 12960, 15558, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=340680704 dispatch_meta_bytes=339584 combine_payload_bytes=340680704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341020288 ingress_bytes=375283712 link_bytes=375283712 token_bytes=681700992 weight_movement_bytes=34603008 total_nvlink_bytes=716304000 weight_to_token_ratio=0.050760 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412162395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75594 payload_rows_remote=59210 dup_rows=22710 dup_rows_remote=22706 dedup_ratio=0.231018 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 6707, 0, 0, 0, 8191, 0, 5905, 0, 346, 0, 0, 0, 38, 1, 0, 0, 0, 163, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5061, 0, 1, 5884, 0, 0, 6300, 0, 0, 0, 0, 0, 0, 7660, 0, 0, 0, 0, 2890, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15788, 12762, 1950, 11160, 11770, 0, 5780] dup_rows_per_dst=[4, 20510, 608, 10, 1578, 0, 0, 0] combine_rows_per_src=[16384, 15788, 12762, 1950, 11160, 11770, 0, 5780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=343408640 dispatch_meta_bytes=335864 combine_payload_bytes=343408640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=343744504 ingress_bytes=378011648 link_bytes=378011648 token_bytes=687153144 weight_movement_bytes=34603008 total_nvlink_bytes=721756152 weight_to_token_ratio=0.050357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422427399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=80134 dup_rows=11366 dup_rows_remote=11366 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 4153, 0, 0, 0, 1541, 3, 0, 7644, 0, 0, 1, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 629, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 2, 1, 63, 0, 0, 0, 624, 31, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6270, 5906, 7582, 6364] payload_rows_per_dst=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] dup_rows_per_dst=[1110, 286, 0, 394, 0, 676, 0, 8900] combine_rows_per_src=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=82446 dup_rows=2148 dup_rows_remote=1394 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 3366, 0, 0, 0, 1393, 3, 0, 7722, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 753, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 38, 0, 0, 0, 690, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5823, 6467, 7736, 6810] payload_rows_per_dst=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] dup_rows_per_dst=[710, 254, 0, 430, 0, 754, 0, 0] combine_rows_per_src=[8814, 15446, 13620, 12722, 5556, 13710, 15472, 10816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96270 payload_rows_remote=79900 dup_rows=2034 dup_rows_remote=2034 dedup_ratio=0.020691 router_tokens_per_expert=[0, 0, 3774, 0, 0, 0, 1507, 1, 0, 7558, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 52, 0, 0, 0, 554, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6222, 6161, 7615, 6770] payload_rows_per_dst=[9632, 15122, 13540, 13342, 16370, 13034, 15230, 0] dup_rows_per_dst=[932, 202, 0, 372, 0, 528, 0, 0] combine_rows_per_src=[9632, 15122, 13540, 13342, 16370, 13034, 15230, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242524160 dispatch_meta_bytes=327664 combine_payload_bytes=242524160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242851824 ingress_bytes=277127168 link_bytes=277127168 token_bytes=485375984 weight_movement_bytes=34603008 total_nvlink_bytes=519978992 weight_to_token_ratio=0.071291 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.042807223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=340680704 dispatch_meta_bytes=339584 combine_payload_bytes=340680704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341020288 ingress_bytes=375283712 link_bytes=375283712 token_bytes=681700992 weight_movement_bytes=34603008 total_nvlink_bytes=716304000 weight_to_token_ratio=0.050760 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412162395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51400 payload_rows_remote=35016 dup_rows=46904 dup_rows_remote=1078 dedup_ratio=0.477132 router_tokens_per_expert=[0, 0, 3407, 0, 0, 0, 1726, 10, 0, 7748, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 706, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 101, 1, 0, 0, 755, 33, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5415, 6695, 7674, 6550] payload_rows_per_dst=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] dup_rows_per_dst=[870, 104, 0, 98, 0, 6, 0, 45826] combine_rows_per_src=[9416, 4772, 4028, 4122, 10900, 1774, 4, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95900 payload_rows_remote=80838 dup_rows=2404 dup_rows_remote=2182 dedup_ratio=0.024455 router_tokens_per_expert=[0, 0, 3587, 0, 0, 0, 1391, 0, 0, 7692, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 725, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 59, 0, 0, 0, 540, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6103, 6356, 7586, 6799] payload_rows_per_dst=[9186, 15062, 13598, 13198, 16376, 13308, 15172, 0] dup_rows_per_dst=[1096, 222, 0, 458, 0, 628, 0, 0] combine_rows_per_src=[9186, 15062, 13598, 13198, 16376, 13308, 15172, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96010 payload_rows_remote=79720 dup_rows=2294 dup_rows_remote=2292 dedup_ratio=0.023336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 49, 0, 4, 0, 256, 3296, 0, 495, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 8168, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8145, 4901, 0, 0, 166, 0, 0, 0, 0, 0, 0, 112, 0, 202, 0, 1, 0] payload_rows_per_dst=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] dup_rows_per_dst=[0, 94, 808, 706, 0, 2, 54, 630] combine_rows_per_src=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328228864 dispatch_meta_bytes=366000 combine_payload_bytes=328228864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328594864 ingress_bytes=362831872 link_bytes=362831872 token_bytes=656823728 weight_movement_bytes=34603008 total_nvlink_bytes=691426736 weight_to_token_ratio=0.052682 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365307124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=327270400 dispatch_meta_bytes=327736 combine_payload_bytes=327270400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=327598136 ingress_bytes=396476416 link_bytes=396476416 token_bytes=654868536 weight_movement_bytes=69206016 total_nvlink_bytes=724074552 weight_to_token_ratio=0.105679 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.491908834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96504 payload_rows_remote=87734 dup_rows=1800 dup_rows_remote=1718 dedup_ratio=0.018311 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 8108, 0, 53, 0, 8, 0, 278, 3561, 0, 587, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 8183, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8114, 4316, 0, 0, 143, 0, 0, 0, 0, 0, 0, 182, 0, 350, 0, 6, 0] payload_rows_per_dst=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] dup_rows_per_dst=[8, 122, 82, 760, 0, 2, 28, 798] combine_rows_per_src=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66436 payload_rows_remote=50120 dup_rows=31868 dup_rows_remote=31074 dedup_ratio=0.324178 router_tokens_per_expert=[1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 8137, 0, 131, 0, 11, 0, 291, 2810, 0, 612, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 8158, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8162, 4864, 0, 0, 143, 0, 0, 0, 0, 0, 0, 86, 0, 307, 0, 4, 0] payload_rows_per_dst=[12, 276, 16384, 674, 16384, 6, 16384, 16316] dup_rows_per_dst=[0, 8, 7316, 0, 13796, 0, 9954, 794] combine_rows_per_src=[12, 276, 16384, 674, 16384, 6, 16384, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=143425536 dispatch_meta_bytes=144376 combine_payload_bytes=143425536 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=143569912 ingress_bytes=316440576 link_bytes=316440576 token_bytes=286995448 weight_movement_bytes=173015040 total_nvlink_bytes=460010488 weight_to_token_ratio=0.602849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190740412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=326533120 dispatch_meta_bytes=328048 combine_payload_bytes=326533120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=326861168 ingress_bytes=430342144 link_bytes=430342144 token_bytes=653394288 weight_movement_bytes=103809024 total_nvlink_bytes=757203312 weight_to_token_ratio=0.158877 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.619342842e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95638 payload_rows_remote=82918 dup_rows=2666 dup_rows_remote=2666 dedup_ratio=0.027120 router_tokens_per_expert=[0, 0, 3765, 0, 0, 0, 1891, 11, 0, 7741, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 745, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 59, 0, 0, 0, 609, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5704, 6380, 7556, 6360] payload_rows_per_dst=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] dup_rows_per_dst=[1324, 268, 0, 354, 0, 720, 0, 0] combine_rows_per_src=[10010, 15484, 12720, 12544, 16356, 13412, 15112, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=331112448 dispatch_meta_bytes=332080 combine_payload_bytes=331112448 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=331444528 ingress_bytes=400318464 link_bytes=400318464 token_bytes=662556976 weight_movement_bytes=69206016 total_nvlink_bytes=731762992 weight_to_token_ratio=0.104453 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506366152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88350 payload_rows_remote=72002 dup_rows=9954 dup_rows_remote=1178 dedup_ratio=0.101257 router_tokens_per_expert=[0, 0, 3185, 0, 0, 0, 1636, 2, 0, 7739, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 71, 0, 0, 0, 558, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5969, 6439, 7476, 6951] payload_rows_per_dst=[16348, 248, 13904, 12968, 16384, 13544, 14952, 2] dup_rows_per_dst=[8776, 0, 16, 488, 0, 674, 0, 0] combine_rows_per_src=[16348, 248, 13904, 12968, 16384, 13544, 14952, 2] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242524160 dispatch_meta_bytes=327664 combine_payload_bytes=242524160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242851824 ingress_bytes=277127168 link_bytes=277127168 token_bytes=485375984 weight_movement_bytes=34603008 total_nvlink_bytes=519978992 weight_to_token_ratio=0.071291 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.042807223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=337698816 dispatch_meta_bytes=335360 combine_payload_bytes=337698816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338034176 ingress_bytes=372301824 link_bytes=372301824 token_bytes=675732992 weight_movement_bytes=34603008 total_nvlink_bytes=710336000 weight_to_token_ratio=0.051208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.400941791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=327270400 dispatch_meta_bytes=327736 combine_payload_bytes=327270400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=327598136 ingress_bytes=396476416 link_bytes=396476416 token_bytes=654868536 weight_movement_bytes=69206016 total_nvlink_bytes=724074552 weight_to_token_ratio=0.105679 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.491908834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=359358464 dispatch_meta_bytes=357808 combine_payload_bytes=359358464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=359716272 ingress_bytes=393961472 link_bytes=393961472 token_bytes=719074736 weight_movement_bytes=34603008 total_nvlink_bytes=753677744 weight_to_token_ratio=0.048122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482445303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=205291520 dispatch_meta_bytes=324776 combine_payload_bytes=205291520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=205616296 ingress_bytes=239894528 link_bytes=239894528 token_bytes=410907816 weight_movement_bytes=34603008 total_nvlink_bytes=445510824 weight_to_token_ratio=0.084211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.027037958e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=80134 dup_rows=11366 dup_rows_remote=11366 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 4153, 0, 0, 0, 1541, 3, 0, 7644, 0, 0, 1, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 629, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 2, 1, 63, 0, 0, 0, 624, 31, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 6270, 5906, 7582, 6364] payload_rows_per_dst=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] dup_rows_per_dst=[1110, 286, 0, 394, 0, 676, 0, 8900] combine_rows_per_src=[10284, 15304, 12728, 13406, 0, 12040, 6804, 16372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96608 payload_rows_remote=83354 dup_rows=1696 dup_rows_remote=932 dedup_ratio=0.017253 router_tokens_per_expert=[3, 0, 4, 1, 1, 0, 2, 0, 1, 0, 0, 8124, 1, 105, 0, 2, 2, 336, 3113, 2, 483, 0, 1, 0, 0, 476, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6533, 8174, 8187, 0, 0, 7, 0, 0, 0, 2, 0, 8169, 4991, 0, 0, 126, 0, 0, 0, 0, 0, 0, 98, 0, 200, 0, 6, 1] payload_rows_per_dst=[16380, 16254, 7794, 13254, 2, 16352, 10214, 16358] dup_rows_per_dst=[16, 212, 80, 764, 0, 4, 20, 600] combine_rows_per_src=[16380, 16254, 7794, 13254, 2, 16352, 10214, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=339632128 dispatch_meta_bytes=342336 combine_payload_bytes=339632128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=339974464 ingress_bytes=374235136 link_bytes=374235136 token_bytes=679606592 weight_movement_bytes=34603008 total_nvlink_bytes=714209600 weight_to_token_ratio=0.050916 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=74076 dup_rows=13086 dup_rows_remote=11212 dedup_ratio=0.133118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8112, 0, 81, 1, 5, 0, 195, 3481, 0, 784, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6476, 8110, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 4844, 0, 0, 209, 0, 0, 0, 0, 0, 0, 45, 0, 112, 0, 8, 0] payload_rows_per_dst=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] dup_rows_per_dst=[0, 0, 8764, 418, 1700, 2, 1874, 328] combine_rows_per_src=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=294920192 dispatch_meta_bytes=292720 combine_payload_bytes=294920192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295212912 ingress_bytes=329523200 link_bytes=329523200 token_bytes=590133104 weight_movement_bytes=34603008 total_nvlink_bytes=624736112 weight_to_token_ratio=0.058636 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.239969273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96930 payload_rows_remote=96930 dup_rows=1374 dup_rows_remote=1374 dedup_ratio=0.013977 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8133, 0, 102, 0, 1, 0, 336, 2916, 0, 482, 0, 0, 0, 0, 366, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7002, 8170, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8179, 4934, 0, 0, 97, 0, 0, 0, 0, 0, 0, 112, 0, 120, 0, 1, 0] payload_rows_per_dst=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] dup_rows_per_dst=[4, 206, 38, 634, 0, 12, 14, 466] combine_rows_per_src=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=341417984 dispatch_meta_bytes=337144 combine_payload_bytes=341417984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341755128 ingress_bytes=376020992 link_bytes=376020992 token_bytes=683173112 weight_movement_bytes=34603008 total_nvlink_bytes=717776120 weight_to_token_ratio=0.050650 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.414936721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66436 payload_rows_remote=50120 dup_rows=31868 dup_rows_remote=31074 dedup_ratio=0.324178 router_tokens_per_expert=[1, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 8137, 0, 131, 0, 11, 0, 291, 2810, 0, 612, 0, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6898, 8158, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 8162, 4864, 0, 0, 143, 0, 0, 0, 0, 0, 0, 86, 0, 307, 0, 4, 0] payload_rows_per_dst=[12, 276, 16384, 674, 16384, 6, 16384, 16316] dup_rows_per_dst=[0, 8, 7316, 0, 13796, 0, 9954, 794] combine_rows_per_src=[12, 276, 16384, 674, 16384, 6, 16384, 16316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94428 payload_rows_remote=80200 dup_rows=3876 dup_rows_remote=3024 dedup_ratio=0.039429 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 782, 2589, 705, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 7926, 0, 73, 0, 0, 0, 315, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 6925, 321, 0, 8149, 1, 0, 2254, 57, 0, 0, 0, 0, 2553] payload_rows_per_dst=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] dup_rows_per_dst=[2, 26, 748, 504, 754, 852, 0, 990] combine_rows_per_src=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328228864 dispatch_meta_bytes=366000 combine_payload_bytes=328228864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328594864 ingress_bytes=362831872 link_bytes=362831872 token_bytes=656823728 weight_movement_bytes=34603008 total_nvlink_bytes=691426736 weight_to_token_ratio=0.052682 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.365307124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88350 payload_rows_remote=72002 dup_rows=9954 dup_rows_remote=1178 dedup_ratio=0.101257 router_tokens_per_expert=[0, 0, 3185, 0, 0, 0, 1636, 2, 0, 7739, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 71, 0, 0, 0, 558, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5969, 6439, 7476, 6951] payload_rows_per_dst=[16348, 248, 13904, 12968, 16384, 13544, 14952, 2] dup_rows_per_dst=[8776, 0, 16, 488, 0, 674, 0, 0] combine_rows_per_src=[16348, 248, 13904, 12968, 16384, 13544, 14952, 2] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97650 payload_rows_remote=81420 dup_rows=654 dup_rows_remote=534 dedup_ratio=0.006653 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8115, 0, 60, 0, 0, 0, 247, 3035, 0, 442, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6869, 8158, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8185, 5116, 0, 0, 165, 0, 0, 0, 0, 0, 0, 165, 0, 141, 0, 5, 0] payload_rows_per_dst=[16384, 16230, 7424, 13802, 16316, 16370, 10502, 622] dup_rows_per_dst=[6, 120, 24, 440, 0, 4, 60, 0] combine_rows_per_src=[16384, 16230, 7424, 13802, 16316, 16370, 10502, 622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=303415296 dispatch_meta_bytes=341152 combine_payload_bytes=303415296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303756448 ingress_bytes=338018304 link_bytes=338018304 token_bytes=607171744 weight_movement_bytes=34603008 total_nvlink_bytes=641774752 weight_to_token_ratio=0.056990 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271935665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=397025280 dispatch_meta_bytes=393216 combine_payload_bytes=397025280 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=397418496 ingress_bytes=535437312 link_bytes=535437312 token_bytes=794443776 weight_movement_bytes=138412032 total_nvlink_bytes=932855808 weight_to_token_ratio=0.174225 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.014807499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=205291520 dispatch_meta_bytes=324776 combine_payload_bytes=205291520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=205616296 ingress_bytes=239894528 link_bytes=239894528 token_bytes=410907816 weight_movement_bytes=34603008 total_nvlink_bytes=445510824 weight_to_token_ratio=0.084211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.027037958e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96010 payload_rows_remote=79720 dup_rows=2294 dup_rows_remote=2292 dedup_ratio=0.023336 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 49, 0, 4, 0, 256, 3296, 0, 495, 0, 0, 0, 0, 437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6605, 8168, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8145, 4901, 0, 0, 166, 0, 0, 0, 0, 0, 0, 112, 0, 202, 0, 1, 0] payload_rows_per_dst=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] dup_rows_per_dst=[0, 94, 808, 706, 0, 2, 54, 630] combine_rows_per_src=[16328, 14806, 8736, 13378, 56, 16290, 10080, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=86668 dup_rows=3934 dup_rows_remote=3060 dedup_ratio=0.040019 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 805, 2773, 710, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 7870, 0, 77, 0, 0, 0, 245, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 7034, 348, 0, 8119, 0, 0, 1664, 55, 0, 0, 0, 0, 2570] payload_rows_per_dst=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] dup_rows_per_dst=[0, 24, 874, 506, 642, 1240, 0, 648] combine_rows_per_src=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96930 payload_rows_remote=96930 dup_rows=1374 dup_rows_remote=1374 dedup_ratio=0.013977 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8133, 0, 102, 0, 1, 0, 336, 2916, 0, 482, 0, 0, 0, 0, 366, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7002, 8170, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 8179, 4934, 0, 0, 97, 0, 0, 0, 0, 0, 0, 112, 0, 120, 0, 1, 0] payload_rows_per_dst=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] dup_rows_per_dst=[4, 206, 38, 634, 0, 12, 14, 466] combine_rows_per_src=[16384, 16266, 7430, 14104, 0, 16358, 10048, 16340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57504 payload_rows_remote=41200 dup_rows=40800 dup_rows_remote=31648 dedup_ratio=0.415039 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 765, 2587, 677, 0, 0, 3, 0, 0, 256, 0, 0, 0, 1, 7947, 0, 82, 0, 0, 0, 354, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 7768, 7101, 402, 0, 8169, 0, 1, 2143, 81, 0, 0, 0, 0, 2556] payload_rows_per_dst=[0, 14, 7416, 15560, 872, 954, 16384, 16304] dup_rows_per_dst=[0, 0, 642, 510, 0, 0, 30496, 9152] combine_rows_per_src=[0, 14, 7416, 15560, 872, 954, 16384, 16304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=328499200 dispatch_meta_bytes=332896 combine_payload_bytes=328499200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328832096 ingress_bytes=363102208 link_bytes=363102208 token_bytes=657331296 weight_movement_bytes=34603008 total_nvlink_bytes=691934304 weight_to_token_ratio=0.052642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366324377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96504 payload_rows_remote=87734 dup_rows=1800 dup_rows_remote=1718 dedup_ratio=0.018311 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 3, 0, 0, 0, 0, 8108, 0, 53, 0, 8, 0, 278, 3561, 0, 587, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6602, 8183, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8114, 4316, 0, 0, 143, 0, 0, 0, 0, 0, 0, 182, 0, 350, 0, 6, 0] payload_rows_per_dst=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] dup_rows_per_dst=[8, 122, 82, 760, 0, 2, 28, 798] combine_rows_per_src=[16384, 16216, 8770, 13372, 4668, 16228, 8890, 11976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=294920192 dispatch_meta_bytes=292720 combine_payload_bytes=294920192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295212912 ingress_bytes=329523200 link_bytes=329523200 token_bytes=590133104 weight_movement_bytes=34603008 total_nvlink_bytes=624736112 weight_to_token_ratio=0.058636 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.239969273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=333496320 dispatch_meta_bytes=327816 combine_payload_bytes=333496320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333824136 ingress_bytes=402702336 link_bytes=402702336 token_bytes=667320456 weight_movement_bytes=69206016 total_nvlink_bytes=736526472 weight_to_token_ratio=0.103707 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515336470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83834 payload_rows_remote=67450 dup_rows=14470 dup_rows_remote=14462 dedup_ratio=0.147196 router_tokens_per_expert=[1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 8156, 0, 235, 1, 2, 1, 348, 3159, 0, 629, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6852, 8181, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 4402, 0, 0, 118, 0, 0, 0, 0, 0, 0, 117, 0, 139, 0, 3, 0] payload_rows_per_dst=[16384, 16312, 8182, 13814, 16380, 3212, 9032, 518] dup_rows_per_dst=[8, 1008, 92, 736, 12618, 0, 8, 0] combine_rows_per_src=[16384, 16312, 8182, 13814, 16380, 3212, 9032, 518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=354992128 dispatch_meta_bytes=358912 combine_payload_bytes=354992128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=355351040 ingress_bytes=424198144 link_bytes=424198144 token_bytes=710343168 weight_movement_bytes=69206016 total_nvlink_bytes=779549184 weight_to_token_ratio=0.097426 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596223465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=326533120 dispatch_meta_bytes=328048 combine_payload_bytes=326533120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=326861168 ingress_bytes=430342144 link_bytes=430342144 token_bytes=653394288 weight_movement_bytes=103809024 total_nvlink_bytes=757203312 weight_to_token_ratio=0.158877 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.619342842e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=397025280 dispatch_meta_bytes=393216 combine_payload_bytes=397025280 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=397418496 ingress_bytes=535437312 link_bytes=535437312 token_bytes=794443776 weight_movement_bytes=138412032 total_nvlink_bytes=932855808 weight_to_token_ratio=0.174225 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.014807499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95024 payload_rows_remote=79276 dup_rows=3280 dup_rows_remote=2758 dedup_ratio=0.033366 router_tokens_per_expert=[0, 0, 1, 0, 1, 0, 0, 1, 10, 0, 0, 0, 1, 0, 1, 0, 3, 0, 7829, 892, 2635, 737, 2, 0, 3, 0, 0, 303, 0, 0, 0, 0, 7902, 0, 42, 0, 0, 0, 270, 0, 1, 494, 0, 3, 0, 1, 0, 2, 2, 0, 8000, 6888, 402, 0, 8137, 6, 1, 2033, 154, 0, 0, 0, 0, 2395] payload_rows_per_dst=[16278, 16002, 7816, 15748, 15830, 14022, 820, 8508] dup_rows_per_dst=[2, 22, 722, 522, 598, 756, 0, 658] combine_rows_per_src=[16278, 16002, 7816, 15748, 15830, 14022, 820, 8508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=359358464 dispatch_meta_bytes=357808 combine_payload_bytes=359358464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=359716272 ingress_bytes=393961472 link_bytes=393961472 token_bytes=719074736 weight_movement_bytes=34603008 total_nvlink_bytes=753677744 weight_to_token_ratio=0.048122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482445303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=168755200 dispatch_meta_bytes=291392 combine_payload_bytes=168755200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=169046592 ingress_bytes=203358208 link_bytes=203358208 token_bytes=337801792 weight_movement_bytes=34603008 total_nvlink_bytes=372404800 weight_to_token_ratio=0.102436 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.652205650e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85218 payload_rows_remote=74076 dup_rows=13086 dup_rows_remote=11212 dedup_ratio=0.133118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8112, 0, 81, 1, 5, 0, 195, 3481, 0, 784, 0, 0, 0, 0, 307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6476, 8110, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 4844, 0, 0, 209, 0, 0, 0, 0, 0, 0, 45, 0, 112, 0, 8, 0] payload_rows_per_dst=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] dup_rows_per_dst=[0, 0, 8764, 418, 1700, 2, 1874, 328] combine_rows_per_src=[6, 174, 16380, 11448, 16384, 13462, 11142, 16222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62042 payload_rows_remote=46292 dup_rows=36262 dup_rows_remote=4742 dedup_ratio=0.368876 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7605, 608, 2690, 625, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 7978, 0, 63, 0, 0, 0, 308, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 6880, 424, 0, 8110, 0, 0, 2644, 131, 0, 0, 0, 0, 2238] payload_rows_per_dst=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] dup_rows_per_dst=[0, 10, 538, 518, 148, 384, 31520, 3144] combine_rows_per_src=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276275200 dispatch_meta_bytes=327648 combine_payload_bytes=276275200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276602848 ingress_bytes=310878208 link_bytes=310878208 token_bytes=552878048 weight_movement_bytes=34603008 total_nvlink_bytes=587481056 weight_to_token_ratio=0.062587 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169809669e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94808 payload_rows_remote=79084 dup_rows=3496 dup_rows_remote=2658 dedup_ratio=0.035563 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7632, 717, 2222, 530, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 7854, 0, 57, 0, 0, 0, 370, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 6801, 317, 0, 8118, 6, 0, 2942, 185, 0, 0, 0, 0, 2566] payload_rows_per_dst=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] dup_rows_per_dst=[2, 12, 174, 530, 838, 692, 0, 1248] combine_rows_per_src=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=324714496 dispatch_meta_bytes=328136 combine_payload_bytes=324714496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325042632 ingress_bytes=359317504 link_bytes=359317504 token_bytes=649757128 weight_movement_bytes=34603008 total_nvlink_bytes=684360136 weight_to_token_ratio=0.053255 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352082840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57504 payload_rows_remote=41200 dup_rows=40800 dup_rows_remote=31648 dedup_ratio=0.415039 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7775, 765, 2587, 677, 0, 0, 3, 0, 0, 256, 0, 0, 0, 1, 7947, 0, 82, 0, 0, 0, 354, 0, 0, 477, 0, 0, 0, 0, 0, 0, 0, 0, 7768, 7101, 402, 0, 8169, 0, 1, 2143, 81, 0, 0, 0, 0, 2556] payload_rows_per_dst=[0, 14, 7416, 15560, 872, 954, 16384, 16304] dup_rows_per_dst=[0, 0, 642, 510, 0, 0, 30496, 9152] combine_rows_per_src=[0, 14, 7416, 15560, 872, 954, 16384, 16304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93286 payload_rows_remote=76902 dup_rows=5018 dup_rows_remote=4512 dedup_ratio=0.051046 router_tokens_per_expert=[0, 4997, 0, 0, 0, 0, 0, 0, 0, 1104, 429, 0, 0, 0, 0, 0, 0, 2902, 0, 0, 0, 0, 0, 462, 89, 8058, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 253, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 4662, 0, 0, 0, 4643, 0] payload_rows_per_dst=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] dup_rows_per_dst=[0, 188, 176, 178, 1052, 506, 0, 2918] combine_rows_per_src=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=189612032 dispatch_meta_bytes=204136 combine_payload_bytes=189612032 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=189816168 ingress_bytes=328024064 link_bytes=328024064 token_bytes=379428200 weight_movement_bytes=138412032 total_nvlink_bytes=517840232 weight_to_token_ratio=0.364791 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234328145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=303415296 dispatch_meta_bytes=341152 combine_payload_bytes=303415296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303756448 ingress_bytes=338018304 link_bytes=338018304 token_bytes=607171744 weight_movement_bytes=34603008 total_nvlink_bytes=641774752 weight_to_token_ratio=0.056990 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271935665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83834 payload_rows_remote=67450 dup_rows=14470 dup_rows_remote=14462 dedup_ratio=0.147196 router_tokens_per_expert=[1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 8156, 0, 235, 1, 2, 1, 348, 3159, 0, 629, 0, 0, 0, 0, 423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6852, 8181, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 4402, 0, 0, 118, 0, 0, 0, 0, 0, 0, 117, 0, 139, 0, 3, 0] payload_rows_per_dst=[16384, 16312, 8182, 13814, 16380, 3212, 9032, 518] dup_rows_per_dst=[8, 1008, 92, 736, 12618, 0, 8, 0] combine_rows_per_src=[16384, 16312, 8182, 13814, 16380, 3212, 9032, 518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88778 payload_rows_remote=78072 dup_rows=9526 dup_rows_remote=6450 dedup_ratio=0.096903 router_tokens_per_expert=[0, 4826, 0, 0, 0, 0, 0, 0, 0, 1048, 560, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 0, 381, 77, 7884, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 337, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 5253, 0, 0, 0, 4603, 0] payload_rows_per_dst=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] dup_rows_per_dst=[826, 2822, 3076, 0, 802, 440, 0, 1560] combine_rows_per_src=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90352 payload_rows_remote=74096 dup_rows=7952 dup_rows_remote=7934 dedup_ratio=0.080892 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7725, 782, 1840, 629, 0, 0, 0, 0, 0, 290, 0, 0, 0, 0, 7933, 0, 46, 0, 0, 0, 394, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 7048, 365, 0, 8153, 26, 0, 3006, 141, 0, 0, 0, 0, 2037] payload_rows_per_dst=[16306, 16256, 15544, 1478, 15886, 14358, 782, 9742] dup_rows_per_dst=[0, 18, 5482, 28, 860, 938, 0, 626] combine_rows_per_src=[16306, 16256, 15544, 1478, 15886, 14358, 782, 9742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=323928064 dispatch_meta_bytes=326968 combine_payload_bytes=323928064 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=324255032 ingress_bytes=427737088 link_bytes=427737088 token_bytes=648183096 weight_movement_bytes=103809024 total_nvlink_bytes=751992120 weight_to_token_ratio=0.160154 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609540226e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=168755200 dispatch_meta_bytes=291392 combine_payload_bytes=168755200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=169046592 ingress_bytes=203358208 link_bytes=203358208 token_bytes=337801792 weight_movement_bytes=34603008 total_nvlink_bytes=372404800 weight_to_token_ratio=0.102436 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.652205650e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94428 payload_rows_remote=80200 dup_rows=3876 dup_rows_remote=3024 dedup_ratio=0.039429 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 782, 2589, 705, 0, 0, 0, 0, 0, 258, 0, 0, 0, 0, 7926, 0, 73, 0, 0, 0, 315, 0, 0, 615, 0, 0, 0, 0, 0, 0, 0, 0, 7878, 6925, 321, 0, 8149, 1, 0, 2254, 57, 0, 0, 0, 0, 2553] payload_rows_per_dst=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] dup_rows_per_dst=[2, 26, 748, 504, 754, 852, 0, 990] combine_rows_per_src=[16298, 15758, 7404, 15484, 15874, 14228, 644, 8738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94808 payload_rows_remote=79084 dup_rows=3496 dup_rows_remote=2658 dedup_ratio=0.035563 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7632, 717, 2222, 530, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 7854, 0, 57, 0, 0, 0, 370, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 6801, 317, 0, 8118, 6, 0, 2942, 185, 0, 0, 0, 0, 2566] payload_rows_per_dst=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] dup_rows_per_dst=[2, 12, 174, 530, 838, 692, 0, 1248] combine_rows_per_src=[16236, 16208, 6766, 15294, 15724, 13796, 646, 10138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314990592 dispatch_meta_bytes=325656 combine_payload_bytes=314990592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=315316248 ingress_bytes=384196608 link_bytes=384196608 token_bytes=630306840 weight_movement_bytes=69206016 total_nvlink_bytes=699512856 weight_to_token_ratio=0.109797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445700906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91922 payload_rows_remote=78172 dup_rows=6382 dup_rows_remote=2368 dedup_ratio=0.064921 router_tokens_per_expert=[0, 5167, 0, 8, 0, 0, 0, 0, 0, 1237, 425, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 378, 112, 7915, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 340, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 4279, 0, 0, 0, 4421, 0] payload_rows_per_dst=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] dup_rows_per_dst=[16, 342, 174, 224, 934, 678, 0, 4014] combine_rows_per_src=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276275200 dispatch_meta_bytes=327648 combine_payload_bytes=276275200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276602848 ingress_bytes=310878208 link_bytes=310878208 token_bytes=552878048 weight_movement_bytes=34603008 total_nvlink_bytes=587481056 weight_to_token_ratio=0.062587 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169809669e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=319782912 dispatch_meta_bytes=338088 combine_payload_bytes=319782912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320121000 ingress_bytes=354385920 link_bytes=354385920 token_bytes=639903912 weight_movement_bytes=34603008 total_nvlink_bytes=674506920 weight_to_token_ratio=0.054075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333525687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303497216 dispatch_meta_bytes=328120 combine_payload_bytes=303497216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303825336 ingress_bytes=338100224 link_bytes=338100224 token_bytes=607322552 weight_movement_bytes=34603008 total_nvlink_bytes=641925560 weight_to_token_ratio=0.056976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272243924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94370 payload_rows_remote=86668 dup_rows=3934 dup_rows_remote=3060 dedup_ratio=0.040019 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7921, 805, 2773, 710, 0, 0, 0, 0, 0, 267, 0, 0, 0, 0, 7870, 0, 77, 0, 0, 0, 245, 0, 0, 825, 0, 0, 0, 0, 0, 0, 0, 0, 7857, 7034, 348, 0, 8119, 0, 0, 1664, 55, 0, 0, 0, 0, 2570] payload_rows_per_dst=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] dup_rows_per_dst=[0, 24, 874, 506, 642, 1240, 0, 648] combine_rows_per_src=[16238, 15714, 7702, 15870, 15742, 14478, 696, 7930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83468 payload_rows_remote=67190 dup_rows=14836 dup_rows_remote=14836 dedup_ratio=0.150920 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 561, 2858, 767, 0, 0, 3, 0, 0, 258, 0, 0, 0, 0, 7918, 0, 73, 0, 0, 0, 294, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 7004, 275, 0, 8139, 8, 0, 2407, 92, 0, 0, 0, 0, 2434] payload_rows_per_dst=[16278, 15830, 16282, 522, 10710, 14268, 566, 9012] dup_rows_per_dst=[0, 36, 12886, 0, 454, 606, 0, 854] combine_rows_per_src=[16278, 15830, 16282, 522, 10710, 14268, 566, 9012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=328499200 dispatch_meta_bytes=332896 combine_payload_bytes=328499200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328832096 ingress_bytes=363102208 link_bytes=363102208 token_bytes=657331296 weight_movement_bytes=34603008 total_nvlink_bytes=691934304 weight_to_token_ratio=0.052642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366324377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=323928064 dispatch_meta_bytes=326968 combine_payload_bytes=323928064 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=324255032 ingress_bytes=427737088 link_bytes=427737088 token_bytes=648183096 weight_movement_bytes=103809024 total_nvlink_bytes=751992120 weight_to_token_ratio=0.160154 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609540226e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90936 payload_rows_remote=90828 dup_rows=7368 dup_rows_remote=7368 dedup_ratio=0.074951 router_tokens_per_expert=[0, 5037, 1, 0, 0, 0, 0, 0, 0, 1332, 361, 1, 1, 2, 1, 0, 1, 3375, 0, 1, 0, 0, 0, 319, 53, 8020, 0, 0, 8187, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1520, 0, 0, 269, 0, 0, 1, 0, 8190, 4, 3, 0, 0, 0, 0, 2, 0, 0, 3746, 0, 4099, 0, 0, 0, 4625, 0] payload_rows_per_dst=[10076, 16318, 7292, 108, 10588, 16382, 16380, 13792] dup_rows_per_dst=[0, 3118, 100, 0, 650, 546, 4, 2950] combine_rows_per_src=[10076, 16318, 7292, 108, 10588, 16382, 16380, 13792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62042 payload_rows_remote=46292 dup_rows=36262 dup_rows_remote=4742 dedup_ratio=0.368876 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7605, 608, 2690, 625, 0, 0, 0, 0, 0, 269, 0, 0, 0, 0, 7978, 0, 63, 0, 0, 0, 308, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 7836, 6880, 424, 0, 8110, 0, 0, 2644, 131, 0, 0, 0, 0, 2238] payload_rows_per_dst=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] dup_rows_per_dst=[0, 10, 538, 518, 148, 384, 31520, 3144] combine_rows_per_src=[664, 2374, 7308, 15230, 3372, 6732, 15750, 10612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=320192512 dispatch_meta_bytes=322160 combine_payload_bytes=320192512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320514672 ingress_bytes=458604544 link_bytes=458604544 token_bytes=640707184 weight_movement_bytes=138412032 total_nvlink_bytes=779119216 weight_to_token_ratio=0.216030 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725691978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=354992128 dispatch_meta_bytes=358912 combine_payload_bytes=354992128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=355351040 ingress_bytes=424198144 link_bytes=424198144 token_bytes=710343168 weight_movement_bytes=69206016 total_nvlink_bytes=779549184 weight_to_token_ratio=0.097426 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.596223465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93468 payload_rows_remote=77084 dup_rows=4836 dup_rows_remote=4836 dedup_ratio=0.049194 router_tokens_per_expert=[0, 4981, 0, 1, 0, 0, 0, 0, 0, 938, 451, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 289, 74, 8026, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 292, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 5272, 0, 0, 0, 4816, 0] payload_rows_per_dst=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] dup_rows_per_dst=[0, 214, 174, 148, 1278, 584, 0, 2438] combine_rows_per_src=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=275210240 dispatch_meta_bytes=328104 combine_payload_bytes=275210240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275538344 ingress_bytes=309813248 link_bytes=309813248 token_bytes=550748584 weight_movement_bytes=34603008 total_nvlink_bytes=585351592 weight_to_token_ratio=0.062829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165802310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=80252 dup_rows=6120 dup_rows_remote=4886 dedup_ratio=0.062256 router_tokens_per_expert=[0, 4529, 0, 0, 0, 0, 0, 0, 0, 1341, 330, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 403, 63, 8104, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 239, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 5088, 0, 0, 0, 4326, 0] payload_rows_per_dst=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] dup_rows_per_dst=[0, 1830, 246, 74, 1234, 478, 0, 2258] combine_rows_per_src=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=189612032 dispatch_meta_bytes=204136 combine_payload_bytes=189612032 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=189816168 ingress_bytes=328024064 link_bytes=328024064 token_bytes=379428200 weight_movement_bytes=138412032 total_nvlink_bytes=517840232 weight_to_token_ratio=0.364791 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234328145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=372031488 dispatch_meta_bytes=392784 combine_payload_bytes=372031488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372424272 ingress_bytes=475840512 link_bytes=475840512 token_bytes=744455760 weight_movement_bytes=103809024 total_nvlink_bytes=848264784 weight_to_token_ratio=0.139443 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.790549538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92146 payload_rows_remote=75762 dup_rows=6158 dup_rows_remote=5984 dedup_ratio=0.062642 router_tokens_per_expert=[3917, 0, 0, 8192, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 2165, 0, 8058, 0, 0, 6929, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5229, 1576] payload_rows_per_dst=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] dup_rows_per_dst=[88, 958, 10, 3476, 0, 174, 0, 1452] combine_rows_per_src=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91922 payload_rows_remote=78172 dup_rows=6382 dup_rows_remote=2368 dedup_ratio=0.064921 router_tokens_per_expert=[0, 5167, 0, 8, 0, 0, 0, 0, 0, 1237, 425, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 378, 112, 7915, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 340, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3308, 0, 4279, 0, 0, 0, 4421, 0] payload_rows_per_dst=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] dup_rows_per_dst=[16, 342, 174, 224, 934, 678, 0, 4014] combine_rows_per_src=[10334, 2982, 7066, 15830, 9200, 16376, 16384, 13750] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315736064 dispatch_meta_bytes=327680 combine_payload_bytes=315736064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316063744 ingress_bytes=350339072 link_bytes=350339072 token_bytes=631799808 weight_movement_bytes=34603008 total_nvlink_bytes=666402816 weight_to_token_ratio=0.054769 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318297723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83468 payload_rows_remote=67190 dup_rows=14836 dup_rows_remote=14836 dedup_ratio=0.150920 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7695, 561, 2858, 767, 0, 0, 3, 0, 0, 258, 0, 0, 0, 0, 7918, 0, 73, 0, 0, 0, 294, 0, 0, 433, 0, 0, 0, 0, 0, 0, 0, 0, 7915, 7004, 275, 0, 8139, 8, 0, 2407, 92, 0, 0, 0, 0, 2434] payload_rows_per_dst=[16278, 15830, 16282, 522, 10710, 14268, 566, 9012] dup_rows_per_dst=[0, 36, 12886, 0, 454, 606, 0, 854] combine_rows_per_src=[16278, 15830, 16282, 522, 10710, 14268, 566, 9012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94798 payload_rows_remote=78414 dup_rows=3506 dup_rows_remote=3502 dedup_ratio=0.035665 router_tokens_per_expert=[4096, 0, 0, 8192, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 1892, 0, 7921, 0, 0, 7362, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5404, 1617] payload_rows_per_dst=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] dup_rows_per_dst=[946, 622, 4, 0, 0, 180, 0, 1754] combine_rows_per_src=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79090 payload_rows_remote=62738 dup_rows=19214 dup_rows_remote=15970 dedup_ratio=0.195455 router_tokens_per_expert=[0, 4455, 0, 0, 0, 0, 0, 0, 0, 1276, 413, 0, 0, 0, 0, 0, 0, 4061, 0, 0, 0, 0, 0, 343, 104, 8109, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2919, 0, 4813, 0, 0, 0, 4428, 0] payload_rows_per_dst=[13254, 16352, 16384, 16384, 11796, 374, 0, 4546] dup_rows_per_dst=[4512, 3244, 8808, 1500, 1150, 0, 0, 0] combine_rows_per_src=[13254, 16352, 16384, 16384, 11796, 374, 0, 4546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=328712192 dispatch_meta_bytes=340552 combine_payload_bytes=328712192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329052744 ingress_bytes=363315200 link_bytes=363315200 token_bytes=657764936 weight_movement_bytes=34603008 total_nvlink_bytes=692367944 weight_to_token_ratio=0.052607 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367125848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310321152 dispatch_meta_bytes=326984 combine_payload_bytes=310321152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310648136 ingress_bytes=344924160 link_bytes=344924160 token_bytes=620969288 weight_movement_bytes=34603008 total_nvlink_bytes=655572296 weight_to_token_ratio=0.055724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93286 payload_rows_remote=76902 dup_rows=5018 dup_rows_remote=4512 dedup_ratio=0.051046 router_tokens_per_expert=[0, 4997, 0, 0, 0, 0, 0, 0, 0, 1104, 429, 0, 0, 0, 0, 0, 0, 2902, 0, 0, 0, 0, 0, 462, 89, 8058, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1564, 0, 0, 253, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3605, 0, 4662, 0, 0, 0, 4643, 0] payload_rows_per_dst=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] dup_rows_per_dst=[0, 188, 176, 178, 1052, 506, 0, 2918] combine_rows_per_src=[9994, 2878, 6552, 16116, 11400, 16384, 16384, 13578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=80252 dup_rows=6120 dup_rows_remote=4886 dedup_ratio=0.062256 router_tokens_per_expert=[0, 4529, 0, 0, 0, 0, 0, 0, 0, 1341, 330, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 0, 0, 0, 403, 63, 8104, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1495, 0, 0, 239, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3363, 0, 5088, 0, 0, 0, 4326, 0] payload_rows_per_dst=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] dup_rows_per_dst=[0, 1830, 246, 74, 1234, 478, 0, 2258] combine_rows_per_src=[9058, 10096, 7534, 7676, 11932, 16384, 16384, 13120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=320192512 dispatch_meta_bytes=322160 combine_payload_bytes=320192512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320514672 ingress_bytes=458604544 link_bytes=458604544 token_bytes=640707184 weight_movement_bytes=138412032 total_nvlink_bytes=779119216 weight_to_token_ratio=0.216030 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725691978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88778 payload_rows_remote=78072 dup_rows=9526 dup_rows_remote=6450 dedup_ratio=0.096903 router_tokens_per_expert=[0, 4826, 0, 0, 0, 0, 0, 0, 0, 1048, 560, 0, 0, 0, 0, 0, 0, 3423, 0, 0, 0, 0, 0, 381, 77, 7884, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1159, 0, 0, 337, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3217, 0, 5253, 0, 0, 0, 4603, 0] payload_rows_per_dst=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] dup_rows_per_dst=[826, 2822, 3076, 0, 802, 440, 0, 1560] combine_rows_per_src=[10400, 16162, 10706, 154, 12022, 12396, 16384, 10554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=275210240 dispatch_meta_bytes=328104 combine_payload_bytes=275210240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275538344 ingress_bytes=309813248 link_bytes=309813248 token_bytes=550748584 weight_movement_bytes=34603008 total_nvlink_bytes=585351592 weight_to_token_ratio=0.062829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165802310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85252 payload_rows_remote=72266 dup_rows=13052 dup_rows_remote=9956 dedup_ratio=0.132772 router_tokens_per_expert=[4031, 0, 0, 8192, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 8191, 0, 15, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 1542, 0, 7989, 0, 0, 7434, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5485, 1599] payload_rows_per_dst=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] dup_rows_per_dst=[0, 0, 0, 7334, 2504, 118, 0, 3096] combine_rows_per_src=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321183744 dispatch_meta_bytes=327664 combine_payload_bytes=321183744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321511408 ingress_bytes=390389760 link_bytes=390389760 token_bytes=642695152 weight_movement_bytes=69206016 total_nvlink_bytes=711901168 weight_to_token_ratio=0.107681 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469005238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=314832 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257289680 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514264528 weight_movement_bytes=34603008 total_nvlink_bytes=548867536 weight_to_token_ratio=0.067286 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74728 payload_rows_remote=60534 dup_rows=23576 dup_rows_remote=17900 dedup_ratio=0.239827 router_tokens_per_expert=[0, 5186, 0, 1, 0, 0, 0, 0, 0, 1081, 434, 0, 0, 0, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 681, 55, 8082, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1498, 0, 0, 196, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 4458, 0, 0, 0, 4748, 0] payload_rows_per_dst=[14194, 16328, 16384, 16384, 11046, 392, 0, 0] dup_rows_per_dst=[5676, 2866, 6690, 7476, 868, 0, 0, 0] combine_rows_per_src=[14194, 16328, 16384, 16384, 11046, 392, 0, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314990592 dispatch_meta_bytes=325656 combine_payload_bytes=314990592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=315316248 ingress_bytes=384196608 link_bytes=384196608 token_bytes=630306840 weight_movement_bytes=69206016 total_nvlink_bytes=699512856 weight_to_token_ratio=0.109797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445700906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=328712192 dispatch_meta_bytes=340552 combine_payload_bytes=328712192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329052744 ingress_bytes=363315200 link_bytes=363315200 token_bytes=657764936 weight_movement_bytes=34603008 total_nvlink_bytes=692367944 weight_to_token_ratio=0.052607 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.367125848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95484 payload_rows_remote=88404 dup_rows=2820 dup_rows_remote=2818 dedup_ratio=0.028687 router_tokens_per_expert=[4183, 0, 0, 8190, 0, 1, 0, 590, 0, 0, 0, 1, 1, 0, 563, 0, 0, 0, 0, 0, 0, 8187, 0, 6, 1, 3, 0, 0, 3537, 0, 0, 0, 0, 1775, 0, 8085, 0, 0, 7091, 0, 0, 2, 1, 110, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 1, 5216, 1604] payload_rows_per_dst=[9408, 14374, 16374, 7080, 3550, 16384, 16174, 12140] dup_rows_per_dst=[140, 938, 12, 2, 0, 224, 0, 1504] combine_rows_per_src=[9408, 14374, 16374, 7080, 3550, 16384, 16174, 12140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93468 payload_rows_remote=77084 dup_rows=4836 dup_rows_remote=4836 dedup_ratio=0.049194 router_tokens_per_expert=[0, 4981, 0, 1, 0, 0, 0, 0, 0, 938, 451, 0, 0, 0, 0, 0, 0, 3269, 0, 0, 0, 0, 0, 289, 74, 8026, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1543, 0, 0, 292, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 5272, 0, 0, 0, 4816, 0] payload_rows_per_dst=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] dup_rows_per_dst=[0, 214, 174, 148, 1278, 584, 0, 2438] combine_rows_per_src=[9964, 2564, 6942, 16052, 12352, 16384, 16384, 12826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296001536 dispatch_meta_bytes=328888 combine_payload_bytes=296001536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296330424 ingress_bytes=365207552 link_bytes=365207552 token_bytes=592331960 weight_movement_bytes=69206016 total_nvlink_bytes=661537976 weight_to_token_ratio=0.116837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374246617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=319782912 dispatch_meta_bytes=338088 combine_payload_bytes=319782912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320121000 ingress_bytes=354385920 link_bytes=354385920 token_bytes=639903912 weight_movement_bytes=34603008 total_nvlink_bytes=674506920 weight_to_token_ratio=0.054075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333525687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87590 payload_rows_remote=71524 dup_rows=10714 dup_rows_remote=10712 dedup_ratio=0.108988 router_tokens_per_expert=[4185, 0, 0, 8192, 0, 0, 0, 754, 3, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 1796, 0, 8033, 0, 0, 7507, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5204, 1333] payload_rows_per_dst=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] dup_rows_per_dst=[182, 362, 2, 6604, 2140, 194, 2, 1228] combine_rows_per_src=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=247947264 dispatch_meta_bytes=313736 combine_payload_bytes=247947264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248261000 ingress_bytes=282550272 link_bytes=282550272 token_bytes=496208264 weight_movement_bytes=34603008 total_nvlink_bytes=530811272 weight_to_token_ratio=0.069735 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.063213926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95728 payload_rows_remote=92804 dup_rows=2576 dup_rows_remote=2576 dedup_ratio=0.026204 router_tokens_per_expert=[3721, 0, 0, 8192, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 1462, 0, 8080, 0, 0, 7412, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5489, 1634] payload_rows_per_dst=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] dup_rows_per_dst=[76, 818, 14, 0, 0, 92, 0, 1576] combine_rows_per_src=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90604 payload_rows_remote=79182 dup_rows=7700 dup_rows_remote=7458 dedup_ratio=0.078328 router_tokens_per_expert=[0, 0, 55, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3057, 0, 0, 0, 1687, 0, 8192, 0, 7, 80, 0, 0, 0, 0, 2, 660, 0, 0, 152, 8192, 0, 0, 8192, 20, 0, 0, 1396, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] dup_rows_per_dst=[106, 752, 1742, 0, 400, 242, 4458, 0] combine_rows_per_src=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=362102784 dispatch_meta_bytes=364888 combine_payload_bytes=362102784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=362467672 ingress_bytes=396705792 link_bytes=396705792 token_bytes=724570456 weight_movement_bytes=34603008 total_nvlink_bytes=759173464 weight_to_token_ratio=0.047757 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.492771958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85252 payload_rows_remote=72266 dup_rows=13052 dup_rows_remote=9956 dedup_ratio=0.132772 router_tokens_per_expert=[4031, 0, 0, 8192, 0, 0, 0, 527, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 8191, 0, 15, 0, 0, 0, 0, 3667, 0, 0, 0, 0, 1542, 0, 7989, 0, 0, 7434, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5485, 1599] payload_rows_per_dst=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] dup_rows_per_dst=[0, 0, 0, 7334, 2504, 118, 0, 3096] combine_rows_per_src=[1054, 836, 6180, 16382, 15448, 16384, 15982, 12986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315736064 dispatch_meta_bytes=327680 combine_payload_bytes=315736064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316063744 ingress_bytes=350339072 link_bytes=350339072 token_bytes=631799808 weight_movement_bytes=34603008 total_nvlink_bytes=666402816 weight_to_token_ratio=0.054769 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318297723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74728 payload_rows_remote=60534 dup_rows=23576 dup_rows_remote=17900 dedup_ratio=0.239827 router_tokens_per_expert=[0, 5186, 0, 1, 0, 0, 0, 0, 0, 1081, 434, 0, 0, 0, 0, 0, 0, 2664, 0, 0, 0, 0, 0, 681, 55, 8082, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1498, 0, 0, 196, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3683, 0, 4458, 0, 0, 0, 4748, 0] payload_rows_per_dst=[14194, 16328, 16384, 16384, 11046, 392, 0, 0] dup_rows_per_dst=[5676, 2866, 6690, 7476, 868, 0, 0, 0] combine_rows_per_src=[14194, 16328, 16384, 16384, 11046, 392, 0, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80776 payload_rows_remote=66184 dup_rows=17528 dup_rows_remote=16348 dedup_ratio=0.178304 router_tokens_per_expert=[3881, 0, 0, 8192, 0, 0, 0, 533, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 3829, 0, 0, 0, 0, 1620, 0, 8067, 0, 0, 7241, 0, 0, 0, 0, 72, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5117, 1755] payload_rows_per_dst=[16384, 14592, 16384, 7658, 14590, 144, 1962, 9062] dup_rows_per_dst=[8828, 1180, 16, 0, 6322, 0, 0, 1182] combine_rows_per_src=[16384, 14592, 16384, 7658, 14590, 144, 1962, 9062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=328944 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293291248 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586253552 weight_movement_bytes=34603008 total_nvlink_bytes=620856560 weight_to_token_ratio=0.059024 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=380125184 dispatch_meta_bytes=381520 combine_payload_bytes=380125184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380506704 ingress_bytes=483934208 link_bytes=483934208 token_bytes=760631888 weight_movement_bytes=103809024 total_nvlink_bytes=864440912 weight_to_token_ratio=0.136477 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821005465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324329472 dispatch_meta_bytes=346560 combine_payload_bytes=324329472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324676032 ingress_bytes=428138496 link_bytes=428138496 token_bytes=649005504 weight_movement_bytes=103809024 total_nvlink_bytes=752814528 weight_to_token_ratio=0.159951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611050692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92146 payload_rows_remote=75762 dup_rows=6158 dup_rows_remote=5984 dedup_ratio=0.062642 router_tokens_per_expert=[3917, 0, 0, 8192, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0, 526, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 0, 0, 3736, 0, 0, 0, 0, 2165, 0, 8058, 0, 0, 6929, 0, 0, 0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5229, 1576] payload_rows_per_dst=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] dup_rows_per_dst=[88, 958, 10, 3476, 0, 174, 0, 1452] combine_rows_per_src=[8824, 13952, 8794, 11588, 4330, 16384, 16116, 12158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91584 payload_rows_remote=85512 dup_rows=6720 dup_rows_remote=6720 dedup_ratio=0.068359 router_tokens_per_expert=[0, 0, 123, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3036, 0, 0, 0, 1633, 0, 8192, 0, 14, 94, 0, 0, 0, 0, 1, 676, 0, 0, 112, 8192, 0, 0, 8192, 52, 0, 0, 1175, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] dup_rows_per_dst=[230, 1650, 0, 3294, 1542, 2, 2, 0] combine_rows_per_src=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95728 payload_rows_remote=92804 dup_rows=2576 dup_rows_remote=2576 dedup_ratio=0.026204 router_tokens_per_expert=[3721, 0, 0, 8192, 0, 0, 0, 517, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 1462, 0, 8080, 0, 0, 7412, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5489, 1634] payload_rows_per_dst=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] dup_rows_per_dst=[76, 818, 14, 0, 0, 92, 0, 1576] combine_rows_per_src=[8400, 14898, 16384, 7908, 2924, 16384, 16160, 12670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=296001536 dispatch_meta_bytes=328888 combine_payload_bytes=296001536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296330424 ingress_bytes=365207552 link_bytes=365207552 token_bytes=592331960 weight_movement_bytes=69206016 total_nvlink_bytes=661537976 weight_to_token_ratio=0.116837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374246617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94798 payload_rows_remote=78414 dup_rows=3506 dup_rows_remote=3502 dedup_ratio=0.035665 router_tokens_per_expert=[4096, 0, 0, 8192, 0, 0, 0, 622, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 3411, 0, 0, 0, 0, 1892, 0, 7921, 0, 0, 7362, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5404, 1617] payload_rows_per_dst=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] dup_rows_per_dst=[946, 622, 4, 0, 0, 180, 0, 1754] combine_rows_per_src=[9866, 14774, 16384, 6822, 3784, 15038, 15842, 12288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=247947264 dispatch_meta_bytes=313736 combine_payload_bytes=247947264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248261000 ingress_bytes=282550272 link_bytes=282550272 token_bytes=496208264 weight_movement_bytes=34603008 total_nvlink_bytes=530811272 weight_to_token_ratio=0.069735 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.063213926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70842 payload_rows_remote=54458 dup_rows=27462 dup_rows_remote=27458 dedup_ratio=0.279358 router_tokens_per_expert=[0, 0, 26, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 1593, 0, 8189, 0, 6, 96, 0, 0, 0, 0, 1, 762, 0, 0, 141, 8192, 0, 0, 8192, 7, 0, 0, 1312, 0, 0, 0, 0, 607, 0, 0, 0, 1, 0, 1, 0, 0, 0] payload_rows_per_dst=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 7598, 0, 0, 16028, 3832, 4] combine_rows_per_src=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=350257152 dispatch_meta_bytes=368928 combine_payload_bytes=350257152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=350626080 ingress_bytes=419463168 link_bytes=419463168 token_bytes=700883232 weight_movement_bytes=69206016 total_nvlink_bytes=770089248 weight_to_token_ratio=0.098741 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578406132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271089664 dispatch_meta_bytes=330128 combine_payload_bytes=271089664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271419792 ingress_bytes=305692672 link_bytes=305692672 token_bytes=542509456 weight_movement_bytes=34603008 total_nvlink_bytes=577112464 weight_to_token_ratio=0.063783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150296914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=58480 dup_rows=23440 dup_rows_remote=14168 dedup_ratio=0.238444 router_tokens_per_expert=[4012, 0, 0, 8192, 0, 0, 0, 624, 3, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 8192, 0, 38, 0, 0, 0, 0, 3533, 0, 0, 0, 0, 1603, 0, 8083, 0, 0, 7497, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 1681] payload_rows_per_dst=[16384, 15066, 16384, 7066, 16382, 220, 0, 3362] dup_rows_per_dst=[9272, 876, 76, 0, 13216, 0, 0, 0] combine_rows_per_src=[16384, 15066, 16384, 7066, 16382, 220, 0, 3362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310321152 dispatch_meta_bytes=326984 combine_payload_bytes=310321152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310648136 ingress_bytes=344924160 link_bytes=344924160 token_bytes=620969288 weight_movement_bytes=34603008 total_nvlink_bytes=655572296 weight_to_token_ratio=0.055724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=380125184 dispatch_meta_bytes=381520 combine_payload_bytes=380125184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380506704 ingress_bytes=483934208 link_bytes=483934208 token_bytes=760631888 weight_movement_bytes=103809024 total_nvlink_bytes=864440912 weight_to_token_ratio=0.136477 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821005465e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87590 payload_rows_remote=71524 dup_rows=10714 dup_rows_remote=10712 dedup_ratio=0.108988 router_tokens_per_expert=[4185, 0, 0, 8192, 0, 0, 0, 754, 3, 0, 0, 0, 0, 0, 531, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 3302, 0, 0, 0, 0, 1796, 0, 8033, 0, 0, 7507, 0, 0, 0, 0, 97, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5204, 1333] payload_rows_per_dst=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] dup_rows_per_dst=[182, 362, 2, 6604, 2140, 194, 2, 1228] combine_rows_per_src=[8452, 4058, 1286, 16384, 13114, 16384, 16066, 11846] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92476 payload_rows_remote=76096 dup_rows=5828 dup_rows_remote=1998 dedup_ratio=0.059285 router_tokens_per_expert=[6, 0, 79, 0, 7823, 0, 0, 1, 1, 1, 0, 2, 221, 1, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3274, 0, 0, 2, 1892, 0, 8185, 0, 26, 83, 0, 0, 0, 1, 7, 613, 2, 0, 126, 8188, 0, 4, 8190, 13, 0, 0, 1552, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 2, 0, 0] payload_rows_per_dst=[15674, 16382, 6548, 16380, 16382, 286, 4440, 16384] dup_rows_per_dst=[144, 446, 0, 3830, 1406, 0, 2, 0] combine_rows_per_src=[15674, 16382, 6548, 16380, 16382, 286, 4440, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321183744 dispatch_meta_bytes=327664 combine_payload_bytes=321183744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321511408 ingress_bytes=390389760 link_bytes=390389760 token_bytes=642695152 weight_movement_bytes=69206016 total_nvlink_bytes=711901168 weight_to_token_ratio=0.107681 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469005238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=223059968 dispatch_meta_bytes=327664 combine_payload_bytes=223059968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223387632 ingress_bytes=257662976 link_bytes=257662976 token_bytes=446447600 weight_movement_bytes=34603008 total_nvlink_bytes=481050608 weight_to_token_ratio=0.077507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.695650351e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74858 payload_rows_remote=58474 dup_rows=23446 dup_rows_remote=19218 dedup_ratio=0.238505 router_tokens_per_expert=[0, 0, 55, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 1697, 0, 8192, 0, 17, 80, 0, 0, 0, 0, 1, 544, 0, 0, 118, 8192, 0, 0, 8192, 16, 0, 0, 1582, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] dup_rows_per_dst=[18, 0, 15076, 396, 0, 3728, 4228, 0] combine_rows_per_src=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=239534080 dispatch_meta_bytes=290592 combine_payload_bytes=239534080 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=239824672 ingress_bytes=343343104 link_bytes=343343104 token_bytes=479358752 weight_movement_bytes=103809024 total_nvlink_bytes=583167776 weight_to_token_ratio=0.216558 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291972459e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92140 payload_rows_remote=75756 dup_rows=6164 dup_rows_remote=5002 dedup_ratio=0.062703 router_tokens_per_expert=[0, 0, 74, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 1694, 0, 8192, 0, 17, 77, 0, 0, 0, 0, 1, 503, 0, 0, 109, 8192, 0, 0, 8192, 16, 0, 0, 1725, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] dup_rows_per_dst=[146, 324, 0, 390, 1162, 0, 4140, 2] combine_rows_per_src=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92582 payload_rows_remote=76568 dup_rows=5722 dup_rows_remote=5654 dedup_ratio=0.058207 router_tokens_per_expert=[0, 0, 4495, 0, 0, 0, 0, 1, 0, 134, 8192, 0, 0, 146, 4785, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 223, 0, 0, 0, 0, 0, 765, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2742, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 71, 7112, 0, 0, 0] payload_rows_per_dst=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] dup_rows_per_dst=[2, 150, 506, 48, 4714, 68, 132, 102] combine_rows_per_src=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292962304 dispatch_meta_bytes=328944 combine_payload_bytes=292962304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293291248 ingress_bytes=327565312 link_bytes=327565312 token_bytes=586253552 weight_movement_bytes=34603008 total_nvlink_bytes=620856560 weight_to_token_ratio=0.059024 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232601898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311689216 dispatch_meta_bytes=312376 combine_payload_bytes=311689216 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312001592 ingress_bytes=380895232 link_bytes=380895232 token_bytes=623690808 weight_movement_bytes=69206016 total_nvlink_bytes=692896824 weight_to_token_ratio=0.110962 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433278093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70842 payload_rows_remote=54458 dup_rows=27462 dup_rows_remote=27458 dedup_ratio=0.279358 router_tokens_per_expert=[0, 0, 26, 0, 7866, 0, 0, 0, 0, 0, 0, 0, 173, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 3799, 0, 0, 0, 1593, 0, 8189, 0, 6, 96, 0, 0, 0, 0, 1, 762, 0, 0, 141, 8192, 0, 0, 8192, 7, 0, 0, 1312, 0, 0, 0, 0, 607, 0, 0, 0, 1, 0, 1, 0, 0, 0] payload_rows_per_dst=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 7598, 0, 0, 16028, 3832, 4] combine_rows_per_src=[52, 346, 16376, 3198, 1718, 16384, 16384, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=58480 dup_rows=23440 dup_rows_remote=14168 dedup_ratio=0.238444 router_tokens_per_expert=[4012, 0, 0, 8192, 0, 0, 0, 624, 3, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 8192, 0, 38, 0, 0, 0, 0, 3533, 0, 0, 0, 0, 1603, 0, 8083, 0, 0, 7497, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5113, 1681] payload_rows_per_dst=[16384, 15066, 16384, 7066, 16382, 220, 0, 3362] dup_rows_per_dst=[9272, 876, 76, 0, 13216, 0, 0, 0] combine_rows_per_src=[16384, 15066, 16384, 7066, 16382, 220, 0, 3362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88822 payload_rows_remote=78104 dup_rows=9482 dup_rows_remote=8872 dedup_ratio=0.096456 router_tokens_per_expert=[0, 0, 3673, 0, 0, 0, 0, 1, 0, 110, 8192, 0, 0, 184, 4664, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 176, 0, 0, 0, 0, 0, 932, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2503, 0, 8192, 0, 0, 8039, 0, 0, 0, 0, 0, 119, 7301, 0, 0, 0] payload_rows_per_dst=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] dup_rows_per_dst=[3360, 256, 610, 4652, 392, 38, 0, 174] combine_rows_per_src=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92352 payload_rows_remote=75970 dup_rows=5952 dup_rows_remote=5556 dedup_ratio=0.060547 router_tokens_per_expert=[0, 0, 62, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 1898, 0, 8192, 0, 19, 93, 0, 0, 0, 0, 0, 584, 0, 0, 119, 8192, 0, 0, 8192, 3, 0, 0, 1620, 0, 0, 0, 0, 666, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15844, 16382, 6402, 16384, 16384, 16384, 4572, 0] dup_rows_per_dst=[124, 396, 0, 3834, 1354, 244, 0, 0] combine_rows_per_src=[15844, 16382, 6402, 16384, 16384, 16384, 4572, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=239509504 dispatch_meta_bytes=310768 combine_payload_bytes=239509504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239820272 ingress_bytes=274112512 link_bytes=274112512 token_bytes=479329776 weight_movement_bytes=34603008 total_nvlink_bytes=513932784 weight_to_token_ratio=0.072190 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.031463315e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=310296576 dispatch_meta_bytes=323032 combine_payload_bytes=310296576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310619608 ingress_bytes=344899584 link_bytes=344899584 token_bytes=620916184 weight_movement_bytes=34603008 total_nvlink_bytes=655519192 weight_to_token_ratio=0.055729 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297829368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313622528 dispatch_meta_bytes=328888 combine_payload_bytes=313622528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313951416 ingress_bytes=348225536 link_bytes=348225536 token_bytes=627573944 weight_movement_bytes=34603008 total_nvlink_bytes=662176952 weight_to_token_ratio=0.055138 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310344658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90604 payload_rows_remote=79182 dup_rows=7700 dup_rows_remote=7458 dedup_ratio=0.078328 router_tokens_per_expert=[0, 0, 55, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3057, 0, 0, 0, 1687, 0, 8192, 0, 7, 80, 0, 0, 0, 0, 2, 660, 0, 0, 152, 8192, 0, 0, 8192, 20, 0, 0, 1396, 0, 0, 0, 0, 833, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] dup_rows_per_dst=[106, 752, 1742, 0, 400, 242, 4458, 0] combine_rows_per_src=[15792, 11990, 9096, 3388, 6148, 11422, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92140 payload_rows_remote=75756 dup_rows=6164 dup_rows_remote=5002 dedup_ratio=0.062703 router_tokens_per_expert=[0, 0, 74, 0, 7922, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3475, 0, 0, 0, 1694, 0, 8192, 0, 17, 77, 0, 0, 0, 0, 1, 503, 0, 0, 109, 8192, 0, 0, 8192, 16, 0, 0, 1725, 0, 0, 0, 0, 608, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] dup_rows_per_dst=[146, 324, 0, 390, 1162, 0, 4140, 2] combine_rows_per_src=[15846, 16384, 6950, 5084, 16384, 250, 14858, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=223059968 dispatch_meta_bytes=327664 combine_payload_bytes=223059968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223387632 ingress_bytes=257662976 link_bytes=257662976 token_bytes=446447600 weight_movement_bytes=34603008 total_nvlink_bytes=481050608 weight_to_token_ratio=0.077507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.695650351e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91584 payload_rows_remote=85512 dup_rows=6720 dup_rows_remote=6720 dedup_ratio=0.068359 router_tokens_per_expert=[0, 0, 123, 0, 7818, 0, 0, 0, 0, 0, 0, 0, 825, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3036, 0, 0, 0, 1633, 0, 8192, 0, 14, 94, 0, 0, 0, 0, 1, 676, 0, 0, 112, 8192, 0, 0, 8192, 52, 0, 0, 1175, 0, 0, 0, 0, 827, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] dup_rows_per_dst=[230, 1650, 0, 3294, 1542, 2, 2, 0] combine_rows_per_src=[15652, 16380, 6072, 16384, 16384, 332, 4002, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75888 payload_rows_remote=61936 dup_rows=22416 dup_rows_remote=22326 dedup_ratio=0.228027 router_tokens_per_expert=[0, 0, 3977, 0, 0, 0, 0, 0, 0, 124, 8192, 0, 0, 63, 5591, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 182, 0, 0, 0, 0, 0, 1054, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2512, 0, 8192, 0, 0, 8125, 0, 0, 0, 0, 0, 65, 6956, 0, 0, 0] payload_rows_per_dst=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] dup_rows_per_dst=[0, 154, 682, 24, 192, 0, 21274, 90] combine_rows_per_src=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=239534080 dispatch_meta_bytes=290592 combine_payload_bytes=239534080 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=239824672 ingress_bytes=343343104 link_bytes=343343104 token_bytes=479358752 weight_movement_bytes=103809024 total_nvlink_bytes=583167776 weight_to_token_ratio=0.216558 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291972459e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=319913984 dispatch_meta_bytes=347904 combine_payload_bytes=319913984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320261888 ingress_bytes=354516992 link_bytes=354516992 token_bytes=640175872 weight_movement_bytes=34603008 total_nvlink_bytes=674778880 weight_to_token_ratio=0.054052 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334018900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=311173120 dispatch_meta_bytes=326104 combine_payload_bytes=311173120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311499224 ingress_bytes=345776128 link_bytes=345776128 token_bytes=622672344 weight_movement_bytes=34603008 total_nvlink_bytes=657275352 weight_to_token_ratio=0.055572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301127732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91946 payload_rows_remote=76202 dup_rows=6358 dup_rows_remote=6208 dedup_ratio=0.064677 router_tokens_per_expert=[0, 0, 85, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 2128, 0, 8192, 0, 19, 83, 0, 0, 0, 0, 1, 619, 0, 0, 78, 8192, 0, 0, 8192, 39, 0, 0, 1790, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15744, 16382, 6060, 16384, 16384, 16384, 4608, 0] dup_rows_per_dst=[150, 264, 0, 4294, 1406, 234, 10, 0] combine_rows_per_src=[15744, 16382, 6060, 16384, 16384, 16384, 4608, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14140497400 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=7074094584 ingress_bytes=8692744192 link_bytes=8692744192 total_nvlink_bytes=15766838776 weight_to_token_ratio=0.115013 prefetch_share_of_ingress=0.067672 dedup_ratio=0.107864 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21209 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.422178112e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324329472 dispatch_meta_bytes=346560 combine_payload_bytes=324329472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324676032 ingress_bytes=428138496 link_bytes=428138496 token_bytes=649005504 weight_movement_bytes=103809024 total_nvlink_bytes=752814528 weight_to_token_ratio=0.159951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611050692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74858 payload_rows_remote=58474 dup_rows=23446 dup_rows_remote=19218 dedup_ratio=0.238505 router_tokens_per_expert=[0, 0, 55, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3753, 0, 0, 0, 1697, 0, 8192, 0, 17, 80, 0, 0, 0, 0, 1, 544, 0, 0, 118, 8192, 0, 0, 8192, 16, 0, 0, 1582, 0, 0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] dup_rows_per_dst=[18, 0, 15076, 396, 0, 3728, 4228, 0] combine_rows_per_src=[2782, 226, 16384, 5064, 1250, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89736 payload_rows_remote=76810 dup_rows=8568 dup_rows_remote=4878 dedup_ratio=0.087158 router_tokens_per_expert=[0, 0, 4139, 0, 0, 0, 0, 2, 0, 223, 8185, 0, 0, 72, 5219, 1, 0, 5, 0, 398, 1, 0, 0, 0, 1, 0, 0, 0, 2452, 168, 0, 0, 0, 2, 0, 797, 1719, 0, 0, 4, 0, 0, 0, 0, 0, 0, 11, 0, 2202, 0, 8189, 1, 0, 8125, 1, 0, 2, 0, 0, 98, 7132, 0, 3, 0] payload_rows_per_dst=[10776, 6220, 16382, 12926, 8328, 16378, 4408, 14318] dup_rows_per_dst=[2182, 134, 796, 3690, 1592, 22, 0, 152] combine_rows_per_src=[10776, 6220, 16382, 12926, 8328, 16378, 4408, 14318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=350257152 dispatch_meta_bytes=368928 combine_payload_bytes=350257152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=350626080 ingress_bytes=419463168 link_bytes=419463168 token_bytes=700883232 weight_movement_bytes=69206016 total_nvlink_bytes=770089248 weight_to_token_ratio=0.098741 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.578406132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=310296576 dispatch_meta_bytes=323032 combine_payload_bytes=310296576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310619608 ingress_bytes=344899584 link_bytes=344899584 token_bytes=620916184 weight_movement_bytes=34603008 total_nvlink_bytes=655519192 weight_to_token_ratio=0.055729 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297829368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=253689856 dispatch_meta_bytes=337048 combine_payload_bytes=253689856 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=254026904 ingress_bytes=322895872 link_bytes=322895872 token_bytes=507716760 weight_movement_bytes=69206016 total_nvlink_bytes=576922776 weight_to_token_ratio=0.136308 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215031171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84848 payload_rows_remote=68464 dup_rows=13456 dup_rows_remote=7536 dedup_ratio=0.136882 router_tokens_per_expert=[0, 0, 3443, 0, 0, 0, 0, 0, 0, 190, 8192, 0, 0, 135, 4919, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 2116, 233, 0, 0, 0, 0, 0, 1202, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2678, 0, 8191, 0, 0, 8092, 1, 0, 0, 0, 0, 140, 7052, 0, 0, 0] payload_rows_per_dst=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] dup_rows_per_dst=[0, 224, 796, 50, 6276, 0, 5920, 190] combine_rows_per_src=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=312123392 dispatch_meta_bytes=329640 combine_payload_bytes=312123392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=312453032 ingress_bytes=415932416 link_bytes=415932416 token_bytes=624576424 weight_movement_bytes=103809024 total_nvlink_bytes=728385448 weight_to_token_ratio=0.166207 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565120196e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91812 payload_rows_remote=75450 dup_rows=6492 dup_rows_remote=1288 dedup_ratio=0.066040 router_tokens_per_expert=[0, 0, 4130, 0, 0, 0, 0, 0, 0, 191, 8192, 0, 0, 64, 5389, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 173, 0, 0, 0, 0, 0, 856, 1755, 0, 0, 0, 1, 0, 0, 0, 0, 0, 40, 0, 2027, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 98, 7089, 0, 0, 0] payload_rows_per_dst=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] dup_rows_per_dst=[0, 174, 872, 46, 5204, 82, 0, 114] combine_rows_per_src=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14539119344 prefetch_bytes=795869184 reduce_bytes=830472192 weight_movement_bytes=1626341376 egress_bytes=7273536240 ingress_bytes=8891924480 link_bytes=8891924480 total_nvlink_bytes=16165460720 weight_to_token_ratio=0.111860 prefetch_share_of_ingress=0.089505 dedup_ratio=0.109488 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=24 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.454765042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=239509504 dispatch_meta_bytes=310768 combine_payload_bytes=239509504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239820272 ingress_bytes=274112512 link_bytes=274112512 token_bytes=479329776 weight_movement_bytes=34603008 total_nvlink_bytes=513932784 weight_to_token_ratio=0.072190 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.031463315e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=314613760 dispatch_meta_bytes=326752 combine_payload_bytes=314613760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314940512 ingress_bytes=349216768 link_bytes=349216768 token_bytes=629554272 weight_movement_bytes=34603008 total_nvlink_bytes=664157280 weight_to_token_ratio=0.054964 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314074584e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75888 payload_rows_remote=61936 dup_rows=22416 dup_rows_remote=22326 dedup_ratio=0.228027 router_tokens_per_expert=[0, 0, 3977, 0, 0, 0, 0, 0, 0, 124, 8192, 0, 0, 63, 5591, 0, 0, 0, 0, 341, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 182, 0, 0, 0, 0, 0, 1054, 1618, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 2512, 0, 8192, 0, 0, 8125, 0, 0, 0, 0, 0, 65, 6956, 0, 0, 0] payload_rows_per_dst=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] dup_rows_per_dst=[0, 154, 682, 24, 192, 0, 21274, 90] combine_rows_per_src=[7954, 11402, 16384, 4620, 5152, 40, 16384, 13952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11469591808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=5738353920 ingress_bytes=7080755200 link_bytes=7080755200 total_nvlink_bytes=12819109120 weight_to_token_ratio=0.117660 prefetch_share_of_ingress=0.097738 dedup_ratio=0.235392 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.158448338e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91946 payload_rows_remote=76202 dup_rows=6358 dup_rows_remote=6208 dedup_ratio=0.064677 router_tokens_per_expert=[0, 0, 85, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 132, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 3030, 0, 0, 0, 2128, 0, 8192, 0, 19, 83, 0, 0, 0, 0, 1, 619, 0, 0, 78, 8192, 0, 0, 8192, 39, 0, 0, 1790, 0, 0, 0, 0, 519, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15744, 16382, 6060, 16384, 16384, 16384, 4608, 0] dup_rows_per_dst=[150, 264, 0, 4294, 1406, 234, 10, 0] combine_rows_per_src=[15744, 16382, 6060, 16384, 16384, 16384, 4608, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92582 payload_rows_remote=76568 dup_rows=5722 dup_rows_remote=5654 dedup_ratio=0.058207 router_tokens_per_expert=[0, 0, 4495, 0, 0, 0, 0, 1, 0, 134, 8192, 0, 0, 146, 4785, 0, 0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 2242, 223, 0, 0, 0, 0, 0, 765, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2742, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 71, 7112, 0, 0, 0] payload_rows_per_dst=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] dup_rows_per_dst=[2, 150, 506, 48, 4714, 68, 132, 102] combine_rows_per_src=[8990, 9980, 16384, 4882, 16346, 16014, 5722, 14264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87414 payload_rows_remote=71030 dup_rows=10890 dup_rows_remote=10286 dedup_ratio=0.110779 router_tokens_per_expert=[0, 0, 4396, 0, 0, 0, 0, 1, 0, 164, 8192, 0, 0, 138, 5108, 0, 0, 2, 0, 379, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 104, 0, 0, 0, 0, 0, 784, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 2475, 0, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 66, 7169, 0, 0, 0] payload_rows_per_dst=[14264, 16384, 760, 16370, 4790, 16384, 6948, 11514] dup_rows_per_dst=[4746, 604, 2, 4438, 124, 18, 890, 68] combine_rows_per_src=[14264, 16384, 760, 16370, 4790, 16384, 6948, 11514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=304000 combine_payload_bytes=280428544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280732544 ingress_bytes=418840576 link_bytes=418840576 token_bytes=561161088 weight_movement_bytes=138412032 total_nvlink_bytes=699573120 weight_to_token_ratio=0.246653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576063368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309043200 dispatch_meta_bytes=306952 combine_payload_bytes=309043200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309350152 ingress_bytes=343646208 link_bytes=343646208 token_bytes=618393352 weight_movement_bytes=34603008 total_nvlink_bytes=652996360 weight_to_token_ratio=0.055956 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293113015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14529617312 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=7268728224 ingress_bytes=8541200384 link_bytes=8541200384 total_nvlink_bytes=15809928608 weight_to_token_ratio=0.088117 prefetch_share_of_ingress=0.081026 dedup_ratio=0.097711 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21463 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.397384757e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91812 payload_rows_remote=75450 dup_rows=6492 dup_rows_remote=1288 dedup_ratio=0.066040 router_tokens_per_expert=[0, 0, 4130, 0, 0, 0, 0, 0, 0, 191, 8192, 0, 0, 64, 5389, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 2347, 173, 0, 0, 0, 0, 0, 856, 1755, 0, 0, 0, 1, 0, 0, 0, 0, 0, 40, 0, 2027, 0, 8192, 0, 0, 8172, 0, 0, 0, 0, 0, 98, 7089, 0, 0, 0] payload_rows_per_dst=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] dup_rows_per_dst=[0, 174, 872, 46, 5204, 82, 0, 114] combine_rows_per_src=[8260, 11114, 16384, 4994, 16362, 16384, 4054, 14260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=253689856 dispatch_meta_bytes=337048 combine_payload_bytes=253689856 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=254026904 ingress_bytes=322895872 link_bytes=322895872 token_bytes=507716760 weight_movement_bytes=69206016 total_nvlink_bytes=576922776 weight_to_token_ratio=0.136308 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215031171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88822 payload_rows_remote=78104 dup_rows=9482 dup_rows_remote=8872 dedup_ratio=0.096456 router_tokens_per_expert=[0, 0, 3673, 0, 0, 0, 0, 1, 0, 110, 8192, 0, 0, 184, 4664, 0, 0, 0, 0, 530, 0, 0, 0, 0, 0, 0, 0, 0, 2241, 176, 0, 0, 0, 0, 0, 932, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 2503, 0, 8192, 0, 0, 8039, 0, 0, 0, 0, 0, 119, 7301, 0, 0, 0] payload_rows_per_dst=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] dup_rows_per_dst=[3360, 256, 610, 4652, 392, 38, 0, 174] combine_rows_per_src=[13316, 6448, 10718, 16260, 6024, 16384, 5006, 14666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=312123392 dispatch_meta_bytes=329640 combine_payload_bytes=312123392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=312453032 ingress_bytes=415932416 link_bytes=415932416 token_bytes=624576424 weight_movement_bytes=103809024 total_nvlink_bytes=728385448 weight_to_token_ratio=0.166207 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565120196e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313622528 dispatch_meta_bytes=328888 combine_payload_bytes=313622528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313951416 ingress_bytes=348225536 link_bytes=348225536 token_bytes=627573944 weight_movement_bytes=34603008 total_nvlink_bytes=662176952 weight_to_token_ratio=0.055138 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310344658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=290938880 dispatch_meta_bytes=325264 combine_payload_bytes=290938880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291264144 ingress_bytes=394747904 link_bytes=394747904 token_bytes=582203024 weight_movement_bytes=103809024 total_nvlink_bytes=686012048 weight_to_token_ratio=0.178304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485404583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12739182128 prefetch_bytes=726663168 reduce_bytes=449839104 weight_movement_bytes=1176502272 egress_bytes=6373326384 ingress_bytes=7542358016 link_bytes=7542358016 total_nvlink_bytes=13915684400 weight_to_token_ratio=0.092353 prefetch_share_of_ingress=0.096344 dedup_ratio=0.178620 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=13 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=20447 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233968957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84848 payload_rows_remote=68464 dup_rows=13456 dup_rows_remote=7536 dedup_ratio=0.136882 router_tokens_per_expert=[0, 0, 3443, 0, 0, 0, 0, 0, 0, 190, 8192, 0, 0, 135, 4919, 0, 0, 0, 0, 398, 0, 0, 0, 0, 0, 0, 0, 0, 2116, 233, 0, 0, 0, 0, 0, 1202, 2134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 2678, 0, 8191, 0, 0, 8092, 1, 0, 0, 0, 0, 140, 7052, 0, 0, 0] payload_rows_per_dst=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] dup_rows_per_dst=[0, 224, 796, 50, 6276, 0, 5920, 190] combine_rows_per_src=[6886, 10264, 16384, 4648, 16016, 72, 16384, 14194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309043200 dispatch_meta_bytes=306952 combine_payload_bytes=309043200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309350152 ingress_bytes=343646208 link_bytes=343646208 token_bytes=618393352 weight_movement_bytes=34603008 total_nvlink_bytes=652996360 weight_to_token_ratio=0.055956 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293113015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14140497400 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=7074094584 ingress_bytes=8692744192 link_bytes=8692744192 total_nvlink_bytes=15766838776 weight_to_token_ratio=0.115013 prefetch_share_of_ingress=0.067672 dedup_ratio=0.107864 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030326 gemm_inner_pad_upper_bound=21209 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.422178112e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=304000 combine_payload_bytes=280428544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=280732544 ingress_bytes=418840576 link_bytes=418840576 token_bytes=561161088 weight_movement_bytes=138412032 total_nvlink_bytes=699573120 weight_to_token_ratio=0.246653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576063368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13302686048 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6655107424 ingress_bytes=8100904960 link_bytes=8100904960 total_nvlink_bytes=14756012384 weight_to_token_ratio=0.109251 prefetch_share_of_ingress=0.081158 dedup_ratio=0.142413 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.325350139e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14535198032 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=7271540048 ingress_bytes=8613175296 link_bytes=8613175296 total_nvlink_bytes=15884715344 weight_to_token_ratio=0.092845 prefetch_share_of_ingress=0.088384 dedup_ratio=0.099342 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030217 gemm_inner_pad_upper_bound=20574 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.409160227e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79378 payload_rows_remote=65838 dup_rows=18926 dup_rows_remote=14898 dedup_ratio=0.192525 router_tokens_per_expert=[0, 0, 4106, 0, 0, 0, 0, 2, 0, 139, 8192, 0, 0, 125, 4676, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 2009, 178, 0, 0, 0, 0, 0, 1106, 2290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 2492, 0, 8192, 0, 0, 8141, 0, 0, 0, 0, 0, 123, 7041, 0, 0, 0] payload_rows_per_dst=[13540, 16384, 644, 16330, 6460, 16384, 9390, 246] dup_rows_per_dst=[4028, 8412, 0, 4326, 332, 36, 1792, 0] combine_rows_per_src=[13540, 16384, 644, 16330, 6460, 16384, 9390, 246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11469591808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=5738353920 ingress_bytes=7080755200 link_bytes=7080755200 total_nvlink_bytes=12819109120 weight_to_token_ratio=0.117660 prefetch_share_of_ingress=0.097738 dedup_ratio=0.235392 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.158448338e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=319913984 dispatch_meta_bytes=347904 combine_payload_bytes=319913984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320261888 ingress_bytes=354516992 link_bytes=354516992 token_bytes=640175872 weight_movement_bytes=34603008 total_nvlink_bytes=674778880 weight_to_token_ratio=0.054052 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.334018900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13302686048 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6655107424 ingress_bytes=8100904960 link_bytes=8100904960 total_nvlink_bytes=14756012384 weight_to_token_ratio=0.109251 prefetch_share_of_ingress=0.081158 dedup_ratio=0.142413 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.325350139e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12739182128 prefetch_bytes=726663168 reduce_bytes=449839104 weight_movement_bytes=1176502272 egress_bytes=6373326384 ingress_bytes=7542358016 link_bytes=7542358016 total_nvlink_bytes=13915684400 weight_to_token_ratio=0.092353 prefetch_share_of_ingress=0.096344 dedup_ratio=0.178620 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=13 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=20447 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233968957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=269672448 dispatch_meta_bytes=322944 combine_payload_bytes=269672448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269995392 ingress_bytes=304275456 link_bytes=304275456 token_bytes=539667840 weight_movement_bytes=34603008 total_nvlink_bytes=574270848 weight_to_token_ratio=0.064119 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144964045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14539119344 prefetch_bytes=795869184 reduce_bytes=830472192 weight_movement_bytes=1626341376 egress_bytes=7273536240 ingress_bytes=8891924480 link_bytes=8891924480 total_nvlink_bytes=16165460720 weight_to_token_ratio=0.111860 prefetch_share_of_ingress=0.089505 dedup_ratio=0.109488 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=24 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.454765042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12119609016 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6063435448 ingress_bytes=7405690880 link_bytes=7405690880 total_nvlink_bytes=13469126328 weight_to_token_ratio=0.111350 prefetch_share_of_ingress=0.093450 dedup_ratio=0.194876 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030761 gemm_inner_pad_upper_bound=20193 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211609503e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 16:55:23] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 791.2 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.478545E+00 | loss scale: 1.0 | grad norm: 7.152 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:07:31] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 793.9 | throughput per GPU (TFLOP/s/GPU): 160.7 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.478545E+00 | loss scale: 1.0 | grad norm: 7.152 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75230 payload_rows_remote=65002 dup_rows=23074 dup_rows_remote=20138 dedup_ratio=0.234721 router_tokens_per_expert=[2835, 535, 241, 1562, 474, 858, 600, 87, 2851, 285, 448, 952, 36, 888, 556, 489, 657, 1186, 369, 658, 360, 237, 493, 915, 1321, 831, 286, 426, 993, 350, 1151, 988, 1146, 576, 795, 1107, 1378, 877, 758, 681, 213, 535, 425, 364, 806, 1505, 827, 185, 166, 480, 500, 1249, 812, 1497, 584, 248, 448, 631, 1317, 584, 224, 654, 1587, 1075] payload_rows_per_dst=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] dup_rows_per_dst=[1404, 1130, 4094, 2820, 2452, 3342, 4896, 2936] combine_rows_per_src=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76654 payload_rows_remote=67172 dup_rows=21650 dup_rows_remote=19282 dedup_ratio=0.220235 router_tokens_per_expert=[2649, 571, 261, 1325, 403, 769, 728, 134, 2807, 327, 428, 842, 41, 967, 575, 435, 787, 1148, 396, 680, 368, 239, 479, 1017, 1243, 974, 318, 434, 1090, 390, 1176, 1030, 1111, 599, 821, 1063, 1336, 798, 786, 747, 312, 574, 437, 320, 868, 1553, 739, 185, 150, 453, 522, 1082, 779, 1649, 552, 292, 379, 618, 1329, 495, 243, 677, 1579, 1073] payload_rows_per_dst=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] dup_rows_per_dst=[3202, 3004, 2094, 3076, 2368, 3364, 1982, 2560] combine_rows_per_src=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76226 payload_rows_remote=65184 dup_rows=22078 dup_rows_remote=18182 dedup_ratio=0.224589 router_tokens_per_expert=[2600, 558, 281, 1467, 477, 800, 713, 153, 2905, 322, 446, 971, 55, 769, 548, 502, 964, 1189, 399, 694, 385, 276, 509, 1016, 1354, 920, 331, 415, 1034, 385, 1161, 987, 1092, 649, 916, 984, 1229, 823, 826, 663, 266, 567, 433, 302, 847, 1494, 806, 232, 161, 418, 548, 1195, 667, 1434, 502, 259, 371, 717, 1246, 488, 255, 598, 1545, 1033] payload_rows_per_dst=[10566, 11042, 8424, 9128, 9602, 9090, 8558, 9816] dup_rows_per_dst=[3532, 3896, 2440, 2144, 2674, 3262, 1810, 2320] combine_rows_per_src=[10566, 11042, 8424, 9128, 9602, 9090, 8558, 9816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76408 payload_rows_remote=67186 dup_rows=21896 dup_rows_remote=18430 dedup_ratio=0.222738 router_tokens_per_expert=[2709, 582, 232, 1598, 448, 750, 686, 115, 3033, 308, 481, 860, 32, 971, 504, 400, 795, 1202, 388, 748, 317, 263, 458, 1049, 1254, 907, 301, 470, 870, 359, 1157, 788, 1266, 562, 849, 1066, 1379, 777, 714, 599, 267, 520, 396, 312, 822, 1578, 851, 219, 143, 431, 552, 1021, 726, 1706, 547, 304, 416, 654, 1325, 533, 218, 639, 1686, 1039] payload_rows_per_dst=[7954, 9982, 11176, 9596, 9308, 9222, 8846, 10324] dup_rows_per_dst=[1904, 3196, 3646, 2616, 2358, 3466, 2014, 2696] combine_rows_per_src=[7954, 9982, 11176, 9596, 9308, 9222, 8846, 10324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75230 payload_rows_remote=65002 dup_rows=23074 dup_rows_remote=20138 dedup_ratio=0.234721 router_tokens_per_expert=[2835, 535, 241, 1562, 474, 858, 600, 87, 2851, 285, 448, 952, 36, 888, 556, 489, 657, 1186, 369, 658, 360, 237, 493, 915, 1321, 831, 286, 426, 993, 350, 1151, 988, 1146, 576, 795, 1107, 1378, 877, 758, 681, 213, 535, 425, 364, 806, 1505, 827, 185, 166, 480, 500, 1249, 812, 1497, 584, 248, 448, 631, 1317, 584, 224, 654, 1587, 1075] payload_rows_per_dst=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] dup_rows_per_dst=[1404, 1130, 4094, 2820, 2452, 3342, 4896, 2936] combine_rows_per_src=[7310, 6178, 11326, 9748, 9428, 9134, 11878, 10228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=66788 dup_rows=21860 dup_rows_remote=19050 dedup_ratio=0.222371 router_tokens_per_expert=[2859, 602, 267, 1604, 456, 707, 569, 113, 3174, 258, 416, 993, 43, 1073, 499, 477, 753, 1167, 320, 690, 310, 260, 516, 1095, 1272, 918, 287, 434, 934, 329, 1154, 905, 1137, 578, 752, 1109, 1292, 774, 682, 681, 263, 602, 428, 292, 722, 1485, 835, 198, 154, 440, 505, 1177, 730, 1825, 520, 231, 380, 684, 1324, 569, 191, 639, 1545, 954] payload_rows_per_dst=[10950, 10346, 8048, 9656, 9266, 8968, 9194, 10016] dup_rows_per_dst=[3404, 3520, 2174, 2810, 2160, 3266, 1970, 2556] combine_rows_per_src=[10950, 10346, 8048, 9656, 9266, 8968, 9194, 10016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75804 payload_rows_remote=64452 dup_rows=22500 dup_rows_remote=18118 dedup_ratio=0.228882 router_tokens_per_expert=[2814, 534, 216, 1717, 337, 805, 502, 119, 2886, 294, 400, 939, 26, 808, 497, 454, 836, 1279, 279, 713, 367, 205, 485, 955, 1374, 939, 292, 453, 905, 326, 1257, 1073, 1161, 643, 854, 1024, 1331, 734, 745, 690, 311, 601, 437, 367, 787, 1542, 885, 226, 141, 449, 554, 1183, 731, 1513, 517, 237, 413, 643, 1275, 585, 187, 601, 1580, 1119] payload_rows_per_dst=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] dup_rows_per_dst=[1246, 1134, 4026, 3256, 2376, 3490, 4382, 2590] combine_rows_per_src=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76654 payload_rows_remote=67172 dup_rows=21650 dup_rows_remote=19282 dedup_ratio=0.220235 router_tokens_per_expert=[2649, 571, 261, 1325, 403, 769, 728, 134, 2807, 327, 428, 842, 41, 967, 575, 435, 787, 1148, 396, 680, 368, 239, 479, 1017, 1243, 974, 318, 434, 1090, 390, 1176, 1030, 1111, 599, 821, 1063, 1336, 798, 786, 747, 312, 574, 437, 320, 868, 1553, 739, 185, 150, 453, 522, 1082, 779, 1649, 552, 292, 379, 618, 1329, 495, 243, 677, 1579, 1073] payload_rows_per_dst=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] dup_rows_per_dst=[3202, 3004, 2094, 3076, 2368, 3364, 1982, 2560] combine_rows_per_src=[10478, 9840, 8134, 10234, 9482, 9284, 8976, 10226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76306 payload_rows_remote=67936 dup_rows=21998 dup_rows_remote=19754 dedup_ratio=0.223775 router_tokens_per_expert=[2699, 559, 283, 1664, 456, 863, 543, 103, 2884, 386, 503, 914, 33, 931, 539, 378, 845, 1279, 400, 650, 375, 282, 475, 1001, 1236, 936, 279, 490, 941, 422, 1250, 1014, 1185, 578, 810, 1089, 1384, 716, 734, 624, 244, 538, 414, 272, 798, 1566, 789, 178, 166, 488, 510, 1250, 730, 1575, 501, 245, 407, 598, 1268, 465, 207, 658, 1547, 1005] payload_rows_per_dst=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] dup_rows_per_dst=[3664, 3072, 2244, 3168, 2238, 3276, 1920, 2416] combine_rows_per_src=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76306 payload_rows_remote=67936 dup_rows=21998 dup_rows_remote=19754 dedup_ratio=0.223775 router_tokens_per_expert=[2699, 559, 283, 1664, 456, 863, 543, 103, 2884, 386, 503, 914, 33, 931, 539, 378, 845, 1279, 400, 650, 375, 282, 475, 1001, 1236, 936, 279, 490, 941, 422, 1250, 1014, 1185, 578, 810, 1089, 1384, 716, 734, 624, 244, 538, 414, 272, 798, 1566, 789, 178, 166, 488, 510, 1250, 730, 1575, 501, 245, 407, 598, 1268, 465, 207, 658, 1547, 1005] payload_rows_per_dst=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] dup_rows_per_dst=[3664, 3072, 2244, 3168, 2238, 3276, 1920, 2416] combine_rows_per_src=[10676, 10064, 8370, 9968, 9234, 9090, 9010, 9894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=340560 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266588752 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532836944 weight_movement_bytes=103809024 total_nvlink_bytes=636645968 weight_to_token_ratio=0.194823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76012 payload_rows_remote=65022 dup_rows=22292 dup_rows_remote=18482 dedup_ratio=0.226766 router_tokens_per_expert=[2892, 585, 178, 1462, 395, 670, 630, 138, 2993, 284, 500, 904, 30, 1025, 496, 374, 726, 1170, 347, 652, 355, 280, 554, 1050, 1149, 809, 297, 478, 983, 376, 1153, 955, 1292, 578, 857, 982, 1456, 808, 784, 607, 253, 627, 424, 311, 740, 1436, 834, 177, 149, 482, 558, 1149, 763, 1930, 512, 215, 420, 661, 1346, 447, 209, 669, 1583, 1003] payload_rows_per_dst=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] dup_rows_per_dst=[3810, 4412, 2174, 1636, 4474, 2324, 2006, 1456] combine_rows_per_src=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=345816 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275482328 ingress_bytes=344342528 link_bytes=344342528 token_bytes=550618840 weight_movement_bytes=69206016 total_nvlink_bytes=619824856 weight_to_token_ratio=0.125688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295733211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=333464 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267327128 ingress_bytes=336199680 link_bytes=336199680 token_bytes=534320792 weight_movement_bytes=69206016 total_nvlink_bytes=603526808 weight_to_token_ratio=0.129521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265092330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275193856 dispatch_meta_bytes=342464 combine_payload_bytes=275193856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275536320 ingress_bytes=309796864 link_bytes=309796864 token_bytes=550730176 weight_movement_bytes=34603008 total_nvlink_bytes=585333184 weight_to_token_ratio=0.062831 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165740659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=340560 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266588752 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532836944 weight_movement_bytes=103809024 total_nvlink_bytes=636645968 weight_to_token_ratio=0.194823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273563648 dispatch_meta_bytes=343352 combine_payload_bytes=273563648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273907000 ingress_bytes=342769664 link_bytes=342769664 token_bytes=547470648 weight_movement_bytes=69206016 total_nvlink_bytes=616676664 weight_to_token_ratio=0.126410 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289814650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=330280 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264325672 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528321064 weight_movement_bytes=34603008 total_nvlink_bytes=562924072 weight_to_token_ratio=0.065496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275136512 dispatch_meta_bytes=345816 combine_payload_bytes=275136512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275482328 ingress_bytes=344342528 link_bytes=344342528 token_bytes=550618840 weight_movement_bytes=69206016 total_nvlink_bytes=619824856 weight_to_token_ratio=0.125688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295733211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76012 payload_rows_remote=65022 dup_rows=22292 dup_rows_remote=18482 dedup_ratio=0.226766 router_tokens_per_expert=[2892, 585, 178, 1462, 395, 670, 630, 138, 2993, 284, 500, 904, 30, 1025, 496, 374, 726, 1170, 347, 652, 355, 280, 554, 1050, 1149, 809, 297, 478, 983, 376, 1153, 955, 1292, 578, 857, 982, 1456, 808, 784, 607, 253, 627, 424, 311, 740, 1436, 834, 177, 149, 482, 558, 1149, 763, 1930, 512, 215, 420, 661, 1346, 447, 209, 669, 1583, 1003] payload_rows_per_dst=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] dup_rows_per_dst=[3810, 4412, 2174, 1636, 4474, 2324, 2006, 1456] combine_rows_per_src=[10990, 11098, 8094, 8466, 11558, 8242, 9510, 8054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278265856 dispatch_meta_bytes=350760 combine_payload_bytes=278265856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278616616 ingress_bytes=312868864 link_bytes=312868864 token_bytes=556882472 weight_movement_bytes=34603008 total_nvlink_bytes=591485480 weight_to_token_ratio=0.062137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177300347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77028 payload_rows_remote=67660 dup_rows=21276 dup_rows_remote=18856 dedup_ratio=0.216431 router_tokens_per_expert=[273, 951, 447, 586, 2195, 88, 4481, 317, 649, 638, 434, 1025, 552, 1162, 49, 1385, 553, 936, 730, 1926, 38, 604, 20, 429, 132, 861, 1021, 2323, 51, 625, 253, 2164, 26, 28, 604, 231, 537, 23, 405, 549, 729, 554, 307, 884, 271, 163, 290, 1247, 2327, 187, 1754, 391, 378, 183, 968, 173, 876, 1092, 1214, 1320, 723, 614, 1290, 916] payload_rows_per_dst=[11170, 9368, 10854, 5190, 11000, 10470, 10136, 8840] dup_rows_per_dst=[3984, 2420, 3946, 696, 1974, 3428, 2586, 2242] combine_rows_per_src=[11170, 9368, 10854, 5190, 11000, 10470, 10136, 8840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=278265856 dispatch_meta_bytes=350760 combine_payload_bytes=278265856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278616616 ingress_bytes=312868864 link_bytes=312868864 token_bytes=556882472 weight_movement_bytes=34603008 total_nvlink_bytes=591485480 weight_to_token_ratio=0.062137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177300347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75804 payload_rows_remote=64452 dup_rows=22500 dup_rows_remote=18118 dedup_ratio=0.228882 router_tokens_per_expert=[2814, 534, 216, 1717, 337, 805, 502, 119, 2886, 294, 400, 939, 26, 808, 497, 454, 836, 1279, 279, 713, 367, 205, 485, 955, 1374, 939, 292, 453, 905, 326, 1257, 1073, 1161, 643, 854, 1024, 1331, 734, 745, 690, 311, 601, 437, 367, 787, 1542, 885, 226, 141, 449, 554, 1183, 731, 1513, 517, 237, 413, 643, 1275, 585, 187, 601, 1580, 1119] payload_rows_per_dst=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] dup_rows_per_dst=[1246, 1134, 4026, 3256, 2376, 3490, 4382, 2590] combine_rows_per_src=[7214, 6390, 11840, 9982, 9326, 9484, 11352, 10216] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=334016 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266664128 ingress_bytes=335536128 link_bytes=335536128 token_bytes=532994240 weight_movement_bytes=69206016 total_nvlink_bytes=602200256 weight_to_token_ratio=0.129844 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77868 payload_rows_remote=67378 dup_rows=20436 dup_rows_remote=16424 dedup_ratio=0.207886 router_tokens_per_expert=[311, 902, 416, 654, 2331, 110, 4806, 282, 730, 654, 369, 913, 527, 1067, 49, 1304, 566, 1069, 958, 1990, 17, 569, 10, 422, 121, 794, 908, 2037, 45, 385, 245, 2526, 23, 15, 686, 194, 515, 25, 252, 501, 765, 511, 313, 818, 232, 157, 228, 1279, 2274, 148, 1461, 494, 430, 161, 841, 115, 1245, 1121, 1260, 1385, 745, 586, 1376, 909] payload_rows_per_dst=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] dup_rows_per_dst=[2178, 2224, 2324, 2848, 2312, 2370, 2168, 4012] combine_rows_per_src=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=330280 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264325672 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528321064 weight_movement_bytes=34603008 total_nvlink_bytes=562924072 weight_to_token_ratio=0.065496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77640 payload_rows_remote=68524 dup_rows=20664 dup_rows_remote=17992 dedup_ratio=0.210205 router_tokens_per_expert=[223, 847, 376, 510, 2182, 126, 4607, 270, 516, 678, 414, 1239, 527, 1033, 37, 1343, 518, 976, 1020, 2052, 11, 657, 16, 343, 123, 721, 904, 2706, 38, 569, 200, 2283, 21, 37, 522, 148, 485, 10, 380, 524, 799, 591, 291, 818, 218, 241, 253, 1269, 2729, 144, 1545, 409, 493, 151, 883, 139, 954, 1072, 1436, 1085, 593, 589, 1414, 844] payload_rows_per_dst=[7420, 9222, 8830, 11690, 11472, 9116, 10086, 9804] dup_rows_per_dst=[1648, 2352, 2356, 3398, 1996, 2672, 2900, 3342] combine_rows_per_src=[7420, 9222, 8830, 11690, 11472, 9116, 10086, 9804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=66344 dup_rows=21028 dup_rows_remote=18996 dedup_ratio=0.213908 router_tokens_per_expert=[294, 885, 540, 553, 2273, 111, 4210, 309, 555, 691, 409, 982, 496, 1199, 69, 1391, 585, 998, 847, 2034, 31, 660, 33, 408, 120, 802, 987, 2475, 74, 472, 288, 2437, 36, 42, 560, 186, 605, 23, 310, 510, 841, 569, 310, 818, 202, 170, 259, 1203, 2565, 170, 1685, 430, 328, 166, 920, 228, 1009, 1081, 1184, 1199, 654, 598, 1239, 834] payload_rows_per_dst=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] dup_rows_per_dst=[2196, 2470, 2312, 3586, 2032, 2502, 2624, 3306] combine_rows_per_src=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77028 payload_rows_remote=67660 dup_rows=21276 dup_rows_remote=18856 dedup_ratio=0.216431 router_tokens_per_expert=[273, 951, 447, 586, 2195, 88, 4481, 317, 649, 638, 434, 1025, 552, 1162, 49, 1385, 553, 936, 730, 1926, 38, 604, 20, 429, 132, 861, 1021, 2323, 51, 625, 253, 2164, 26, 28, 604, 231, 537, 23, 405, 549, 729, 554, 307, 884, 271, 163, 290, 1247, 2327, 187, 1754, 391, 378, 183, 968, 173, 876, 1092, 1214, 1320, 723, 614, 1290, 916] payload_rows_per_dst=[11170, 9368, 10854, 5190, 11000, 10470, 10136, 8840] dup_rows_per_dst=[3984, 2420, 3946, 696, 1974, 3428, 2586, 2242] combine_rows_per_src=[11170, 9368, 10854, 5190, 11000, 10470, 10136, 8840] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77484 payload_rows_remote=65840 dup_rows=20820 dup_rows_remote=17306 dedup_ratio=0.211792 router_tokens_per_expert=[252, 821, 373, 626, 2189, 121, 4379, 253, 686, 721, 351, 1056, 547, 1103, 50, 1391, 532, 931, 797, 2123, 31, 604, 25, 465, 126, 752, 975, 2559, 31, 375, 201, 2560, 25, 29, 585, 212, 549, 27, 332, 537, 799, 517, 317, 837, 187, 214, 240, 1259, 2528, 156, 1547, 443, 491, 183, 982, 237, 1029, 962, 1289, 1270, 593, 591, 1237, 942] payload_rows_per_dst=[7476, 9292, 8886, 11644, 11086, 8936, 10324, 9840] dup_rows_per_dst=[1794, 2518, 2130, 3514, 2264, 2278, 2810, 3512] combine_rows_per_src=[7476, 9292, 8886, 11644, 11086, 8936, 10324, 9840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77868 payload_rows_remote=67378 dup_rows=20436 dup_rows_remote=16424 dedup_ratio=0.207886 router_tokens_per_expert=[311, 902, 416, 654, 2331, 110, 4806, 282, 730, 654, 369, 913, 527, 1067, 49, 1304, 566, 1069, 958, 1990, 17, 569, 10, 422, 121, 794, 908, 2037, 45, 385, 245, 2526, 23, 15, 686, 194, 515, 25, 252, 501, 765, 511, 313, 818, 232, 157, 228, 1279, 2274, 148, 1461, 494, 430, 161, 841, 115, 1245, 1121, 1260, 1385, 745, 586, 1376, 909] payload_rows_per_dst=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] dup_rows_per_dst=[2178, 2224, 2324, 2848, 2312, 2370, 2168, 4012] combine_rows_per_src=[7834, 9002, 8878, 11274, 11722, 8988, 9680, 10490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77340 payload_rows_remote=67264 dup_rows=20964 dup_rows_remote=18202 dedup_ratio=0.213257 router_tokens_per_expert=[217, 855, 488, 630, 2102, 104, 4708, 292, 625, 713, 364, 1015, 502, 1240, 82, 1379, 497, 952, 690, 1905, 22, 611, 21, 388, 161, 692, 839, 2243, 62, 608, 190, 2484, 14, 20, 589, 295, 446, 27, 392, 546, 785, 575, 280, 909, 242, 156, 231, 1204, 2430, 156, 1771, 467, 436, 154, 885, 120, 888, 1018, 1369, 1511, 591, 606, 1433, 925] payload_rows_per_dst=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] dup_rows_per_dst=[1880, 2320, 1774, 3324, 2366, 2556, 2762, 3982] combine_rows_per_src=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275980288 dispatch_meta_bytes=335208 combine_payload_bytes=275980288 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=276315496 ingress_bytes=414392320 link_bytes=414392320 token_bytes=552295784 weight_movement_bytes=138412032 total_nvlink_bytes=690707816 weight_to_token_ratio=0.250612 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.559324939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78060 payload_rows_remote=68362 dup_rows=20244 dup_rows_remote=17060 dedup_ratio=0.205933 router_tokens_per_expert=[236, 867, 339, 597, 2282, 100, 4510, 320, 539, 562, 453, 1173, 487, 1011, 60, 1488, 506, 1001, 1097, 2169, 21, 708, 27, 428, 144, 759, 950, 2402, 43, 467, 251, 2257, 19, 19, 512, 168, 535, 16, 382, 501, 834, 518, 282, 792, 226, 166, 244, 1080, 2719, 155, 1833, 406, 424, 155, 856, 125, 1186, 1059, 1172, 1062, 602, 533, 1383, 934] payload_rows_per_dst=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] dup_rows_per_dst=[2420, 2416, 3184, 2228, 2086, 3248, 2920, 1742] combine_rows_per_src=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=66344 dup_rows=21028 dup_rows_remote=18996 dedup_ratio=0.213908 router_tokens_per_expert=[294, 885, 540, 553, 2273, 111, 4210, 309, 555, 691, 409, 982, 496, 1199, 69, 1391, 585, 998, 847, 2034, 31, 660, 33, 408, 120, 802, 987, 2475, 74, 472, 288, 2437, 36, 42, 560, 186, 605, 23, 310, 510, 841, 569, 310, 818, 202, 170, 259, 1203, 2565, 170, 1685, 430, 328, 166, 920, 228, 1009, 1081, 1184, 1199, 654, 598, 1239, 834] payload_rows_per_dst=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] dup_rows_per_dst=[2196, 2470, 2312, 3586, 2032, 2502, 2624, 3306] combine_rows_per_src=[7734, 9114, 8880, 11724, 10932, 8720, 10360, 9812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277135360 dispatch_meta_bytes=346064 combine_payload_bytes=277135360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277481424 ingress_bytes=311738368 link_bytes=311738368 token_bytes=554616784 weight_movement_bytes=34603008 total_nvlink_bytes=589219792 weight_to_token_ratio=0.062391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173046382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71460 payload_rows_remote=57728 dup_rows=26844 dup_rows_remote=16496 dedup_ratio=0.273071 router_tokens_per_expert=[315, 935, 370, 632, 2325, 95, 4544, 297, 602, 704, 429, 979, 451, 1080, 45, 1479, 571, 974, 1061, 2036, 31, 643, 11, 374, 134, 845, 927, 2527, 50, 396, 253, 2226, 23, 33, 520, 129, 534, 12, 276, 531, 783, 579, 287, 791, 217, 182, 270, 1265, 2718, 156, 1670, 397, 451, 172, 862, 194, 1052, 1125, 1229, 989, 591, 602, 1337, 834] payload_rows_per_dst=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] dup_rows_per_dst=[10348, 5186, 4720, 920, 124, 3426, 1290, 830] combine_rows_per_src=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280674304 dispatch_meta_bytes=346064 combine_payload_bytes=280674304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=281020368 ingress_bytes=349880320 link_bytes=349880320 token_bytes=561694672 weight_movement_bytes=69206016 total_nvlink_bytes=630900688 weight_to_token_ratio=0.123209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316571476e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78060 payload_rows_remote=68362 dup_rows=20244 dup_rows_remote=17060 dedup_ratio=0.205933 router_tokens_per_expert=[236, 867, 339, 597, 2282, 100, 4510, 320, 539, 562, 453, 1173, 487, 1011, 60, 1488, 506, 1001, 1097, 2169, 21, 708, 27, 428, 144, 759, 950, 2402, 43, 467, 251, 2257, 19, 19, 512, 168, 535, 16, 382, 501, 834, 518, 282, 792, 226, 166, 244, 1080, 2719, 155, 1833, 406, 424, 155, 856, 125, 1186, 1059, 1172, 1062, 602, 533, 1383, 934] payload_rows_per_dst=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] dup_rows_per_dst=[2420, 2416, 3184, 2228, 2086, 3248, 2920, 1742] combine_rows_per_src=[8994, 9130, 9698, 9418, 11238, 10066, 10426, 9090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269680640 dispatch_meta_bytes=332584 combine_payload_bytes=269680640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270013224 ingress_bytes=373489664 link_bytes=373489664 token_bytes=539693864 weight_movement_bytes=103809024 total_nvlink_bytes=643502888 weight_to_token_ratio=0.192348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405411537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275980288 dispatch_meta_bytes=335208 combine_payload_bytes=275980288 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=276315496 ingress_bytes=414392320 link_bytes=414392320 token_bytes=552295784 weight_movement_bytes=138412032 total_nvlink_bytes=690707816 weight_to_token_ratio=0.250612 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.559324939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275513344 dispatch_meta_bytes=341864 combine_payload_bytes=275513344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275855208 ingress_bytes=344719360 link_bytes=344719360 token_bytes=551368552 weight_movement_bytes=69206016 total_nvlink_bytes=620574568 weight_to_token_ratio=0.125517 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297151199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271745024 dispatch_meta_bytes=341360 combine_payload_bytes=271745024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272086384 ingress_bytes=306348032 link_bytes=306348032 token_bytes=543831408 weight_movement_bytes=34603008 total_nvlink_bytes=578434416 weight_to_token_ratio=0.063628 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152762981e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71460 payload_rows_remote=57728 dup_rows=26844 dup_rows_remote=16496 dedup_ratio=0.273071 router_tokens_per_expert=[315, 935, 370, 632, 2325, 95, 4544, 297, 602, 704, 429, 979, 451, 1080, 45, 1479, 571, 974, 1061, 2036, 31, 643, 11, 374, 134, 845, 927, 2527, 50, 396, 253, 2226, 23, 33, 520, 129, 534, 12, 276, 531, 783, 579, 287, 791, 217, 182, 270, 1265, 2718, 156, 1670, 397, 451, 172, 862, 194, 1052, 1125, 1229, 989, 591, 602, 1337, 834] payload_rows_per_dst=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] dup_rows_per_dst=[10348, 5186, 4720, 920, 124, 3426, 1290, 830] combine_rows_per_src=[13732, 11750, 11134, 5616, 3992, 10454, 7156, 7626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=341688 combine_payload_bytes=280010752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280352440 ingress_bytes=314613760 link_bytes=314613760 token_bytes=560363192 weight_movement_bytes=34603008 total_nvlink_bytes=594966200 weight_to_token_ratio=0.061751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183866250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277135360 dispatch_meta_bytes=346064 combine_payload_bytes=277135360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277481424 ingress_bytes=311738368 link_bytes=311738368 token_bytes=554616784 weight_movement_bytes=34603008 total_nvlink_bytes=589219792 weight_to_token_ratio=0.062391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173046382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269680640 dispatch_meta_bytes=332584 combine_payload_bytes=269680640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270013224 ingress_bytes=373489664 link_bytes=373489664 token_bytes=539693864 weight_movement_bytes=103809024 total_nvlink_bytes=643502888 weight_to_token_ratio=0.192348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405411537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65826 dup_rows=24010 dup_rows_remote=22016 dedup_ratio=0.244242 router_tokens_per_expert=[422, 300, 450, 1204, 903, 57, 1104, 710, 1156, 814, 1797, 835, 643, 1437, 165, 181, 3219, 1183, 4621, 565, 654, 39, 1393, 1434, 395, 952, 616, 157, 136, 1208, 972, 743, 235, 334, 944, 1277, 277, 523, 771, 199, 1039, 561, 19, 312, 303, 876, 297, 104, 2255, 1087, 441, 855, 107, 1013, 1348, 268, 596, 876, 456, 44, 209, 44, 22, 995] payload_rows_per_dst=[10152, 8468, 10458, 10096, 12252, 6224, 8258, 8386] dup_rows_per_dst=[3742, 1994, 4542, 3328, 6110, 798, 1980, 1516] combine_rows_per_src=[10152, 8468, 10458, 10096, 12252, 6224, 8258, 8386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77412 payload_rows_remote=65182 dup_rows=20892 dup_rows_remote=16748 dedup_ratio=0.212524 router_tokens_per_expert=[458, 265, 379, 1015, 747, 44, 1530, 818, 959, 1102, 1917, 915, 671, 1377, 103, 207, 3464, 932, 4893, 413, 466, 20, 1671, 1607, 462, 774, 558, 262, 170, 1242, 830, 689, 249, 287, 1081, 1107, 402, 419, 587, 176, 994, 515, 14, 317, 264, 788, 300, 102, 2241, 1105, 427, 874, 62, 906, 1295, 268, 652, 999, 376, 29, 258, 72, 8, 1018] payload_rows_per_dst=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] dup_rows_per_dst=[1990, 4008, 1722, 1784, 1128, 4144, 3628, 2488] combine_rows_per_src=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280010752 dispatch_meta_bytes=341688 combine_payload_bytes=280010752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280352440 ingress_bytes=314613760 link_bytes=314613760 token_bytes=560363192 weight_movement_bytes=34603008 total_nvlink_bytes=594966200 weight_to_token_ratio=0.061751 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183866250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=236453888 dispatch_meta_bytes=296896 combine_payload_bytes=236453888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236750784 ingress_bytes=305659904 link_bytes=305659904 token_bytes=473204672 weight_movement_bytes=69206016 total_nvlink_bytes=542410688 weight_to_token_ratio=0.146250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150173611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77322 payload_rows_remote=66680 dup_rows=20982 dup_rows_remote=17300 dedup_ratio=0.213440 router_tokens_per_expert=[472, 316, 463, 1307, 875, 46, 1340, 708, 1043, 1045, 2009, 836, 687, 1442, 134, 236, 3486, 866, 4506, 490, 499, 25, 1647, 1537, 458, 751, 495, 195, 192, 1171, 820, 897, 255, 296, 1032, 1206, 244, 521, 727, 204, 920, 466, 14, 289, 277, 657, 323, 103, 2183, 1182, 398, 884, 94, 935, 1189, 332, 646, 1098, 400, 41, 228, 44, 7, 963] payload_rows_per_dst=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] dup_rows_per_dst=[3486, 2782, 1596, 3682, 1534, 3474, 1816, 2612] combine_rows_per_src=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77340 payload_rows_remote=67264 dup_rows=20964 dup_rows_remote=18202 dedup_ratio=0.213257 router_tokens_per_expert=[217, 855, 488, 630, 2102, 104, 4708, 292, 625, 713, 364, 1015, 502, 1240, 82, 1379, 497, 952, 690, 1905, 22, 611, 21, 388, 161, 692, 839, 2243, 62, 608, 190, 2484, 14, 20, 589, 295, 446, 27, 392, 546, 785, 575, 280, 909, 242, 156, 231, 1204, 2430, 156, 1771, 467, 436, 154, 885, 120, 888, 1018, 1369, 1511, 591, 606, 1433, 925] payload_rows_per_dst=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] dup_rows_per_dst=[1880, 2320, 1774, 3324, 2366, 2556, 2762, 3982] combine_rows_per_src=[7496, 9520, 8398, 11234, 11708, 9074, 10076, 9834] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75874 payload_rows_remote=65076 dup_rows=22430 dup_rows_remote=19920 dedup_ratio=0.228170 router_tokens_per_expert=[567, 340, 419, 1330, 1014, 71, 1156, 623, 1133, 888, 1990, 792, 735, 1516, 114, 206, 3258, 943, 4610, 398, 640, 45, 1411, 1512, 405, 795, 498, 146, 171, 1100, 818, 943, 233, 293, 923, 1088, 283, 513, 699, 214, 923, 488, 15, 247, 284, 760, 329, 118, 2503, 1182, 441, 850, 142, 1057, 1185, 399, 537, 1113, 385, 31, 238, 50, 15, 1027] payload_rows_per_dst=[8848, 9164, 6890, 8328, 8242, 11892, 11712, 10798] dup_rows_per_dst=[2192, 2562, 1342, 1424, 1916, 3656, 6828, 2510] combine_rows_per_src=[8848, 9164, 6890, 8328, 8242, 11892, 11712, 10798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76752 payload_rows_remote=65756 dup_rows=21552 dup_rows_remote=17362 dedup_ratio=0.219238 router_tokens_per_expert=[535, 322, 476, 1195, 834, 68, 1308, 720, 1143, 1060, 1849, 710, 701, 1313, 131, 163, 3511, 904, 4522, 498, 691, 25, 1495, 1535, 439, 848, 523, 138, 166, 1199, 885, 777, 186, 299, 886, 1160, 309, 566, 693, 194, 1076, 491, 7, 322, 262, 706, 290, 94, 2696, 1259, 396, 811, 109, 937, 1099, 286, 629, 808, 423, 23, 254, 55, 16, 1126] payload_rows_per_dst=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] dup_rows_per_dst=[2156, 3716, 1754, 1774, 1514, 3944, 4190, 2504] combine_rows_per_src=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269623296 dispatch_meta_bytes=351368 combine_payload_bytes=269623296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269974664 ingress_bytes=338829312 link_bytes=338829312 token_bytes=539597960 weight_movement_bytes=69206016 total_nvlink_bytes=608803976 weight_to_token_ratio=0.128255 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274987423e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76262 payload_rows_remote=68926 dup_rows=22042 dup_rows_remote=20446 dedup_ratio=0.224223 router_tokens_per_expert=[426, 335, 457, 1078, 703, 71, 1214, 741, 1031, 1147, 1468, 857, 719, 1407, 150, 249, 3204, 953, 4520, 549, 725, 48, 1432, 1688, 367, 839, 631, 211, 173, 1182, 959, 784, 227, 373, 913, 1135, 337, 441, 852, 188, 1016, 498, 33, 295, 314, 847, 346, 141, 2210, 1207, 464, 907, 87, 982, 1269, 368, 593, 754, 486, 50, 283, 59, 20, 1139] payload_rows_per_dst=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] dup_rows_per_dst=[1986, 3868, 2058, 3844, 1596, 4030, 2096, 2564] combine_rows_per_src=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=327720 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313192 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534298664 weight_movement_bytes=34603008 total_nvlink_bytes=568901672 weight_to_token_ratio=0.064763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273121280 dispatch_meta_bytes=335920 combine_payload_bytes=273121280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273457200 ingress_bytes=307724288 link_bytes=307724288 token_bytes=546578480 weight_movement_bytes=34603008 total_nvlink_bytes=581181488 weight_to_token_ratio=0.063308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157941722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77412 payload_rows_remote=65182 dup_rows=20892 dup_rows_remote=16748 dedup_ratio=0.212524 router_tokens_per_expert=[458, 265, 379, 1015, 747, 44, 1530, 818, 959, 1102, 1917, 915, 671, 1377, 103, 207, 3464, 932, 4893, 413, 466, 20, 1671, 1607, 462, 774, 558, 262, 170, 1242, 830, 689, 249, 287, 1081, 1107, 402, 419, 587, 176, 994, 515, 14, 317, 264, 788, 300, 102, 2241, 1105, 427, 874, 62, 906, 1295, 268, 652, 999, 376, 29, 258, 72, 8, 1018] payload_rows_per_dst=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] dup_rows_per_dst=[1990, 4008, 1722, 1784, 1128, 4144, 3628, 2488] combine_rows_per_src=[8522, 10494, 8496, 8612, 7488, 12230, 10306, 11264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76262 payload_rows_remote=68926 dup_rows=22042 dup_rows_remote=20446 dedup_ratio=0.224223 router_tokens_per_expert=[426, 335, 457, 1078, 703, 71, 1214, 741, 1031, 1147, 1468, 857, 719, 1407, 150, 249, 3204, 953, 4520, 549, 725, 48, 1432, 1688, 367, 839, 631, 211, 173, 1182, 959, 784, 227, 373, 913, 1135, 337, 441, 852, 188, 1016, 498, 33, 295, 314, 847, 346, 141, 2210, 1207, 464, 907, 87, 982, 1269, 368, 593, 754, 486, 50, 283, 59, 20, 1139] payload_rows_per_dst=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] dup_rows_per_dst=[1986, 3868, 2058, 3844, 1596, 4030, 2096, 2564] combine_rows_per_src=[8064, 10188, 8732, 10868, 7336, 11990, 8472, 10612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77322 payload_rows_remote=66680 dup_rows=20982 dup_rows_remote=17300 dedup_ratio=0.213440 router_tokens_per_expert=[472, 316, 463, 1307, 875, 46, 1340, 708, 1043, 1045, 2009, 836, 687, 1442, 134, 236, 3486, 866, 4506, 490, 499, 25, 1647, 1537, 458, 751, 495, 195, 192, 1171, 820, 897, 255, 296, 1032, 1206, 244, 521, 727, 204, 920, 466, 14, 289, 277, 657, 323, 103, 2183, 1182, 398, 884, 94, 935, 1189, 332, 646, 1098, 400, 41, 228, 44, 7, 963] payload_rows_per_dst=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] dup_rows_per_dst=[3486, 2782, 1596, 3682, 1534, 3474, 1816, 2612] combine_rows_per_src=[10086, 9564, 8532, 10642, 7450, 11622, 8212, 11214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65826 dup_rows=24010 dup_rows_remote=22016 dedup_ratio=0.244242 router_tokens_per_expert=[422, 300, 450, 1204, 903, 57, 1104, 710, 1156, 814, 1797, 835, 643, 1437, 165, 181, 3219, 1183, 4621, 565, 654, 39, 1393, 1434, 395, 952, 616, 157, 136, 1208, 972, 743, 235, 334, 944, 1277, 277, 523, 771, 199, 1039, 561, 19, 312, 303, 876, 297, 104, 2255, 1087, 441, 855, 107, 1013, 1348, 268, 596, 876, 456, 44, 209, 44, 22, 995] payload_rows_per_dst=[10152, 8468, 10458, 10096, 12252, 6224, 8258, 8386] dup_rows_per_dst=[3742, 1994, 4542, 3328, 6110, 798, 1980, 1516] combine_rows_per_src=[10152, 8468, 10458, 10096, 12252, 6224, 8258, 8386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=275513344 dispatch_meta_bytes=341864 combine_payload_bytes=275513344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275855208 ingress_bytes=344719360 link_bytes=344719360 token_bytes=551368552 weight_movement_bytes=69206016 total_nvlink_bytes=620574568 weight_to_token_ratio=0.125517 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297151199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=339984 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266891280 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533442576 weight_movement_bytes=34603008 total_nvlink_bytes=568045584 weight_to_token_ratio=0.064867 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=332472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269669048 ingress_bytes=373145600 link_bytes=373145600 token_bytes=539005624 weight_movement_bytes=103809024 total_nvlink_bytes=642814648 weight_to_token_ratio=0.192594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404116852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=67284 dup_rows=22272 dup_rows_remote=20040 dedup_ratio=0.226562 router_tokens_per_expert=[308, 403, 312, 1013, 698, 53, 1480, 723, 1164, 913, 1845, 919, 625, 1440, 99, 275, 3685, 1052, 4542, 487, 665, 46, 1656, 1584, 487, 1075, 585, 159, 150, 1324, 1033, 558, 181, 233, 876, 1033, 426, 477, 380, 164, 1033, 439, 11, 255, 245, 879, 269, 114, 2596, 1152, 302, 974, 81, 813, 1275, 274, 552, 921, 456, 42, 195, 52, 18, 1076] payload_rows_per_dst=[10226, 8836, 8748, 11194, 11912, 5846, 7938, 11332] dup_rows_per_dst=[3444, 2034, 2232, 4740, 4712, 644, 1804, 2662] combine_rows_per_src=[10226, 8836, 8748, 11194, 11912, 5846, 7938, 11332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=357488 combine_payload_bytes=282320896 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=282678384 ingress_bytes=351526912 link_bytes=351526912 token_bytes=564999280 weight_movement_bytes=69206016 total_nvlink_bytes=634205296 weight_to_token_ratio=0.122489 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322767470e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73114 payload_rows_remote=62328 dup_rows=25190 dup_rows_remote=21326 dedup_ratio=0.256246 router_tokens_per_expert=[458, 367, 445, 1198, 686, 57, 1423, 793, 1035, 970, 1898, 866, 641, 1261, 114, 214, 3616, 949, 4920, 469, 489, 36, 1826, 1593, 321, 836, 550, 153, 189, 1233, 836, 788, 238, 293, 935, 999, 415, 462, 591, 168, 924, 429, 6, 264, 262, 799, 316, 107, 2352, 1048, 546, 937, 73, 882, 1179, 369, 644, 1099, 301, 41, 225, 45, 12, 961] payload_rows_per_dst=[10786, 8310, 13344, 8226, 12266, 5686, 8282, 6214] dup_rows_per_dst=[3864, 1892, 9316, 1586, 5776, 528, 1786, 442] combine_rows_per_src=[10786, 8310, 13344, 8226, 12266, 5686, 8282, 6214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=327720 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313192 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534298664 weight_movement_bytes=34603008 total_nvlink_bytes=568901672 weight_to_token_ratio=0.064763 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282320896 dispatch_meta_bytes=357488 combine_payload_bytes=282320896 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=282678384 ingress_bytes=351526912 link_bytes=351526912 token_bytes=564999280 weight_movement_bytes=69206016 total_nvlink_bytes=634205296 weight_to_token_ratio=0.122489 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322767470e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273121280 dispatch_meta_bytes=335920 combine_payload_bytes=273121280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273457200 ingress_bytes=307724288 link_bytes=307724288 token_bytes=546578480 weight_movement_bytes=34603008 total_nvlink_bytes=581181488 weight_to_token_ratio=0.063308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157941722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269623296 dispatch_meta_bytes=351368 combine_payload_bytes=269623296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269974664 ingress_bytes=338829312 link_bytes=338829312 token_bytes=539597960 weight_movement_bytes=69206016 total_nvlink_bytes=608803976 weight_to_token_ratio=0.128255 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274987423e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275595264 dispatch_meta_bytes=349296 combine_payload_bytes=275595264 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275944560 ingress_bytes=448610304 link_bytes=448610304 token_bytes=551539824 weight_movement_bytes=173015040 total_nvlink_bytes=724554864 weight_to_token_ratio=0.313695 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688084458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77078 payload_rows_remote=65024 dup_rows=21226 dup_rows_remote=17222 dedup_ratio=0.215922 router_tokens_per_expert=[10, 45, 3, 1737, 1261, 2839, 393, 228, 1179, 778, 383, 189, 472, 185, 937, 245, 378, 299, 1, 10, 775, 174, 410, 118, 3259, 1419, 30, 694, 859, 338, 1298, 1403, 11, 141, 2782, 591, 4632, 173, 1207, 263, 711, 428, 1032, 607, 203, 293, 5, 168, 746, 487, 1111, 395, 3661, 131, 978, 165, 722, 51, 65, 1254, 22, 1392, 156, 2220] payload_rows_per_dst=[9968, 12054, 11290, 8934, 8636, 9830, 7058, 9308] dup_rows_per_dst=[3064, 4004, 2304, 3148, 1700, 3582, 968, 2456] combine_rows_per_src=[9968, 12054, 11290, 8934, 8636, 9830, 7058, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=255295488 dispatch_meta_bytes=334616 combine_payload_bytes=255295488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255630104 ingress_bytes=289898496 link_bytes=289898496 token_bytes=510925592 weight_movement_bytes=34603008 total_nvlink_bytes=545528600 weight_to_token_ratio=0.067726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.090864702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77408 payload_rows_remote=68250 dup_rows=20896 dup_rows_remote=17514 dedup_ratio=0.212565 router_tokens_per_expert=[13, 50, 0, 1789, 1281, 3180, 387, 124, 647, 701, 269, 165, 530, 148, 1002, 176, 362, 314, 5, 1, 1014, 133, 299, 127, 3318, 1504, 5, 807, 1049, 365, 1288, 1252, 14, 122, 2565, 557, 5199, 158, 1094, 268, 631, 406, 936, 616, 285, 186, 1, 299, 824, 443, 1070, 475, 3822, 169, 862, 174, 571, 52, 91, 1058, 37, 1587, 68, 2207] payload_rows_per_dst=[10254, 11734, 12104, 9158, 8210, 9884, 7012, 9052] dup_rows_per_dst=[3394, 3186, 2804, 3382, 1346, 3472, 1022, 2290] combine_rows_per_src=[10254, 11734, 12104, 9158, 8210, 9884, 7012, 9052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=65730 dup_rows=23312 dup_rows_remote=20070 dedup_ratio=0.237142 router_tokens_per_expert=[18, 20, 3, 1580, 1388, 2682, 741, 126, 896, 699, 339, 125, 407, 126, 1095, 113, 350, 323, 1, 2, 959, 131, 148, 137, 2997, 1537, 20, 773, 1430, 378, 1468, 1297, 4, 106, 2535, 466, 5077, 154, 1070, 226, 576, 453, 744, 763, 286, 251, 2, 180, 742, 362, 1417, 404, 4218, 148, 805, 121, 534, 34, 109, 909, 36, 1537, 61, 2513] payload_rows_per_dst=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] dup_rows_per_dst=[3634, 1254, 2436, 3316, 1226, 3242, 5812, 2392] combine_rows_per_src=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76752 payload_rows_remote=65756 dup_rows=21552 dup_rows_remote=17362 dedup_ratio=0.219238 router_tokens_per_expert=[535, 322, 476, 1195, 834, 68, 1308, 720, 1143, 1060, 1849, 710, 701, 1313, 131, 163, 3511, 904, 4522, 498, 691, 25, 1495, 1535, 439, 848, 523, 138, 166, 1199, 885, 777, 186, 299, 886, 1160, 309, 566, 693, 194, 1076, 491, 7, 322, 262, 706, 290, 94, 2696, 1259, 396, 811, 109, 937, 1099, 286, 629, 808, 423, 23, 254, 55, 16, 1126] payload_rows_per_dst=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] dup_rows_per_dst=[2156, 3716, 1754, 1774, 1514, 3944, 4190, 2504] combine_rows_per_src=[8760, 10424, 8542, 8176, 7072, 11596, 10996, 11186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70756 payload_rows_remote=59840 dup_rows=27548 dup_rows_remote=22964 dedup_ratio=0.280233 router_tokens_per_expert=[0, 20, 1, 2003, 1133, 2990, 280, 150, 743, 743, 312, 214, 457, 126, 901, 183, 302, 311, 2, 4, 871, 155, 351, 121, 3198, 1826, 6, 801, 766, 332, 1154, 1628, 9, 119, 2939, 694, 4901, 92, 918, 239, 632, 374, 899, 504, 205, 228, 10, 170, 991, 415, 1161, 417, 3834, 144, 928, 241, 607, 67, 96, 1292, 26, 1371, 119, 2426] payload_rows_per_dst=[6474, 6060, 3860, 6896, 14376, 9348, 12826, 10916] dup_rows_per_dst=[700, 1298, 374, 2478, 7934, 3092, 7088, 4584] combine_rows_per_src=[6474, 6060, 3860, 6896, 14376, 9348, 12826, 10916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72354 payload_rows_remote=59916 dup_rows=25950 dup_rows_remote=19646 dedup_ratio=0.263977 router_tokens_per_expert=[3, 43, 4, 1716, 1413, 3241, 489, 52, 839, 727, 292, 148, 394, 118, 981, 136, 372, 286, 2, 11, 817, 154, 255, 124, 3363, 1472, 13, 881, 1045, 405, 1493, 1314, 3, 174, 2557, 483, 5179, 250, 895, 195, 581, 445, 926, 641, 204, 191, 4, 203, 898, 391, 1120, 430, 3901, 128, 857, 174, 658, 57, 66, 1011, 50, 1317, 121, 2439] payload_rows_per_dst=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] dup_rows_per_dst=[2016, 1184, 1310, 2840, 6426, 3512, 6304, 2358] combine_rows_per_src=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=65730 dup_rows=23312 dup_rows_remote=20070 dedup_ratio=0.237142 router_tokens_per_expert=[18, 20, 3, 1580, 1388, 2682, 741, 126, 896, 699, 339, 125, 407, 126, 1095, 113, 350, 323, 1, 2, 959, 131, 148, 137, 2997, 1537, 20, 773, 1430, 378, 1468, 1297, 4, 106, 2535, 466, 5077, 154, 1070, 226, 576, 453, 744, 763, 286, 251, 2, 180, 742, 362, 1417, 404, 4218, 148, 805, 121, 534, 34, 109, 909, 36, 1537, 61, 2513] payload_rows_per_dst=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] dup_rows_per_dst=[3634, 1254, 2436, 3316, 1226, 3242, 5812, 2392] combine_rows_per_src=[9482, 6346, 11820, 8644, 7896, 9262, 12468, 9074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77408 payload_rows_remote=68250 dup_rows=20896 dup_rows_remote=17514 dedup_ratio=0.212565 router_tokens_per_expert=[13, 50, 0, 1789, 1281, 3180, 387, 124, 647, 701, 269, 165, 530, 148, 1002, 176, 362, 314, 5, 1, 1014, 133, 299, 127, 3318, 1504, 5, 807, 1049, 365, 1288, 1252, 14, 122, 2565, 557, 5199, 158, 1094, 268, 631, 406, 936, 616, 285, 186, 1, 299, 824, 443, 1070, 475, 3822, 169, 862, 174, 571, 52, 91, 1058, 37, 1587, 68, 2207] payload_rows_per_dst=[10254, 11734, 12104, 9158, 8210, 9884, 7012, 9052] dup_rows_per_dst=[3394, 3186, 2804, 3382, 1346, 3472, 1022, 2290] combine_rows_per_src=[10254, 11734, 12104, 9158, 8210, 9884, 7012, 9052] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76272 payload_rows_remote=68104 dup_rows=22032 dup_rows_remote=20546 dedup_ratio=0.224121 router_tokens_per_expert=[43, 61, 7, 1494, 1332, 2637, 603, 154, 1087, 824, 304, 281, 450, 123, 953, 284, 497, 330, 5, 6, 837, 182, 327, 134, 3256, 1334, 17, 801, 940, 368, 1569, 1391, 13, 195, 2354, 505, 4592, 225, 1281, 254, 737, 397, 919, 679, 207, 229, 5, 147, 798, 455, 1042, 380, 3614, 152, 841, 177, 659, 40, 100, 1170, 58, 1797, 149, 2350] payload_rows_per_dst=[9224, 10142, 11400, 9374, 8168, 9820, 8462, 9682] dup_rows_per_dst=[3438, 2996, 2420, 3466, 1486, 3332, 1930, 2964] combine_rows_per_src=[9224, 10142, 11400, 9374, 8168, 9820, 8462, 9682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=328984 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667288 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533005592 weight_movement_bytes=34603008 total_nvlink_bytes=567608600 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=343056 combine_payload_bytes=279552000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279895056 ingress_bytes=383361024 link_bytes=383361024 token_bytes=559447056 weight_movement_bytes=103809024 total_nvlink_bytes=663256080 weight_to_token_ratio=0.185556 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.442556670e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269230080 dispatch_meta_bytes=343200 combine_payload_bytes=269230080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269573280 ingress_bytes=303833088 link_bytes=303833088 token_bytes=538803360 weight_movement_bytes=34603008 total_nvlink_bytes=573406368 weight_to_token_ratio=0.064222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143299450e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77078 payload_rows_remote=65024 dup_rows=21226 dup_rows_remote=17222 dedup_ratio=0.215922 router_tokens_per_expert=[10, 45, 3, 1737, 1261, 2839, 393, 228, 1179, 778, 383, 189, 472, 185, 937, 245, 378, 299, 1, 10, 775, 174, 410, 118, 3259, 1419, 30, 694, 859, 338, 1298, 1403, 11, 141, 2782, 591, 4632, 173, 1207, 263, 711, 428, 1032, 607, 203, 293, 5, 168, 746, 487, 1111, 395, 3661, 131, 978, 165, 722, 51, 65, 1254, 22, 1392, 156, 2220] payload_rows_per_dst=[9968, 12054, 11290, 8934, 8636, 9830, 7058, 9308] dup_rows_per_dst=[3064, 4004, 2304, 3148, 1700, 3582, 968, 2456] combine_rows_per_src=[9968, 12054, 11290, 8934, 8636, 9830, 7058, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76460 payload_rows_remote=64754 dup_rows=21844 dup_rows_remote=19552 dedup_ratio=0.222209 router_tokens_per_expert=[22, 26, 4, 1558, 1593, 3171, 653, 119, 919, 705, 365, 108, 433, 107, 1111, 259, 495, 311, 9, 2, 792, 130, 176, 160, 2977, 1407, 6, 902, 1261, 400, 1487, 1186, 5, 159, 2381, 331, 4924, 211, 1238, 224, 606, 418, 1004, 653, 178, 281, 4, 204, 656, 363, 1183, 448, 3962, 160, 650, 110, 545, 47, 85, 895, 114, 1580, 189, 2490] payload_rows_per_dst=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] dup_rows_per_dst=[4136, 3686, 2292, 3670, 1336, 3244, 740, 2740] combine_rows_per_src=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=245104640 dispatch_meta_bytes=331216 combine_payload_bytes=245104640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245435856 ingress_bytes=279707648 link_bytes=279707648 token_bytes=490540496 weight_movement_bytes=34603008 total_nvlink_bytes=525143504 weight_to_token_ratio=0.070541 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.052517361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76620 payload_rows_remote=66336 dup_rows=21684 dup_rows_remote=17514 dedup_ratio=0.220581 router_tokens_per_expert=[11, 13, 3, 1452, 1515, 2824, 563, 112, 1069, 736, 363, 58, 425, 121, 1106, 173, 469, 343, 1, 5, 1025, 147, 162, 129, 3033, 1415, 7, 858, 909, 415, 1395, 1372, 4, 142, 2758, 424, 5046, 189, 1011, 251, 600, 453, 691, 664, 268, 242, 2, 201, 725, 352, 1299, 408, 3960, 165, 733, 206, 542, 31, 99, 1108, 32, 1830, 116, 2371] payload_rows_per_dst=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] dup_rows_per_dst=[4170, 3626, 2692, 4644, 1440, 1684, 824, 2604] combine_rows_per_src=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=332472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269669048 ingress_bytes=373145600 link_bytes=373145600 token_bytes=539005624 weight_movement_bytes=103809024 total_nvlink_bytes=642814648 weight_to_token_ratio=0.192594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404116852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=245415936 dispatch_meta_bytes=318248 combine_payload_bytes=245415936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245734184 ingress_bytes=314621952 link_bytes=314621952 token_bytes=491150120 weight_movement_bytes=69206016 total_nvlink_bytes=560356136 weight_to_token_ratio=0.140906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183897076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76460 payload_rows_remote=64754 dup_rows=21844 dup_rows_remote=19552 dedup_ratio=0.222209 router_tokens_per_expert=[22, 26, 4, 1558, 1593, 3171, 653, 119, 919, 705, 365, 108, 433, 107, 1111, 259, 495, 311, 9, 2, 792, 130, 176, 160, 2977, 1407, 6, 902, 1261, 400, 1487, 1186, 5, 159, 2381, 331, 4924, 211, 1238, 224, 606, 418, 1004, 653, 178, 281, 4, 204, 656, 363, 1183, 448, 3962, 160, 650, 110, 545, 47, 85, 895, 114, 1580, 189, 2490] payload_rows_per_dst=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] dup_rows_per_dst=[4136, 3686, 2292, 3670, 1336, 3244, 740, 2740] combine_rows_per_src=[10156, 12252, 11706, 9628, 7762, 9406, 6400, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=354600 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279308584 ingress_bytes=348160000 link_bytes=348160000 token_bytes=558262568 weight_movement_bytes=69206016 total_nvlink_bytes=627468584 weight_to_token_ratio=0.123967 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310098051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269230080 dispatch_meta_bytes=343200 combine_payload_bytes=269230080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269573280 ingress_bytes=303833088 link_bytes=303833088 token_bytes=538803360 weight_movement_bytes=34603008 total_nvlink_bytes=573406368 weight_to_token_ratio=0.064222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143299450e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76620 payload_rows_remote=66336 dup_rows=21684 dup_rows_remote=17514 dedup_ratio=0.220581 router_tokens_per_expert=[11, 13, 3, 1452, 1515, 2824, 563, 112, 1069, 736, 363, 58, 425, 121, 1106, 173, 469, 343, 1, 5, 1025, 147, 162, 129, 3033, 1415, 7, 858, 909, 415, 1395, 1372, 4, 142, 2758, 424, 5046, 189, 1011, 251, 600, 453, 691, 664, 268, 242, 2, 201, 725, 352, 1299, 408, 3960, 165, 733, 206, 542, 31, 99, 1108, 32, 1830, 116, 2371] payload_rows_per_dst=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] dup_rows_per_dst=[4170, 3626, 2692, 4644, 1440, 1684, 824, 2604] combine_rows_per_src=[10284, 12396, 11962, 11264, 8118, 5990, 6952, 9654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=337224 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265569608 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530801992 weight_movement_bytes=34603008 total_nvlink_bytes=565405000 weight_to_token_ratio=0.065190 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76312 payload_rows_remote=63800 dup_rows=21992 dup_rows_remote=17154 dedup_ratio=0.223714 router_tokens_per_expert=[2247, 2852, 1480, 101, 44, 0, 0, 363, 324, 1605, 2, 679, 132, 1898, 407, 64, 1046, 242, 42, 505, 696, 270, 711, 1105, 1345, 588, 3, 1360, 4, 1611, 10, 3754, 516, 1665, 924, 107, 2466, 0, 963, 402, 25, 941, 419, 1171, 0, 484, 194, 1975, 449, 3110, 259, 353, 1008, 275, 703, 543, 1036, 17, 283, 799, 595, 1542, 316, 122] payload_rows_per_dst=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] dup_rows_per_dst=[3986, 3058, 1456, 4838, 2346, 1766, 3024, 1518] combine_rows_per_src=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75422 payload_rows_remote=63104 dup_rows=22882 dup_rows_remote=17020 dedup_ratio=0.232768 router_tokens_per_expert=[2778, 2379, 1105, 122, 24, 0, 0, 279, 337, 1466, 1, 645, 88, 2307, 437, 62, 1010, 185, 117, 406, 730, 233, 652, 856, 1128, 589, 2, 946, 1, 1480, 3, 4125, 736, 2045, 1049, 84, 2306, 0, 848, 758, 12, 906, 365, 1032, 1, 596, 216, 2146, 715, 3216, 157, 354, 1016, 280, 389, 467, 1168, 8, 364, 771, 611, 1698, 211, 134] payload_rows_per_dst=[7812, 8738, 7030, 9380, 10662, 8816, 10666, 12318] dup_rows_per_dst=[804, 1948, 1348, 3314, 4990, 1732, 2884, 5862] combine_rows_per_src=[7812, 8738, 7030, 9380, 10662, 8816, 10666, 12318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75788 payload_rows_remote=64464 dup_rows=22516 dup_rows_remote=19428 dedup_ratio=0.229045 router_tokens_per_expert=[2145, 2569, 1180, 162, 213, 3, 0, 394, 347, 1710, 0, 619, 100, 1811, 397, 56, 928, 240, 163, 361, 877, 288, 761, 1561, 1313, 660, 0, 1167, 1, 1498, 6, 3963, 545, 1802, 569, 112, 2166, 4, 738, 532, 22, 1180, 406, 1298, 0, 700, 207, 2010, 872, 2872, 130, 387, 1027, 364, 697, 533, 1124, 11, 330, 695, 497, 1403, 295, 131] payload_rows_per_dst=[9782, 11324, 12814, 7758, 7200, 10518, 8856, 7536] dup_rows_per_dst=[3550, 3088, 5470, 1532, 1404, 3472, 2564, 1436] combine_rows_per_src=[9782, 11324, 12814, 7758, 7200, 10518, 8856, 7536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74610 payload_rows_remote=65820 dup_rows=23694 dup_rows_remote=21452 dedup_ratio=0.241028 router_tokens_per_expert=[1995, 2736, 1267, 81, 51, 0, 0, 318, 292, 1640, 0, 643, 123, 1788, 394, 40, 845, 446, 39, 625, 745, 250, 848, 1480, 919, 653, 8, 1090, 2, 1721, 1, 3709, 513, 1616, 636, 145, 2712, 0, 1031, 720, 11, 1172, 387, 1727, 0, 470, 229, 1520, 442, 3225, 271, 350, 1083, 249, 711, 496, 891, 10, 346, 918, 552, 1309, 535, 126] payload_rows_per_dst=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] dup_rows_per_dst=[3234, 1770, 2016, 1504, 4452, 2242, 3136, 5340] combine_rows_per_src=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74726 payload_rows_remote=64882 dup_rows=23578 dup_rows_remote=20764 dedup_ratio=0.239848 router_tokens_per_expert=[2103, 2586, 1145, 58, 124, 3, 0, 315, 202, 1510, 0, 686, 93, 1518, 375, 73, 958, 274, 20, 563, 695, 238, 788, 1340, 1321, 629, 2, 979, 1, 1661, 6, 3655, 444, 1750, 728, 157, 3296, 3, 859, 815, 19, 1671, 341, 1460, 1, 518, 234, 1837, 540, 2748, 199, 318, 835, 271, 869, 549, 990, 4, 487, 842, 469, 1415, 449, 113] payload_rows_per_dst=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] dup_rows_per_dst=[1520, 1504, 1686, 3010, 5038, 2562, 2814, 5444] combine_rows_per_src=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279552000 dispatch_meta_bytes=343056 combine_payload_bytes=279552000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279895056 ingress_bytes=383361024 link_bytes=383361024 token_bytes=559447056 weight_movement_bytes=103809024 total_nvlink_bytes=663256080 weight_to_token_ratio=0.185556 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.442556670e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=354600 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279308584 ingress_bytes=348160000 link_bytes=348160000 token_bytes=558262568 weight_movement_bytes=69206016 total_nvlink_bytes=627468584 weight_to_token_ratio=0.123967 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310098051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=328984 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667288 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533005592 weight_movement_bytes=34603008 total_nvlink_bytes=567608600 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=337224 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265569608 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530801992 weight_movement_bytes=34603008 total_nvlink_bytes=565405000 weight_to_token_ratio=0.065190 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271712256 dispatch_meta_bytes=335400 combine_payload_bytes=271712256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=272047656 ingress_bytes=375521280 link_bytes=375521280 token_bytes=543759912 weight_movement_bytes=103809024 total_nvlink_bytes=647568936 weight_to_token_ratio=0.190910 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413056345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75422 payload_rows_remote=63104 dup_rows=22882 dup_rows_remote=17020 dedup_ratio=0.232768 router_tokens_per_expert=[2778, 2379, 1105, 122, 24, 0, 0, 279, 337, 1466, 1, 645, 88, 2307, 437, 62, 1010, 185, 117, 406, 730, 233, 652, 856, 1128, 589, 2, 946, 1, 1480, 3, 4125, 736, 2045, 1049, 84, 2306, 0, 848, 758, 12, 906, 365, 1032, 1, 596, 216, 2146, 715, 3216, 157, 354, 1016, 280, 389, 467, 1168, 8, 364, 771, 611, 1698, 211, 134] payload_rows_per_dst=[7812, 8738, 7030, 9380, 10662, 8816, 10666, 12318] dup_rows_per_dst=[804, 1948, 1348, 3314, 4990, 1732, 2884, 5862] combine_rows_per_src=[7812, 8738, 7030, 9380, 10662, 8816, 10666, 12318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72354 payload_rows_remote=59916 dup_rows=25950 dup_rows_remote=19646 dedup_ratio=0.263977 router_tokens_per_expert=[3, 43, 4, 1716, 1413, 3241, 489, 52, 839, 727, 292, 148, 394, 118, 981, 136, 372, 286, 2, 11, 817, 154, 255, 124, 3363, 1472, 13, 881, 1045, 405, 1493, 1314, 3, 174, 2557, 483, 5179, 250, 895, 195, 581, 445, 926, 641, 204, 191, 4, 203, 898, 391, 1120, 430, 3901, 128, 857, 174, 658, 57, 66, 1011, 50, 1317, 121, 2439] payload_rows_per_dst=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] dup_rows_per_dst=[2016, 1184, 1310, 2840, 6426, 3512, 6304, 2358] combine_rows_per_src=[7910, 6086, 7530, 7462, 12244, 9604, 12438, 9080] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74610 payload_rows_remote=65820 dup_rows=23694 dup_rows_remote=21452 dedup_ratio=0.241028 router_tokens_per_expert=[1995, 2736, 1267, 81, 51, 0, 0, 318, 292, 1640, 0, 643, 123, 1788, 394, 40, 845, 446, 39, 625, 745, 250, 848, 1480, 919, 653, 8, 1090, 2, 1721, 1, 3709, 513, 1616, 636, 145, 2712, 0, 1031, 720, 11, 1172, 387, 1727, 0, 470, 229, 1520, 442, 3225, 271, 350, 1083, 249, 711, 496, 891, 10, 346, 918, 552, 1309, 535, 126] payload_rows_per_dst=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] dup_rows_per_dst=[3234, 1770, 2016, 1504, 4452, 2242, 3136, 5340] combine_rows_per_src=[9662, 8070, 8540, 7284, 10294, 8790, 10518, 11452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76312 payload_rows_remote=63800 dup_rows=21992 dup_rows_remote=17154 dedup_ratio=0.223714 router_tokens_per_expert=[2247, 2852, 1480, 101, 44, 0, 0, 363, 324, 1605, 2, 679, 132, 1898, 407, 64, 1046, 242, 42, 505, 696, 270, 711, 1105, 1345, 588, 3, 1360, 4, 1611, 10, 3754, 516, 1665, 924, 107, 2466, 0, 963, 402, 25, 941, 419, 1171, 0, 484, 194, 1975, 449, 3110, 259, 353, 1008, 275, 703, 543, 1036, 17, 283, 799, 595, 1542, 316, 122] payload_rows_per_dst=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] dup_rows_per_dst=[3986, 3058, 1456, 4838, 2346, 1766, 3024, 1518] combine_rows_per_src=[10188, 10798, 7778, 12512, 8106, 8652, 10376, 7902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258473984 dispatch_meta_bytes=320496 combine_payload_bytes=258473984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258794480 ingress_bytes=293076992 link_bytes=293076992 token_bytes=517268464 weight_movement_bytes=34603008 total_nvlink_bytes=551871472 weight_to_token_ratio=0.066896 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102825126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64780 dup_rows=23102 dup_rows_remote=19218 dedup_ratio=0.235006 router_tokens_per_expert=[1505, 2909, 1448, 141, 296, 7, 0, 259, 246, 1565, 6, 673, 108, 1277, 406, 77, 857, 374, 19, 595, 684, 367, 749, 1497, 1305, 623, 22, 1197, 9, 1629, 11, 3437, 551, 1670, 439, 201, 2913, 12, 846, 521, 11, 1600, 450, 1276, 3, 550, 338, 2161, 630, 2470, 367, 356, 1072, 349, 819, 620, 843, 39, 419, 866, 563, 1347, 419, 133] payload_rows_per_dst=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] dup_rows_per_dst=[3468, 1382, 1834, 4282, 3884, 2738, 3400, 2114] combine_rows_per_src=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=245415936 dispatch_meta_bytes=318248 combine_payload_bytes=245415936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245734184 ingress_bytes=314621952 link_bytes=314621952 token_bytes=491150120 weight_movement_bytes=69206016 total_nvlink_bytes=560356136 weight_to_token_ratio=0.140906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183897076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75788 payload_rows_remote=64464 dup_rows=22516 dup_rows_remote=19428 dedup_ratio=0.229045 router_tokens_per_expert=[2145, 2569, 1180, 162, 213, 3, 0, 394, 347, 1710, 0, 619, 100, 1811, 397, 56, 928, 240, 163, 361, 877, 288, 761, 1561, 1313, 660, 0, 1167, 1, 1498, 6, 3963, 545, 1802, 569, 112, 2166, 4, 738, 532, 22, 1180, 406, 1298, 0, 700, 207, 2010, 872, 2872, 130, 387, 1027, 364, 697, 533, 1124, 11, 330, 695, 497, 1403, 295, 131] payload_rows_per_dst=[9782, 11324, 12814, 7758, 7200, 10518, 8856, 7536] dup_rows_per_dst=[3550, 3088, 5470, 1532, 1404, 3472, 2564, 1436] combine_rows_per_src=[9782, 11324, 12814, 7758, 7200, 10518, 8856, 7536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75032 payload_rows_remote=64640 dup_rows=23272 dup_rows_remote=19296 dedup_ratio=0.236735 router_tokens_per_expert=[1467, 2644, 1500, 86, 202, 0, 1, 345, 246, 1811, 0, 751, 154, 1304, 424, 52, 944, 471, 12, 879, 627, 355, 690, 1508, 1263, 614, 18, 978, 3, 1743, 10, 3556, 334, 1416, 402, 297, 3246, 0, 908, 522, 4, 1869, 389, 1790, 0, 401, 248, 1823, 377, 2590, 236, 287, 962, 284, 876, 614, 1030, 9, 329, 886, 453, 1336, 521, 55] payload_rows_per_dst=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] dup_rows_per_dst=[3102, 4282, 3976, 3238, 1292, 2802, 2816, 1764] combine_rows_per_src=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73576 payload_rows_remote=63474 dup_rows=24728 dup_rows_remote=20638 dedup_ratio=0.251546 router_tokens_per_expert=[1971, 2922, 1707, 85, 44, 0, 0, 367, 296, 1822, 0, 682, 148, 1597, 382, 40, 896, 474, 24, 664, 810, 260, 631, 1472, 1124, 624, 4, 1289, 5, 1557, 4, 3493, 451, 1313, 614, 232, 2985, 0, 1090, 457, 13, 1298, 424, 1392, 0, 397, 210, 1697, 478, 2917, 313, 308, 1058, 321, 865, 559, 875, 6, 330, 793, 737, 1119, 379, 127] payload_rows_per_dst=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] dup_rows_per_dst=[4090, 4320, 5536, 1656, 2674, 3642, 1394, 1416] combine_rows_per_src=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64780 dup_rows=23102 dup_rows_remote=19218 dedup_ratio=0.235006 router_tokens_per_expert=[1505, 2909, 1448, 141, 296, 7, 0, 259, 246, 1565, 6, 673, 108, 1277, 406, 77, 857, 374, 19, 595, 684, 367, 749, 1497, 1305, 623, 22, 1197, 9, 1629, 11, 3437, 551, 1670, 439, 201, 2913, 12, 846, 521, 11, 1600, 450, 1276, 3, 550, 338, 2161, 630, 2470, 367, 356, 1072, 349, 819, 620, 843, 39, 419, 866, 563, 1347, 419, 133] payload_rows_per_dst=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] dup_rows_per_dst=[3468, 1382, 1834, 4282, 3884, 2738, 3400, 2114] combine_rows_per_src=[9662, 7334, 8450, 11320, 10422, 10040, 9966, 8008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261324800 dispatch_meta_bytes=323816 combine_payload_bytes=261324800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261648616 ingress_bytes=365133824 link_bytes=365133824 token_bytes=522973416 weight_movement_bytes=103809024 total_nvlink_bytes=626782440 weight_to_token_ratio=0.198498 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269598720 dispatch_meta_bytes=349088 combine_payload_bytes=269598720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269947808 ingress_bytes=304201728 link_bytes=304201728 token_bytes=539546528 weight_movement_bytes=34603008 total_nvlink_bytes=574149536 weight_to_token_ratio=0.064134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144686612e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258473984 dispatch_meta_bytes=320496 combine_payload_bytes=258473984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258794480 ingress_bytes=293076992 link_bytes=293076992 token_bytes=517268464 weight_movement_bytes=34603008 total_nvlink_bytes=551871472 weight_to_token_ratio=0.066896 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102825126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=335568 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264380112 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528424656 weight_movement_bytes=34603008 total_nvlink_bytes=563027664 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=342584 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266099256 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531855928 weight_movement_bytes=103809024 total_nvlink_bytes=635664952 weight_to_token_ratio=0.195183 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75032 payload_rows_remote=64640 dup_rows=23272 dup_rows_remote=19296 dedup_ratio=0.236735 router_tokens_per_expert=[1467, 2644, 1500, 86, 202, 0, 1, 345, 246, 1811, 0, 751, 154, 1304, 424, 52, 944, 471, 12, 879, 627, 355, 690, 1508, 1263, 614, 18, 978, 3, 1743, 10, 3556, 334, 1416, 402, 297, 3246, 0, 908, 522, 4, 1869, 389, 1790, 0, 401, 248, 1823, 377, 2590, 236, 287, 962, 284, 876, 614, 1030, 9, 329, 886, 453, 1336, 521, 55] payload_rows_per_dst=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] dup_rows_per_dst=[3102, 4282, 3976, 3238, 1292, 2802, 2816, 1764] combine_rows_per_src=[9388, 11694, 10392, 9736, 6466, 10246, 9636, 7474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261324800 dispatch_meta_bytes=323816 combine_payload_bytes=261324800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261648616 ingress_bytes=365133824 link_bytes=365133824 token_bytes=522973416 weight_movement_bytes=103809024 total_nvlink_bytes=626782440 weight_to_token_ratio=0.198498 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=335992 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265674872 ingress_bytes=334544896 link_bytes=334544896 token_bytes=531013752 weight_movement_bytes=69206016 total_nvlink_bytes=600219768 weight_to_token_ratio=0.130328 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258865511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=335568 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264380112 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528424656 weight_movement_bytes=34603008 total_nvlink_bytes=563027664 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=335744 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265101184 ingress_bytes=299368448 link_bytes=299368448 token_bytes=529866624 weight_movement_bytes=34603008 total_nvlink_bytes=564469632 weight_to_token_ratio=0.065305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126499369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73576 payload_rows_remote=63474 dup_rows=24728 dup_rows_remote=20638 dedup_ratio=0.251546 router_tokens_per_expert=[1971, 2922, 1707, 85, 44, 0, 0, 367, 296, 1822, 0, 682, 148, 1597, 382, 40, 896, 474, 24, 664, 810, 260, 631, 1472, 1124, 624, 4, 1289, 5, 1557, 4, 3493, 451, 1313, 614, 232, 2985, 0, 1090, 457, 13, 1298, 424, 1392, 0, 397, 210, 1697, 478, 2917, 313, 308, 1058, 321, 865, 559, 875, 6, 330, 793, 737, 1119, 379, 127] payload_rows_per_dst=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] dup_rows_per_dst=[4090, 4320, 5536, 1656, 2674, 3642, 1394, 1416] combine_rows_per_src=[10102, 11584, 11912, 7558, 8008, 10686, 6410, 7316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=67354 dup_rows=21632 dup_rows_remote=19340 dedup_ratio=0.220052 router_tokens_per_expert=[113, 2743, 1419, 1777, 1000, 1901, 60, 76, 489, 520, 81, 1508, 3383, 122, 36, 2415, 576, 3, 321, 702, 26, 38, 503, 806, 35, 1600, 596, 149, 69, 0, 503, 1848, 391, 336, 48, 1191, 132, 1, 864, 36, 1, 241, 4, 176, 2356, 1677, 67, 1283, 1090, 490, 562, 727, 85, 4, 1536, 356, 287, 2938, 522, 1953, 2398, 1068, 738, 176] payload_rows_per_dst=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] dup_rows_per_dst=[1290, 2102, 2390, 4698, 3918, 2292, 1382, 3560] combine_rows_per_src=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78636 payload_rows_remote=70324 dup_rows=19668 dup_rows_remote=18638 dedup_ratio=0.200073 router_tokens_per_expert=[116, 1955, 1501, 1369, 919, 2001, 52, 79, 405, 696, 103, 1168, 3172, 228, 35, 2106, 800, 3, 238, 613, 136, 19, 400, 976, 44, 1398, 561, 96, 67, 12, 304, 2189, 409, 326, 48, 1568, 147, 2, 909, 6, 1, 293, 6, 177, 2293, 1830, 100, 1022, 915, 469, 446, 399, 109, 6, 1787, 469, 462, 3512, 498, 2351, 3008, 961, 755, 107] payload_rows_per_dst=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] dup_rows_per_dst=[1530, 4356, 3268, 1030, 2170, 2400, 3422, 1492] combine_rows_per_src=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76102 payload_rows_remote=62202 dup_rows=22202 dup_rows_remote=14450 dedup_ratio=0.225850 router_tokens_per_expert=[110, 1612, 1462, 1792, 264, 1849, 9, 114, 455, 803, 154, 1105, 2958, 332, 54, 1909, 698, 13, 263, 804, 67, 21, 278, 1301, 95, 1010, 638, 100, 154, 94, 398, 2269, 326, 382, 152, 1896, 123, 26, 681, 5, 3, 452, 5, 128, 2417, 1542, 106, 1024, 1361, 436, 481, 257, 234, 4, 2659, 441, 478, 3097, 534, 1875, 3053, 756, 894, 139] payload_rows_per_dst=[6914, 7814, 6366, 11480, 10692, 9100, 9836, 13900] dup_rows_per_dst=[588, 1810, 524, 3952, 3412, 2254, 1910, 7752] combine_rows_per_src=[6914, 7814, 6366, 11480, 10692, 9100, 9836, 13900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76354 payload_rows_remote=66022 dup_rows=21950 dup_rows_remote=19570 dedup_ratio=0.223287 router_tokens_per_expert=[133, 1255, 1606, 1942, 738, 1878, 23, 156, 383, 780, 97, 1066, 3550, 352, 73, 1903, 769, 6, 403, 677, 24, 36, 343, 1301, 147, 1256, 625, 158, 118, 158, 301, 2101, 463, 433, 145, 1725, 195, 83, 609, 4, 3, 363, 7, 137, 2036, 1916, 115, 1127, 1388, 513, 468, 255, 354, 17, 1964, 406, 472, 3110, 570, 1633, 2499, 786, 785, 213] payload_rows_per_dst=[11854, 10332, 10128, 8490, 8000, 10362, 11682, 5506] dup_rows_per_dst=[4794, 2380, 3210, 1238, 1824, 3890, 4046, 568] combine_rows_per_src=[11854, 10332, 10128, 8490, 8000, 10362, 11682, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76302 payload_rows_remote=68528 dup_rows=22002 dup_rows_remote=20972 dedup_ratio=0.223816 router_tokens_per_expert=[102, 1874, 1526, 1669, 1002, 2142, 83, 47, 392, 529, 97, 1296, 3578, 127, 139, 2214, 703, 39, 244, 685, 104, 21, 703, 711, 27, 1177, 546, 107, 69, 0, 421, 2246, 520, 169, 51, 1458, 114, 0, 1073, 17, 9, 282, 7, 110, 2012, 1498, 84, 1194, 1026, 489, 510, 313, 212, 8, 1355, 489, 427, 3531, 501, 1918, 3042, 1141, 842, 130] payload_rows_per_dst=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] dup_rows_per_dst=[986, 1772, 2884, 4562, 3906, 2048, 1030, 4814] combine_rows_per_src=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=336448 combine_payload_bytes=259989504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=260325952 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520315456 weight_movement_bytes=69206016 total_nvlink_bytes=589521472 weight_to_token_ratio=0.133008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76102 payload_rows_remote=62202 dup_rows=22202 dup_rows_remote=14450 dedup_ratio=0.225850 router_tokens_per_expert=[110, 1612, 1462, 1792, 264, 1849, 9, 114, 455, 803, 154, 1105, 2958, 332, 54, 1909, 698, 13, 263, 804, 67, 21, 278, 1301, 95, 1010, 638, 100, 154, 94, 398, 2269, 326, 382, 152, 1896, 123, 26, 681, 5, 3, 452, 5, 128, 2417, 1542, 106, 1024, 1361, 436, 481, 257, 234, 4, 2659, 441, 478, 3097, 534, 1875, 3053, 756, 894, 139] payload_rows_per_dst=[6914, 7814, 6366, 11480, 10692, 9100, 9836, 13900] dup_rows_per_dst=[588, 1810, 524, 3952, 3412, 2254, 1910, 7752] combine_rows_per_src=[6914, 7814, 6366, 11480, 10692, 9100, 9836, 13900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74726 payload_rows_remote=64882 dup_rows=23578 dup_rows_remote=20764 dedup_ratio=0.239848 router_tokens_per_expert=[2103, 2586, 1145, 58, 124, 3, 0, 315, 202, 1510, 0, 686, 93, 1518, 375, 73, 958, 274, 20, 563, 695, 238, 788, 1340, 1321, 629, 2, 979, 1, 1661, 6, 3655, 444, 1750, 728, 157, 3296, 3, 859, 815, 19, 1671, 341, 1460, 1, 518, 234, 1837, 540, 2748, 199, 318, 835, 271, 869, 549, 990, 4, 487, 842, 469, 1415, 449, 113] payload_rows_per_dst=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] dup_rows_per_dst=[1520, 1504, 1686, 3010, 5038, 2562, 2814, 5444] combine_rows_per_src=[7954, 7410, 8066, 9382, 11066, 9600, 9844, 11404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77320 payload_rows_remote=68138 dup_rows=20984 dup_rows_remote=18374 dedup_ratio=0.213460 router_tokens_per_expert=[119, 1884, 1478, 1693, 1124, 1884, 50, 122, 392, 625, 86, 1335, 3064, 135, 166, 2175, 671, 93, 458, 490, 108, 49, 413, 764, 62, 1528, 552, 149, 32, 102, 301, 1744, 496, 320, 36, 1643, 137, 3, 968, 26, 5, 299, 16, 248, 2455, 1917, 105, 982, 1004, 530, 506, 496, 139, 10, 1551, 474, 239, 3645, 509, 1786, 2938, 955, 672, 194] payload_rows_per_dst=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] dup_rows_per_dst=[1980, 2108, 3240, 3830, 2610, 2538, 1210, 3468] combine_rows_per_src=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275881984 dispatch_meta_bytes=346776 combine_payload_bytes=275881984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276228760 ingress_bytes=310484992 link_bytes=310484992 token_bytes=552110744 weight_movement_bytes=34603008 total_nvlink_bytes=586713752 weight_to_token_ratio=0.062674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168330029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=288047104 dispatch_meta_bytes=355848 combine_payload_bytes=288047104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288402952 ingress_bytes=322650112 link_bytes=322650112 token_bytes=576450056 weight_movement_bytes=34603008 total_nvlink_bytes=611053064 weight_to_token_ratio=0.060028 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214106396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76672 payload_rows_remote=67354 dup_rows=21632 dup_rows_remote=19340 dedup_ratio=0.220052 router_tokens_per_expert=[113, 2743, 1419, 1777, 1000, 1901, 60, 76, 489, 520, 81, 1508, 3383, 122, 36, 2415, 576, 3, 321, 702, 26, 38, 503, 806, 35, 1600, 596, 149, 69, 0, 503, 1848, 391, 336, 48, 1191, 132, 1, 864, 36, 1, 241, 4, 176, 2356, 1677, 67, 1283, 1090, 490, 562, 727, 85, 4, 1536, 356, 287, 2938, 522, 1953, 2398, 1068, 738, 176] payload_rows_per_dst=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] dup_rows_per_dst=[1290, 2102, 2390, 4698, 3918, 2292, 1382, 3560] combine_rows_per_src=[7600, 8240, 9436, 11668, 11368, 9318, 8318, 10724] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=254779392 dispatch_meta_bytes=306608 combine_payload_bytes=254779392 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=255086000 ingress_bytes=393191424 link_bytes=393191424 token_bytes=509865392 weight_movement_bytes=138412032 total_nvlink_bytes=648277424 weight_to_token_ratio=0.271468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479547674e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=342584 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266099256 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531855928 weight_movement_bytes=103809024 total_nvlink_bytes=635664952 weight_to_token_ratio=0.195183 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76354 payload_rows_remote=66022 dup_rows=21950 dup_rows_remote=19570 dedup_ratio=0.223287 router_tokens_per_expert=[133, 1255, 1606, 1942, 738, 1878, 23, 156, 383, 780, 97, 1066, 3550, 352, 73, 1903, 769, 6, 403, 677, 24, 36, 343, 1301, 147, 1256, 625, 158, 118, 158, 301, 2101, 463, 433, 145, 1725, 195, 83, 609, 4, 3, 363, 7, 137, 2036, 1916, 115, 1127, 1388, 513, 468, 255, 354, 17, 1964, 406, 472, 3110, 570, 1633, 2499, 786, 785, 213] payload_rows_per_dst=[11854, 10332, 10128, 8490, 8000, 10362, 11682, 5506] dup_rows_per_dst=[4794, 2380, 3210, 1238, 1824, 3890, 4046, 568] combine_rows_per_src=[11854, 10332, 10128, 8490, 8000, 10362, 11682, 5506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77320 payload_rows_remote=68138 dup_rows=20984 dup_rows_remote=18374 dedup_ratio=0.213460 router_tokens_per_expert=[119, 1884, 1478, 1693, 1124, 1884, 50, 122, 392, 625, 86, 1335, 3064, 135, 166, 2175, 671, 93, 458, 490, 108, 49, 413, 764, 62, 1528, 552, 149, 32, 102, 301, 1744, 496, 320, 36, 1643, 137, 3, 968, 26, 5, 299, 16, 248, 2455, 1917, 105, 982, 1004, 530, 506, 496, 139, 10, 1551, 474, 239, 3645, 509, 1786, 2938, 955, 672, 194] payload_rows_per_dst=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] dup_rows_per_dst=[1980, 2108, 3240, 3830, 2610, 2538, 1210, 3468] combine_rows_per_src=[10194, 8316, 10142, 10642, 9182, 9516, 8210, 11118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78636 payload_rows_remote=70324 dup_rows=19668 dup_rows_remote=18638 dedup_ratio=0.200073 router_tokens_per_expert=[116, 1955, 1501, 1369, 919, 2001, 52, 79, 405, 696, 103, 1168, 3172, 228, 35, 2106, 800, 3, 238, 613, 136, 19, 400, 976, 44, 1398, 561, 96, 67, 12, 304, 2189, 409, 326, 48, 1568, 147, 2, 909, 6, 1, 293, 6, 177, 2293, 1830, 100, 1022, 915, 469, 446, 399, 109, 6, 1787, 469, 462, 3512, 498, 2351, 3008, 961, 755, 107] payload_rows_per_dst=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] dup_rows_per_dst=[1530, 4356, 3268, 1030, 2170, 2400, 3422, 1492] combine_rows_per_src=[10544, 11470, 10126, 8312, 8570, 9044, 11382, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78078 payload_rows_remote=67396 dup_rows=20226 dup_rows_remote=16842 dedup_ratio=0.205750 router_tokens_per_expert=[106, 2553, 1567, 1495, 1443, 2121, 82, 97, 361, 474, 68, 1396, 3364, 87, 103, 2611, 720, 24, 467, 963, 98, 42, 752, 773, 24, 1591, 565, 96, 53, 4, 324, 1783, 632, 63, 41, 1287, 143, 1, 1017, 12, 21, 201, 7, 146, 2337, 1805, 109, 987, 874, 424, 472, 306, 282, 15, 1057, 342, 447, 3194, 518, 1593, 2795, 1020, 655, 142] payload_rows_per_dst=[11264, 12192, 10682, 7900, 8772, 8946, 10768, 7554] dup_rows_per_dst=[2558, 4736, 3384, 980, 2726, 2280, 2366, 1196] combine_rows_per_src=[11264, 12192, 10682, 7900, 8772, 8946, 10768, 7554] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=342368 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270768480 ingress_bytes=374235136 link_bytes=374235136 token_bytes=541194592 weight_movement_bytes=103809024 total_nvlink_bytes=645003616 weight_to_token_ratio=0.191815 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280690688 dispatch_meta_bytes=358000 combine_payload_bytes=280690688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281048688 ingress_bytes=315293696 link_bytes=315293696 token_bytes=561739376 weight_movement_bytes=34603008 total_nvlink_bytes=596342384 weight_to_token_ratio=0.061600 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186424795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=62930 dup_rows=22972 dup_rows_remote=17038 dedup_ratio=0.233683 router_tokens_per_expert=[127, 2103, 1515, 1634, 1263, 1831, 28, 112, 485, 647, 56, 1262, 3506, 158, 28, 2658, 762, 1, 286, 987, 37, 34, 334, 833, 29, 1488, 581, 82, 55, 0, 265, 1371, 448, 58, 24, 1309, 135, 1, 1021, 13, 4, 223, 2, 169, 2630, 1853, 93, 1403, 873, 509, 590, 603, 93, 3, 1614, 433, 288, 3100, 578, 1897, 2766, 1076, 650, 135] payload_rows_per_dst=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] dup_rows_per_dst=[5934, 2456, 2752, 808, 2212, 4752, 3572, 486] combine_rows_per_src=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275881984 dispatch_meta_bytes=346776 combine_payload_bytes=275881984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276228760 ingress_bytes=310484992 link_bytes=310484992 token_bytes=552110744 weight_movement_bytes=34603008 total_nvlink_bytes=586713752 weight_to_token_ratio=0.062674 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168330029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279093248 dispatch_meta_bytes=346048 combine_payload_bytes=279093248 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279439296 ingress_bytes=348299264 link_bytes=348299264 token_bytes=558532544 weight_movement_bytes=69206016 total_nvlink_bytes=627738560 weight_to_token_ratio=0.123907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310622090e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=342368 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270768480 ingress_bytes=374235136 link_bytes=374235136 token_bytes=541194592 weight_movement_bytes=103809024 total_nvlink_bytes=645003616 weight_to_token_ratio=0.191815 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408216688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=288047104 dispatch_meta_bytes=355848 combine_payload_bytes=288047104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288402952 ingress_bytes=322650112 link_bytes=322650112 token_bytes=576450056 weight_movement_bytes=34603008 total_nvlink_bytes=611053064 weight_to_token_ratio=0.060028 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214106396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276054016 dispatch_meta_bytes=336952 combine_payload_bytes=276054016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276390968 ingress_bytes=310657024 link_bytes=310657024 token_bytes=552444984 weight_movement_bytes=34603008 total_nvlink_bytes=587047992 weight_to_token_ratio=0.062636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168977371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=62930 dup_rows=22972 dup_rows_remote=17038 dedup_ratio=0.233683 router_tokens_per_expert=[127, 2103, 1515, 1634, 1263, 1831, 28, 112, 485, 647, 56, 1262, 3506, 158, 28, 2658, 762, 1, 286, 987, 37, 34, 334, 833, 29, 1488, 581, 82, 55, 0, 265, 1371, 448, 58, 24, 1309, 135, 1, 1021, 13, 4, 223, 2, 169, 2630, 1853, 93, 1403, 873, 509, 590, 603, 93, 3, 1614, 433, 288, 3100, 578, 1897, 2766, 1076, 650, 135] payload_rows_per_dst=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] dup_rows_per_dst=[5934, 2456, 2752, 808, 2212, 4752, 3572, 486] combine_rows_per_src=[12402, 10404, 9996, 6934, 8012, 11220, 11396, 4968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257761280 dispatch_meta_bytes=319872 combine_payload_bytes=257761280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258081152 ingress_bytes=361570304 link_bytes=361570304 token_bytes=515842432 weight_movement_bytes=103809024 total_nvlink_bytes=619651456 weight_to_token_ratio=0.201242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360559945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74736 payload_rows_remote=61448 dup_rows=23568 dup_rows_remote=15570 dedup_ratio=0.239746 router_tokens_per_expert=[589, 1114, 114, 6, 154, 154, 606, 4220, 1358, 41, 77, 154, 404, 77, 5080, 96, 4, 17, 25, 1128, 239, 55, 119, 14, 30, 941, 1387, 0, 1693, 123, 38, 2942, 364, 547, 255, 13, 2268, 421, 0, 163, 3, 961, 1733, 153, 49, 243, 100, 267, 1428, 264, 40, 10, 1814, 35, 3, 4497, 112, 368, 24, 3652, 373, 1711, 2200, 2082] payload_rows_per_dst=[5184, 10114, 7668, 7162, 7072, 12956, 11292, 13288] dup_rows_per_dst=[416, 2614, 1176, 1262, 990, 4222, 4890, 7998] combine_rows_per_src=[5184, 10114, 7668, 7162, 7072, 12956, 11292, 13288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76302 payload_rows_remote=68528 dup_rows=22002 dup_rows_remote=20972 dedup_ratio=0.223816 router_tokens_per_expert=[102, 1874, 1526, 1669, 1002, 2142, 83, 47, 392, 529, 97, 1296, 3578, 127, 139, 2214, 703, 39, 244, 685, 104, 21, 703, 711, 27, 1177, 546, 107, 69, 0, 421, 2246, 520, 169, 51, 1458, 114, 0, 1073, 17, 9, 282, 7, 110, 2012, 1498, 84, 1194, 1026, 489, 510, 313, 212, 8, 1355, 489, 427, 3531, 501, 1918, 3042, 1141, 842, 130] payload_rows_per_dst=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] dup_rows_per_dst=[986, 1772, 2884, 4562, 3906, 2048, 1030, 4814] combine_rows_per_src=[7872, 7816, 9776, 11780, 10930, 8344, 7774, 12010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=251691008 dispatch_meta_bytes=308072 combine_payload_bytes=251691008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251999080 ingress_bytes=355500032 link_bytes=355500032 token_bytes=503690088 weight_movement_bytes=103809024 total_nvlink_bytes=607499112 weight_to_token_ratio=0.206097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337718000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280690688 dispatch_meta_bytes=358000 combine_payload_bytes=280690688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281048688 ingress_bytes=315293696 link_bytes=315293696 token_bytes=561739376 weight_movement_bytes=34603008 total_nvlink_bytes=596342384 weight_to_token_ratio=0.061600 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186424795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77656 payload_rows_remote=64860 dup_rows=20648 dup_rows_remote=16340 dedup_ratio=0.210042 router_tokens_per_expert=[523, 1038, 361, 9, 76, 46, 314, 4077, 910, 12, 961, 136, 407, 947, 5018, 194, 10, 2, 44, 2084, 104, 5, 493, 68, 15, 846, 1680, 0, 1279, 356, 54, 2150, 179, 119, 1277, 55, 1809, 159, 0, 34, 30, 1576, 775, 185, 26, 303, 105, 534, 1566, 252, 138, 16, 2579, 248, 12, 1269, 266, 482, 10, 3693, 434, 1286, 3393, 2123] payload_rows_per_dst=[10750, 6142, 9936, 7936, 10752, 12796, 9062, 10282] dup_rows_per_dst=[2138, 992, 3070, 1112, 3298, 4308, 3098, 2632] combine_rows_per_src=[10750, 6142, 9936, 7936, 10752, 12796, 9062, 10282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=66642 dup_rows=21526 dup_rows_remote=18422 dedup_ratio=0.218974 router_tokens_per_expert=[525, 813, 408, 8, 147, 56, 578, 3884, 1209, 3, 918, 147, 438, 681, 3958, 328, 13, 6, 29, 2659, 191, 5, 427, 61, 43, 704, 1699, 0, 1206, 471, 54, 2407, 183, 190, 1290, 32, 1771, 219, 6, 76, 99, 1865, 1273, 233, 107, 361, 113, 542, 1357, 325, 264, 47, 2181, 91, 14, 1446, 311, 465, 31, 3838, 466, 1190, 2853, 1837] payload_rows_per_dst=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] dup_rows_per_dst=[2212, 1326, 3786, 2824, 3104, 4720, 2800, 754] combine_rows_per_src=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78336 payload_rows_remote=65980 dup_rows=19968 dup_rows_remote=17804 dedup_ratio=0.203125 router_tokens_per_expert=[457, 1302, 188, 2, 172, 118, 742, 3455, 1289, 17, 233, 150, 328, 20, 5071, 152, 7, 22, 25, 1530, 276, 8, 247, 20, 22, 874, 1634, 0, 1407, 432, 62, 2677, 372, 285, 613, 19, 2506, 593, 3, 176, 4, 1086, 1748, 145, 40, 328, 53, 371, 1347, 403, 112, 9, 2100, 43, 25, 3110, 170, 779, 22, 3479, 399, 1590, 2247, 2036] payload_rows_per_dst=[10434, 12356, 9138, 10894, 10278, 6164, 9998, 9074] dup_rows_per_dst=[2438, 2164, 2090, 3322, 3350, 1386, 4300, 918] combine_rows_per_src=[10434, 12356, 9138, 10894, 10278, 6164, 9998, 9074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77662 payload_rows_remote=66024 dup_rows=20642 dup_rows_remote=17014 dedup_ratio=0.209981 router_tokens_per_expert=[796, 1349, 346, 19, 122, 59, 411, 4172, 1312, 16, 383, 152, 460, 336, 4659, 99, 8, 9, 38, 1563, 309, 2, 470, 81, 40, 773, 1717, 0, 1739, 424, 37, 2903, 211, 166, 955, 25, 1871, 330, 1, 116, 5, 1224, 946, 228, 24, 256, 130, 458, 1270, 272, 84, 23, 2299, 104, 12, 2473, 199, 353, 11, 3687, 380, 1553, 2686, 1996] payload_rows_per_dst=[11672, 11816, 9632, 11638, 9832, 5510, 9450, 8112] dup_rows_per_dst=[2876, 2622, 2702, 3628, 2890, 1428, 3624, 872] combine_rows_per_src=[11672, 11816, 9632, 11638, 9832, 5510, 9450, 8112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74736 payload_rows_remote=61448 dup_rows=23568 dup_rows_remote=15570 dedup_ratio=0.239746 router_tokens_per_expert=[589, 1114, 114, 6, 154, 154, 606, 4220, 1358, 41, 77, 154, 404, 77, 5080, 96, 4, 17, 25, 1128, 239, 55, 119, 14, 30, 941, 1387, 0, 1693, 123, 38, 2942, 364, 547, 255, 13, 2268, 421, 0, 163, 3, 961, 1733, 153, 49, 243, 100, 267, 1428, 264, 40, 10, 1814, 35, 3, 4497, 112, 368, 24, 3652, 373, 1711, 2200, 2082] payload_rows_per_dst=[5184, 10114, 7668, 7162, 7072, 12956, 11292, 13288] dup_rows_per_dst=[416, 2614, 1176, 1262, 990, 4222, 4890, 7998] combine_rows_per_src=[5184, 10114, 7668, 7162, 7072, 12956, 11292, 13288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257761280 dispatch_meta_bytes=319872 combine_payload_bytes=257761280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258081152 ingress_bytes=361570304 link_bytes=361570304 token_bytes=515842432 weight_movement_bytes=103809024 total_nvlink_bytes=619651456 weight_to_token_ratio=0.201242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.360559945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77536 payload_rows_remote=67872 dup_rows=20768 dup_rows_remote=17642 dedup_ratio=0.211263 router_tokens_per_expert=[618, 635, 887, 4, 113, 44, 439, 3677, 564, 9, 1201, 120, 573, 1184, 5217, 281, 3, 6, 51, 2391, 107, 12, 485, 80, 28, 735, 2223, 0, 1037, 223, 46, 2032, 214, 128, 1401, 39, 1578, 114, 1, 17, 49, 1782, 484, 252, 51, 426, 179, 598, 1298, 249, 153, 25, 2838, 91, 19, 817, 234, 1115, 18, 3260, 390, 1106, 3506, 1695] payload_rows_per_dst=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] dup_rows_per_dst=[2240, 4782, 3126, 2894, 3234, 1528, 2144, 820] combine_rows_per_src=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77334 payload_rows_remote=66114 dup_rows=20970 dup_rows_remote=18162 dedup_ratio=0.213318 router_tokens_per_expert=[520, 983, 472, 4, 108, 38, 455, 4110, 846, 5, 694, 138, 381, 679, 5241, 137, 6, 3, 26, 1951, 168, 64, 504, 45, 16, 676, 1480, 1, 1227, 446, 42, 2471, 207, 177, 997, 18, 1810, 203, 1, 32, 9, 1735, 910, 244, 27, 305, 126, 477, 1558, 243, 71, 23, 2483, 53, 9, 1680, 233, 470, 10, 3859, 449, 1520, 3073, 2203] payload_rows_per_dst=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] dup_rows_per_dst=[2808, 3366, 3046, 4814, 2988, 1676, 1140, 1132] combine_rows_per_src=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76482 payload_rows_remote=66944 dup_rows=21822 dup_rows_remote=18626 dedup_ratio=0.221985 router_tokens_per_expert=[349, 867, 339, 11, 83, 52, 469, 4178, 985, 7, 1070, 115, 491, 878, 5027, 202, 14, 6, 47, 2189, 104, 49, 452, 62, 33, 615, 2177, 0, 1017, 425, 58, 2398, 190, 116, 1370, 24, 1704, 238, 0, 39, 13, 1335, 981, 319, 16, 236, 163, 472, 1330, 299, 84, 25, 2550, 125, 20, 1934, 259, 759, 21, 3273, 435, 1365, 2903, 1785] payload_rows_per_dst=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] dup_rows_per_dst=[1958, 1302, 2786, 920, 1658, 4262, 3196, 5740] combine_rows_per_src=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78336 payload_rows_remote=65980 dup_rows=19968 dup_rows_remote=17804 dedup_ratio=0.203125 router_tokens_per_expert=[457, 1302, 188, 2, 172, 118, 742, 3455, 1289, 17, 233, 150, 328, 20, 5071, 152, 7, 22, 25, 1530, 276, 8, 247, 20, 22, 874, 1634, 0, 1407, 432, 62, 2677, 372, 285, 613, 19, 2506, 593, 3, 176, 4, 1086, 1748, 145, 40, 328, 53, 371, 1347, 403, 112, 9, 2100, 43, 25, 3110, 170, 779, 22, 3479, 399, 1590, 2247, 2036] payload_rows_per_dst=[10434, 12356, 9138, 10894, 10278, 6164, 9998, 9074] dup_rows_per_dst=[2438, 2164, 2090, 3322, 3350, 1386, 4300, 918] combine_rows_per_src=[10434, 12356, 9138, 10894, 10278, 6164, 9998, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=324800 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265991360 ingress_bytes=300269568 link_bytes=300269568 token_bytes=531657920 weight_movement_bytes=34603008 total_nvlink_bytes=566260928 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129890211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=66642 dup_rows=21526 dup_rows_remote=18422 dedup_ratio=0.218974 router_tokens_per_expert=[525, 813, 408, 8, 147, 56, 578, 3884, 1209, 3, 918, 147, 438, 681, 3958, 328, 13, 6, 29, 2659, 191, 5, 427, 61, 43, 704, 1699, 0, 1206, 471, 54, 2407, 183, 190, 1290, 32, 1771, 219, 6, 76, 99, 1865, 1273, 233, 107, 361, 113, 542, 1357, 325, 264, 47, 2181, 91, 14, 1446, 311, 465, 31, 3838, 466, 1190, 2853, 1837] payload_rows_per_dst=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] dup_rows_per_dst=[2212, 1326, 3786, 2824, 3104, 4720, 2800, 754] combine_rows_per_src=[10626, 6122, 10672, 10344, 10136, 12382, 8650, 7846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=332152 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270766456 ingress_bytes=339640320 link_bytes=339640320 token_bytes=541200760 weight_movement_bytes=69206016 total_nvlink_bytes=610406776 weight_to_token_ratio=0.127875 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278039181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=251691008 dispatch_meta_bytes=308072 combine_payload_bytes=251691008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251999080 ingress_bytes=355500032 link_bytes=355500032 token_bytes=503690088 weight_movement_bytes=103809024 total_nvlink_bytes=607499112 weight_to_token_ratio=0.206097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337718000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77536 payload_rows_remote=67872 dup_rows=20768 dup_rows_remote=17642 dedup_ratio=0.211263 router_tokens_per_expert=[618, 635, 887, 4, 113, 44, 439, 3677, 564, 9, 1201, 120, 573, 1184, 5217, 281, 3, 6, 51, 2391, 107, 12, 485, 80, 28, 735, 2223, 0, 1037, 223, 46, 2032, 214, 128, 1401, 39, 1578, 114, 1, 17, 49, 1782, 484, 252, 51, 426, 179, 598, 1298, 249, 153, 25, 2838, 91, 19, 817, 234, 1115, 18, 3260, 390, 1106, 3506, 1695] payload_rows_per_dst=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] dup_rows_per_dst=[2240, 4782, 3126, 2894, 3234, 1528, 2144, 820] combine_rows_per_src=[10594, 13516, 9664, 9754, 10762, 6114, 8836, 8296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274202624 dispatch_meta_bytes=342280 combine_payload_bytes=274202624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274544904 ingress_bytes=343408640 link_bytes=343408640 token_bytes=548747528 weight_movement_bytes=69206016 total_nvlink_bytes=617953544 weight_to_token_ratio=0.126116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292219066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270254080 dispatch_meta_bytes=335136 combine_payload_bytes=270254080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270589216 ingress_bytes=339460096 link_bytes=339460096 token_bytes=540843296 weight_movement_bytes=69206016 total_nvlink_bytes=610049312 weight_to_token_ratio=0.127959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277361012e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=272965632 dispatch_meta_bytes=340256 combine_payload_bytes=272965632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273305888 ingress_bytes=307568640 link_bytes=307568640 token_bytes=546271520 weight_movement_bytes=34603008 total_nvlink_bytes=580874528 weight_to_token_ratio=0.063344 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157356031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77334 payload_rows_remote=66114 dup_rows=20970 dup_rows_remote=18162 dedup_ratio=0.213318 router_tokens_per_expert=[520, 983, 472, 4, 108, 38, 455, 4110, 846, 5, 694, 138, 381, 679, 5241, 137, 6, 3, 26, 1951, 168, 64, 504, 45, 16, 676, 1480, 1, 1227, 446, 42, 2471, 207, 177, 997, 18, 1810, 203, 1, 32, 9, 1735, 910, 244, 27, 305, 126, 477, 1558, 243, 71, 23, 2483, 53, 9, 1680, 233, 470, 10, 3859, 449, 1520, 3073, 2203] payload_rows_per_dst=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] dup_rows_per_dst=[2808, 3366, 3046, 4814, 2988, 1676, 1140, 1132] combine_rows_per_src=[11220, 12876, 10206, 12070, 10048, 5990, 6286, 8638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=272965632 dispatch_meta_bytes=340256 combine_payload_bytes=272965632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273305888 ingress_bytes=307568640 link_bytes=307568640 token_bytes=546271520 weight_movement_bytes=34603008 total_nvlink_bytes=580874528 weight_to_token_ratio=0.063344 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157356031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270254080 dispatch_meta_bytes=335136 combine_payload_bytes=270254080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270589216 ingress_bytes=339460096 link_bytes=339460096 token_bytes=540843296 weight_movement_bytes=69206016 total_nvlink_bytes=610049312 weight_to_token_ratio=0.127959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277361012e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270434304 dispatch_meta_bytes=332152 combine_payload_bytes=270434304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270766456 ingress_bytes=339640320 link_bytes=339640320 token_bytes=541200760 weight_movement_bytes=69206016 total_nvlink_bytes=610406776 weight_to_token_ratio=0.127875 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278039181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278003712 dispatch_meta_bytes=342056 combine_payload_bytes=278003712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278345768 ingress_bytes=312606720 link_bytes=312606720 token_bytes=556349480 weight_movement_bytes=34603008 total_nvlink_bytes=590952488 weight_to_token_ratio=0.062197 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176313920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76286 payload_rows_remote=67478 dup_rows=22018 dup_rows_remote=20074 dedup_ratio=0.223979 router_tokens_per_expert=[255, 337, 468, 127, 495, 3451, 1, 1207, 264, 688, 414, 4222, 136, 30, 367, 1011, 1141, 1473, 21, 166, 1158, 209, 3, 1690, 259, 135, 37, 419, 852, 686, 294, 18, 589, 187, 0, 208, 1324, 190, 996, 15, 343, 2, 2170, 68, 215, 112, 1117, 10, 548, 387, 26, 159, 1724, 14, 1192, 1816, 4231, 1, 1339, 0, 5219, 1255, 1439, 222] payload_rows_per_dst=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] dup_rows_per_dst=[2742, 2554, 3116, 2102, 2632, 1944, 3758, 3170] combine_rows_per_src=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77490 payload_rows_remote=63664 dup_rows=20814 dup_rows_remote=18038 dedup_ratio=0.211731 router_tokens_per_expert=[269, 374, 863, 55, 499, 4048, 32, 1292, 190, 378, 318, 4562, 184, 7, 386, 897, 910, 876, 32, 175, 856, 309, 5, 1583, 258, 127, 92, 355, 1399, 421, 257, 1, 497, 328, 4, 168, 1165, 69, 802, 12, 360, 0, 2356, 86, 93, 110, 714, 47, 268, 435, 45, 198, 1777, 18, 1026, 1481, 4312, 0, 2699, 0, 5391, 1387, 1007, 287] payload_rows_per_dst=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] dup_rows_per_dst=[3806, 2684, 1830, 2776, 3432, 2624, 3354, 308] combine_rows_per_src=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66752 payload_rows_remote=51302 dup_rows=31552 dup_rows_remote=11932 dedup_ratio=0.320964 router_tokens_per_expert=[253, 386, 2270, 53, 323, 4154, 14, 563, 205, 227, 154, 3806, 352, 0, 368, 681, 718, 745, 209, 172, 495, 269, 2, 1247, 213, 28, 175, 141, 1016, 380, 227, 0, 634, 659, 0, 127, 401, 66, 363, 3, 351, 4, 3587, 73, 22, 160, 356, 598, 105, 486, 11, 40, 1838, 8, 700, 1044, 4947, 0, 4197, 2, 6102, 1576, 655, 191] payload_rows_per_dst=[6450, 3198, 12042, 4200, 3956, 10254, 11202, 15450] dup_rows_per_dst=[1274, 776, 3284, 160, 550, 1458, 4430, 19620] combine_rows_per_src=[6450, 3198, 12042, 4200, 3956, 10254, 11202, 15450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=337104 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271140048 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541942992 weight_movement_bytes=69206016 total_nvlink_bytes=611149008 weight_to_token_ratio=0.127700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69738 payload_rows_remote=60522 dup_rows=28566 dup_rows_remote=23620 dedup_ratio=0.290588 router_tokens_per_expert=[247, 339, 528, 128, 355, 3959, 2, 1117, 267, 554, 317, 4732, 106, 90, 389, 1065, 1152, 1167, 12, 169, 720, 277, 16, 1875, 216, 192, 26, 394, 977, 458, 212, 0, 445, 163, 1, 100, 1398, 121, 988, 13, 354, 3, 2248, 32, 143, 67, 890, 46, 711, 409, 186, 177, 1779, 27, 1268, 1618, 4279, 7, 1975, 0, 5099, 1256, 1062, 229] payload_rows_per_dst=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] dup_rows_per_dst=[2386, 1032, 6628, 270, 1432, 2102, 4946, 9770] combine_rows_per_src=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276389888 dispatch_meta_bytes=350208 combine_payload_bytes=276389888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276740096 ingress_bytes=310992896 link_bytes=310992896 token_bytes=553129984 weight_movement_bytes=34603008 total_nvlink_bytes=587732992 weight_to_token_ratio=0.062559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170241231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77088 payload_rows_remote=68598 dup_rows=21216 dup_rows_remote=19672 dedup_ratio=0.215820 router_tokens_per_expert=[257, 377, 2300, 28, 366, 3545, 16, 644, 179, 257, 229, 3206, 323, 2, 309, 512, 650, 656, 497, 227, 225, 214, 1, 1300, 278, 44, 181, 114, 1076, 327, 228, 0, 836, 476, 0, 97, 563, 36, 467, 5, 309, 0, 3883, 86, 52, 147, 526, 366, 46, 540, 51, 166, 2242, 2, 756, 1384, 4958, 1, 3912, 0, 6126, 1723, 604, 224] payload_rows_per_dst=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] dup_rows_per_dst=[3942, 1544, 1242, 2436, 3058, 5542, 3202, 250] combine_rows_per_src=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260767744 dispatch_meta_bytes=326808 combine_payload_bytes=260767744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261094552 ingress_bytes=295370752 link_bytes=295370752 token_bytes=521862296 weight_movement_bytes=34603008 total_nvlink_bytes=556465304 weight_to_token_ratio=0.066307 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111456360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76482 payload_rows_remote=66944 dup_rows=21822 dup_rows_remote=18626 dedup_ratio=0.221985 router_tokens_per_expert=[349, 867, 339, 11, 83, 52, 469, 4178, 985, 7, 1070, 115, 491, 878, 5027, 202, 14, 6, 47, 2189, 104, 49, 452, 62, 33, 615, 2177, 0, 1017, 425, 58, 2398, 190, 116, 1370, 24, 1704, 238, 0, 39, 13, 1335, 981, 319, 16, 236, 163, 472, 1330, 299, 84, 25, 2550, 125, 20, 1934, 259, 759, 21, 3273, 435, 1365, 2903, 1785] payload_rows_per_dst=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] dup_rows_per_dst=[1958, 1302, 2786, 920, 1658, 4262, 3196, 5740] combine_rows_per_src=[10738, 6194, 9606, 7730, 7912, 12862, 9538, 11902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=210132992 dispatch_meta_bytes=252936 combine_payload_bytes=210132992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=210385928 ingress_bytes=383148032 link_bytes=383148032 token_bytes=420518920 weight_movement_bytes=173015040 total_nvlink_bytes=593533960 weight_to_token_ratio=0.411432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.441755198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76286 payload_rows_remote=67478 dup_rows=22018 dup_rows_remote=20074 dedup_ratio=0.223979 router_tokens_per_expert=[255, 337, 468, 127, 495, 3451, 1, 1207, 264, 688, 414, 4222, 136, 30, 367, 1011, 1141, 1473, 21, 166, 1158, 209, 3, 1690, 259, 135, 37, 419, 852, 686, 294, 18, 589, 187, 0, 208, 1324, 190, 996, 15, 343, 2, 2170, 68, 215, 112, 1117, 10, 548, 387, 26, 159, 1724, 14, 1192, 1816, 4231, 1, 1339, 0, 5219, 1255, 1439, 222] payload_rows_per_dst=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] dup_rows_per_dst=[2742, 2554, 3116, 2102, 2632, 1944, 3758, 3170] combine_rows_per_src=[9940, 10692, 9624, 11854, 8176, 8808, 7974, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274202624 dispatch_meta_bytes=342280 combine_payload_bytes=274202624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274544904 ingress_bytes=343408640 link_bytes=343408640 token_bytes=548747528 weight_movement_bytes=69206016 total_nvlink_bytes=617953544 weight_to_token_ratio=0.126116 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292219066e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77372 payload_rows_remote=65600 dup_rows=20932 dup_rows_remote=16170 dedup_ratio=0.212931 router_tokens_per_expert=[261, 412, 468, 73, 413, 3225, 1, 879, 159, 660, 311, 4660, 204, 7, 326, 1226, 1189, 918, 16, 202, 606, 269, 12, 2082, 291, 153, 199, 249, 1253, 566, 252, 33, 711, 184, 1, 146, 1513, 69, 1583, 34, 317, 1, 2197, 73, 89, 148, 938, 5, 509, 381, 133, 251, 1866, 23, 1202, 1510, 4026, 0, 1964, 1, 4965, 1208, 1187, 342] payload_rows_per_dst=[9340, 12208, 8374, 13208, 11772, 9440, 7918, 5112] dup_rows_per_dst=[2124, 2898, 2214, 2714, 4762, 2024, 3832, 364] combine_rows_per_src=[9340, 12208, 8374, 13208, 11772, 9440, 7918, 5112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77088 payload_rows_remote=68598 dup_rows=21216 dup_rows_remote=19672 dedup_ratio=0.215820 router_tokens_per_expert=[257, 377, 2300, 28, 366, 3545, 16, 644, 179, 257, 229, 3206, 323, 2, 309, 512, 650, 656, 497, 227, 225, 214, 1, 1300, 278, 44, 181, 114, 1076, 327, 228, 0, 836, 476, 0, 97, 563, 36, 467, 5, 309, 0, 3883, 86, 52, 147, 526, 366, 46, 540, 51, 166, 2242, 2, 756, 1384, 4958, 1, 3912, 0, 6126, 1723, 604, 224] payload_rows_per_dst=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] dup_rows_per_dst=[3942, 1544, 1242, 2436, 3058, 5542, 3202, 250] combine_rows_per_src=[11124, 8490, 6298, 14312, 11818, 13020, 7172, 4854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77490 payload_rows_remote=63664 dup_rows=20814 dup_rows_remote=18038 dedup_ratio=0.211731 router_tokens_per_expert=[269, 374, 863, 55, 499, 4048, 32, 1292, 190, 378, 318, 4562, 184, 7, 386, 897, 910, 876, 32, 175, 856, 309, 5, 1583, 258, 127, 92, 355, 1399, 421, 257, 1, 497, 328, 4, 168, 1165, 69, 802, 12, 360, 0, 2356, 86, 93, 110, 714, 47, 268, 435, 45, 198, 1777, 18, 1026, 1481, 4312, 0, 2699, 0, 5391, 1387, 1007, 287] payload_rows_per_dst=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] dup_rows_per_dst=[3806, 2684, 1830, 2776, 3432, 2624, 3354, 308] combine_rows_per_src=[11058, 11160, 7662, 13826, 11282, 10306, 7142, 5054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=247898112 dispatch_meta_bytes=336568 combine_payload_bytes=247898112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248234680 ingress_bytes=282501120 link_bytes=282501120 token_bytes=496132792 weight_movement_bytes=34603008 total_nvlink_bytes=530735800 weight_to_token_ratio=0.069745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.063028971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77628 payload_rows_remote=68786 dup_rows=20676 dup_rows_remote=18220 dedup_ratio=0.210327 router_tokens_per_expert=[248, 329, 365, 57, 593, 3453, 1, 1119, 180, 827, 501, 4790, 155, 26, 367, 887, 1375, 1207, 8, 112, 986, 353, 6, 1602, 248, 169, 12, 452, 1103, 511, 297, 2, 431, 74, 3, 125, 1397, 118, 1130, 11, 360, 3, 1901, 64, 197, 141, 1086, 20, 420, 302, 69, 195, 1939, 41, 1335, 1836, 4498, 0, 1198, 0, 5229, 1358, 1045, 285] payload_rows_per_dst=[9746, 12290, 8842, 13456, 11380, 8344, 8404, 5166] dup_rows_per_dst=[2584, 3176, 2456, 2590, 4194, 1596, 3870, 210] combine_rows_per_src=[9746, 12290, 8842, 13456, 11380, 8344, 8404, 5166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=353080 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281330488 ingress_bytes=350183424 link_bytes=350183424 token_bytes=562307896 weight_movement_bytes=69206016 total_nvlink_bytes=631513912 weight_to_token_ratio=0.123075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317712032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73854 payload_rows_remote=63840 dup_rows=24450 dup_rows_remote=21828 dedup_ratio=0.248718 router_tokens_per_expert=[242, 396, 667, 103, 410, 3600, 3, 897, 210, 487, 324, 4217, 170, 15, 344, 799, 1051, 1425, 18, 73, 924, 241, 3, 1805, 252, 99, 45, 449, 1387, 510, 324, 3, 382, 61, 0, 181, 1469, 122, 814, 7, 338, 0, 2517, 82, 119, 137, 705, 21, 458, 461, 50, 158, 1865, 9, 1142, 1799, 4381, 1, 1795, 0, 5754, 1239, 1353, 239] payload_rows_per_dst=[10014, 13390, 8572, 8386, 10994, 9246, 7936, 5316] dup_rows_per_dst=[2622, 7668, 2508, 1334, 3840, 2182, 3948, 348] combine_rows_per_src=[10014, 13390, 8572, 8386, 10994, 9246, 7936, 5316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276389888 dispatch_meta_bytes=350208 combine_payload_bytes=276389888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276740096 ingress_bytes=310992896 link_bytes=310992896 token_bytes=553129984 weight_movement_bytes=34603008 total_nvlink_bytes=587732992 weight_to_token_ratio=0.062559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.170241231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268697600 dispatch_meta_bytes=327080 combine_payload_bytes=268697600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269024680 ingress_bytes=303300608 link_bytes=303300608 token_bytes=537722280 weight_movement_bytes=34603008 total_nvlink_bytes=572325288 weight_to_token_ratio=0.064351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141295770e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281747456 dispatch_meta_bytes=348024 combine_payload_bytes=281747456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282095480 ingress_bytes=316350464 link_bytes=316350464 token_bytes=563842936 weight_movement_bytes=34603008 total_nvlink_bytes=598445944 weight_to_token_ratio=0.061370 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190401328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73854 payload_rows_remote=63840 dup_rows=24450 dup_rows_remote=21828 dedup_ratio=0.248718 router_tokens_per_expert=[242, 396, 667, 103, 410, 3600, 3, 897, 210, 487, 324, 4217, 170, 15, 344, 799, 1051, 1425, 18, 73, 924, 241, 3, 1805, 252, 99, 45, 449, 1387, 510, 324, 3, 382, 61, 0, 181, 1469, 122, 814, 7, 338, 0, 2517, 82, 119, 137, 705, 21, 458, 461, 50, 158, 1865, 9, 1142, 1799, 4381, 1, 1795, 0, 5754, 1239, 1353, 239] payload_rows_per_dst=[10014, 13390, 8572, 8386, 10994, 9246, 7936, 5316] dup_rows_per_dst=[2622, 7668, 2508, 1334, 3840, 2182, 3948, 348] combine_rows_per_src=[10014, 13390, 8572, 8386, 10994, 9246, 7936, 5316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75976 payload_rows_remote=66952 dup_rows=22328 dup_rows_remote=19998 dedup_ratio=0.227132 router_tokens_per_expert=[66, 26, 266, 1881, 52, 334, 170, 0, 230, 313, 7, 294, 156, 1, 305, 355, 1594, 942, 1172, 1575, 0, 720, 1734, 925, 1154, 1001, 316, 2317, 0, 1, 80, 861, 1631, 313, 0, 84, 2, 742, 93, 2525, 950, 0, 936, 58, 24, 8, 613, 1354, 36, 50, 72, 0, 2448, 6830, 153, 23, 4415, 706, 158, 180, 2348, 477, 739, 2336] payload_rows_per_dst=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] dup_rows_per_dst=[2996, 540, 2516, 2668, 1478, 2330, 6664, 3136] combine_rows_per_src=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77902 payload_rows_remote=68332 dup_rows=20402 dup_rows_remote=17214 dedup_ratio=0.207540 router_tokens_per_expert=[51, 54, 372, 1662, 27, 349, 198, 0, 267, 401, 11, 281, 771, 2, 238, 289, 1464, 610, 1068, 1881, 1, 692, 1942, 957, 1228, 784, 314, 3138, 0, 1, 79, 835, 1039, 478, 1, 114, 9, 577, 107, 3273, 1570, 0, 873, 66, 6, 72, 810, 828, 16, 110, 124, 0, 2080, 6722, 103, 126, 4252, 1004, 170, 149, 2403, 267, 500, 1336] payload_rows_per_dst=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] dup_rows_per_dst=[2902, 3938, 3658, 3188, 1466, 2862, 190, 2198] combine_rows_per_src=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77228 payload_rows_remote=66384 dup_rows=21076 dup_rows_remote=16980 dedup_ratio=0.214396 router_tokens_per_expert=[42, 7, 402, 216, 35, 798, 189, 0, 279, 594, 33, 32, 2984, 1, 462, 44, 1536, 76, 504, 1828, 0, 615, 2030, 1168, 1377, 820, 180, 3726, 12, 0, 152, 952, 527, 433, 8, 97, 12, 519, 236, 4546, 2356, 2, 1215, 90, 4, 297, 1009, 63, 1, 130, 73, 0, 2048, 5063, 157, 149, 3072, 943, 224, 182, 3741, 76, 290, 495] payload_rows_per_dst=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] dup_rows_per_dst=[682, 922, 792, 3964, 1602, 3372, 5646, 4096] combine_rows_per_src=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260767744 dispatch_meta_bytes=326808 combine_payload_bytes=260767744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261094552 ingress_bytes=295370752 link_bytes=295370752 token_bytes=521862296 weight_movement_bytes=34603008 total_nvlink_bytes=556465304 weight_to_token_ratio=0.066307 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111456360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=353080 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281330488 ingress_bytes=350183424 link_bytes=350183424 token_bytes=562307896 weight_movement_bytes=69206016 total_nvlink_bytes=631513912 weight_to_token_ratio=0.123075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317712032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=59796 dup_rows=22862 dup_rows_remote=14626 dedup_ratio=0.232564 router_tokens_per_expert=[72, 41, 258, 1348, 120, 328, 205, 0, 225, 410, 5, 269, 251, 8, 255, 478, 1548, 910, 1072, 2172, 1, 638, 1586, 957, 716, 937, 384, 2540, 2, 1, 81, 776, 1025, 342, 0, 75, 5, 693, 105, 3140, 1288, 1, 841, 49, 14, 35, 680, 1113, 20, 96, 90, 0, 2418, 6935, 146, 64, 4336, 711, 238, 171, 2869, 560, 654, 1844] payload_rows_per_dst=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] dup_rows_per_dst=[2434, 554, 2164, 2320, 1378, 2364, 8236, 3412] combine_rows_per_src=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77228 payload_rows_remote=66384 dup_rows=21076 dup_rows_remote=16980 dedup_ratio=0.214396 router_tokens_per_expert=[42, 7, 402, 216, 35, 798, 189, 0, 279, 594, 33, 32, 2984, 1, 462, 44, 1536, 76, 504, 1828, 0, 615, 2030, 1168, 1377, 820, 180, 3726, 12, 0, 152, 952, 527, 433, 8, 97, 12, 519, 236, 4546, 2356, 2, 1215, 90, 4, 297, 1009, 63, 1, 130, 73, 0, 2048, 5063, 157, 149, 3072, 943, 224, 182, 3741, 76, 290, 495] payload_rows_per_dst=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] dup_rows_per_dst=[682, 922, 792, 3964, 1602, 3372, 5646, 4096] combine_rows_per_src=[5802, 7936, 7006, 10474, 11154, 10760, 13252, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281747456 dispatch_meta_bytes=348024 combine_payload_bytes=281747456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282095480 ingress_bytes=316350464 link_bytes=316350464 token_bytes=563842936 weight_movement_bytes=34603008 total_nvlink_bytes=598445944 weight_to_token_ratio=0.061370 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190401328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=261488640 dispatch_meta_bytes=342672 combine_payload_bytes=261488640 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=261831312 ingress_bytes=330694656 link_bytes=330694656 token_bytes=523319952 weight_movement_bytes=69206016 total_nvlink_bytes=592525968 weight_to_token_ratio=0.132244 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244377367e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69738 payload_rows_remote=60522 dup_rows=28566 dup_rows_remote=23620 dedup_ratio=0.290588 router_tokens_per_expert=[247, 339, 528, 128, 355, 3959, 2, 1117, 267, 554, 317, 4732, 106, 90, 389, 1065, 1152, 1167, 12, 169, 720, 277, 16, 1875, 216, 192, 26, 394, 977, 458, 212, 0, 445, 163, 1, 100, 1398, 121, 988, 13, 354, 3, 2248, 32, 143, 67, 890, 46, 711, 409, 186, 177, 1779, 27, 1268, 1618, 4279, 7, 1975, 0, 5099, 1256, 1062, 229] payload_rows_per_dst=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] dup_rows_per_dst=[2386, 1032, 6628, 270, 1432, 2102, 4946, 9770] combine_rows_per_src=[9152, 4544, 13612, 4680, 5026, 9414, 9216, 14094] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77326 payload_rows_remote=67246 dup_rows=20978 dup_rows_remote=19004 dedup_ratio=0.213399 router_tokens_per_expert=[70, 24, 373, 1639, 160, 576, 213, 0, 268, 407, 3, 302, 531, 10, 389, 270, 1575, 778, 922, 1476, 8, 696, 2120, 1008, 869, 1168, 325, 2828, 7, 6, 86, 818, 1040, 480, 1, 86, 28, 694, 130, 3568, 1297, 0, 1047, 126, 13, 25, 711, 841, 39, 274, 79, 2, 2195, 6521, 160, 84, 3965, 791, 194, 233, 2471, 213, 682, 1237] payload_rows_per_dst=[10954, 9662, 9250, 9184, 10080, 9552, 9240, 9404] dup_rows_per_dst=[3086, 2564, 3676, 3030, 1974, 2808, 1602, 2238] combine_rows_per_src=[10954, 9662, 9250, 9184, 10080, 9552, 9240, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274235392 dispatch_meta_bytes=347800 combine_payload_bytes=274235392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274583192 ingress_bytes=308838400 link_bytes=308838400 token_bytes=548818584 weight_movement_bytes=34603008 total_nvlink_bytes=583421592 weight_to_token_ratio=0.063050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162134036e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75976 payload_rows_remote=66952 dup_rows=22328 dup_rows_remote=19998 dedup_ratio=0.227132 router_tokens_per_expert=[66, 26, 266, 1881, 52, 334, 170, 0, 230, 313, 7, 294, 156, 1, 305, 355, 1594, 942, 1172, 1575, 0, 720, 1734, 925, 1154, 1001, 316, 2317, 0, 1, 80, 861, 1631, 313, 0, 84, 2, 742, 93, 2525, 950, 0, 936, 58, 24, 8, 613, 1354, 36, 50, 72, 0, 2448, 6830, 153, 23, 4415, 706, 158, 180, 2348, 477, 739, 2336] payload_rows_per_dst=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] dup_rows_per_dst=[2996, 540, 2516, 2668, 1478, 2330, 6664, 3136] combine_rows_per_src=[11424, 2782, 8326, 8792, 9302, 9024, 15574, 10752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77902 payload_rows_remote=68332 dup_rows=20402 dup_rows_remote=17214 dedup_ratio=0.207540 router_tokens_per_expert=[51, 54, 372, 1662, 27, 349, 198, 0, 267, 401, 11, 281, 771, 2, 238, 289, 1464, 610, 1068, 1881, 1, 692, 1942, 957, 1228, 784, 314, 3138, 0, 1, 79, 835, 1039, 478, 1, 114, 9, 577, 107, 3273, 1570, 0, 873, 66, 6, 72, 810, 828, 16, 110, 124, 0, 2080, 6722, 103, 126, 4252, 1004, 170, 149, 2403, 267, 500, 1336] payload_rows_per_dst=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] dup_rows_per_dst=[2902, 3938, 3658, 3188, 1466, 2862, 190, 2198] combine_rows_per_src=[11028, 14026, 9688, 9570, 9730, 9472, 4928, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75870 payload_rows_remote=63402 dup_rows=22434 dup_rows_remote=17464 dedup_ratio=0.228210 router_tokens_per_expert=[46, 71, 474, 287, 38, 877, 193, 0, 265, 659, 47, 54, 2981, 2, 473, 46, 1657, 57, 299, 1057, 0, 670, 2495, 1133, 1186, 648, 147, 3969, 12, 0, 244, 911, 319, 565, 0, 190, 25, 759, 242, 4757, 2642, 6, 1252, 146, 0, 463, 1231, 25, 3, 139, 84, 0, 2043, 4192, 141, 251, 2226, 1050, 135, 248, 4222, 111, 276, 411] payload_rows_per_dst=[7222, 12468, 7856, 10488, 12238, 11818, 5210, 8570] dup_rows_per_dst=[1202, 4970, 1890, 3746, 3260, 4702, 112, 2552] combine_rows_per_src=[7222, 12468, 7856, 10488, 12238, 11818, 5210, 8570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=342184 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280230056 ingress_bytes=349093888 link_bytes=349093888 token_bytes=560117928 weight_movement_bytes=69206016 total_nvlink_bytes=629323944 weight_to_token_ratio=0.123556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313612196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271908864 dispatch_meta_bytes=333456 combine_payload_bytes=271908864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=272242320 ingress_bytes=375717888 link_bytes=375717888 token_bytes=544151184 weight_movement_bytes=103809024 total_nvlink_bytes=647960208 weight_to_token_ratio=0.190772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=244924416 dispatch_meta_bytes=297688 combine_payload_bytes=244924416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245222104 ingress_bytes=314130432 link_bytes=314130432 token_bytes=490146520 weight_movement_bytes=69206016 total_nvlink_bytes=559352536 weight_to_token_ratio=0.141195 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182047526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77682 payload_rows_remote=67518 dup_rows=20622 dup_rows_remote=15762 dedup_ratio=0.209778 router_tokens_per_expert=[59, 81, 238, 1783, 45, 351, 198, 0, 190, 423, 10, 310, 140, 5, 457, 420, 1539, 976, 1065, 1708, 0, 954, 1739, 1270, 724, 1211, 287, 2725, 1, 1, 82, 725, 846, 424, 0, 95, 4, 979, 38, 2308, 838, 0, 969, 83, 28, 14, 603, 1517, 26, 129, 139, 0, 2651, 6773, 165, 20, 4174, 677, 207, 205, 2425, 328, 718, 2052] payload_rows_per_dst=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] dup_rows_per_dst=[2788, 3594, 4860, 2770, 1034, 2356, 298, 2922] combine_rows_per_src=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271908864 dispatch_meta_bytes=333456 combine_payload_bytes=271908864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=272242320 ingress_bytes=375717888 link_bytes=375717888 token_bytes=544151184 weight_movement_bytes=103809024 total_nvlink_bytes=647960208 weight_to_token_ratio=0.190772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413796165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=247898112 dispatch_meta_bytes=336568 combine_payload_bytes=247898112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248234680 ingress_bytes=282501120 link_bytes=282501120 token_bytes=496132792 weight_movement_bytes=34603008 total_nvlink_bytes=530735800 weight_to_token_ratio=0.069745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.063028971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=275439616 dispatch_meta_bytes=345000 combine_payload_bytes=275439616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275784616 ingress_bytes=310042624 link_bytes=310042624 token_bytes=551224232 weight_movement_bytes=34603008 total_nvlink_bytes=585827240 weight_to_token_ratio=0.062775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166665434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274235392 dispatch_meta_bytes=347800 combine_payload_bytes=274235392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274583192 ingress_bytes=308838400 link_bytes=308838400 token_bytes=548818584 weight_movement_bytes=34603008 total_nvlink_bytes=583421592 weight_to_token_ratio=0.063050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162134036e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=342184 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280230056 ingress_bytes=349093888 link_bytes=349093888 token_bytes=560117928 weight_movement_bytes=69206016 total_nvlink_bytes=629323944 weight_to_token_ratio=0.123556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313612196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=259694592 dispatch_meta_bytes=323464 combine_payload_bytes=259694592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260018056 ingress_bytes=294297600 link_bytes=294297600 token_bytes=519712648 weight_movement_bytes=34603008 total_nvlink_bytes=554315656 weight_to_token_ratio=0.066581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107418176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77682 payload_rows_remote=67518 dup_rows=20622 dup_rows_remote=15762 dedup_ratio=0.209778 router_tokens_per_expert=[59, 81, 238, 1783, 45, 351, 198, 0, 190, 423, 10, 310, 140, 5, 457, 420, 1539, 976, 1065, 1708, 0, 954, 1739, 1270, 724, 1211, 287, 2725, 1, 1, 82, 725, 846, 424, 0, 95, 4, 979, 38, 2308, 838, 0, 969, 83, 28, 14, 603, 1517, 26, 129, 139, 0, 2651, 6773, 165, 20, 4174, 677, 207, 205, 2425, 328, 718, 2052] payload_rows_per_dst=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] dup_rows_per_dst=[2788, 3594, 4860, 2770, 1034, 2356, 298, 2922] combine_rows_per_src=[11070, 13862, 10164, 8742, 8354, 9226, 5962, 10302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76786 payload_rows_remote=65698 dup_rows=21518 dup_rows_remote=18874 dedup_ratio=0.218892 router_tokens_per_expert=[71, 83, 287, 1534, 31, 375, 236, 0, 238, 442, 2, 251, 456, 1, 508, 272, 1473, 658, 919, 1826, 1, 709, 1539, 1419, 1483, 1137, 297, 2754, 1, 1, 81, 1049, 803, 449, 0, 103, 1, 659, 123, 3128, 1308, 1, 765, 111, 9, 25, 625, 641, 24, 63, 95, 0, 2376, 6449, 262, 52, 4249, 726, 299, 179, 2829, 337, 556, 1771] payload_rows_per_dst=[11088, 13478, 11966, 8420, 12108, 7176, 5552, 6998] dup_rows_per_dst=[2644, 3760, 6404, 2394, 3754, 1632, 192, 738] combine_rows_per_src=[11088, 13478, 11966, 8420, 12108, 7176, 5552, 6998] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=276553728 dispatch_meta_bytes=333120 combine_payload_bytes=276553728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276886848 ingress_bytes=380362752 link_bytes=380362752 token_bytes=553440576 weight_movement_bytes=103809024 total_nvlink_bytes=657249600 weight_to_token_ratio=0.187570 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431274414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76786 payload_rows_remote=65698 dup_rows=21518 dup_rows_remote=18874 dedup_ratio=0.218892 router_tokens_per_expert=[71, 83, 287, 1534, 31, 375, 236, 0, 238, 442, 2, 251, 456, 1, 508, 272, 1473, 658, 919, 1826, 1, 709, 1539, 1419, 1483, 1137, 297, 2754, 1, 1, 81, 1049, 803, 449, 0, 103, 1, 659, 123, 3128, 1308, 1, 765, 111, 9, 25, 625, 641, 24, 63, 95, 0, 2376, 6449, 262, 52, 4249, 726, 299, 179, 2829, 337, 556, 1771] payload_rows_per_dst=[11088, 13478, 11966, 8420, 12108, 7176, 5552, 6998] dup_rows_per_dst=[2644, 3760, 6404, 2394, 3754, 1632, 192, 738] combine_rows_per_src=[11088, 13478, 11966, 8420, 12108, 7176, 5552, 6998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=269099008 dispatch_meta_bytes=338288 combine_payload_bytes=269099008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269437296 ingress_bytes=303702016 link_bytes=303702016 token_bytes=538536304 weight_movement_bytes=34603008 total_nvlink_bytes=573139312 weight_to_token_ratio=0.064254 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142806236e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77692 payload_rows_remote=67252 dup_rows=20612 dup_rows_remote=18772 dedup_ratio=0.209676 router_tokens_per_expert=[0, 408, 416, 4, 1498, 1861, 45, 72, 52, 1794, 14, 93, 47, 1, 6, 1397, 360, 16, 7592, 2346, 207, 538, 1039, 601, 954, 1092, 51, 329, 21, 5795, 209, 2137, 1346, 306, 101, 821, 3, 30, 3494, 39, 507, 13, 984, 277, 14, 401, 523, 3, 3495, 0, 215, 517, 654, 0, 8, 6, 1004, 14, 207, 2, 341, 20, 2621, 191] payload_rows_per_dst=[7716, 6064, 8104, 14032, 10440, 15348, 8456, 7532] dup_rows_per_dst=[892, 744, 2792, 7144, 1840, 4598, 1334, 1268] combine_rows_per_src=[7716, 6064, 8104, 14032, 10440, 15348, 8456, 7532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=59796 dup_rows=22862 dup_rows_remote=14626 dedup_ratio=0.232564 router_tokens_per_expert=[72, 41, 258, 1348, 120, 328, 205, 0, 225, 410, 5, 269, 251, 8, 255, 478, 1548, 910, 1072, 2172, 1, 638, 1586, 957, 716, 937, 384, 2540, 2, 1, 81, 776, 1025, 342, 0, 75, 5, 693, 105, 3140, 1288, 1, 841, 49, 14, 35, 680, 1113, 20, 96, 90, 0, 2418, 6935, 146, 64, 4336, 711, 238, 171, 2869, 560, 654, 1844] payload_rows_per_dst=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] dup_rows_per_dst=[2434, 554, 2164, 2320, 1378, 2364, 8236, 3412] combine_rows_per_src=[10982, 3248, 8088, 8554, 9392, 8850, 15646, 10682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71698 payload_rows_remote=55966 dup_rows=26606 dup_rows_remote=11824 dedup_ratio=0.270650 router_tokens_per_expert=[1, 35, 30, 0, 2297, 221, 13, 38, 55, 2845, 128, 209, 14, 1, 4, 767, 487, 47, 8039, 189, 308, 1553, 246, 362, 692, 99, 12, 79, 0, 5899, 71, 1650, 2565, 292, 90, 2510, 5, 1, 262, 12, 131, 2, 3485, 302, 5, 437, 1628, 43, 2950, 0, 65, 64, 237, 0, 1, 6, 2823, 0, 134, 6, 120, 4, 4557, 24] payload_rows_per_dst=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] dup_rows_per_dst=[122, 912, 1300, 578, 1790, 2050, 5072, 14782] combine_rows_per_src=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78492 payload_rows_remote=63206 dup_rows=19812 dup_rows_remote=16784 dedup_ratio=0.201538 router_tokens_per_expert=[0, 340, 255, 0, 885, 2404, 8, 9, 5, 1660, 12, 45, 23, 1, 12, 2030, 252, 10, 7446, 3235, 160, 486, 901, 1198, 1228, 423, 35, 225, 0, 6066, 97, 3279, 1014, 380, 91, 358, 0, 1, 3449, 5, 152, 0, 729, 241, 11, 244, 334, 0, 4320, 0, 58, 601, 415, 0, 12, 1, 1036, 0, 125, 5, 283, 3, 2407, 147] payload_rows_per_dst=[7090, 6788, 7194, 14126, 11062, 15286, 10018, 6928] dup_rows_per_dst=[712, 788, 1578, 7352, 3246, 3028, 2024, 1084] combine_rows_per_src=[7090, 6788, 7194, 14126, 11062, 15286, 10018, 6928] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=65752 dup_rows=21472 dup_rows_remote=17642 dedup_ratio=0.218424 router_tokens_per_expert=[0, 281, 234, 0, 1970, 1409, 6, 14, 22, 2181, 26, 123, 10, 1, 10, 1671, 253, 3, 7915, 1507, 274, 650, 705, 718, 907, 566, 23, 238, 31, 6453, 84, 2663, 1521, 441, 86, 830, 1, 5, 2096, 7, 143, 0, 1338, 230, 4, 302, 822, 4, 3552, 1, 95, 434, 691, 0, 10, 3, 1533, 3, 238, 9, 186, 3, 3582, 34] payload_rows_per_dst=[10808, 7086, 16366, 11080, 8700, 5128, 8358, 9306] dup_rows_per_dst=[4040, 1002, 7684, 3830, 1274, 558, 1214, 1870] combine_rows_per_src=[10808, 7086, 16366, 11080, 8700, 5128, 8358, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77148 payload_rows_remote=60948 dup_rows=21156 dup_rows_remote=12400 dedup_ratio=0.215210 router_tokens_per_expert=[0, 0, 7, 0, 2311, 100, 6, 13, 43, 3256, 288, 243, 24, 283, 0, 293, 1046, 18, 8048, 26, 321, 1712, 162, 309, 612, 3, 0, 81, 0, 4788, 80, 1009, 3073, 290, 118, 2443, 2, 1, 214, 5, 83, 0, 3525, 318, 0, 505, 2369, 79, 2851, 0, 71, 0, 355, 0, 8, 40, 2333, 0, 131, 4, 114, 3, 5122, 13] payload_rows_per_dst=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] dup_rows_per_dst=[2820, 8756, 1422, 360, 1782, 2508, 126, 3382] combine_rows_per_src=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71698 payload_rows_remote=55966 dup_rows=26606 dup_rows_remote=11824 dedup_ratio=0.270650 router_tokens_per_expert=[1, 35, 30, 0, 2297, 221, 13, 38, 55, 2845, 128, 209, 14, 1, 4, 767, 487, 47, 8039, 189, 308, 1553, 246, 362, 692, 99, 12, 79, 0, 5899, 71, 1650, 2565, 292, 90, 2510, 5, 1, 262, 12, 131, 2, 3485, 302, 5, 437, 1628, 43, 2950, 0, 65, 64, 237, 0, 1, 6, 2823, 0, 134, 6, 120, 4, 4557, 24] payload_rows_per_dst=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] dup_rows_per_dst=[122, 912, 1300, 578, 1790, 2050, 5072, 14782] combine_rows_per_src=[5148, 7134, 4706, 4628, 10062, 10916, 13372, 15732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77022 payload_rows_remote=62488 dup_rows=21282 dup_rows_remote=13972 dedup_ratio=0.216492 router_tokens_per_expert=[0, 386, 328, 1, 914, 2040, 33, 29, 54, 1975, 13, 49, 21, 0, 11, 2420, 185, 6, 7441, 2769, 181, 539, 923, 1073, 1026, 774, 87, 287, 61, 5713, 80, 3658, 898, 328, 88, 445, 0, 27, 3408, 7, 201, 0, 583, 241, 6, 342, 457, 2, 3948, 0, 138, 680, 433, 0, 9, 1, 817, 1, 274, 1, 304, 1, 2209, 226] payload_rows_per_dst=[6856, 7828, 5164, 9430, 11174, 15350, 14534, 6686] dup_rows_per_dst=[606, 1258, 650, 2516, 4766, 3196, 7310, 980] combine_rows_per_src=[6856, 7828, 5164, 9430, 11174, 15350, 14534, 6686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77692 payload_rows_remote=67252 dup_rows=20612 dup_rows_remote=18772 dedup_ratio=0.209676 router_tokens_per_expert=[0, 408, 416, 4, 1498, 1861, 45, 72, 52, 1794, 14, 93, 47, 1, 6, 1397, 360, 16, 7592, 2346, 207, 538, 1039, 601, 954, 1092, 51, 329, 21, 5795, 209, 2137, 1346, 306, 101, 821, 3, 30, 3494, 39, 507, 13, 984, 277, 14, 401, 523, 3, 3495, 0, 215, 517, 654, 0, 8, 6, 1004, 14, 207, 2, 341, 20, 2621, 191] payload_rows_per_dst=[7716, 6064, 8104, 14032, 10440, 15348, 8456, 7532] dup_rows_per_dst=[892, 744, 2792, 7144, 1840, 4598, 1334, 1268] combine_rows_per_src=[7716, 6064, 8104, 14032, 10440, 15348, 8456, 7532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=275464192 dispatch_meta_bytes=344096 combine_payload_bytes=275464192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275808288 ingress_bytes=310067200 link_bytes=310067200 token_bytes=551272480 weight_movement_bytes=34603008 total_nvlink_bytes=585875488 weight_to_token_ratio=0.062769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166757911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=269099008 dispatch_meta_bytes=338288 combine_payload_bytes=269099008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269437296 ingress_bytes=303702016 link_bytes=303702016 token_bytes=538536304 weight_movement_bytes=34603008 total_nvlink_bytes=573139312 weight_to_token_ratio=0.064254 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142806236e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=244924416 dispatch_meta_bytes=297688 combine_payload_bytes=244924416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245222104 ingress_bytes=314130432 link_bytes=314130432 token_bytes=490146520 weight_movement_bytes=69206016 total_nvlink_bytes=559352536 weight_to_token_ratio=0.141195 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.182047526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229236736 dispatch_meta_bytes=271160 combine_payload_bytes=229236736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=229507896 ingress_bytes=263839744 link_bytes=263839744 token_bytes=458744632 weight_movement_bytes=34603008 total_nvlink_bytes=493347640 weight_to_token_ratio=0.075430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.928077158e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258891776 dispatch_meta_bytes=319960 combine_payload_bytes=258891776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259211736 ingress_bytes=293494784 link_bytes=293494784 token_bytes=518103512 weight_movement_bytes=34603008 total_nvlink_bytes=552706520 weight_to_token_ratio=0.066788 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.104397244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77148 payload_rows_remote=60948 dup_rows=21156 dup_rows_remote=12400 dedup_ratio=0.215210 router_tokens_per_expert=[0, 0, 7, 0, 2311, 100, 6, 13, 43, 3256, 288, 243, 24, 283, 0, 293, 1046, 18, 8048, 26, 321, 1712, 162, 309, 612, 3, 0, 81, 0, 4788, 80, 1009, 3073, 290, 118, 2443, 2, 1, 214, 5, 83, 0, 3525, 318, 0, 505, 2369, 79, 2851, 0, 71, 0, 355, 0, 8, 40, 2333, 0, 131, 4, 114, 3, 5122, 13] payload_rows_per_dst=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] dup_rows_per_dst=[2820, 8756, 1422, 360, 1782, 2508, 126, 3382] combine_rows_per_src=[11630, 16200, 5766, 3210, 10510, 11250, 6524, 12058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=269320192 dispatch_meta_bytes=333576 combine_payload_bytes=269320192 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=269653768 ingress_bytes=373129216 link_bytes=373129216 token_bytes=538973960 weight_movement_bytes=103809024 total_nvlink_bytes=642782984 weight_to_token_ratio=0.192605 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404055200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=229236736 dispatch_meta_bytes=271160 combine_payload_bytes=229236736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=229507896 ingress_bytes=263839744 link_bytes=263839744 token_bytes=458744632 weight_movement_bytes=34603008 total_nvlink_bytes=493347640 weight_to_token_ratio=0.075430 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.928077158e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=255950848 dispatch_meta_bytes=305840 combine_payload_bytes=255950848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256256688 ingress_bytes=290553856 link_bytes=290553856 token_bytes=512207536 weight_movement_bytes=34603008 total_nvlink_bytes=546810544 weight_to_token_ratio=0.067557 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093330769e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=275464192 dispatch_meta_bytes=344096 combine_payload_bytes=275464192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275808288 ingress_bytes=310067200 link_bytes=310067200 token_bytes=551272480 weight_movement_bytes=34603008 total_nvlink_bytes=585875488 weight_to_token_ratio=0.062769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166757911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73404 payload_rows_remote=57692 dup_rows=24900 dup_rows_remote=14278 dedup_ratio=0.253296 router_tokens_per_expert=[1, 256, 258, 2, 788, 2496, 29, 25, 18, 1331, 13, 20, 33, 14, 12, 1708, 333, 1, 7471, 3903, 136, 467, 1374, 966, 1300, 911, 129, 269, 24, 5755, 160, 3230, 987, 417, 177, 281, 2, 9, 3876, 2, 212, 3, 519, 173, 8, 306, 312, 0, 4075, 0, 110, 487, 338, 0, 5, 0, 622, 0, 91, 9, 410, 0, 2066, 222] payload_rows_per_dst=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] dup_rows_per_dst=[6130, 1428, 10622, 2934, 1588, 44, 1302, 852] combine_rows_per_src=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249643008 dispatch_meta_bytes=293392 combine_payload_bytes=249643008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249936400 ingress_bytes=284246016 link_bytes=284246016 token_bytes=499579408 weight_movement_bytes=34603008 total_nvlink_bytes=534182416 weight_to_token_ratio=0.069264 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069594875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=236306432 dispatch_meta_bytes=287880 combine_payload_bytes=236306432 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=236594312 ingress_bytes=409321472 link_bytes=409321472 token_bytes=472900744 weight_movement_bytes=173015040 total_nvlink_bytes=645915784 weight_to_token_ratio=0.365859 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540243746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80762 payload_rows_remote=66688 dup_rows=17542 dup_rows_remote=14454 dedup_ratio=0.178446 router_tokens_per_expert=[3551, 385, 2176, 178, 1, 34, 952, 0, 729, 0, 28, 23, 342, 248, 0, 0, 1647, 0, 442, 7, 757, 89, 3, 9, 2735, 7, 463, 248, 0, 317, 554, 38, 1107, 3370, 1360, 4292, 282, 3471, 29, 0, 62, 17, 6315, 353, 0, 1605, 229, 0, 0, 624, 666, 0, 39, 2279, 183, 0, 1313, 4012, 0, 107, 0, 834, 488, 152] payload_rows_per_dst=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] dup_rows_per_dst=[374, 1554, 2266, 1198, 6354, 3088, 500, 2208] combine_rows_per_src=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73404 payload_rows_remote=57692 dup_rows=24900 dup_rows_remote=14278 dedup_ratio=0.253296 router_tokens_per_expert=[1, 256, 258, 2, 788, 2496, 29, 25, 18, 1331, 13, 20, 33, 14, 12, 1708, 333, 1, 7471, 3903, 136, 467, 1374, 966, 1300, 911, 129, 269, 24, 5755, 160, 3230, 987, 417, 177, 281, 2, 9, 3876, 2, 212, 3, 519, 173, 8, 306, 312, 0, 4075, 0, 110, 487, 338, 0, 5, 0, 622, 0, 91, 9, 410, 0, 2066, 222] payload_rows_per_dst=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] dup_rows_per_dst=[6130, 1428, 10622, 2934, 1588, 44, 1302, 852] combine_rows_per_src=[13090, 7838, 15712, 9112, 9914, 3022, 8728, 5988] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75584 payload_rows_remote=61520 dup_rows=22720 dup_rows_remote=16140 dedup_ratio=0.231120 router_tokens_per_expert=[0, 169, 176, 2, 1943, 1631, 20, 13, 48, 2409, 25, 114, 19, 4, 13, 1767, 349, 5, 7840, 1486, 231, 724, 806, 857, 995, 274, 28, 259, 0, 6368, 91, 2503, 1877, 418, 101, 556, 1, 0, 2114, 11, 246, 1, 1497, 275, 5, 395, 592, 0, 4107, 0, 158, 430, 265, 0, 4, 2, 1457, 0, 259, 16, 171, 2, 2950, 73] payload_rows_per_dst=[14064, 15922, 7228, 7000, 8628, 5618, 8790, 8334] dup_rows_per_dst=[6580, 8556, 2016, 972, 1528, 404, 1142, 1522] combine_rows_per_src=[14064, 15922, 7228, 7000, 8628, 5618, 8790, 8334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81022 payload_rows_remote=72284 dup_rows=17282 dup_rows_remote=15666 dedup_ratio=0.175802 router_tokens_per_expert=[4302, 925, 1993, 184, 261, 61, 1490, 45, 604, 0, 18, 88, 143, 361, 8, 0, 646, 0, 143, 33, 898, 117, 3, 24, 2877, 0, 729, 419, 0, 312, 728, 112, 1087, 3528, 503, 3299, 99, 3070, 151, 0, 78, 4, 5076, 376, 0, 2109, 218, 0, 0, 315, 1994, 1, 86, 924, 207, 2, 1478, 4905, 0, 89, 1, 798, 760, 470] payload_rows_per_dst=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] dup_rows_per_dst=[1358, 4254, 1456, 1616, 1906, 2336, 326, 4030] combine_rows_per_src=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79540 payload_rows_remote=73686 dup_rows=18764 dup_rows_remote=18462 dedup_ratio=0.190877 router_tokens_per_expert=[2858, 1082, 2257, 205, 101, 186, 1430, 0, 716, 0, 79, 91, 310, 313, 1, 3, 729, 2, 805, 59, 609, 92, 7, 4, 3539, 4, 317, 309, 0, 352, 482, 211, 1049, 4239, 2028, 4010, 286, 2753, 36, 0, 33, 24, 5372, 372, 1, 2005, 164, 0, 0, 597, 802, 0, 19, 1434, 226, 0, 1104, 3227, 0, 184, 0, 1296, 473, 265] payload_rows_per_dst=[9074, 9272, 9046, 9086, 13424, 13048, 5854, 10736] dup_rows_per_dst=[1448, 1774, 1284, 1342, 7358, 2894, 302, 2362] combine_rows_per_src=[9074, 9272, 9046, 9086, 13424, 13048, 5854, 10736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81470 payload_rows_remote=66182 dup_rows=16834 dup_rows_remote=11400 dedup_ratio=0.171244 router_tokens_per_expert=[5304, 598, 1958, 14, 288, 20, 1164, 376, 190, 1, 0, 299, 205, 552, 72, 0, 251, 0, 24, 440, 1539, 129, 5, 371, 1502, 4, 1372, 230, 0, 9, 1759, 2, 1450, 2471, 74, 1870, 5, 1563, 310, 2, 34, 0, 2717, 73, 26, 2891, 472, 0, 14, 115, 5017, 0, 311, 309, 277, 112, 603, 7401, 0, 3, 0, 96, 1708, 550] payload_rows_per_dst=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] dup_rows_per_dst=[1078, 614, 3092, 974, 2512, 1996, 1134, 5434] combine_rows_per_src=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251985920 dispatch_meta_bytes=310640 combine_payload_bytes=251985920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252296560 ingress_bytes=286588928 link_bytes=286588928 token_bytes=504282480 weight_movement_bytes=34603008 total_nvlink_bytes=538885488 weight_to_token_ratio=0.068618 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.078411064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=236306432 dispatch_meta_bytes=287880 combine_payload_bytes=236306432 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=236594312 ingress_bytes=409321472 link_bytes=409321472 token_bytes=472900744 weight_movement_bytes=173015040 total_nvlink_bytes=645915784 weight_to_token_ratio=0.365859 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.540243746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81146 payload_rows_remote=69028 dup_rows=17158 dup_rows_remote=12636 dedup_ratio=0.174540 router_tokens_per_expert=[3738, 1192, 2017, 254, 195, 90, 1834, 7, 917, 0, 43, 94, 370, 461, 0, 8, 1020, 0, 268, 20, 841, 176, 0, 19, 3042, 2, 715, 334, 4, 511, 512, 248, 1181, 3813, 898, 3850, 149, 2328, 101, 3, 94, 26, 4628, 310, 14, 1838, 361, 9, 0, 400, 1145, 1, 59, 1561, 141, 0, 1349, 3789, 13, 87, 4, 1419, 307, 342] payload_rows_per_dst=[9544, 9598, 10348, 8972, 12118, 12406, 6194, 11966] dup_rows_per_dst=[1634, 2194, 1816, 1764, 4522, 2154, 420, 2654] combine_rows_per_src=[9544, 9598, 10348, 8972, 12118, 12406, 6194, 11966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273154048 dispatch_meta_bytes=324568 combine_payload_bytes=273154048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273478616 ingress_bytes=376963072 link_bytes=376963072 token_bytes=546632664 weight_movement_bytes=103809024 total_nvlink_bytes=650441688 weight_to_token_ratio=0.189906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418481692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77022 payload_rows_remote=62488 dup_rows=21282 dup_rows_remote=13972 dedup_ratio=0.216492 router_tokens_per_expert=[0, 386, 328, 1, 914, 2040, 33, 29, 54, 1975, 13, 49, 21, 0, 11, 2420, 185, 6, 7441, 2769, 181, 539, 923, 1073, 1026, 774, 87, 287, 61, 5713, 80, 3658, 898, 328, 88, 445, 0, 27, 3408, 7, 201, 0, 583, 241, 6, 342, 457, 2, 3948, 0, 138, 680, 433, 0, 9, 1, 817, 1, 274, 1, 304, 1, 2209, 226] payload_rows_per_dst=[6856, 7828, 5164, 9430, 11174, 15350, 14534, 6686] dup_rows_per_dst=[606, 1258, 650, 2516, 4766, 3196, 7310, 980] combine_rows_per_src=[6856, 7828, 5164, 9430, 11174, 15350, 14534, 6686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81470 payload_rows_remote=66182 dup_rows=16834 dup_rows_remote=11400 dedup_ratio=0.171244 router_tokens_per_expert=[5304, 598, 1958, 14, 288, 20, 1164, 376, 190, 1, 0, 299, 205, 552, 72, 0, 251, 0, 24, 440, 1539, 129, 5, 371, 1502, 4, 1372, 230, 0, 9, 1759, 2, 1450, 2471, 74, 1870, 5, 1563, 310, 2, 34, 0, 2717, 73, 26, 2891, 472, 0, 14, 115, 5017, 0, 311, 309, 277, 112, 603, 7401, 0, 3, 0, 96, 1708, 550] payload_rows_per_dst=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] dup_rows_per_dst=[1078, 614, 3092, 974, 2512, 1996, 1134, 5434] combine_rows_per_src=[7758, 5764, 13034, 8782, 9238, 10430, 11176, 15288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80762 payload_rows_remote=66688 dup_rows=17542 dup_rows_remote=14454 dedup_ratio=0.178446 router_tokens_per_expert=[3551, 385, 2176, 178, 1, 34, 952, 0, 729, 0, 28, 23, 342, 248, 0, 0, 1647, 0, 442, 7, 757, 89, 3, 9, 2735, 7, 463, 248, 0, 317, 554, 38, 1107, 3370, 1360, 4292, 282, 3471, 29, 0, 62, 17, 6315, 353, 0, 1605, 229, 0, 0, 624, 666, 0, 39, 2279, 183, 0, 1313, 4012, 0, 107, 0, 834, 488, 152] payload_rows_per_dst=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] dup_rows_per_dst=[374, 1554, 2266, 1198, 6354, 3088, 500, 2208] combine_rows_per_src=[7078, 9770, 10744, 7526, 12884, 14074, 7082, 11604] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73466 payload_rows_remote=65866 dup_rows=24838 dup_rows_remote=23890 dedup_ratio=0.252665 router_tokens_per_expert=[4745, 529, 2372, 1, 468, 4, 1688, 1214, 43, 0, 0, 410, 119, 680, 284, 0, 135, 0, 1, 212, 1672, 140, 6, 644, 1080, 2, 1601, 457, 0, 1, 2183, 0, 592, 1141, 7, 1597, 0, 1231, 330, 3, 12, 0, 1791, 72, 10, 2886, 779, 0, 16, 124, 5799, 0, 485, 416, 209, 143, 635, 7735, 0, 0, 0, 17, 1717, 714] payload_rows_per_dst=[11124, 7600, 12338, 11264, 1836, 7236, 16198, 5870] dup_rows_per_dst=[3890, 948, 2772, 2966, 28, 282, 13656, 296] combine_rows_per_src=[11124, 7600, 12338, 11264, 1836, 7236, 16198, 5870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251985920 dispatch_meta_bytes=310640 combine_payload_bytes=251985920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252296560 ingress_bytes=286588928 link_bytes=286588928 token_bytes=504282480 weight_movement_bytes=34603008 total_nvlink_bytes=538885488 weight_to_token_ratio=0.068618 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.078411064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81022 payload_rows_remote=72284 dup_rows=17282 dup_rows_remote=15666 dedup_ratio=0.175802 router_tokens_per_expert=[4302, 925, 1993, 184, 261, 61, 1490, 45, 604, 0, 18, 88, 143, 361, 8, 0, 646, 0, 143, 33, 898, 117, 3, 24, 2877, 0, 729, 419, 0, 312, 728, 112, 1087, 3528, 503, 3299, 99, 3070, 151, 0, 78, 4, 5076, 376, 0, 2109, 218, 0, 0, 315, 1994, 1, 86, 924, 207, 2, 1478, 4905, 0, 89, 1, 798, 760, 470] payload_rows_per_dst=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] dup_rows_per_dst=[1358, 4254, 1456, 1616, 1906, 2336, 326, 4030] combine_rows_per_src=[8860, 11844, 10876, 8738, 9100, 11900, 6732, 12972] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296075264 dispatch_meta_bytes=351800 combine_payload_bytes=296075264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296427064 ingress_bytes=330678272 link_bytes=330678272 token_bytes=592502328 weight_movement_bytes=34603008 total_nvlink_bytes=627105336 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244315716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=282738688 dispatch_meta_bytes=326656 combine_payload_bytes=282738688 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=283065344 ingress_bytes=421150720 link_bytes=421150720 token_bytes=565804032 weight_movement_bytes=138412032 total_nvlink_bytes=704216064 weight_to_token_ratio=0.244629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584756254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=301817856 dispatch_meta_bytes=368592 combine_payload_bytes=301817856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302186448 ingress_bytes=336420864 link_bytes=336420864 token_bytes=604004304 weight_movement_bytes=34603008 total_nvlink_bytes=638607312 weight_to_token_ratio=0.057289 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=255950848 dispatch_meta_bytes=305840 combine_payload_bytes=255950848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256256688 ingress_bytes=290553856 link_bytes=290553856 token_bytes=512207536 weight_movement_bytes=34603008 total_nvlink_bytes=546810544 weight_to_token_ratio=0.067557 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093330769e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=271081472 dispatch_meta_bytes=310328 combine_payload_bytes=271081472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271391800 ingress_bytes=340287488 link_bytes=340287488 token_bytes=542473272 weight_movement_bytes=69206016 total_nvlink_bytes=611679288 weight_to_token_ratio=0.127575 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280474422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79058 payload_rows_remote=70178 dup_rows=19246 dup_rows_remote=17754 dedup_ratio=0.195780 router_tokens_per_expert=[2754, 858, 2037, 214, 52, 193, 1084, 0, 841, 0, 15, 21, 77, 282, 0, 0, 1285, 0, 232, 15, 737, 133, 6, 24, 2818, 3, 419, 309, 2, 342, 409, 123, 1227, 3551, 2289, 4285, 133, 2924, 31, 1, 83, 10, 6079, 402, 6, 1645, 536, 1, 0, 750, 498, 0, 10, 2594, 211, 3, 1184, 3915, 0, 56, 3, 903, 317, 220] payload_rows_per_dst=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] dup_rows_per_dst=[3740, 4828, 1492, 3144, 2366, 160, 2924, 592] combine_rows_per_src=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=282738688 dispatch_meta_bytes=326656 combine_payload_bytes=282738688 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=283065344 ingress_bytes=421150720 link_bytes=421150720 token_bytes=565804032 weight_movement_bytes=138412032 total_nvlink_bytes=704216064 weight_to_token_ratio=0.244629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584756254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=273154048 dispatch_meta_bytes=324568 combine_payload_bytes=273154048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273478616 ingress_bytes=376963072 link_bytes=376963072 token_bytes=546632664 weight_movement_bytes=103809024 total_nvlink_bytes=650441688 weight_to_token_ratio=0.189906 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418481692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269787136 dispatch_meta_bytes=359024 combine_payload_bytes=269787136 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270146160 ingress_bytes=338993152 link_bytes=338993152 token_bytes=539933296 weight_movement_bytes=69206016 total_nvlink_bytes=609139312 weight_to_token_ratio=0.128175 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275603940e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287449088 dispatch_meta_bytes=351728 combine_payload_bytes=287449088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287800816 ingress_bytes=322052096 link_bytes=322052096 token_bytes=575249904 weight_movement_bytes=34603008 total_nvlink_bytes=609852912 weight_to_token_ratio=0.060153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211856110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81502 payload_rows_remote=66054 dup_rows=16802 dup_rows_remote=11296 dedup_ratio=0.170919 router_tokens_per_expert=[298, 0, 0, 44, 744, 288, 47, 2763, 44, 6375, 196, 178, 142, 2399, 0, 1091, 3437, 173, 0, 0, 20, 387, 373, 86, 336, 0, 5557, 618, 60, 281, 5, 14, 521, 0, 0, 1621, 108, 722, 207, 578, 7573, 33, 120, 1, 221, 3506, 311, 0, 723, 65, 1357, 2861, 601, 140, 263, 214, 885, 44, 6, 2, 284, 91, 16, 122] payload_rows_per_dst=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] dup_rows_per_dst=[848, 42, 1974, 1262, 2618, 5506, 2480, 2072] combine_rows_per_src=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296075264 dispatch_meta_bytes=351800 combine_payload_bytes=296075264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296427064 ingress_bytes=330678272 link_bytes=330678272 token_bytes=592502328 weight_movement_bytes=34603008 total_nvlink_bytes=627105336 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244315716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73976 payload_rows_remote=60994 dup_rows=24328 dup_rows_remote=16950 dedup_ratio=0.247477 router_tokens_per_expert=[4530, 781, 2439, 234, 108, 40, 1068, 8, 486, 1, 13, 97, 185, 306, 2, 0, 723, 0, 112, 67, 795, 125, 52, 26, 3046, 3, 763, 317, 0, 236, 852, 3, 873, 2977, 426, 3420, 39, 3502, 41, 2, 77, 9, 4764, 344, 0, 2049, 1125, 0, 0, 343, 1702, 0, 93, 1704, 217, 2, 1179, 5230, 0, 68, 0, 747, 570, 231] payload_rows_per_dst=[12982, 11520, 7920, 13760, 2554, 8770, 11086, 5384] dup_rows_per_dst=[7378, 3454, 940, 6208, 208, 2324, 3610, 206] combine_rows_per_src=[12982, 11520, 7920, 13760, 2554, 8770, 11086, 5384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82390 payload_rows_remote=70082 dup_rows=15914 dup_rows_remote=15054 dedup_ratio=0.161886 router_tokens_per_expert=[442, 0, 46, 61, 351, 163, 78, 1645, 4, 6834, 139, 104, 180, 3757, 0, 1279, 2919, 165, 0, 0, 115, 169, 463, 152, 289, 0, 5718, 374, 6, 190, 0, 7, 533, 0, 0, 864, 39, 2549, 141, 477, 6224, 85, 364, 0, 144, 4328, 1483, 0, 906, 23, 954, 1566, 331, 69, 80, 1026, 860, 24, 0, 3, 142, 99, 74, 114] payload_rows_per_dst=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] dup_rows_per_dst=[4186, 1694, 3968, 860, 1170, 48, 2074, 1914] combine_rows_per_src=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69354 payload_rows_remote=55598 dup_rows=28950 dup_rows_remote=28626 dedup_ratio=0.294495 router_tokens_per_expert=[768, 0, 2, 83, 53, 63, 119, 1506, 54, 6838, 192, 4, 10, 5905, 1, 1253, 777, 72, 2, 0, 84, 3, 759, 0, 275, 0, 5628, 39, 2, 142, 3, 0, 288, 0, 0, 99, 3, 6379, 8, 374, 1560, 70, 1394, 0, 91, 5255, 4921, 1, 239, 9, 85, 293, 34, 81, 34, 3095, 50, 3, 0, 0, 3, 5, 56, 85] payload_rows_per_dst=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] dup_rows_per_dst=[356, 20, 418, 332, 10610, 14646, 2244, 324] combine_rows_per_src=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77398 payload_rows_remote=62380 dup_rows=20906 dup_rows_remote=10714 dedup_ratio=0.212667 router_tokens_per_expert=[254, 0, 105, 44, 1204, 384, 25, 2274, 72, 6141, 87, 243, 286, 2687, 4, 1114, 2898, 144, 0, 0, 40, 518, 356, 185, 510, 0, 5386, 694, 26, 263, 3, 16, 454, 0, 0, 1017, 150, 759, 194, 671, 7386, 18, 157, 6, 154, 2600, 383, 2, 966, 63, 1621, 2504, 346, 130, 223, 286, 2252, 52, 3, 14, 200, 442, 16, 120] payload_rows_per_dst=[7332, 3560, 7470, 12390, 9324, 8112, 15018, 14192] dup_rows_per_dst=[1248, 52, 812, 1406, 2540, 368, 10192, 4288] combine_rows_per_src=[7332, 3560, 7470, 12390, 9324, 8112, 15018, 14192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79058 payload_rows_remote=70178 dup_rows=19246 dup_rows_remote=17754 dedup_ratio=0.195780 router_tokens_per_expert=[2754, 858, 2037, 214, 52, 193, 1084, 0, 841, 0, 15, 21, 77, 282, 0, 0, 1285, 0, 232, 15, 737, 133, 6, 24, 2818, 3, 419, 309, 2, 342, 409, 123, 1227, 3551, 2289, 4285, 133, 2924, 31, 1, 83, 10, 6079, 402, 6, 1645, 536, 1, 0, 750, 498, 0, 10, 2594, 211, 3, 1184, 3915, 0, 56, 3, 903, 317, 220] payload_rows_per_dst=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] dup_rows_per_dst=[3740, 4828, 1492, 3144, 2366, 160, 2924, 592] combine_rows_per_src=[10984, 13316, 8880, 10964, 11896, 5206, 11110, 6702] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=270557184 dispatch_meta_bytes=309400 combine_payload_bytes=270557184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270866584 ingress_bytes=408969216 link_bytes=408969216 token_bytes=541423768 weight_movement_bytes=138412032 total_nvlink_bytes=679835800 weight_to_token_ratio=0.255645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538918235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=249831424 dispatch_meta_bytes=311776 combine_payload_bytes=249831424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250143200 ingress_bytes=319037440 link_bytes=319037440 token_bytes=499974624 weight_movement_bytes=69206016 total_nvlink_bytes=569180640 weight_to_token_ratio=0.138419 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200512202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287449088 dispatch_meta_bytes=351728 combine_payload_bytes=287449088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287800816 ingress_bytes=322052096 link_bytes=322052096 token_bytes=575249904 weight_movement_bytes=34603008 total_nvlink_bytes=609852912 weight_to_token_ratio=0.060153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211856110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81244 payload_rows_remote=74588 dup_rows=17060 dup_rows_remote=16120 dedup_ratio=0.173543 router_tokens_per_expert=[389, 9, 66, 62, 478, 502, 105, 1995, 27, 6133, 122, 193, 290, 3565, 1, 1364, 3029, 129, 0, 2, 106, 186, 455, 698, 349, 6, 5332, 717, 41, 388, 35, 40, 603, 0, 1, 881, 71, 1391, 201, 649, 6599, 57, 196, 1, 165, 2980, 823, 0, 936, 62, 1038, 2050, 277, 159, 112, 456, 1791, 114, 19, 25, 151, 300, 19, 211] payload_rows_per_dst=[7832, 9170, 11640, 12262, 6656, 11364, 8616, 13704] dup_rows_per_dst=[2274, 1952, 3530, 1554, 940, 1424, 1564, 3822] combine_rows_per_src=[7832, 9170, 11640, 12262, 6656, 11364, 8616, 13704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79540 payload_rows_remote=73686 dup_rows=18764 dup_rows_remote=18462 dedup_ratio=0.190877 router_tokens_per_expert=[2858, 1082, 2257, 205, 101, 186, 1430, 0, 716, 0, 79, 91, 310, 313, 1, 3, 729, 2, 805, 59, 609, 92, 7, 4, 3539, 4, 317, 309, 0, 352, 482, 211, 1049, 4239, 2028, 4010, 286, 2753, 36, 0, 33, 24, 5372, 372, 1, 2005, 164, 0, 0, 597, 802, 0, 19, 1434, 226, 0, 1104, 3227, 0, 184, 0, 1296, 473, 265] payload_rows_per_dst=[9074, 9272, 9046, 9086, 13424, 13048, 5854, 10736] dup_rows_per_dst=[1448, 1774, 1284, 1342, 7358, 2894, 302, 2362] combine_rows_per_src=[9074, 9272, 9046, 9086, 13424, 13048, 5854, 10736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69354 payload_rows_remote=55598 dup_rows=28950 dup_rows_remote=28626 dedup_ratio=0.294495 router_tokens_per_expert=[768, 0, 2, 83, 53, 63, 119, 1506, 54, 6838, 192, 4, 10, 5905, 1, 1253, 777, 72, 2, 0, 84, 3, 759, 0, 275, 0, 5628, 39, 2, 142, 3, 0, 288, 0, 0, 99, 3, 6379, 8, 374, 1560, 70, 1394, 0, 91, 5255, 4921, 1, 239, 9, 85, 293, 34, 81, 34, 3095, 50, 3, 0, 0, 3, 5, 56, 85] payload_rows_per_dst=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] dup_rows_per_dst=[356, 20, 418, 332, 10610, 14646, 2244, 324] combine_rows_per_src=[4832, 3008, 2976, 5514, 15502, 15150, 8616, 13756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81502 payload_rows_remote=66054 dup_rows=16802 dup_rows_remote=11296 dedup_ratio=0.170919 router_tokens_per_expert=[298, 0, 0, 44, 744, 288, 47, 2763, 44, 6375, 196, 178, 142, 2399, 0, 1091, 3437, 173, 0, 0, 20, 387, 373, 86, 336, 0, 5557, 618, 60, 281, 5, 14, 521, 0, 0, 1621, 108, 722, 207, 578, 7573, 33, 120, 1, 221, 3506, 311, 0, 723, 65, 1357, 2861, 601, 140, 263, 214, 885, 44, 6, 2, 284, 91, 16, 122] payload_rows_per_dst=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] dup_rows_per_dst=[848, 42, 1974, 1262, 2618, 5506, 2480, 2072] combine_rows_per_src=[7520, 3260, 9554, 12480, 9694, 15448, 9968, 13578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78208 payload_rows_remote=62822 dup_rows=20096 dup_rows_remote=7018 dedup_ratio=0.204427 router_tokens_per_expert=[663, 0, 0, 169, 62, 56, 128, 1258, 47, 6349, 344, 7, 1, 6388, 0, 1096, 70, 106, 0, 0, 127, 1, 867, 1, 382, 39, 4734, 1, 1, 111, 0, 0, 264, 0, 0, 0, 0, 7247, 2, 407, 416, 139, 2506, 0, 68, 4669, 6532, 0, 404, 0, 9, 99, 0, 104, 0, 3151, 18, 1, 0, 0, 0, 0, 28, 80] payload_rows_per_dst=[4820, 15386, 10980, 9974, 14508, 15436, 6852, 252] dup_rows_per_dst=[684, 13078, 702, 562, 1332, 3054, 682, 2] combine_rows_per_src=[4820, 15386, 10980, 9974, 14508, 15436, 6852, 252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82390 payload_rows_remote=70082 dup_rows=15914 dup_rows_remote=15054 dedup_ratio=0.161886 router_tokens_per_expert=[442, 0, 46, 61, 351, 163, 78, 1645, 4, 6834, 139, 104, 180, 3757, 0, 1279, 2919, 165, 0, 0, 115, 169, 463, 152, 289, 0, 5718, 374, 6, 190, 0, 7, 533, 0, 0, 864, 39, 2549, 141, 477, 6224, 85, 364, 0, 144, 4328, 1483, 0, 906, 23, 954, 1566, 331, 69, 80, 1026, 860, 24, 0, 3, 142, 99, 74, 114] payload_rows_per_dst=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] dup_rows_per_dst=[4186, 1694, 3968, 860, 1170, 48, 2074, 1914] combine_rows_per_src=[13834, 9232, 12654, 12308, 8036, 4104, 7836, 14386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305512448 dispatch_meta_bytes=362832 combine_payload_bytes=305512448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305875280 ingress_bytes=340115456 link_bytes=340115456 token_bytes=611387728 weight_movement_bytes=34603008 total_nvlink_bytes=645990736 weight_to_token_ratio=0.056597 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279827079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=249831424 dispatch_meta_bytes=311776 combine_payload_bytes=249831424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250143200 ingress_bytes=319037440 link_bytes=319037440 token_bytes=499974624 weight_movement_bytes=69206016 total_nvlink_bytes=569180640 weight_to_token_ratio=0.138419 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200512202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287055872 dispatch_meta_bytes=340544 combine_payload_bytes=287055872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=287396416 ingress_bytes=356261888 link_bytes=356261888 token_bytes=574452288 weight_movement_bytes=69206016 total_nvlink_bytes=643658304 weight_to_token_ratio=0.120473 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340584803e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=227729408 dispatch_meta_bytes=336896 combine_payload_bytes=227729408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228066304 ingress_bytes=262332416 link_bytes=262332416 token_bytes=455795712 weight_movement_bytes=34603008 total_nvlink_bytes=490398720 weight_to_token_ratio=0.075918 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.871357619e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=255508480 dispatch_meta_bytes=292376 combine_payload_bytes=255508480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255800856 ingress_bytes=290111488 link_bytes=290111488 token_bytes=511309336 weight_movement_bytes=34603008 total_nvlink_bytes=545912344 weight_to_token_ratio=0.067675 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091666173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77782 payload_rows_remote=66536 dup_rows=20522 dup_rows_remote=17426 dedup_ratio=0.208761 router_tokens_per_expert=[239, 0, 22, 96, 788, 346, 85, 2346, 33, 5742, 98, 524, 221, 3025, 0, 863, 2682, 102, 0, 0, 69, 373, 512, 492, 503, 0, 5566, 550, 33, 340, 30, 11, 811, 0, 0, 1270, 127, 366, 210, 584, 7717, 1, 164, 10, 225, 2941, 93, 0, 1514, 49, 1285, 3299, 313, 88, 198, 106, 1646, 54, 10, 0, 154, 96, 6, 124] payload_rows_per_dst=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] dup_rows_per_dst=[7756, 1796, 3096, 1546, 962, 2, 2678, 2686] combine_rows_per_src=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305512448 dispatch_meta_bytes=362832 combine_payload_bytes=305512448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305875280 ingress_bytes=340115456 link_bytes=340115456 token_bytes=611387728 weight_movement_bytes=34603008 total_nvlink_bytes=645990736 weight_to_token_ratio=0.056597 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279827079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=301817856 dispatch_meta_bytes=368592 combine_payload_bytes=301817856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302186448 ingress_bytes=336420864 link_bytes=336420864 token_bytes=604004304 weight_movement_bytes=34603008 total_nvlink_bytes=638607312 weight_to_token_ratio=0.057289 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=227729408 dispatch_meta_bytes=336896 combine_payload_bytes=227729408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228066304 ingress_bytes=262332416 link_bytes=262332416 token_bytes=455795712 weight_movement_bytes=34603008 total_nvlink_bytes=490398720 weight_to_token_ratio=0.075918 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.871357619e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=270557184 dispatch_meta_bytes=309400 combine_payload_bytes=270557184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270866584 ingress_bytes=408969216 link_bytes=408969216 token_bytes=541423768 weight_movement_bytes=138412032 total_nvlink_bytes=679835800 weight_to_token_ratio=0.255645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.538918235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=257318912 dispatch_meta_bytes=279360 combine_payload_bytes=257318912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=257598272 ingress_bytes=361127936 link_bytes=361127936 token_bytes=514917184 weight_movement_bytes=103809024 total_nvlink_bytes=618726208 weight_to_token_ratio=0.201603 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287055872 dispatch_meta_bytes=340544 combine_payload_bytes=287055872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=287396416 ingress_bytes=356261888 link_bytes=356261888 token_bytes=574452288 weight_movement_bytes=69206016 total_nvlink_bytes=643658304 weight_to_token_ratio=0.120473 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340584803e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73702 payload_rows_remote=59744 dup_rows=24602 dup_rows_remote=19300 dedup_ratio=0.250264 router_tokens_per_expert=[425, 0, 2, 73, 336, 188, 113, 2069, 94, 7243, 349, 157, 95, 3687, 0, 1503, 2960, 348, 1, 0, 71, 161, 559, 25, 501, 1, 5869, 466, 42, 153, 3, 2, 500, 0, 0, 759, 194, 2027, 126, 713, 6424, 79, 271, 0, 121, 4517, 858, 0, 899, 40, 603, 1530, 280, 80, 48, 749, 472, 10, 0, 0, 128, 24, 9, 195] payload_rows_per_dst=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] dup_rows_per_dst=[5302, 11532, 4314, 1106, 1066, 6, 1252, 24] combine_rows_per_src=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77782 payload_rows_remote=66536 dup_rows=20522 dup_rows_remote=17426 dedup_ratio=0.208761 router_tokens_per_expert=[239, 0, 22, 96, 788, 346, 85, 2346, 33, 5742, 98, 524, 221, 3025, 0, 863, 2682, 102, 0, 0, 69, 373, 512, 492, 503, 0, 5566, 550, 33, 340, 30, 11, 811, 0, 0, 1270, 127, 366, 210, 584, 7717, 1, 164, 10, 225, 2941, 93, 0, 1514, 49, 1285, 3299, 313, 88, 198, 106, 1646, 54, 10, 0, 154, 96, 6, 124] payload_rows_per_dst=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] dup_rows_per_dst=[7756, 1796, 3096, 1546, 962, 2, 2678, 2686] combine_rows_per_src=[15522, 9058, 11246, 12520, 5774, 984, 11026, 11652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=244711424 dispatch_meta_bytes=316176 combine_payload_bytes=244711424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245027600 ingress_bytes=279314432 link_bytes=279314432 token_bytes=489739024 weight_movement_bytes=34603008 total_nvlink_bytes=524342032 weight_to_token_ratio=0.070656 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051037721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77892 payload_rows_remote=65862 dup_rows=20412 dup_rows_remote=15992 dedup_ratio=0.207642 router_tokens_per_expert=[343, 0, 62, 4, 2074, 681, 793, 109, 0, 1715, 163, 42, 344, 105, 747, 16, 6237, 33, 3576, 486, 601, 2723, 83, 17, 0, 4274, 1, 552, 518, 1835, 1521, 100, 2761, 435, 898, 44, 346, 0, 8, 157, 35, 515, 16, 400, 3, 1, 346, 3356, 1711, 0, 0, 1491, 358, 1, 61, 1558, 580, 4, 326, 2852, 9, 1, 78, 1046] payload_rows_per_dst=[7320, 13864, 6912, 12868, 12030, 8200, 8388, 8310] dup_rows_per_dst=[812, 4874, 974, 4734, 4420, 1144, 1972, 1482] combine_rows_per_src=[7320, 13864, 6912, 12868, 12030, 8200, 8388, 8310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=272531456 dispatch_meta_bytes=335848 combine_payload_bytes=272531456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272867304 ingress_bytes=307134464 link_bytes=307134464 token_bytes=545398760 weight_movement_bytes=34603008 total_nvlink_bytes=580001768 weight_to_token_ratio=0.063445 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155722262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77398 payload_rows_remote=62380 dup_rows=20906 dup_rows_remote=10714 dedup_ratio=0.212667 router_tokens_per_expert=[254, 0, 105, 44, 1204, 384, 25, 2274, 72, 6141, 87, 243, 286, 2687, 4, 1114, 2898, 144, 0, 0, 40, 518, 356, 185, 510, 0, 5386, 694, 26, 263, 3, 16, 454, 0, 0, 1017, 150, 759, 194, 671, 7386, 18, 157, 6, 154, 2600, 383, 2, 966, 63, 1621, 2504, 346, 130, 223, 286, 2252, 52, 3, 14, 200, 442, 16, 120] payload_rows_per_dst=[7332, 3560, 7470, 12390, 9324, 8112, 15018, 14192] dup_rows_per_dst=[1248, 52, 812, 1406, 2540, 368, 10192, 4288] combine_rows_per_src=[7332, 3560, 7470, 12390, 9324, 8112, 15018, 14192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77252 payload_rows_remote=61258 dup_rows=21052 dup_rows_remote=15024 dedup_ratio=0.214152 router_tokens_per_expert=[141, 0, 86, 1, 93, 218, 3514, 66, 0, 125, 5, 0, 42, 0, 50, 1, 1612, 0, 5469, 92, 81, 555, 91, 4, 0, 7534, 0, 105, 3175, 1998, 93, 0, 59, 201, 4527, 0, 736, 1, 0, 156, 2, 31, 1, 678, 0, 2, 257, 7268, 27, 0, 0, 3253, 211, 0, 82, 3032, 228, 0, 66, 122, 1, 3, 0, 3057] payload_rows_per_dst=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] dup_rows_per_dst=[360, 56, 390, 1280, 1036, 6510, 5392, 6028] combine_rows_per_src=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77092 payload_rows_remote=70196 dup_rows=21212 dup_rows_remote=20348 dedup_ratio=0.215780 router_tokens_per_expert=[277, 0, 63, 1, 2285, 446, 605, 27, 0, 2574, 129, 26, 662, 77, 912, 15, 7148, 12, 3507, 313, 511, 2332, 24, 13, 0, 3536, 0, 422, 392, 1123, 2136, 36, 3227, 610, 292, 7, 311, 0, 0, 125, 74, 573, 7, 169, 1, 0, 388, 2668, 2673, 3, 0, 1304, 222, 0, 96, 1451, 287, 0, 612, 3579, 0, 0, 2, 867] payload_rows_per_dst=[6876, 8714, 12118, 11998, 11800, 6896, 9586, 9104] dup_rows_per_dst=[532, 4112, 4552, 3292, 4358, 864, 1912, 1590] combine_rows_per_src=[6876, 8714, 12118, 11998, 11800, 6896, 9586, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73702 payload_rows_remote=59744 dup_rows=24602 dup_rows_remote=19300 dedup_ratio=0.250264 router_tokens_per_expert=[425, 0, 2, 73, 336, 188, 113, 2069, 94, 7243, 349, 157, 95, 3687, 0, 1503, 2960, 348, 1, 0, 71, 161, 559, 25, 501, 1, 5869, 466, 42, 153, 3, 2, 500, 0, 0, 759, 194, 2027, 126, 713, 6424, 79, 271, 0, 121, 4517, 858, 0, 899, 40, 603, 1530, 280, 80, 48, 749, 472, 10, 0, 0, 128, 24, 9, 195] payload_rows_per_dst=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] dup_rows_per_dst=[5302, 11532, 4314, 1106, 1066, 6, 1252, 24] combine_rows_per_src=[13958, 15464, 12970, 12228, 7572, 2652, 7206, 1652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82274 payload_rows_remote=81440 dup_rows=16030 dup_rows_remote=16018 dedup_ratio=0.163066 router_tokens_per_expert=[358, 0, 70, 0, 3, 366, 3811, 62, 0, 4, 0, 0, 2, 0, 0, 0, 417, 0, 4895, 36, 158, 440, 177, 8, 0, 7765, 0, 99, 3418, 2635, 8, 0, 10, 0, 4671, 0, 829, 2, 0, 151, 0, 1, 0, 692, 0, 0, 262, 7562, 7, 0, 0, 3488, 285, 0, 7, 2567, 125, 0, 28, 85, 25, 24, 0, 3599] payload_rows_per_dst=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] dup_rows_per_dst=[8652, 12, 456, 1758, 1324, 1392, 2290, 146] combine_rows_per_src=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269770752 dispatch_meta_bytes=327416 combine_payload_bytes=269770752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270098168 ingress_bytes=304373760 link_bytes=304373760 token_bytes=539868920 weight_movement_bytes=34603008 total_nvlink_bytes=574471928 weight_to_token_ratio=0.064095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145333955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79124 payload_rows_remote=68142 dup_rows=19180 dup_rows_remote=15396 dedup_ratio=0.195109 router_tokens_per_expert=[288, 0, 55, 0, 971, 692, 1227, 66, 0, 1289, 64, 2, 208, 31, 741, 5, 5967, 17, 4768, 404, 282, 1708, 124, 16, 0, 5124, 1, 643, 792, 2410, 971, 51, 1944, 399, 1556, 2, 400, 0, 1, 129, 29, 302, 17, 451, 0, 2, 307, 4694, 1459, 2, 0, 1662, 200, 0, 46, 2429, 520, 0, 308, 2053, 0, 0, 12, 1311] payload_rows_per_dst=[9696, 12814, 5426, 10982, 12500, 10490, 9854, 7362] dup_rows_per_dst=[2120, 3800, 890, 3784, 4684, 1114, 1742, 1046] combine_rows_per_src=[9696, 12814, 5426, 10982, 12500, 10490, 9854, 7362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77252 payload_rows_remote=61258 dup_rows=21052 dup_rows_remote=15024 dedup_ratio=0.214152 router_tokens_per_expert=[141, 0, 86, 1, 93, 218, 3514, 66, 0, 125, 5, 0, 42, 0, 50, 1, 1612, 0, 5469, 92, 81, 555, 91, 4, 0, 7534, 0, 105, 3175, 1998, 93, 0, 59, 201, 4527, 0, 736, 1, 0, 156, 2, 31, 1, 678, 0, 2, 257, 7268, 27, 0, 0, 3253, 211, 0, 82, 3032, 228, 0, 66, 122, 1, 3, 0, 3057] payload_rows_per_dst=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] dup_rows_per_dst=[360, 56, 390, 1280, 1036, 6510, 5392, 6028] combine_rows_per_src=[7878, 390, 4480, 9462, 10324, 15698, 13026, 15994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78892 payload_rows_remote=68418 dup_rows=19412 dup_rows_remote=17290 dedup_ratio=0.197469 router_tokens_per_expert=[196, 0, 36, 0, 1988, 518, 598, 43, 0, 2557, 123, 32, 610, 63, 965, 20, 6789, 15, 3069, 352, 506, 2202, 12, 1, 0, 3416, 5, 466, 411, 1476, 2461, 142, 3052, 388, 508, 11, 299, 0, 0, 185, 37, 1066, 43, 231, 0, 0, 287, 2040, 3058, 1, 0, 1412, 212, 0, 50, 1565, 376, 0, 477, 3983, 0, 0, 13, 786] payload_rows_per_dst=[6398, 7578, 14136, 9946, 8780, 9296, 10474, 12284] dup_rows_per_dst=[360, 1162, 5618, 1762, 1756, 2600, 2122, 4032] combine_rows_per_src=[6398, 7578, 14136, 9946, 8780, 9296, 10474, 12284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287522816 dispatch_meta_bytes=362176 combine_payload_bytes=287522816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287884992 ingress_bytes=322125824 link_bytes=322125824 token_bytes=575407808 weight_movement_bytes=34603008 total_nvlink_bytes=610010816 weight_to_token_ratio=0.060136 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212133542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77892 payload_rows_remote=65862 dup_rows=20412 dup_rows_remote=15992 dedup_ratio=0.207642 router_tokens_per_expert=[343, 0, 62, 4, 2074, 681, 793, 109, 0, 1715, 163, 42, 344, 105, 747, 16, 6237, 33, 3576, 486, 601, 2723, 83, 17, 0, 4274, 1, 552, 518, 1835, 1521, 100, 2761, 435, 898, 44, 346, 0, 8, 157, 35, 515, 16, 400, 3, 1, 346, 3356, 1711, 0, 0, 1491, 358, 1, 61, 1558, 580, 4, 326, 2852, 9, 1, 78, 1046] payload_rows_per_dst=[7320, 13864, 6912, 12868, 12030, 8200, 8388, 8310] dup_rows_per_dst=[812, 4874, 974, 4734, 4420, 1144, 1972, 1482] combine_rows_per_src=[7320, 13864, 6912, 12868, 12030, 8200, 8388, 8310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82274 payload_rows_remote=81440 dup_rows=16030 dup_rows_remote=16018 dedup_ratio=0.163066 router_tokens_per_expert=[358, 0, 70, 0, 3, 366, 3811, 62, 0, 4, 0, 0, 2, 0, 0, 0, 417, 0, 4895, 36, 158, 440, 177, 8, 0, 7765, 0, 99, 3418, 2635, 8, 0, 10, 0, 4671, 0, 829, 2, 0, 151, 0, 1, 0, 692, 0, 0, 262, 7562, 7, 0, 0, 3488, 285, 0, 7, 2567, 125, 0, 28, 85, 25, 24, 0, 3599] payload_rows_per_dst=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] dup_rows_per_dst=[8652, 12, 456, 1758, 1324, 1392, 2290, 146] combine_rows_per_src=[16218, 834, 10972, 10562, 10002, 15642, 10418, 7626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=244711424 dispatch_meta_bytes=316176 combine_payload_bytes=244711424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245027600 ingress_bytes=279314432 link_bytes=279314432 token_bytes=489739024 weight_movement_bytes=34603008 total_nvlink_bytes=524342032 weight_to_token_ratio=0.070656 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051037721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279109632 dispatch_meta_bytes=334152 combine_payload_bytes=279109632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279443784 ingress_bytes=382918656 link_bytes=382918656 token_bytes=558553416 weight_movement_bytes=103809024 total_nvlink_bytes=662362440 weight_to_token_ratio=0.185853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440892075e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=250912768 dispatch_meta_bytes=305128 combine_payload_bytes=250912768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251217896 ingress_bytes=285515776 link_bytes=285515776 token_bytes=502130664 weight_movement_bytes=34603008 total_nvlink_bytes=536733672 weight_to_token_ratio=0.068912 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074372879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280240128 dispatch_meta_bytes=342832 combine_payload_bytes=280240128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280582960 ingress_bytes=314843136 link_bytes=314843136 token_bytes=560823088 weight_movement_bytes=34603008 total_nvlink_bytes=595426096 weight_to_token_ratio=0.061700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184729374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77216 payload_rows_remote=65852 dup_rows=21088 dup_rows_remote=17028 dedup_ratio=0.214518 router_tokens_per_expert=[238, 0, 44, 0, 2265, 458, 482, 19, 2, 3153, 150, 20, 317, 67, 900, 10, 7205, 3, 3563, 197, 1190, 2722, 36, 1, 0, 3216, 1, 384, 407, 1398, 2137, 59, 2432, 233, 445, 36, 226, 0, 6, 155, 91, 802, 50, 93, 1, 0, 332, 1956, 2255, 4, 0, 1314, 198, 0, 124, 1032, 343, 0, 547, 5002, 0, 1, 29, 801] payload_rows_per_dst=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] dup_rows_per_dst=[2682, 8020, 4060, 920, 738, 720, 1734, 2214] combine_rows_per_src=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269770752 dispatch_meta_bytes=327416 combine_payload_bytes=269770752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270098168 ingress_bytes=304373760 link_bytes=304373760 token_bytes=539868920 weight_movement_bytes=34603008 total_nvlink_bytes=574471928 weight_to_token_ratio=0.064095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145333955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=255508480 dispatch_meta_bytes=292376 combine_payload_bytes=255508480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255800856 ingress_bytes=290111488 link_bytes=290111488 token_bytes=511309336 weight_movement_bytes=34603008 total_nvlink_bytes=545912344 weight_to_token_ratio=0.067675 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091666173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287522816 dispatch_meta_bytes=362176 combine_payload_bytes=287522816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287884992 ingress_bytes=322125824 link_bytes=322125824 token_bytes=575407808 weight_movement_bytes=34603008 total_nvlink_bytes=610010816 weight_to_token_ratio=0.060136 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212133542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=250912768 dispatch_meta_bytes=305128 combine_payload_bytes=250912768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251217896 ingress_bytes=285515776 link_bytes=285515776 token_bytes=502130664 weight_movement_bytes=34603008 total_nvlink_bytes=536733672 weight_to_token_ratio=0.068912 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074372879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=333578240 dispatch_meta_bytes=389832 combine_payload_bytes=333578240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333968072 ingress_bytes=368181248 link_bytes=368181248 token_bytes=667546312 weight_movement_bytes=34603008 total_nvlink_bytes=702149320 weight_to_token_ratio=0.051836 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385436395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79314 payload_rows_remote=67314 dup_rows=18990 dup_rows_remote=16722 dedup_ratio=0.193176 router_tokens_per_expert=[1923, 951, 3963, 6397, 633, 15, 39, 5829, 2374, 1487, 455, 107, 0, 0, 615, 34, 40, 1158, 0, 479, 22, 55, 0, 1550, 538, 1, 70, 188, 0, 0, 1353, 786, 134, 4, 0, 9, 1159, 474, 402, 0, 4872, 60, 1, 1061, 1359, 2, 0, 377, 555, 3651, 0, 2, 0, 388, 0, 0, 1, 3, 0, 458, 2490, 569, 0, 59] payload_rows_per_dst=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] dup_rows_per_dst=[84, 5474, 5052, 1036, 2218, 2268, 1124, 1734] combine_rows_per_src=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269729792 dispatch_meta_bytes=331520 combine_payload_bytes=269729792 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=270061312 ingress_bytes=442744832 link_bytes=442744832 token_bytes=539791104 weight_movement_bytes=173015040 total_nvlink_bytes=712806144 weight_to_token_ratio=0.320522 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.666013159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80624 payload_rows_remote=67394 dup_rows=17680 dup_rows_remote=13272 dedup_ratio=0.179850 router_tokens_per_expert=[468, 0, 78, 0, 654, 720, 1358, 25, 1, 1144, 129, 16, 168, 78, 734, 4, 5949, 20, 4705, 328, 560, 1898, 137, 6, 0, 5516, 2, 425, 657, 2128, 663, 1, 1558, 395, 1256, 14, 734, 1, 0, 154, 23, 168, 4, 388, 0, 3, 294, 5024, 813, 6, 0, 1601, 253, 0, 158, 2806, 615, 1, 280, 2454, 0, 3, 2, 1572] payload_rows_per_dst=[13230, 13116, 12184, 6434, 7098, 10832, 9406, 8324] dup_rows_per_dst=[4408, 3330, 3772, 670, 1126, 976, 1868, 1530] combine_rows_per_src=[13230, 13116, 12184, 6434, 7098, 10832, 9406, 8324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81060 payload_rows_remote=68736 dup_rows=17244 dup_rows_remote=12632 dedup_ratio=0.175415 router_tokens_per_expert=[3414, 1636, 2978, 6274, 1583, 20, 8, 3337, 1330, 1548, 326, 481, 0, 0, 415, 35, 27, 474, 0, 220, 17, 13, 0, 510, 428, 0, 97, 106, 0, 0, 2537, 1205, 41, 0, 0, 99, 887, 376, 735, 0, 4095, 186, 15, 603, 1685, 4, 0, 1827, 656, 5276, 0, 6, 9, 386, 0, 0, 21, 8, 3, 767, 1748, 628, 0, 72] payload_rows_per_dst=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] dup_rows_per_dst=[524, 3378, 2100, 4612, 1368, 1146, 1470, 2646] combine_rows_per_src=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81302 payload_rows_remote=72046 dup_rows=17002 dup_rows_remote=15308 dedup_ratio=0.172953 router_tokens_per_expert=[3632, 5505, 778, 4042, 3152, 1, 2, 543, 164, 680, 20, 1824, 13, 5, 284, 19, 1, 58, 180, 5, 0, 5, 0, 51, 12, 0, 48, 25, 2, 0, 3744, 1229, 25, 3, 4, 678, 419, 15, 1165, 1, 2281, 238, 1, 46, 1696, 6, 0, 5936, 609, 7346, 5, 119, 0, 687, 5, 0, 0, 8, 87, 774, 365, 608, 0, 1] payload_rows_per_dst=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] dup_rows_per_dst=[2804, 1176, 224, 1312, 1364, 5322, 3106, 1694] combine_rows_per_src=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79258 payload_rows_remote=72248 dup_rows=19046 dup_rows_remote=17966 dedup_ratio=0.193746 router_tokens_per_expert=[2867, 833, 3383, 5996, 1143, 36, 64, 5866, 2503, 1584, 391, 32, 0, 0, 559, 30, 89, 1134, 0, 416, 30, 37, 0, 1058, 981, 1, 69, 163, 0, 0, 1313, 804, 271, 14, 0, 16, 825, 191, 472, 2, 4987, 124, 30, 1318, 1348, 1, 0, 284, 607, 3000, 2, 1, 82, 353, 0, 0, 32, 22, 0, 867, 2107, 613, 0, 201] payload_rows_per_dst=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] dup_rows_per_dst=[398, 4870, 3810, 788, 1714, 3532, 1080, 2854] combine_rows_per_src=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275718144 dispatch_meta_bytes=336144 combine_payload_bytes=275718144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276054288 ingress_bytes=344924160 link_bytes=344924160 token_bytes=551772432 weight_movement_bytes=69206016 total_nvlink_bytes=620978448 weight_to_token_ratio=0.125425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279109632 dispatch_meta_bytes=334152 combine_payload_bytes=279109632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=279443784 ingress_bytes=382918656 link_bytes=382918656 token_bytes=558553416 weight_movement_bytes=103809024 total_nvlink_bytes=662362440 weight_to_token_ratio=0.185853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440892075e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77216 payload_rows_remote=65852 dup_rows=21088 dup_rows_remote=17028 dedup_ratio=0.214518 router_tokens_per_expert=[238, 0, 44, 0, 2265, 458, 482, 19, 2, 3153, 150, 20, 317, 67, 900, 10, 7205, 3, 3563, 197, 1190, 2722, 36, 1, 0, 3216, 1, 384, 407, 1398, 2137, 59, 2432, 233, 445, 36, 226, 0, 6, 155, 91, 802, 50, 93, 1, 0, 332, 1956, 2255, 4, 0, 1314, 198, 0, 124, 1032, 343, 0, 547, 5002, 0, 1, 29, 801] payload_rows_per_dst=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] dup_rows_per_dst=[2682, 8020, 4060, 920, 738, 720, 1734, 2214] combine_rows_per_src=[10762, 15628, 11364, 7852, 6328, 5930, 8120, 11232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=68064 dup_rows=18160 dup_rows_remote=16676 dedup_ratio=0.184733 router_tokens_per_expert=[2804, 1025, 3602, 5514, 1528, 60, 54, 4714, 1671, 1513, 319, 168, 0, 0, 562, 27, 70, 1150, 0, 653, 54, 57, 0, 752, 725, 8, 115, 111, 0, 0, 1052, 667, 231, 11, 12, 13, 835, 196, 768, 2, 3532, 251, 58, 879, 1663, 2, 1, 670, 899, 5117, 3, 3, 47, 411, 0, 0, 143, 26, 3, 825, 2845, 596, 0, 135] payload_rows_per_dst=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] dup_rows_per_dst=[616, 4168, 3654, 2246, 1484, 962, 1774, 3256] combine_rows_per_src=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79968 payload_rows_remote=73648 dup_rows=18336 dup_rows_remote=18148 dedup_ratio=0.186523 router_tokens_per_expert=[4323, 5506, 552, 1786, 2521, 3, 0, 27, 8, 193, 1, 2750, 53, 1, 236, 12, 0, 6, 344, 0, 0, 0, 0, 9, 0, 1, 8, 0, 0, 0, 4275, 871, 4, 0, 0, 1961, 391, 1, 1344, 0, 1576, 601, 0, 5, 1633, 1, 0, 7366, 552, 7611, 0, 201, 0, 952, 1, 0, 0, 2, 152, 790, 22, 497, 0, 3] payload_rows_per_dst=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] dup_rows_per_dst=[6372, 188, 14, 2852, 584, 3962, 2604, 1760] combine_rows_per_src=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281542656 dispatch_meta_bytes=325472 combine_payload_bytes=281542656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281868128 ingress_bytes=316145664 link_bytes=316145664 token_bytes=563410784 weight_movement_bytes=34603008 total_nvlink_bytes=598013792 weight_to_token_ratio=0.061417 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.189630682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276045824 dispatch_meta_bytes=322664 combine_payload_bytes=276045824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276368488 ingress_bytes=310648832 link_bytes=310648832 token_bytes=552414312 weight_movement_bytes=34603008 total_nvlink_bytes=587017320 weight_to_token_ratio=0.062640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168946546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=68064 dup_rows=18160 dup_rows_remote=16676 dedup_ratio=0.184733 router_tokens_per_expert=[2804, 1025, 3602, 5514, 1528, 60, 54, 4714, 1671, 1513, 319, 168, 0, 0, 562, 27, 70, 1150, 0, 653, 54, 57, 0, 752, 725, 8, 115, 111, 0, 0, 1052, 667, 231, 11, 12, 13, 835, 196, 768, 2, 3532, 251, 58, 879, 1663, 2, 1, 670, 899, 5117, 3, 3, 47, 411, 0, 0, 143, 26, 3, 825, 2845, 596, 0, 135] payload_rows_per_dst=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] dup_rows_per_dst=[616, 4168, 3654, 2246, 1484, 962, 1774, 3256] combine_rows_per_src=[4718, 11556, 12846, 10174, 12080, 6086, 11186, 11498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=269729792 dispatch_meta_bytes=331520 combine_payload_bytes=269729792 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=270061312 ingress_bytes=442744832 link_bytes=442744832 token_bytes=539791104 weight_movement_bytes=173015040 total_nvlink_bytes=712806144 weight_to_token_ratio=0.320522 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.666013159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295100416 dispatch_meta_bytes=349416 combine_payload_bytes=295100416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295449832 ingress_bytes=329703424 link_bytes=329703424 token_bytes=590550248 weight_movement_bytes=34603008 total_nvlink_bytes=625153256 weight_to_token_ratio=0.058595 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240647441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295927808 dispatch_meta_bytes=360856 combine_payload_bytes=295927808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296288664 ingress_bytes=365133824 link_bytes=365133824 token_bytes=592216472 weight_movement_bytes=69206016 total_nvlink_bytes=661422488 weight_to_token_ratio=0.116859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81090 payload_rows_remote=67362 dup_rows=17214 dup_rows_remote=12686 dedup_ratio=0.175110 router_tokens_per_expert=[2385, 751, 4503, 5997, 597, 30, 15, 5652, 2153, 1504, 447, 27, 0, 0, 709, 14, 46, 998, 0, 455, 8, 51, 0, 1573, 552, 0, 64, 100, 0, 0, 1317, 941, 215, 30, 0, 14, 1037, 225, 555, 0, 4844, 179, 14, 587, 1497, 2, 0, 299, 588, 3701, 0, 2, 40, 518, 0, 0, 23, 33, 1, 807, 2188, 608, 0, 256] payload_rows_per_dst=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] dup_rows_per_dst=[372, 2954, 4528, 3284, 1676, 494, 1244, 2662] combine_rows_per_src=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78892 payload_rows_remote=68418 dup_rows=19412 dup_rows_remote=17290 dedup_ratio=0.197469 router_tokens_per_expert=[196, 0, 36, 0, 1988, 518, 598, 43, 0, 2557, 123, 32, 610, 63, 965, 20, 6789, 15, 3069, 352, 506, 2202, 12, 1, 0, 3416, 5, 466, 411, 1476, 2461, 142, 3052, 388, 508, 11, 299, 0, 0, 185, 37, 1066, 43, 231, 0, 0, 287, 2040, 3058, 1, 0, 1412, 212, 0, 50, 1565, 376, 0, 477, 3983, 0, 0, 13, 786] payload_rows_per_dst=[6398, 7578, 14136, 9946, 8780, 9296, 10474, 12284] dup_rows_per_dst=[360, 1162, 5618, 1762, 1756, 2600, 2122, 4032] combine_rows_per_src=[6398, 7578, 14136, 9946, 8780, 9296, 10474, 12284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79314 payload_rows_remote=67314 dup_rows=18990 dup_rows_remote=16722 dedup_ratio=0.193176 router_tokens_per_expert=[1923, 951, 3963, 6397, 633, 15, 39, 5829, 2374, 1487, 455, 107, 0, 0, 615, 34, 40, 1158, 0, 479, 22, 55, 0, 1550, 538, 1, 70, 188, 0, 0, 1353, 786, 134, 4, 0, 9, 1159, 474, 402, 0, 4872, 60, 1, 1061, 1359, 2, 0, 377, 555, 3651, 0, 2, 0, 388, 0, 0, 1, 3, 0, 458, 2490, 569, 0, 59] payload_rows_per_dst=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] dup_rows_per_dst=[84, 5474, 5052, 1036, 2218, 2268, 1124, 1734] combine_rows_per_src=[3192, 12596, 14350, 6032, 13804, 12000, 8068, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81302 payload_rows_remote=72046 dup_rows=17002 dup_rows_remote=15308 dedup_ratio=0.172953 router_tokens_per_expert=[3632, 5505, 778, 4042, 3152, 1, 2, 543, 164, 680, 20, 1824, 13, 5, 284, 19, 1, 58, 180, 5, 0, 5, 0, 51, 12, 0, 48, 25, 2, 0, 3744, 1229, 25, 3, 4, 678, 419, 15, 1165, 1, 2281, 238, 1, 46, 1696, 6, 0, 5936, 609, 7346, 5, 119, 0, 687, 5, 0, 0, 8, 87, 774, 365, 608, 0, 1] payload_rows_per_dst=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] dup_rows_per_dst=[2804, 1176, 224, 1312, 1364, 5322, 3106, 1694] combine_rows_per_src=[11526, 6398, 8460, 8808, 7332, 14182, 15340, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79968 payload_rows_remote=73648 dup_rows=18336 dup_rows_remote=18148 dedup_ratio=0.186523 router_tokens_per_expert=[4323, 5506, 552, 1786, 2521, 3, 0, 27, 8, 193, 1, 2750, 53, 1, 236, 12, 0, 6, 344, 0, 0, 0, 0, 9, 0, 1, 8, 0, 0, 0, 4275, 871, 4, 0, 0, 1961, 391, 1, 1344, 0, 1576, 601, 0, 5, 1633, 1, 0, 7366, 552, 7611, 0, 201, 0, 952, 1, 0, 0, 2, 152, 790, 22, 497, 0, 3] payload_rows_per_dst=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] dup_rows_per_dst=[6372, 188, 14, 2852, 584, 3962, 2604, 1760] combine_rows_per_src=[14144, 6320, 924, 10610, 6872, 15250, 16030, 9818] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278790144 dispatch_meta_bytes=338960 combine_payload_bytes=278790144 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=279129104 ingress_bytes=347996160 link_bytes=347996160 token_bytes=557919248 weight_movement_bytes=69206016 total_nvlink_bytes=627125264 weight_to_token_ratio=0.124043 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309481534e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81060 payload_rows_remote=68736 dup_rows=17244 dup_rows_remote=12632 dedup_ratio=0.175415 router_tokens_per_expert=[3414, 1636, 2978, 6274, 1583, 20, 8, 3337, 1330, 1548, 326, 481, 0, 0, 415, 35, 27, 474, 0, 220, 17, 13, 0, 510, 428, 0, 97, 106, 0, 0, 2537, 1205, 41, 0, 0, 99, 887, 376, 735, 0, 4095, 186, 15, 603, 1685, 4, 0, 1827, 656, 5276, 0, 6, 9, 386, 0, 0, 21, 8, 3, 767, 1748, 628, 0, 72] payload_rows_per_dst=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] dup_rows_per_dst=[524, 3378, 2100, 4612, 1368, 1146, 1470, 2646] combine_rows_per_src=[5970, 10848, 12970, 12324, 9582, 7494, 11196, 10676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280240128 dispatch_meta_bytes=342832 combine_payload_bytes=280240128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280582960 ingress_bytes=314843136 link_bytes=314843136 token_bytes=560823088 weight_movement_bytes=34603008 total_nvlink_bytes=595426096 weight_to_token_ratio=0.061700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184729374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275718144 dispatch_meta_bytes=336144 combine_payload_bytes=275718144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276054288 ingress_bytes=344924160 link_bytes=344924160 token_bytes=551772432 weight_movement_bytes=69206016 total_nvlink_bytes=620978448 weight_to_token_ratio=0.125425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297921845e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295100416 dispatch_meta_bytes=349416 combine_payload_bytes=295100416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295449832 ingress_bytes=329703424 link_bytes=329703424 token_bytes=590550248 weight_movement_bytes=34603008 total_nvlink_bytes=625153256 weight_to_token_ratio=0.058595 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240647441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=301662208 dispatch_meta_bytes=367184 combine_payload_bytes=301662208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302029392 ingress_bytes=336265216 link_bytes=336265216 token_bytes=603691600 weight_movement_bytes=34603008 total_nvlink_bytes=638294608 weight_to_token_ratio=0.057319 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68034 payload_rows_remote=52442 dup_rows=30270 dup_rows_remote=9874 dedup_ratio=0.307922 router_tokens_per_expert=[3113, 2868, 3891, 5917, 1376, 11, 0, 3732, 948, 863, 115, 713, 0, 0, 323, 14, 9, 323, 0, 71, 12, 9, 0, 543, 211, 3, 139, 29, 0, 0, 2357, 1056, 46, 0, 0, 139, 522, 213, 771, 0, 4205, 321, 5, 155, 1824, 24, 0, 1817, 632, 5978, 18, 2, 14, 424, 0, 0, 1, 17, 28, 796, 1908, 612, 0, 34] payload_rows_per_dst=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] dup_rows_per_dst=[20396, 788, 184, 3986, 742, 1162, 1134, 1878] combine_rows_per_src=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81090 payload_rows_remote=67362 dup_rows=17214 dup_rows_remote=12686 dedup_ratio=0.175110 router_tokens_per_expert=[2385, 751, 4503, 5997, 597, 30, 15, 5652, 2153, 1504, 447, 27, 0, 0, 709, 14, 46, 998, 0, 455, 8, 51, 0, 1573, 552, 0, 64, 100, 0, 0, 1317, 941, 215, 30, 0, 14, 1037, 225, 555, 0, 4844, 179, 14, 587, 1497, 2, 0, 299, 588, 3701, 0, 2, 40, 518, 0, 0, 23, 33, 1, 807, 2188, 608, 0, 256] payload_rows_per_dst=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] dup_rows_per_dst=[372, 2954, 4528, 3284, 1676, 494, 1244, 2662] combine_rows_per_src=[7842, 10332, 13728, 12352, 13780, 4662, 8454, 9940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281542656 dispatch_meta_bytes=325472 combine_payload_bytes=281542656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281868128 ingress_bytes=316145664 link_bytes=316145664 token_bytes=563410784 weight_movement_bytes=34603008 total_nvlink_bytes=598013792 weight_to_token_ratio=0.061417 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.189630682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76946 payload_rows_remote=65086 dup_rows=21358 dup_rows_remote=17330 dedup_ratio=0.217265 router_tokens_per_expert=[1, 1, 11, 978, 2065, 825, 0, 77, 27, 817, 0, 6959, 0, 0, 4, 7514, 130, 2649, 18, 384, 41, 4, 1212, 0, 475, 409, 35, 1464, 419, 1, 233, 863, 396, 561, 8, 269, 9, 23, 520, 2, 477, 6, 651, 650, 41, 914, 3915, 78, 793, 696, 38, 737, 608, 1044, 2751, 208, 232, 2021, 934, 62, 254, 281, 0, 2357] payload_rows_per_dst=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] dup_rows_per_dst=[826, 2, 280, 7084, 3108, 4028, 2750, 3280] combine_rows_per_src=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214802432 dispatch_meta_bytes=249264 combine_payload_bytes=214802432 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=215051696 ingress_bytes=422420480 link_bytes=422420480 token_bytes=429854128 weight_movement_bytes=207618048 total_nvlink_bytes=637472176 weight_to_token_ratio=0.482997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589534259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78438 payload_rows_remote=63614 dup_rows=19866 dup_rows_remote=16324 dedup_ratio=0.202087 router_tokens_per_expert=[3, 1, 28, 662, 1123, 1139, 0, 74, 86, 1709, 0, 7095, 0, 2, 13, 7446, 81, 3007, 10, 819, 102, 5, 2047, 0, 199, 165, 58, 699, 731, 8, 220, 926, 837, 619, 58, 167, 10, 205, 192, 0, 463, 11, 494, 768, 95, 1101, 3343, 87, 815, 472, 42, 785, 415, 976, 1875, 304, 220, 2011, 858, 39, 238, 336, 48, 2810] payload_rows_per_dst=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] dup_rows_per_dst=[460, 138, 2750, 4468, 3542, 2930, 1854, 3724] combine_rows_per_src=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=275914752 dispatch_meta_bytes=320192 combine_payload_bytes=275914752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276234944 ingress_bytes=310517760 link_bytes=310517760 token_bytes=552149696 weight_movement_bytes=34603008 total_nvlink_bytes=586752704 weight_to_token_ratio=0.062670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168453332e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79258 payload_rows_remote=72248 dup_rows=19046 dup_rows_remote=17966 dedup_ratio=0.193746 router_tokens_per_expert=[2867, 833, 3383, 5996, 1143, 36, 64, 5866, 2503, 1584, 391, 32, 0, 0, 559, 30, 89, 1134, 0, 416, 30, 37, 0, 1058, 981, 1, 69, 163, 0, 0, 1313, 804, 271, 14, 0, 16, 825, 191, 472, 2, 4987, 124, 30, 1318, 1348, 1, 0, 284, 607, 3000, 2, 1, 82, 353, 0, 0, 32, 22, 0, 867, 2107, 613, 0, 201] payload_rows_per_dst=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] dup_rows_per_dst=[398, 4870, 3810, 788, 1714, 3532, 1080, 2854] combine_rows_per_src=[3754, 12094, 13710, 5874, 13600, 12652, 7010, 10564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76946 payload_rows_remote=65086 dup_rows=21358 dup_rows_remote=17330 dedup_ratio=0.217265 router_tokens_per_expert=[1, 1, 11, 978, 2065, 825, 0, 77, 27, 817, 0, 6959, 0, 0, 4, 7514, 130, 2649, 18, 384, 41, 4, 1212, 0, 475, 409, 35, 1464, 419, 1, 233, 863, 396, 561, 8, 269, 9, 23, 520, 2, 477, 6, 651, 650, 41, 914, 3915, 78, 793, 696, 38, 737, 608, 1044, 2751, 208, 232, 2021, 934, 62, 254, 281, 0, 2357] payload_rows_per_dst=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] dup_rows_per_dst=[826, 2, 280, 7084, 3108, 4028, 2750, 3280] combine_rows_per_src=[7090, 1694, 6172, 15742, 14386, 11860, 11000, 9002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=260562944 dispatch_meta_bytes=319752 combine_payload_bytes=260562944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260882696 ingress_bytes=295165952 link_bytes=295165952 token_bytes=521445640 weight_movement_bytes=34603008 total_nvlink_bytes=556048648 weight_to_token_ratio=0.066360 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110685715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83570 payload_rows_remote=72964 dup_rows=14734 dup_rows_remote=10550 dedup_ratio=0.149882 router_tokens_per_expert=[0, 0, 3416, 39, 362, 419, 0, 511, 16, 3541, 1, 1623, 8, 0, 0, 8169, 0, 570, 24, 4695, 0, 1, 6030, 0, 15, 18, 0, 922, 15, 0, 14, 26, 4885, 428, 0, 13, 9, 371, 10, 1, 70, 0, 41, 9, 253, 2673, 302, 180, 479, 290, 13, 181, 66, 1936, 151, 17, 10, 3517, 96, 4, 198, 17, 0, 2497] payload_rows_per_dst=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] dup_rows_per_dst=[654, 0, 494, 2014, 2230, 4726, 432, 4184] combine_rows_per_src=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71562 payload_rows_remote=63282 dup_rows=26742 dup_rows_remote=26288 dedup_ratio=0.272034 router_tokens_per_expert=[0, 0, 5327, 4, 300, 149, 6, 526, 0, 3907, 0, 459, 0, 0, 1, 7941, 0, 34, 1, 6536, 1, 0, 6017, 0, 2, 1, 0, 258, 12, 0, 1, 2, 5741, 60, 0, 0, 0, 949, 0, 0, 2, 0, 0, 9, 179, 2662, 67, 181, 191, 106, 14, 125, 1, 2772, 3, 0, 2, 3116, 0, 0, 121, 0, 0, 1366] payload_rows_per_dst=[16288, 8280, 15112, 552, 11892, 5950, 10306, 3182] dup_rows_per_dst=[12218, 454, 10066, 0, 1608, 250, 1742, 404] combine_rows_per_src=[16288, 8280, 15112, 552, 11892, 5950, 10306, 3182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70320 payload_rows_remote=65756 dup_rows=27984 dup_rows_remote=27612 dedup_ratio=0.284668 router_tokens_per_expert=[0, 1, 330, 618, 700, 981, 0, 76, 35, 2041, 1, 6437, 0, 0, 4, 7745, 163, 2379, 5, 1528, 34, 6, 3752, 0, 107, 95, 53, 1068, 350, 4, 194, 597, 1539, 677, 6, 147, 8, 233, 112, 0, 178, 3, 332, 431, 165, 1037, 1785, 89, 784, 560, 45, 617, 372, 1246, 1289, 150, 98, 3084, 762, 16, 226, 163, 0, 3694] payload_rows_per_dst=[5120, 16294, 11116, 4564, 6278, 6978, 8668, 11302] dup_rows_per_dst=[292, 14032, 4618, 372, 1366, 1062, 1458, 4784] combine_rows_per_src=[5120, 16294, 11116, 4564, 6278, 6978, 8668, 11302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68034 payload_rows_remote=52442 dup_rows=30270 dup_rows_remote=9874 dedup_ratio=0.307922 router_tokens_per_expert=[3113, 2868, 3891, 5917, 1376, 11, 0, 3732, 948, 863, 115, 713, 0, 0, 323, 14, 9, 323, 0, 71, 12, 9, 0, 543, 211, 3, 139, 29, 0, 0, 2357, 1056, 46, 0, 0, 139, 522, 213, 771, 0, 4205, 321, 5, 155, 1824, 24, 0, 1817, 632, 5978, 18, 2, 14, 424, 0, 0, 1, 17, 28, 796, 1908, 612, 0, 34] payload_rows_per_dst=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] dup_rows_per_dst=[20396, 788, 184, 3986, 742, 1162, 1134, 1878] combine_rows_per_src=[15592, 5164, 1750, 12014, 10104, 7130, 7484, 8796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83570 payload_rows_remote=72964 dup_rows=14734 dup_rows_remote=10550 dedup_ratio=0.149882 router_tokens_per_expert=[0, 0, 3416, 39, 362, 419, 0, 511, 16, 3541, 1, 1623, 8, 0, 0, 8169, 0, 570, 24, 4695, 0, 1, 6030, 0, 15, 18, 0, 922, 15, 0, 14, 26, 4885, 428, 0, 13, 9, 371, 10, 1, 70, 0, 41, 9, 253, 2673, 302, 180, 479, 290, 13, 181, 66, 1936, 151, 17, 10, 3517, 96, 4, 198, 17, 0, 2497] payload_rows_per_dst=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] dup_rows_per_dst=[654, 0, 494, 2014, 2230, 4726, 432, 4184] combine_rows_per_src=[8840, 7132, 10086, 16344, 12450, 12278, 5834, 10606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=295927808 dispatch_meta_bytes=360856 combine_payload_bytes=295927808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296288664 ingress_bytes=365133824 link_bytes=365133824 token_bytes=592216472 weight_movement_bytes=69206016 total_nvlink_bytes=661422488 weight_to_token_ratio=0.116859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373969184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=329664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266921920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533514176 weight_movement_bytes=34603008 total_nvlink_bytes=568117184 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78438 payload_rows_remote=63614 dup_rows=19866 dup_rows_remote=16324 dedup_ratio=0.202087 router_tokens_per_expert=[3, 1, 28, 662, 1123, 1139, 0, 74, 86, 1709, 0, 7095, 0, 2, 13, 7446, 81, 3007, 10, 819, 102, 5, 2047, 0, 199, 165, 58, 699, 731, 8, 220, 926, 837, 619, 58, 167, 10, 205, 192, 0, 463, 11, 494, 768, 95, 1101, 3343, 87, 815, 472, 42, 785, 415, 976, 1875, 304, 220, 2011, 858, 39, 238, 336, 48, 2810] payload_rows_per_dst=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] dup_rows_per_dst=[460, 138, 2750, 4468, 3542, 2930, 1854, 3724] combine_rows_per_src=[5600, 4676, 9296, 15242, 14824, 9890, 9514, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76842 payload_rows_remote=66444 dup_rows=21462 dup_rows_remote=18736 dedup_ratio=0.218323 router_tokens_per_expert=[4, 1, 4, 1235, 1213, 726, 0, 75, 35, 1354, 1, 6976, 0, 0, 7, 7378, 84, 2729, 19, 343, 20, 7, 1402, 0, 390, 225, 152, 941, 503, 23, 173, 1265, 416, 752, 16, 311, 10, 85, 320, 2, 564, 3, 762, 1027, 22, 561, 3856, 76, 595, 390, 24, 1173, 905, 621, 2600, 254, 220, 2003, 649, 15, 264, 320, 3, 3048] payload_rows_per_dst=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] dup_rows_per_dst=[566, 16, 232, 6276, 3354, 4550, 2726, 3742] combine_rows_per_src=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71562 payload_rows_remote=63282 dup_rows=26742 dup_rows_remote=26288 dedup_ratio=0.272034 router_tokens_per_expert=[0, 0, 5327, 4, 300, 149, 6, 526, 0, 3907, 0, 459, 0, 0, 1, 7941, 0, 34, 1, 6536, 1, 0, 6017, 0, 2, 1, 0, 258, 12, 0, 1, 2, 5741, 60, 0, 0, 0, 949, 0, 0, 2, 0, 0, 9, 179, 2662, 67, 181, 191, 106, 14, 125, 1, 2772, 3, 0, 2, 3116, 0, 0, 121, 0, 0, 1366] payload_rows_per_dst=[16288, 8280, 15112, 552, 11892, 5950, 10306, 3182] dup_rows_per_dst=[12218, 454, 10066, 0, 1608, 250, 1742, 404] combine_rows_per_src=[16288, 8280, 15112, 552, 11892, 5950, 10306, 3182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=373472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269710048 ingress_bytes=303939584 link_bytes=303939584 token_bytes=539046624 weight_movement_bytes=34603008 total_nvlink_bytes=573649632 weight_to_token_ratio=0.064193 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143700185e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=214802432 dispatch_meta_bytes=249264 combine_payload_bytes=214802432 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=215051696 ingress_bytes=422420480 link_bytes=422420480 token_bytes=429854128 weight_movement_bytes=207618048 total_nvlink_bytes=637472176 weight_to_token_ratio=0.482997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589534259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=298860544 dispatch_meta_bytes=334056 combine_payload_bytes=298860544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299194600 ingress_bytes=368066560 link_bytes=368066560 token_bytes=598055144 weight_movement_bytes=69206016 total_nvlink_bytes=667261160 weight_to_token_ratio=0.115718 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385004833e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=260562944 dispatch_meta_bytes=319752 combine_payload_bytes=260562944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260882696 ingress_bytes=295165952 link_bytes=295165952 token_bytes=521445640 weight_movement_bytes=34603008 total_nvlink_bytes=556048648 weight_to_token_ratio=0.066360 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110685715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=60148 dup_rows=21984 dup_rows_remote=16510 dedup_ratio=0.223633 router_tokens_per_expert=[0, 4, 585, 281, 733, 1107, 0, 75, 7, 1559, 0, 5547, 0, 0, 2, 8038, 15, 2877, 16, 1652, 39, 2, 2953, 0, 21, 164, 1, 799, 508, 7, 236, 103, 2036, 617, 0, 68, 3, 27, 62, 0, 19, 0, 177, 75, 192, 1021, 2610, 109, 1519, 987, 50, 606, 123, 1906, 835, 92, 50, 3694, 613, 3, 273, 129, 0, 3925] payload_rows_per_dst=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] dup_rows_per_dst=[5474, 4788, 3170, 372, 150, 780, 6434, 816] combine_rows_per_src=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73448 payload_rows_remote=64908 dup_rows=24856 dup_rows_remote=23266 dedup_ratio=0.252848 router_tokens_per_expert=[0, 5, 5, 1183, 837, 1057, 0, 72, 68, 1220, 0, 6669, 0, 0, 18, 7868, 16, 3448, 36, 466, 73, 18, 1007, 1, 181, 285, 15, 761, 846, 4, 297, 683, 587, 464, 37, 203, 3, 0, 315, 0, 201, 3, 557, 572, 54, 564, 4660, 89, 722, 670, 78, 701, 118, 1045, 2506, 159, 280, 3168, 652, 239, 300, 258, 6, 2802] payload_rows_per_dst=[9576, 15314, 8540, 5418, 2976, 11066, 9862, 10696] dup_rows_per_dst=[3178, 9936, 1590, 726, 242, 2334, 2136, 4714] combine_rows_per_src=[9576, 15314, 8540, 5418, 2976, 11066, 9862, 10696] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=259203072 dispatch_meta_bytes=358280 combine_payload_bytes=259203072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259561352 ingress_bytes=397615104 link_bytes=397615104 token_bytes=518764424 weight_movement_bytes=138412032 total_nvlink_bytes=657176456 weight_to_token_ratio=0.266811 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.496193626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272154624 dispatch_meta_bytes=340720 combine_payload_bytes=272154624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272495344 ingress_bytes=306757632 link_bytes=306757632 token_bytes=544649968 weight_movement_bytes=34603008 total_nvlink_bytes=579252976 weight_to_token_ratio=0.063533 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154304273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82084 payload_rows_remote=70998 dup_rows=16220 dup_rows_remote=13924 dedup_ratio=0.164998 router_tokens_per_expert=[1858, 101, 46, 123, 2345, 204, 0, 0, 0, 232, 264, 148, 3933, 0, 963, 369, 0, 1112, 0, 4723, 0, 600, 183, 0, 1204, 0, 509, 2528, 0, 1414, 1135, 844, 3441, 231, 445, 117, 158, 419, 37, 885, 0, 341, 335, 69, 966, 284, 1252, 3, 3747, 0, 451, 705, 211, 1143, 1093, 640, 1083, 5990, 125, 0, 27, 0, 7, 109] payload_rows_per_dst=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] dup_rows_per_dst=[2502, 1302, 1782, 2036, 108, 2296, 3766, 2428] combine_rows_per_src=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=269336576 dispatch_meta_bytes=373472 combine_payload_bytes=269336576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269710048 ingress_bytes=303939584 link_bytes=303939584 token_bytes=539046624 weight_movement_bytes=34603008 total_nvlink_bytes=573649632 weight_to_token_ratio=0.064193 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143700185e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=77388 dup_rows=16082 dup_rows_remote=15716 dedup_ratio=0.163595 router_tokens_per_expert=[1286, 206, 61, 363, 2158, 406, 0, 0, 0, 206, 391, 89, 4481, 7, 873, 355, 0, 727, 85, 4615, 0, 737, 248, 7, 1646, 0, 602, 2621, 1, 2443, 468, 687, 4382, 463, 701, 384, 451, 473, 40, 789, 0, 367, 557, 257, 651, 260, 1266, 66, 3041, 0, 306, 541, 550, 732, 1581, 602, 731, 3763, 185, 0, 12, 0, 122, 110] payload_rows_per_dst=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] dup_rows_per_dst=[2718, 1252, 1850, 2184, 366, 3072, 3096, 1544] combine_rows_per_src=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=246366208 dispatch_meta_bytes=306632 combine_payload_bytes=246366208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246672840 ingress_bytes=280969216 link_bytes=280969216 token_bytes=493039048 weight_movement_bytes=34603008 total_nvlink_bytes=527642056 weight_to_token_ratio=0.070183 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057264540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=352696 combine_payload_bytes=265863168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266215864 ingress_bytes=369672192 link_bytes=369672192 token_bytes=532079032 weight_movement_bytes=103809024 total_nvlink_bytes=635888056 weight_to_token_ratio=0.195101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391046697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76842 payload_rows_remote=66444 dup_rows=21462 dup_rows_remote=18736 dedup_ratio=0.218323 router_tokens_per_expert=[4, 1, 4, 1235, 1213, 726, 0, 75, 35, 1354, 1, 6976, 0, 0, 7, 7378, 84, 2729, 19, 343, 20, 7, 1402, 0, 390, 225, 152, 941, 503, 23, 173, 1265, 416, 752, 16, 311, 10, 85, 320, 2, 564, 3, 762, 1027, 22, 561, 3856, 76, 595, 390, 24, 1173, 905, 621, 2600, 254, 220, 2003, 649, 15, 264, 320, 3, 3048] payload_rows_per_dst=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] dup_rows_per_dst=[566, 16, 232, 6276, 3354, 4550, 2726, 3742] combine_rows_per_src=[5950, 2778, 6172, 15824, 14422, 11996, 10398, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316981248 dispatch_meta_bytes=372416 combine_payload_bytes=316981248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317353664 ingress_bytes=420790272 link_bytes=420790272 token_bytes=634334912 weight_movement_bytes=103809024 total_nvlink_bytes=738143936 weight_to_token_ratio=0.163650 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583399917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82084 payload_rows_remote=70998 dup_rows=16220 dup_rows_remote=13924 dedup_ratio=0.164998 router_tokens_per_expert=[1858, 101, 46, 123, 2345, 204, 0, 0, 0, 232, 264, 148, 3933, 0, 963, 369, 0, 1112, 0, 4723, 0, 600, 183, 0, 1204, 0, 509, 2528, 0, 1414, 1135, 844, 3441, 231, 445, 117, 158, 419, 37, 885, 0, 341, 335, 69, 966, 284, 1252, 3, 3747, 0, 451, 705, 211, 1143, 1093, 640, 1083, 5990, 125, 0, 27, 0, 7, 109] payload_rows_per_dst=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] dup_rows_per_dst=[2502, 1302, 1782, 2036, 108, 2296, 3766, 2428] combine_rows_per_src=[9260, 10516, 11454, 10824, 3586, 11086, 12214, 13144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73312 payload_rows_remote=62186 dup_rows=24992 dup_rows_remote=21660 dedup_ratio=0.254232 router_tokens_per_expert=[207, 462, 110, 1188, 349, 880, 2, 1, 1, 24, 696, 1, 6974, 0, 43, 23, 0, 16, 0, 671, 0, 86, 166, 752, 5857, 0, 1, 1777, 3, 4333, 16, 109, 8048, 1122, 4322, 14, 3719, 63, 139, 28, 0, 167, 231, 1, 49, 2, 68, 143, 1979, 1, 438, 53, 567, 39, 48, 256, 70, 224, 1279, 1253, 4, 1, 3, 73] payload_rows_per_dst=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] dup_rows_per_dst=[852, 416, 202, 17230, 1108, 1152, 700, 3332] combine_rows_per_src=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73528 payload_rows_remote=59778 dup_rows=24776 dup_rows_remote=24362 dedup_ratio=0.252035 router_tokens_per_expert=[5, 385, 98, 1442, 34, 787, 0, 0, 23, 1, 428, 0, 6628, 0, 1, 1, 0, 0, 0, 77, 1, 4, 48, 1301, 6434, 0, 3, 687, 10, 3697, 4, 1, 8180, 2133, 4787, 0, 4593, 0, 2, 2, 0, 417, 41, 1, 1, 2, 3, 173, 1114, 0, 802, 7, 450, 8, 9, 336, 2, 24, 1114, 2728, 0, 56, 1, 66] payload_rows_per_dst=[14534, 13750, 9132, 1406, 14760, 6704, 3694, 9548] dup_rows_per_dst=[3836, 414, 1124, 4, 16790, 500, 650, 1458] combine_rows_per_src=[14534, 13750, 9132, 1406, 14760, 6704, 3694, 9548] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272154624 dispatch_meta_bytes=340720 combine_payload_bytes=272154624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272495344 ingress_bytes=306757632 link_bytes=306757632 token_bytes=544649968 weight_movement_bytes=34603008 total_nvlink_bytes=579252976 weight_to_token_ratio=0.063533 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154304273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83712 payload_rows_remote=74414 dup_rows=14592 dup_rows_remote=13496 dedup_ratio=0.148438 router_tokens_per_expert=[1277, 181, 125, 385, 1236, 495, 0, 0, 0, 242, 394, 200, 5637, 5, 695, 145, 0, 236, 29, 4073, 0, 465, 115, 31, 3136, 0, 354, 2793, 4, 2397, 438, 716, 5904, 559, 1001, 110, 941, 312, 83, 834, 0, 360, 364, 72, 495, 160, 602, 39, 3487, 0, 395, 427, 586, 718, 1167, 626, 427, 2904, 623, 2, 4, 0, 0, 146] payload_rows_per_dst=[10788, 13346, 10270, 9298, 5346, 13854, 11682, 9128] dup_rows_per_dst=[2882, 1290, 2638, 1096, 332, 2138, 3130, 1086] combine_rows_per_src=[10788, 13346, 10270, 9298, 5346, 13854, 11682, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76320 payload_rows_remote=60148 dup_rows=21984 dup_rows_remote=16510 dedup_ratio=0.223633 router_tokens_per_expert=[0, 4, 585, 281, 733, 1107, 0, 75, 7, 1559, 0, 5547, 0, 0, 2, 8038, 15, 2877, 16, 1652, 39, 2, 2953, 0, 21, 164, 1, 799, 508, 7, 236, 103, 2036, 617, 0, 68, 3, 27, 62, 0, 19, 0, 177, 75, 192, 1021, 2610, 109, 1519, 987, 50, 606, 123, 1906, 835, 92, 50, 3694, 613, 3, 273, 129, 0, 3925] payload_rows_per_dst=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] dup_rows_per_dst=[5474, 4788, 3170, 372, 150, 780, 6434, 816] combine_rows_per_src=[16172, 12770, 8610, 3306, 5476, 7626, 13190, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=290807808 dispatch_meta_bytes=339688 combine_payload_bytes=290807808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291147496 ingress_bytes=325410816 link_bytes=325410816 token_bytes=581955304 weight_movement_bytes=34603008 total_nvlink_bytes=616558312 weight_to_token_ratio=0.059460 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224494703e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73312 payload_rows_remote=62186 dup_rows=24992 dup_rows_remote=21660 dedup_ratio=0.254232 router_tokens_per_expert=[207, 462, 110, 1188, 349, 880, 2, 1, 1, 24, 696, 1, 6974, 0, 43, 23, 0, 16, 0, 671, 0, 86, 166, 752, 5857, 0, 1, 1777, 3, 4333, 16, 109, 8048, 1122, 4322, 14, 3719, 63, 139, 28, 0, 167, 231, 1, 49, 2, 68, 143, 1979, 1, 438, 53, 567, 39, 48, 256, 70, 224, 1279, 1253, 4, 1, 3, 73] payload_rows_per_dst=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] dup_rows_per_dst=[852, 416, 202, 17230, 1108, 1152, 700, 3332] combine_rows_per_src=[5638, 5674, 3180, 15610, 9062, 16266, 6756, 11126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71796 payload_rows_remote=61144 dup_rows=26508 dup_rows_remote=22526 dedup_ratio=0.269653 router_tokens_per_expert=[973, 488, 109, 131, 2490, 725, 0, 0, 0, 89, 459, 28, 5931, 0, 309, 131, 0, 186, 2, 3306, 0, 713, 89, 37, 2401, 0, 128, 2556, 4, 3060, 238, 763, 7207, 1008, 2340, 83, 1533, 554, 44, 347, 0, 685, 350, 24, 258, 93, 369, 91, 3048, 0, 601, 140, 403, 369, 490, 728, 237, 1867, 712, 3, 2, 0, 105, 115] payload_rows_per_dst=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] dup_rows_per_dst=[3982, 576, 3542, 622, 16762, 298, 448, 278] combine_rows_per_src=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82222 payload_rows_remote=77388 dup_rows=16082 dup_rows_remote=15716 dedup_ratio=0.163595 router_tokens_per_expert=[1286, 206, 61, 363, 2158, 406, 0, 0, 0, 206, 391, 89, 4481, 7, 873, 355, 0, 727, 85, 4615, 0, 737, 248, 7, 1646, 0, 602, 2621, 1, 2443, 468, 687, 4382, 463, 701, 384, 451, 473, 40, 789, 0, 367, 557, 257, 651, 260, 1266, 66, 3041, 0, 306, 541, 550, 732, 1581, 602, 731, 3763, 185, 0, 12, 0, 122, 110] payload_rows_per_dst=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] dup_rows_per_dst=[2718, 1252, 1850, 2184, 366, 3072, 3096, 1544] combine_rows_per_src=[9534, 11552, 10988, 11460, 4834, 12540, 11610, 9704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73528 payload_rows_remote=59778 dup_rows=24776 dup_rows_remote=24362 dedup_ratio=0.252035 router_tokens_per_expert=[5, 385, 98, 1442, 34, 787, 0, 0, 23, 1, 428, 0, 6628, 0, 1, 1, 0, 0, 0, 77, 1, 4, 48, 1301, 6434, 0, 3, 687, 10, 3697, 4, 1, 8180, 2133, 4787, 0, 4593, 0, 2, 2, 0, 417, 41, 1, 1, 2, 3, 173, 1114, 0, 802, 7, 450, 8, 9, 336, 2, 24, 1114, 2728, 0, 56, 1, 66] payload_rows_per_dst=[14534, 13750, 9132, 1406, 14760, 6704, 3694, 9548] dup_rows_per_dst=[3836, 414, 1124, 4, 16790, 500, 650, 1458] combine_rows_per_src=[14534, 13750, 9132, 1406, 14760, 6704, 3694, 9548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304799744 dispatch_meta_bytes=351640 combine_payload_bytes=304799744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305151384 ingress_bytes=408608768 link_bytes=408608768 token_bytes=609951128 weight_movement_bytes=103809024 total_nvlink_bytes=713760152 weight_to_token_ratio=0.170192 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.537561898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=246366208 dispatch_meta_bytes=306632 combine_payload_bytes=246366208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246672840 ingress_bytes=280969216 link_bytes=280969216 token_bytes=493039048 weight_movement_bytes=34603008 total_nvlink_bytes=527642056 weight_to_token_ratio=0.070183 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057264540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81556 payload_rows_remote=68170 dup_rows=16748 dup_rows_remote=10822 dedup_ratio=0.170369 router_tokens_per_expert=[2018, 14, 18, 183, 1808, 294, 1, 1, 0, 254, 278, 100, 3591, 9, 1552, 441, 0, 751, 16, 4291, 0, 686, 121, 0, 1068, 0, 958, 2165, 2, 1564, 752, 809, 2958, 128, 395, 332, 104, 491, 24, 1170, 0, 312, 340, 206, 1281, 248, 944, 0, 4185, 0, 221, 799, 571, 1535, 1666, 679, 816, 5846, 66, 0, 4, 0, 2, 84] payload_rows_per_dst=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] dup_rows_per_dst=[1996, 1546, 1376, 1894, 226, 2046, 5926, 1738] combine_rows_per_src=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=254713856 dispatch_meta_bytes=335384 combine_payload_bytes=254713856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255049240 ingress_bytes=289316864 link_bytes=289316864 token_bytes=509763096 weight_movement_bytes=34603008 total_nvlink_bytes=544366104 weight_to_token_ratio=0.067881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.088676067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316981248 dispatch_meta_bytes=372416 combine_payload_bytes=316981248 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=317353664 ingress_bytes=420790272 link_bytes=420790272 token_bytes=634334912 weight_movement_bytes=103809024 total_nvlink_bytes=738143936 weight_to_token_ratio=0.163650 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.583399917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79822 payload_rows_remote=66230 dup_rows=18482 dup_rows_remote=12906 dedup_ratio=0.188009 router_tokens_per_expert=[1705, 167, 28, 103, 2507, 297, 0, 0, 0, 256, 361, 71, 4313, 0, 845, 378, 0, 1039, 0, 5499, 0, 838, 168, 0, 351, 0, 269, 1883, 9, 2040, 438, 1040, 3332, 439, 665, 266, 152, 1113, 6, 617, 0, 404, 492, 174, 569, 215, 994, 9, 3621, 1, 328, 733, 219, 1017, 1437, 760, 1014, 5723, 119, 0, 18, 2, 2, 106] payload_rows_per_dst=[8426, 10952, 13592, 6762, 4832, 10192, 12056, 13010] dup_rows_per_dst=[1890, 1496, 5576, 516, 354, 2186, 4176, 2288] combine_rows_per_src=[8426, 10952, 13592, 6762, 4832, 10192, 12056, 13010] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=244850688 dispatch_meta_bytes=336560 combine_payload_bytes=244850688 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=245187248 ingress_bytes=314056704 link_bytes=314056704 token_bytes=490037936 weight_movement_bytes=69206016 total_nvlink_bytes=559243952 weight_to_token_ratio=0.141226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181770094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=279224320 dispatch_meta_bytes=315968 combine_payload_bytes=279224320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279540288 ingress_bytes=348430336 link_bytes=348430336 token_bytes=558764608 weight_movement_bytes=69206016 total_nvlink_bytes=627970624 weight_to_token_ratio=0.123855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311115303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=74714 dup_rows=17760 dup_rows_remote=17748 dedup_ratio=0.180664 router_tokens_per_expert=[0, 9, 651, 235, 1, 6, 0, 0, 1100, 21, 2425, 1, 1764, 1, 442, 5, 1518, 750, 1236, 41, 16, 550, 5, 2627, 0, 591, 273, 1286, 6721, 0, 0, 968, 4114, 0, 0, 0, 2198, 350, 872, 0, 2, 0, 7709, 2574, 0, 344, 0, 1, 11, 485, 3530, 0, 254, 9, 0, 9, 20, 51, 2407, 0, 742, 138, 1, 88] payload_rows_per_dst=[13668, 9330, 7368, 5578, 12126, 5830, 10714, 15930] dup_rows_per_dst=[1578, 2188, 864, 658, 2942, 12, 3136, 6382] combine_rows_per_src=[13668, 9330, 7368, 5578, 12126, 5830, 10714, 15930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304799744 dispatch_meta_bytes=351640 combine_payload_bytes=304799744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305151384 ingress_bytes=408608768 link_bytes=408608768 token_bytes=609951128 weight_movement_bytes=103809024 total_nvlink_bytes=713760152 weight_to_token_ratio=0.170192 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.537561898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79980 payload_rows_remote=67928 dup_rows=18324 dup_rows_remote=15258 dedup_ratio=0.186401 router_tokens_per_expert=[0, 6, 50, 70, 8, 13, 16, 6, 743, 43, 1443, 0, 1622, 55, 948, 9, 1740, 952, 811, 119, 56, 865, 4, 3549, 0, 507, 140, 909, 6177, 19, 0, 1498, 2945, 6, 3, 0, 2672, 351, 1806, 3, 1, 19, 7129, 1867, 0, 969, 0, 10, 2, 469, 3683, 0, 377, 14, 4, 20, 20, 219, 2969, 2, 1045, 52, 0, 117] payload_rows_per_dst=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] dup_rows_per_dst=[244, 1330, 868, 782, 3066, 82, 4304, 7648] combine_rows_per_src=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250445824 dispatch_meta_bytes=334680 combine_payload_bytes=250445824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250780504 ingress_bytes=285048832 link_bytes=285048832 token_bytes=501226328 weight_movement_bytes=34603008 total_nvlink_bytes=535829336 weight_to_token_ratio=0.069037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072615807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=271278080 dispatch_meta_bytes=316544 combine_payload_bytes=271278080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271594624 ingress_bytes=305881088 link_bytes=305881088 token_bytes=542872704 weight_movement_bytes=34603008 total_nvlink_bytes=577475712 weight_to_token_ratio=0.063741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151005909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=306028544 dispatch_meta_bytes=369848 combine_payload_bytes=306028544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306398392 ingress_bytes=375234560 link_bytes=375234560 token_bytes=612426936 weight_movement_bytes=69206016 total_nvlink_bytes=681632952 weight_to_token_ratio=0.113003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411977440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84232 payload_rows_remote=72916 dup_rows=14072 dup_rows_remote=11964 dedup_ratio=0.143148 router_tokens_per_expert=[4, 27, 82, 59, 8, 25, 1, 0, 652, 4, 1626, 0, 1271, 83, 1221, 3, 1743, 1115, 2034, 65, 80, 477, 12, 4403, 0, 316, 11, 1026, 6737, 201, 0, 1741, 2785, 0, 0, 0, 3417, 239, 2307, 0, 2, 0, 5299, 880, 0, 2235, 15, 0, 0, 493, 3085, 0, 480, 3, 10, 66, 27, 22, 1832, 0, 733, 138, 0, 57] payload_rows_per_dst=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] dup_rows_per_dst=[372, 1308, 2224, 2108, 1018, 1690, 3752, 1600] combine_rows_per_src=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71796 payload_rows_remote=61144 dup_rows=26508 dup_rows_remote=22526 dedup_ratio=0.269653 router_tokens_per_expert=[973, 488, 109, 131, 2490, 725, 0, 0, 0, 89, 459, 28, 5931, 0, 309, 131, 0, 186, 2, 3306, 0, 713, 89, 37, 2401, 0, 128, 2556, 4, 3060, 238, 763, 7207, 1008, 2340, 83, 1533, 554, 44, 347, 0, 685, 350, 24, 258, 93, 369, 91, 3048, 0, 601, 140, 403, 369, 490, 728, 237, 1867, 712, 3, 2, 0, 105, 115] payload_rows_per_dst=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] dup_rows_per_dst=[3982, 576, 3542, 622, 16762, 298, 448, 278] combine_rows_per_src=[10652, 13318, 11244, 6756, 15566, 3442, 5014, 5804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=332744 combine_payload_bytes=278233088 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278565832 ingress_bytes=382042112 link_bytes=382042112 token_bytes=556798920 weight_movement_bytes=103809024 total_nvlink_bytes=660607944 weight_to_token_ratio=0.186439 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437593711e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81556 payload_rows_remote=68170 dup_rows=16748 dup_rows_remote=10822 dedup_ratio=0.170369 router_tokens_per_expert=[2018, 14, 18, 183, 1808, 294, 1, 1, 0, 254, 278, 100, 3591, 9, 1552, 441, 0, 751, 16, 4291, 0, 686, 121, 0, 1068, 0, 958, 2165, 2, 1564, 752, 809, 2958, 128, 395, 332, 104, 491, 24, 1170, 0, 312, 340, 206, 1281, 248, 944, 0, 4185, 0, 221, 799, 571, 1535, 1666, 679, 816, 5846, 66, 0, 4, 0, 2, 84] payload_rows_per_dst=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] dup_rows_per_dst=[1996, 1546, 1376, 1894, 226, 2046, 5926, 1738] combine_rows_per_src=[8814, 10904, 10354, 10606, 4272, 10532, 13386, 12688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80544 payload_rows_remote=74714 dup_rows=17760 dup_rows_remote=17748 dedup_ratio=0.180664 router_tokens_per_expert=[0, 9, 651, 235, 1, 6, 0, 0, 1100, 21, 2425, 1, 1764, 1, 442, 5, 1518, 750, 1236, 41, 16, 550, 5, 2627, 0, 591, 273, 1286, 6721, 0, 0, 968, 4114, 0, 0, 0, 2198, 350, 872, 0, 2, 0, 7709, 2574, 0, 344, 0, 1, 11, 485, 3530, 0, 254, 9, 0, 9, 20, 51, 2407, 0, 742, 138, 1, 88] payload_rows_per_dst=[13668, 9330, 7368, 5578, 12126, 5830, 10714, 15930] dup_rows_per_dst=[1578, 2188, 864, 658, 2942, 12, 3136, 6382] combine_rows_per_src=[13668, 9330, 7368, 5578, 12126, 5830, 10714, 15930] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84246 payload_rows_remote=81150 dup_rows=14058 dup_rows_remote=13808 dedup_ratio=0.143005 router_tokens_per_expert=[299, 98, 22, 45, 18, 148, 0, 0, 178, 2, 483, 0, 37, 158, 4648, 0, 702, 844, 3905, 435, 144, 1, 1, 7065, 0, 125, 20, 14, 7541, 2528, 0, 748, 195, 16, 1, 10, 3206, 8, 4202, 0, 5, 5, 382, 146, 0, 7060, 174, 0, 1, 77, 1175, 0, 679, 0, 151, 159, 655, 0, 603, 0, 9, 17, 0, 7] payload_rows_per_dst=[1252, 10298, 11060, 16086, 12182, 14412, 15860, 3096] dup_rows_per_dst=[8, 714, 1004, 5866, 3094, 368, 2754, 250] combine_rows_per_src=[1252, 10298, 11060, 16086, 12182, 14412, 15860, 3096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79980 payload_rows_remote=67928 dup_rows=18324 dup_rows_remote=15258 dedup_ratio=0.186401 router_tokens_per_expert=[0, 6, 50, 70, 8, 13, 16, 6, 743, 43, 1443, 0, 1622, 55, 948, 9, 1740, 952, 811, 119, 56, 865, 4, 3549, 0, 507, 140, 909, 6177, 19, 0, 1498, 2945, 6, 3, 0, 2672, 351, 1806, 3, 1, 19, 7129, 1867, 0, 969, 0, 10, 2, 469, 3683, 0, 377, 14, 4, 20, 20, 219, 2969, 2, 1045, 52, 0, 117] payload_rows_per_dst=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] dup_rows_per_dst=[244, 1330, 868, 782, 3066, 82, 4304, 7648] combine_rows_per_src=[12448, 8396, 8226, 5818, 12052, 5650, 11932, 15458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=279224320 dispatch_meta_bytes=315968 combine_payload_bytes=279224320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279540288 ingress_bytes=348430336 link_bytes=348430336 token_bytes=558764608 weight_movement_bytes=69206016 total_nvlink_bytes=627970624 weight_to_token_ratio=0.123855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311115303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83542 payload_rows_remote=71342 dup_rows=14762 dup_rows_remote=12108 dedup_ratio=0.150167 router_tokens_per_expert=[289, 78, 0, 0, 28, 299, 0, 0, 82, 0, 432, 0, 6, 272, 4891, 0, 602, 328, 3371, 47, 188, 4, 0, 7366, 0, 2, 1, 0, 7220, 5256, 0, 104, 10, 0, 0, 0, 3034, 0, 3387, 0, 0, 0, 28, 5, 0, 7888, 348, 0, 0, 25, 396, 0, 589, 0, 1020, 190, 1319, 0, 41, 0, 3, 1, 0, 2] payload_rows_per_dst=[14602, 12200, 13880, 13424, 6794, 15832, 4080, 2730] dup_rows_per_dst=[1226, 2654, 6444, 3370, 0, 706, 360, 2] combine_rows_per_src=[14602, 12200, 13880, 13424, 6794, 15832, 4080, 2730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=298663936 dispatch_meta_bytes=339520 combine_payload_bytes=298663936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299003456 ingress_bytes=367869952 link_bytes=367869952 token_bytes=597667392 weight_movement_bytes=69206016 total_nvlink_bytes=666873408 weight_to_token_ratio=0.115794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384265013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79022 payload_rows_remote=70310 dup_rows=19282 dup_rows_remote=17192 dedup_ratio=0.196147 router_tokens_per_expert=[0, 1, 341, 165, 4, 4, 7, 3, 694, 76, 2336, 0, 2807, 8, 378, 6, 1550, 669, 783, 61, 19, 1148, 4, 1848, 0, 471, 175, 1581, 6172, 3, 0, 1160, 4781, 0, 0, 0, 2313, 367, 887, 0, 1, 15, 7777, 2560, 0, 472, 0, 1, 11, 489, 2881, 0, 142, 12, 0, 18, 13, 54, 2050, 0, 1583, 170, 0, 81] payload_rows_per_dst=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] dup_rows_per_dst=[708, 2322, 818, 2210, 3644, 42, 2090, 7448] combine_rows_per_src=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250445824 dispatch_meta_bytes=334680 combine_payload_bytes=250445824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250780504 ingress_bytes=285048832 link_bytes=285048832 token_bytes=501226328 weight_movement_bytes=34603008 total_nvlink_bytes=535829336 weight_to_token_ratio=0.069037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072615807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84232 payload_rows_remote=72916 dup_rows=14072 dup_rows_remote=11964 dedup_ratio=0.143148 router_tokens_per_expert=[4, 27, 82, 59, 8, 25, 1, 0, 652, 4, 1626, 0, 1271, 83, 1221, 3, 1743, 1115, 2034, 65, 80, 477, 12, 4403, 0, 316, 11, 1026, 6737, 201, 0, 1741, 2785, 0, 0, 0, 3417, 239, 2307, 0, 2, 0, 5299, 880, 0, 2235, 15, 0, 0, 493, 3085, 0, 480, 3, 10, 66, 27, 22, 1832, 0, 733, 138, 0, 57] payload_rows_per_dst=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] dup_rows_per_dst=[372, 1308, 2224, 2108, 1018, 1690, 3752, 1600] combine_rows_per_src=[13514, 8412, 10294, 11316, 9644, 9694, 11862, 9496] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=306028544 dispatch_meta_bytes=369848 combine_payload_bytes=306028544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306398392 ingress_bytes=375234560 link_bytes=375234560 token_bytes=612426936 weight_movement_bytes=69206016 total_nvlink_bytes=681632952 weight_to_token_ratio=0.113003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411977440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84382 payload_rows_remote=68810 dup_rows=13922 dup_rows_remote=13108 dedup_ratio=0.141622 router_tokens_per_expert=[1, 97, 42, 105, 10, 170, 1, 0, 538, 22, 1323, 0, 199, 86, 2018, 2, 1133, 1119, 2537, 101, 203, 37, 8, 5486, 0, 270, 21, 317, 7767, 465, 4, 1723, 1199, 1, 0, 0, 2792, 90, 3085, 0, 1, 0, 3925, 517, 0, 3834, 13, 2, 0, 826, 3191, 0, 677, 59, 102, 101, 34, 5, 2608, 0, 215, 5, 0, 65] payload_rows_per_dst=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] dup_rows_per_dst=[814, 5558, 1726, 1512, 328, 2946, 974, 64] combine_rows_per_src=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=332390400 dispatch_meta_bytes=379832 combine_payload_bytes=332390400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332770232 ingress_bytes=366993408 link_bytes=366993408 token_bytes=665160632 weight_movement_bytes=34603008 total_nvlink_bytes=699763640 weight_to_token_ratio=0.052022 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380966649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82002 payload_rows_remote=70080 dup_rows=16302 dup_rows_remote=12334 dedup_ratio=0.165833 router_tokens_per_expert=[0, 12, 388, 286, 1, 32, 8, 0, 588, 57, 1991, 0, 1582, 0, 534, 6, 1849, 918, 1064, 161, 50, 519, 2, 3382, 0, 671, 316, 486, 7032, 1, 0, 1388, 3570, 1, 0, 1, 1852, 324, 1480, 13, 0, 1, 7789, 2613, 0, 565, 0, 5, 30, 564, 3468, 0, 259, 89, 2, 6, 11, 92, 2560, 0, 405, 36, 1, 91] payload_rows_per_dst=[14234, 8032, 11922, 8304, 9436, 15794, 8038, 6242] dup_rows_per_dst=[1284, 1484, 3968, 1124, 1342, 6152, 798, 150] combine_rows_per_src=[14234, 8032, 11922, 8304, 9436, 15794, 8038, 6242] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=292216832 dispatch_meta_bytes=333800 combine_payload_bytes=292216832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292550632 ingress_bytes=326819840 link_bytes=326819840 token_bytes=584767464 weight_movement_bytes=34603008 total_nvlink_bytes=619370472 weight_to_token_ratio=0.059174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229796747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=278233088 dispatch_meta_bytes=332744 combine_payload_bytes=278233088 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=278565832 ingress_bytes=382042112 link_bytes=382042112 token_bytes=556798920 weight_movement_bytes=103809024 total_nvlink_bytes=660607944 weight_to_token_ratio=0.186439 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437593711e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75544 payload_rows_remote=67180 dup_rows=22760 dup_rows_remote=21502 dedup_ratio=0.231527 router_tokens_per_expert=[1274, 13, 75, 101, 224, 66, 0, 1730, 685, 229, 0, 0, 339, 353, 5, 1, 1180, 600, 1499, 815, 1434, 0, 49, 410, 759, 779, 116, 0, 1434, 0, 10, 6375, 1668, 948, 81, 0, 2916, 0, 392, 4478, 453, 524, 6, 1667, 903, 377, 1450, 1098, 0, 662, 0, 260, 2984, 39, 189, 16, 564, 0, 1354, 219, 161, 4040, 226, 922] payload_rows_per_dst=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] dup_rows_per_dst=[670, 2128, 2934, 4976, 6912, 1258, 2484, 1398] combine_rows_per_src=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287989760 dispatch_meta_bytes=350008 combine_payload_bytes=287989760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288339768 ingress_bytes=322592768 link_bytes=322592768 token_bytes=576329528 weight_movement_bytes=34603008 total_nvlink_bytes=610932536 weight_to_token_ratio=0.060040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213890615e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76556 payload_rows_remote=64716 dup_rows=21748 dup_rows_remote=17820 dedup_ratio=0.221232 router_tokens_per_expert=[938, 27, 157, 155, 569, 78, 0, 1690, 1293, 144, 99, 0, 344, 201, 17, 31, 897, 606, 881, 1236, 1490, 2, 158, 290, 778, 463, 207, 0, 1382, 55, 26, 5908, 1329, 1068, 194, 0, 1933, 24, 587, 4216, 776, 705, 81, 2181, 1126, 204, 1683, 898, 0, 431, 6, 316, 1805, 152, 375, 34, 1282, 24, 649, 178, 83, 5193, 190, 1307] payload_rows_per_dst=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] dup_rows_per_dst=[1950, 3474, 2960, 4266, 3928, 1532, 2172, 1466] combine_rows_per_src=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=298663936 dispatch_meta_bytes=339520 combine_payload_bytes=298663936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299003456 ingress_bytes=367869952 link_bytes=367869952 token_bytes=597667392 weight_movement_bytes=69206016 total_nvlink_bytes=666873408 weight_to_token_ratio=0.115794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384265013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281845760 dispatch_meta_bytes=327672 combine_payload_bytes=281845760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282173432 ingress_bytes=316448768 link_bytes=316448768 token_bytes=564019192 weight_movement_bytes=34603008 total_nvlink_bytes=598622200 weight_to_token_ratio=0.061351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190771238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79022 payload_rows_remote=70310 dup_rows=19282 dup_rows_remote=17192 dedup_ratio=0.196147 router_tokens_per_expert=[0, 1, 341, 165, 4, 4, 7, 3, 694, 76, 2336, 0, 2807, 8, 378, 6, 1550, 669, 783, 61, 19, 1148, 4, 1848, 0, 471, 175, 1581, 6172, 3, 0, 1160, 4781, 0, 0, 0, 2313, 367, 887, 0, 1, 15, 7777, 2560, 0, 472, 0, 1, 11, 489, 2881, 0, 142, 12, 0, 18, 13, 54, 2050, 0, 1583, 170, 0, 81] payload_rows_per_dst=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] dup_rows_per_dst=[708, 2322, 818, 2210, 3644, 42, 2090, 7448] combine_rows_per_src=[6780, 10288, 7650, 10476, 13052, 6056, 8712, 16008] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287047680 dispatch_meta_bytes=329656 combine_payload_bytes=287047680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287377336 ingress_bytes=390856704 link_bytes=390856704 token_bytes=574425016 weight_movement_bytes=103809024 total_nvlink_bytes=678234040 weight_to_token_ratio=0.180718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.470762311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275169280 dispatch_meta_bytes=354728 combine_payload_bytes=275169280 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275524008 ingress_bytes=344375296 link_bytes=344375296 token_bytes=550693288 weight_movement_bytes=69206016 total_nvlink_bytes=619899304 weight_to_token_ratio=0.125671 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295856514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79110 payload_rows_remote=65760 dup_rows=19194 dup_rows_remote=16074 dedup_ratio=0.195251 router_tokens_per_expert=[670, 11, 131, 129, 1234, 31, 0, 824, 1555, 109, 1, 0, 167, 131, 4, 10, 960, 514, 386, 478, 823, 3, 168, 194, 668, 287, 68, 0, 1111, 107, 34, 6419, 1769, 2313, 48, 0, 1500, 0, 454, 4060, 772, 684, 25, 4167, 841, 56, 1913, 502, 0, 248, 73, 162, 1545, 433, 66, 25, 1886, 0, 673, 393, 85, 6099, 184, 949] payload_rows_per_dst=[12076, 12634, 5420, 13350, 10000, 8974, 10202, 6454] dup_rows_per_dst=[2104, 3518, 2550, 3120, 2168, 1216, 2632, 1886] combine_rows_per_src=[12076, 12634, 5420, 13350, 10000, 8974, 10202, 6454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265076736 dispatch_meta_bytes=330144 combine_payload_bytes=265076736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265406880 ingress_bytes=403488768 link_bytes=403488768 token_bytes=530483616 weight_movement_bytes=138412032 total_nvlink_bytes=668895648 weight_to_token_ratio=0.260917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518295750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70186 payload_rows_remote=53990 dup_rows=28118 dup_rows_remote=10152 dedup_ratio=0.286031 router_tokens_per_expert=[42, 1, 817, 48, 7044, 3, 1, 50, 1317, 7, 1, 5, 11, 28, 6, 3, 76, 308, 12, 65, 1, 0, 673, 65, 18, 70, 10, 0, 35, 176, 14, 6380, 144, 4851, 16, 0, 90, 1, 2, 401, 192, 617, 58, 7359, 119, 48, 560, 6, 4, 4, 435, 7, 171, 3379, 28, 24, 4687, 0, 54, 87, 41, 7998, 26, 456] payload_rows_per_dst=[14538, 2736, 2300, 12990, 2924, 3010, 15492, 16196] dup_rows_per_dst=[1474, 20, 100, 416, 622, 190, 7330, 17966] combine_rows_per_src=[14538, 2736, 2300, 12990, 2924, 3010, 15492, 16196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84382 payload_rows_remote=68810 dup_rows=13922 dup_rows_remote=13108 dedup_ratio=0.141622 router_tokens_per_expert=[1, 97, 42, 105, 10, 170, 1, 0, 538, 22, 1323, 0, 199, 86, 2018, 2, 1133, 1119, 2537, 101, 203, 37, 8, 5486, 0, 270, 21, 317, 7767, 465, 4, 1723, 1199, 1, 0, 0, 2792, 90, 3085, 0, 1, 0, 3925, 517, 0, 3834, 13, 2, 0, 826, 3191, 0, 677, 59, 102, 101, 34, 5, 2608, 0, 215, 5, 0, 65] payload_rows_per_dst=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] dup_rows_per_dst=[814, 5558, 1726, 1512, 328, 2946, 974, 64] combine_rows_per_src=[15572, 13790, 9988, 9672, 7584, 13038, 8938, 5800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75544 payload_rows_remote=67180 dup_rows=22760 dup_rows_remote=21502 dedup_ratio=0.231527 router_tokens_per_expert=[1274, 13, 75, 101, 224, 66, 0, 1730, 685, 229, 0, 0, 339, 353, 5, 1, 1180, 600, 1499, 815, 1434, 0, 49, 410, 759, 779, 116, 0, 1434, 0, 10, 6375, 1668, 948, 81, 0, 2916, 0, 392, 4478, 453, 524, 6, 1667, 903, 377, 1450, 1098, 0, 662, 0, 260, 2984, 39, 189, 16, 564, 0, 1354, 219, 161, 4040, 226, 922] payload_rows_per_dst=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] dup_rows_per_dst=[670, 2128, 2934, 4976, 6912, 1258, 2484, 1398] combine_rows_per_src=[6296, 9176, 9040, 13970, 14054, 8364, 9150, 5494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=287989760 dispatch_meta_bytes=350008 combine_payload_bytes=287989760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288339768 ingress_bytes=322592768 link_bytes=322592768 token_bytes=576329528 weight_movement_bytes=34603008 total_nvlink_bytes=610932536 weight_to_token_ratio=0.060040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213890615e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76556 payload_rows_remote=64716 dup_rows=21748 dup_rows_remote=17820 dedup_ratio=0.221232 router_tokens_per_expert=[938, 27, 157, 155, 569, 78, 0, 1690, 1293, 144, 99, 0, 344, 201, 17, 31, 897, 606, 881, 1236, 1490, 2, 158, 290, 778, 463, 207, 0, 1382, 55, 26, 5908, 1329, 1068, 194, 0, 1933, 24, 587, 4216, 776, 705, 81, 2181, 1126, 204, 1683, 898, 0, 431, 6, 316, 1805, 152, 375, 34, 1282, 24, 649, 178, 83, 5193, 190, 1307] payload_rows_per_dst=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] dup_rows_per_dst=[1950, 3474, 2960, 4266, 3928, 1532, 2172, 1466] combine_rows_per_src=[8212, 11170, 8160, 13372, 11840, 9414, 8428, 5960] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91358 payload_rows_remote=74996 dup_rows=6946 dup_rows_remote=5648 dedup_ratio=0.070658 router_tokens_per_expert=[7, 0, 1231, 1, 8088, 0, 3, 3, 648, 0, 0, 0, 2, 0, 0, 0, 3, 260, 3, 1, 0, 0, 1052, 1, 6, 5, 0, 0, 2, 195, 25, 4852, 18, 4499, 1, 0, 10, 0, 1, 33, 66, 598, 115, 7772, 1, 2, 50, 0, 0, 1, 540, 0, 5, 5225, 0, 25, 5284, 0, 5, 0, 2, 8180, 0, 331] payload_rows_per_dst=[16268, 16362, 11670, 460, 9028, 15602, 11362, 10606] dup_rows_per_dst=[2464, 1298, 674, 6, 30, 1606, 230, 638] combine_rows_per_src=[16268, 16362, 11670, 460, 9028, 15602, 11362, 10606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269352960 dispatch_meta_bytes=327336 combine_payload_bytes=269352960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269680296 ingress_bytes=338558976 link_bytes=338558976 token_bytes=539033256 weight_movement_bytes=69206016 total_nvlink_bytes=608239272 weight_to_token_ratio=0.128389 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273970170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79110 payload_rows_remote=65760 dup_rows=19194 dup_rows_remote=16074 dedup_ratio=0.195251 router_tokens_per_expert=[670, 11, 131, 129, 1234, 31, 0, 824, 1555, 109, 1, 0, 167, 131, 4, 10, 960, 514, 386, 478, 823, 3, 168, 194, 668, 287, 68, 0, 1111, 107, 34, 6419, 1769, 2313, 48, 0, 1500, 0, 454, 4060, 772, 684, 25, 4167, 841, 56, 1913, 502, 0, 248, 73, 162, 1545, 433, 66, 25, 1886, 0, 673, 393, 85, 6099, 184, 949] payload_rows_per_dst=[12076, 12634, 5420, 13350, 10000, 8974, 10202, 6454] dup_rows_per_dst=[2104, 3518, 2550, 3120, 2168, 1216, 2632, 1886] combine_rows_per_src=[12076, 12634, 5420, 13350, 10000, 8974, 10202, 6454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=221143040 dispatch_meta_bytes=256568 combine_payload_bytes=221143040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221399608 ingress_bytes=290349056 link_bytes=290349056 token_bytes=442542648 weight_movement_bytes=69206016 total_nvlink_bytes=511748664 weight_to_token_ratio=0.156383 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.092560123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281845760 dispatch_meta_bytes=327672 combine_payload_bytes=281845760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282173432 ingress_bytes=316448768 link_bytes=316448768 token_bytes=564019192 weight_movement_bytes=34603008 total_nvlink_bytes=598622200 weight_to_token_ratio=0.061351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190771238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72488 payload_rows_remote=64706 dup_rows=25816 dup_rows_remote=23740 dedup_ratio=0.262614 router_tokens_per_expert=[1862, 41, 18, 102, 219, 58, 0, 1479, 441, 195, 1, 0, 299, 233, 8, 44, 2134, 534, 1047, 766, 1742, 2, 46, 114, 1188, 618, 249, 0, 2318, 9, 8, 5978, 2049, 782, 74, 0, 2875, 8, 733, 4556, 274, 502, 16, 1751, 711, 455, 1606, 1333, 1, 866, 0, 291, 1886, 24, 97, 13, 530, 5, 1067, 146, 102, 3696, 186, 764] payload_rows_per_dst=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] dup_rows_per_dst=[916, 52, 3530, 6910, 7826, 1278, 2076, 3228] combine_rows_per_src=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265076736 dispatch_meta_bytes=330144 combine_payload_bytes=265076736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265406880 ingress_bytes=403488768 link_bytes=403488768 token_bytes=530483616 weight_movement_bytes=138412032 total_nvlink_bytes=668895648 weight_to_token_ratio=0.260917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518295750e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275169280 dispatch_meta_bytes=354728 combine_payload_bytes=275169280 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275524008 ingress_bytes=344375296 link_bytes=344375296 token_bytes=550693288 weight_movement_bytes=69206016 total_nvlink_bytes=619899304 weight_to_token_ratio=0.125671 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295856514e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77370 payload_rows_remote=67318 dup_rows=20934 dup_rows_remote=18768 dedup_ratio=0.212952 router_tokens_per_expert=[312, 16, 889, 229, 2914, 17, 0, 585, 1895, 54, 0, 0, 119, 109, 1, 0, 290, 930, 301, 119, 199, 0, 364, 582, 83, 115, 25, 0, 361, 179, 76, 6293, 613, 2403, 93, 0, 681, 2, 42, 1147, 919, 845, 25, 5670, 908, 88, 2083, 90, 0, 137, 173, 52, 2165, 1229, 78, 99, 2724, 0, 462, 509, 101, 7243, 248, 1266] payload_rows_per_dst=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] dup_rows_per_dst=[2166, 4142, 3922, 138, 658, 6738, 762, 2408] combine_rows_per_src=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76796 payload_rows_remote=61666 dup_rows=21508 dup_rows_remote=15068 dedup_ratio=0.218791 router_tokens_per_expert=[879, 57, 289, 308, 355, 38, 0, 1106, 1068, 270, 0, 2, 471, 158, 5, 2, 869, 1038, 754, 498, 312, 0, 88, 724, 1012, 1521, 120, 0, 1308, 0, 61, 6502, 843, 1006, 92, 0, 1791, 0, 126, 3206, 507, 872, 64, 2487, 1526, 308, 1504, 866, 43, 961, 10, 445, 3647, 68, 335, 40, 890, 14, 865, 225, 167, 5240, 260, 929] payload_rows_per_dst=[10188, 11506, 15130, 6646, 6906, 11618, 9306, 5496] dup_rows_per_dst=[2288, 2926, 6440, 1398, 810, 4650, 1792, 1204] combine_rows_per_src=[10188, 11506, 15130, 6646, 6906, 11618, 9306, 5496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79660 payload_rows_remote=66740 dup_rows=18644 dup_rows_remote=10772 dedup_ratio=0.189657 router_tokens_per_expert=[105, 0, 0, 1481, 784, 0, 2, 0, 71, 3359, 99, 644, 687, 2, 147, 49, 2593, 0, 4806, 0, 0, 41, 753, 693, 2217, 0, 0, 390, 0, 1307, 61, 0, 526, 14, 4586, 710, 30, 0, 2, 1254, 4924, 945, 147, 52, 1005, 2163, 5829, 1160, 0, 580, 396, 3, 0, 192, 0, 4, 0, 27, 103, 477, 47, 2851, 395, 439] payload_rows_per_dst=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] dup_rows_per_dst=[758, 1100, 2282, 1548, 2980, 7872, 1032, 1072] combine_rows_per_src=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307183616 dispatch_meta_bytes=322576 combine_payload_bytes=307183616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307506192 ingress_bytes=341786624 link_bytes=341786624 token_bytes=614689808 weight_movement_bytes=34603008 total_nvlink_bytes=649292816 weight_to_token_ratio=0.056293 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286115550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=353784 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265389560 ingress_bytes=299638784 link_bytes=299638784 token_bytes=530425336 weight_movement_bytes=34603008 total_nvlink_bytes=565028344 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127516621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82034 payload_rows_remote=71738 dup_rows=16270 dup_rows_remote=14444 dedup_ratio=0.165507 router_tokens_per_expert=[119, 0, 50, 658, 1285, 0, 6, 0, 54, 2703, 87, 570, 1554, 26, 374, 2, 2724, 0, 3525, 0, 21, 14, 1160, 1095, 2356, 0, 0, 765, 0, 298, 116, 0, 414, 63, 3229, 1740, 124, 0, 4, 487, 5196, 442, 216, 158, 1351, 1768, 4799, 2359, 7, 843, 485, 15, 0, 333, 1, 8, 7, 24, 283, 584, 225, 3576, 442, 407] payload_rows_per_dst=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] dup_rows_per_dst=[908, 1636, 1732, 2138, 1826, 5196, 1110, 1724] combine_rows_per_src=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84950 payload_rows_remote=69716 dup_rows=13354 dup_rows_remote=11018 dedup_ratio=0.135844 router_tokens_per_expert=[246, 0, 323, 625, 898, 0, 0, 0, 22, 3895, 51, 432, 2373, 0, 108, 0, 2784, 0, 2344, 0, 1, 0, 3545, 1056, 1518, 0, 0, 342, 0, 745, 210, 0, 282, 129, 2170, 3178, 112, 0, 3, 531, 5970, 374, 273, 607, 1167, 941, 3068, 2560, 2, 1627, 227, 1, 0, 608, 0, 0, 2, 10, 130, 595, 494, 2152, 238, 183] payload_rows_per_dst=[9462, 11116, 11202, 15234, 11056, 9968, 10106, 6806] dup_rows_per_dst=[1812, 2646, 1168, 2336, 1754, 1876, 960, 802] combine_rows_per_src=[9462, 11116, 11202, 15234, 11056, 9968, 10106, 6806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269352960 dispatch_meta_bytes=327336 combine_payload_bytes=269352960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269680296 ingress_bytes=338558976 link_bytes=338558976 token_bytes=539033256 weight_movement_bytes=69206016 total_nvlink_bytes=608239272 weight_to_token_ratio=0.128389 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273970170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72488 payload_rows_remote=64706 dup_rows=25816 dup_rows_remote=23740 dedup_ratio=0.262614 router_tokens_per_expert=[1862, 41, 18, 102, 219, 58, 0, 1479, 441, 195, 1, 0, 299, 233, 8, 44, 2134, 534, 1047, 766, 1742, 2, 46, 114, 1188, 618, 249, 0, 2318, 9, 8, 5978, 2049, 782, 74, 0, 2875, 8, 733, 4556, 274, 502, 16, 1751, 711, 455, 1606, 1333, 1, 866, 0, 291, 1886, 24, 97, 13, 530, 5, 1067, 146, 102, 3696, 186, 764] payload_rows_per_dst=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] dup_rows_per_dst=[916, 52, 3530, 6910, 7826, 1278, 2076, 3228] combine_rows_per_src=[6642, 2462, 9240, 13826, 14328, 8516, 7782, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275734528 dispatch_meta_bytes=344344 combine_payload_bytes=275734528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276078872 ingress_bytes=310337536 link_bytes=310337536 token_bytes=551813400 weight_movement_bytes=34603008 total_nvlink_bytes=586416408 weight_to_token_ratio=0.062708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167775164e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=252583936 dispatch_meta_bytes=306936 combine_payload_bytes=252583936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252890872 ingress_bytes=287186944 link_bytes=287186944 token_bytes=505474808 weight_movement_bytes=34603008 total_nvlink_bytes=540077816 weight_to_token_ratio=0.068456 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.080661350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273367040 dispatch_meta_bytes=310048 combine_payload_bytes=273367040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=273677088 ingress_bytes=411779072 link_bytes=411779072 token_bytes=547044128 weight_movement_bytes=138412032 total_nvlink_bytes=685456160 weight_to_token_ratio=0.253018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.549491497e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=293838848 dispatch_meta_bytes=344728 combine_payload_bytes=293838848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=294183576 ingress_bytes=363044864 link_bytes=363044864 token_bytes=588022424 weight_movement_bytes=69206016 total_nvlink_bytes=657228440 weight_to_token_ratio=0.117693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366108596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78054 payload_rows_remote=62238 dup_rows=20250 dup_rows_remote=10618 dedup_ratio=0.205994 router_tokens_per_expert=[51, 0, 5331, 47, 81, 7, 0, 0, 0, 1085, 5, 4, 4153, 0, 24, 11, 2002, 2, 174, 0, 0, 0, 6915, 333, 0, 1, 0, 7, 0, 52, 234, 0, 3, 361, 76, 5533, 213, 0, 0, 96, 7805, 16, 127, 3314, 179, 12, 159, 783, 2, 4319, 86, 1, 7, 610, 0, 12, 1, 11, 3, 114, 4697, 61, 30, 2] payload_rows_per_dst=[14702, 9552, 7860, 586, 11728, 8144, 9666, 15816] dup_rows_per_dst=[6154, 1012, 822, 2, 1184, 718, 726, 9632] combine_rows_per_src=[14702, 9552, 7860, 586, 11728, 8144, 9666, 15816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77370 payload_rows_remote=67318 dup_rows=20934 dup_rows_remote=18768 dedup_ratio=0.212952 router_tokens_per_expert=[312, 16, 889, 229, 2914, 17, 0, 585, 1895, 54, 0, 0, 119, 109, 1, 0, 290, 930, 301, 119, 199, 0, 364, 582, 83, 115, 25, 0, 361, 179, 76, 6293, 613, 2403, 93, 0, 681, 2, 42, 1147, 919, 845, 25, 5670, 908, 88, 2083, 90, 0, 137, 173, 52, 2165, 1229, 78, 99, 2724, 0, 462, 509, 101, 7243, 248, 1266] payload_rows_per_dst=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] dup_rows_per_dst=[2166, 4142, 3922, 138, 658, 6738, 762, 2408] combine_rows_per_src=[10052, 14700, 14234, 2460, 6270, 14338, 7104, 8212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285556736 dispatch_meta_bytes=322936 combine_payload_bytes=285556736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285879672 ingress_bytes=320159744 link_bytes=320159744 token_bytes=571436408 weight_movement_bytes=34603008 total_nvlink_bytes=606039416 weight_to_token_ratio=0.060554 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204735342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82034 payload_rows_remote=71738 dup_rows=16270 dup_rows_remote=14444 dedup_ratio=0.165507 router_tokens_per_expert=[119, 0, 50, 658, 1285, 0, 6, 0, 54, 2703, 87, 570, 1554, 26, 374, 2, 2724, 0, 3525, 0, 21, 14, 1160, 1095, 2356, 0, 0, 765, 0, 298, 116, 0, 414, 63, 3229, 1740, 124, 0, 4, 487, 5196, 442, 216, 158, 1351, 1768, 4799, 2359, 7, 843, 485, 15, 0, 333, 1, 8, 7, 24, 283, 584, 225, 3576, 442, 407] payload_rows_per_dst=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] dup_rows_per_dst=[908, 1636, 1732, 2138, 1826, 5196, 1110, 1724] combine_rows_per_src=[5648, 9104, 13026, 11462, 10296, 11254, 11872, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79660 payload_rows_remote=66740 dup_rows=18644 dup_rows_remote=10772 dedup_ratio=0.189657 router_tokens_per_expert=[105, 0, 0, 1481, 784, 0, 2, 0, 71, 3359, 99, 644, 687, 2, 147, 49, 2593, 0, 4806, 0, 0, 41, 753, 693, 2217, 0, 0, 390, 0, 1307, 61, 0, 526, 14, 4586, 710, 30, 0, 2, 1254, 4924, 945, 147, 52, 1005, 2163, 5829, 1160, 0, 580, 396, 3, 0, 192, 0, 4, 0, 27, 103, 477, 47, 2851, 395, 439] payload_rows_per_dst=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] dup_rows_per_dst=[758, 1100, 2282, 1548, 2980, 7872, 1032, 1072] combine_rows_per_src=[5492, 9016, 13984, 6402, 11264, 12920, 12976, 7606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=353784 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265389560 ingress_bytes=299638784 link_bytes=299638784 token_bytes=530425336 weight_movement_bytes=34603008 total_nvlink_bytes=565028344 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127516621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82580 payload_rows_remote=74718 dup_rows=15724 dup_rows_remote=14524 dedup_ratio=0.159953 router_tokens_per_expert=[5, 0, 7035, 5, 21, 0, 0, 0, 0, 90, 0, 1, 2470, 0, 0, 2, 1252, 0, 8, 0, 0, 0, 7488, 104, 0, 0, 0, 1, 0, 8, 40, 0, 1, 755, 10, 7109, 134, 0, 0, 3, 7769, 1, 15, 2755, 22, 5, 11, 51, 0, 5264, 5, 0, 0, 584, 0, 0, 0, 0, 1, 1, 6113, 7, 1, 5] payload_rows_per_dst=[16350, 7862, 2712, 15552, 11040, 5678, 11138, 12248] dup_rows_per_dst=[12758, 1200, 26, 84, 1048, 10, 590, 8] combine_rows_per_src=[16350, 7862, 2712, 15552, 11040, 5678, 11138, 12248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=254926848 dispatch_meta_bytes=291424 combine_payload_bytes=254926848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=255218272 ingress_bytes=289529856 link_bytes=289529856 token_bytes=510145120 weight_movement_bytes=34603008 total_nvlink_bytes=544748128 weight_to_token_ratio=0.067830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.089477539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80584 payload_rows_remote=67826 dup_rows=17720 dup_rows_remote=17018 dedup_ratio=0.180257 router_tokens_per_expert=[121, 0, 0, 908, 931, 0, 0, 0, 62, 2131, 66, 1384, 523, 2, 349, 3, 1657, 0, 5341, 0, 13, 30, 602, 776, 3378, 0, 0, 771, 0, 714, 41, 0, 444, 4, 4870, 449, 15, 0, 1, 1119, 3877, 713, 133, 21, 1122, 1673, 5704, 1516, 7, 468, 439, 5, 0, 106, 0, 1, 0, 15, 399, 554, 29, 4707, 334, 624] payload_rows_per_dst=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] dup_rows_per_dst=[726, 702, 618, 1662, 4986, 4400, 702, 3924] combine_rows_per_src=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273367040 dispatch_meta_bytes=310048 combine_payload_bytes=273367040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=273677088 ingress_bytes=411779072 link_bytes=411779072 token_bytes=547044128 weight_movement_bytes=138412032 total_nvlink_bytes=685456160 weight_to_token_ratio=0.253018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.549491497e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84950 payload_rows_remote=69716 dup_rows=13354 dup_rows_remote=11018 dedup_ratio=0.135844 router_tokens_per_expert=[246, 0, 323, 625, 898, 0, 0, 0, 22, 3895, 51, 432, 2373, 0, 108, 0, 2784, 0, 2344, 0, 1, 0, 3545, 1056, 1518, 0, 0, 342, 0, 745, 210, 0, 282, 129, 2170, 3178, 112, 0, 3, 531, 5970, 374, 273, 607, 1167, 941, 3068, 2560, 2, 1627, 227, 1, 0, 608, 0, 0, 2, 10, 130, 595, 494, 2152, 238, 183] payload_rows_per_dst=[9462, 11116, 11202, 15234, 11056, 9968, 10106, 6806] dup_rows_per_dst=[1812, 2646, 1168, 2336, 1754, 1876, 960, 802] combine_rows_per_src=[9462, 11116, 11202, 15234, 11056, 9968, 10106, 6806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78536 payload_rows_remote=69516 dup_rows=19768 dup_rows_remote=18560 dedup_ratio=0.201090 router_tokens_per_expert=[150, 0, 625, 402, 550, 0, 0, 0, 31, 4221, 22, 151, 3002, 5, 31, 9, 4699, 0, 1241, 0, 0, 5, 3387, 986, 384, 0, 0, 98, 0, 206, 254, 0, 130, 160, 988, 5311, 179, 1, 0, 256, 7208, 224, 364, 1143, 1339, 613, 1311, 969, 3, 3131, 184, 2, 0, 1208, 0, 5, 0, 36, 21, 734, 2076, 707, 340, 50] payload_rows_per_dst=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] dup_rows_per_dst=[1208, 10582, 2852, 938, 644, 1318, 1184, 1042] combine_rows_per_src=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=275734528 dispatch_meta_bytes=344344 combine_payload_bytes=275734528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276078872 ingress_bytes=310337536 link_bytes=310337536 token_bytes=551813400 weight_movement_bytes=34603008 total_nvlink_bytes=586416408 weight_to_token_ratio=0.062708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167775164e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=293838848 dispatch_meta_bytes=344728 combine_payload_bytes=293838848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=294183576 ingress_bytes=363044864 link_bytes=363044864 token_bytes=588022424 weight_movement_bytes=69206016 total_nvlink_bytes=657228440 weight_to_token_ratio=0.117693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366108596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306044928 dispatch_meta_bytes=356968 combine_payload_bytes=306044928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306401896 ingress_bytes=340647936 link_bytes=340647936 token_bytes=612446824 weight_movement_bytes=34603008 total_nvlink_bytes=647049832 weight_to_token_ratio=0.056500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281830759e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81586 payload_rows_remote=70222 dup_rows=16718 dup_rows_remote=13088 dedup_ratio=0.170064 router_tokens_per_expert=[973, 112, 58, 0, 285, 295, 3612, 0, 0, 1638, 1734, 1903, 0, 3456, 128, 3164, 1550, 42, 1355, 2666, 57, 14, 478, 489, 0, 82, 3319, 384, 1821, 210, 222, 0, 0, 24, 10, 1982, 672, 1049, 7, 3765, 38, 292, 12, 184, 271, 0, 68, 12, 46, 6750, 0, 695, 0, 83, 56, 299, 213, 0, 0, 7, 1, 194, 2266, 109] payload_rows_per_dst=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] dup_rows_per_dst=[1288, 2454, 1354, 2110, 3456, 3630, 1914, 512] combine_rows_per_src=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84146 payload_rows_remote=70406 dup_rows=14158 dup_rows_remote=10776 dedup_ratio=0.144023 router_tokens_per_expert=[29, 0, 0, 2209, 1881, 0, 0, 0, 79, 3020, 66, 351, 1056, 28, 43, 84, 3418, 0, 3987, 0, 0, 1, 335, 894, 1388, 0, 0, 261, 0, 890, 36, 0, 347, 1, 3319, 1557, 40, 0, 29, 978, 5851, 462, 157, 48, 1309, 2919, 5657, 627, 6, 609, 676, 6, 1, 351, 0, 27, 17, 45, 53, 291, 132, 2689, 590, 302] payload_rows_per_dst=[7564, 8340, 13740, 14674, 10586, 8992, 12992, 7258] dup_rows_per_dst=[1344, 1114, 3382, 2178, 1956, 1530, 1674, 980] combine_rows_per_src=[7564, 8340, 13740, 14674, 10586, 8992, 12992, 7258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277815296 dispatch_meta_bytes=339376 combine_payload_bytes=277815296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278154672 ingress_bytes=312418304 link_bytes=312418304 token_bytes=555969968 weight_movement_bytes=34603008 total_nvlink_bytes=590572976 weight_to_token_ratio=0.062239 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175604926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80806 payload_rows_remote=70146 dup_rows=17498 dup_rows_remote=14318 dedup_ratio=0.177999 router_tokens_per_expert=[947, 57, 25, 1, 351, 195, 3951, 0, 52, 1225, 1595, 2502, 5, 4577, 407, 2528, 2531, 51, 702, 2865, 109, 0, 629, 572, 1, 121, 4058, 507, 1852, 162, 456, 0, 0, 94, 72, 1605, 477, 1094, 26, 3552, 264, 94, 35, 64, 389, 0, 12, 2, 105, 4972, 0, 499, 0, 254, 62, 117, 282, 0, 0, 13, 12, 122, 1227, 673] payload_rows_per_dst=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] dup_rows_per_dst=[1004, 2904, 1136, 2812, 3180, 3984, 1328, 1150] combine_rows_per_src=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306044928 dispatch_meta_bytes=356968 combine_payload_bytes=306044928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306401896 ingress_bytes=340647936 link_bytes=340647936 token_bytes=612446824 weight_movement_bytes=34603008 total_nvlink_bytes=647049832 weight_to_token_ratio=0.056500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281830759e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77510 payload_rows_remote=66758 dup_rows=20794 dup_rows_remote=18910 dedup_ratio=0.211528 router_tokens_per_expert=[1393, 44, 6, 0, 203, 208, 5157, 0, 7, 947, 1139, 1310, 4, 4392, 1038, 2583, 3854, 8, 650, 2133, 114, 0, 1284, 397, 0, 168, 4005, 136, 915, 119, 1531, 1, 0, 239, 36, 1242, 212, 1001, 8, 2058, 195, 148, 12, 60, 603, 0, 0, 2, 116, 5502, 0, 296, 0, 825, 17, 212, 196, 0, 0, 81, 2, 227, 906, 1210] payload_rows_per_dst=[11978, 6874, 8558, 10752, 9594, 11932, 7416, 10406] dup_rows_per_dst=[2044, 2016, 1728, 1884, 5034, 4058, 1484, 2546] combine_rows_per_src=[11978, 6874, 8558, 10752, 9594, 11932, 7416, 10406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=333240 combine_payload_bytes=287629312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287962552 ingress_bytes=356835328 link_bytes=356835328 token_bytes=575591864 weight_movement_bytes=69206016 total_nvlink_bytes=644797880 weight_to_token_ratio=0.120235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342742612e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285556736 dispatch_meta_bytes=322936 combine_payload_bytes=285556736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285879672 ingress_bytes=320159744 link_bytes=320159744 token_bytes=571436408 weight_movement_bytes=34603008 total_nvlink_bytes=606039416 weight_to_token_ratio=0.060554 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.204735342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=284737536 dispatch_meta_bytes=352304 combine_payload_bytes=284737536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285089840 ingress_bytes=319340544 link_bytes=319340544 token_bytes=569827376 weight_movement_bytes=34603008 total_nvlink_bytes=604430384 weight_to_token_ratio=0.060725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201652758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80584 payload_rows_remote=67826 dup_rows=17720 dup_rows_remote=17018 dedup_ratio=0.180257 router_tokens_per_expert=[121, 0, 0, 908, 931, 0, 0, 0, 62, 2131, 66, 1384, 523, 2, 349, 3, 1657, 0, 5341, 0, 13, 30, 602, 776, 3378, 0, 0, 771, 0, 714, 41, 0, 444, 4, 4870, 449, 15, 0, 1, 1119, 3877, 713, 133, 21, 1122, 1673, 5704, 1516, 7, 468, 439, 5, 0, 106, 0, 1, 0, 15, 399, 554, 29, 4707, 334, 624] payload_rows_per_dst=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] dup_rows_per_dst=[726, 702, 618, 1662, 4986, 4400, 702, 3924] combine_rows_per_src=[4398, 8338, 10662, 8146, 13172, 9982, 12758, 13128] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=288382976 dispatch_meta_bytes=324728 combine_payload_bytes=288382976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288707704 ingress_bytes=392192000 link_bytes=392192000 token_bytes=577090680 weight_movement_bytes=103809024 total_nvlink_bytes=680899704 weight_to_token_ratio=0.179883 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475786922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287318016 dispatch_meta_bytes=337856 combine_payload_bytes=287318016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287655872 ingress_bytes=321921024 link_bytes=321921024 token_bytes=574973888 weight_movement_bytes=34603008 total_nvlink_bytes=609576896 weight_to_token_ratio=0.060182 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211362897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78966 payload_rows_remote=63098 dup_rows=19338 dup_rows_remote=7590 dedup_ratio=0.196716 router_tokens_per_expert=[443, 14, 7, 0, 1, 13, 7224, 2, 9, 98, 67, 47, 4, 2298, 5225, 161, 7803, 30, 12, 324, 252, 0, 1953, 10, 5, 3, 623, 5, 13, 12, 6461, 0, 11, 539, 0, 95, 72, 28, 0, 158, 319, 8, 0, 29, 347, 0, 21, 27, 231, 1170, 5, 85, 1, 6866, 5, 11, 10, 0, 1, 367, 0, 429, 35, 5163] payload_rows_per_dst=[9680, 12394, 4886, 13628, 1674, 5958, 14878, 15868] dup_rows_per_dst=[196, 3372, 276, 616, 132, 462, 2536, 11748] combine_rows_per_src=[9680, 12394, 4886, 13628, 1674, 5958, 14878, 15868] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81586 payload_rows_remote=70222 dup_rows=16718 dup_rows_remote=13088 dedup_ratio=0.170064 router_tokens_per_expert=[973, 112, 58, 0, 285, 295, 3612, 0, 0, 1638, 1734, 1903, 0, 3456, 128, 3164, 1550, 42, 1355, 2666, 57, 14, 478, 489, 0, 82, 3319, 384, 1821, 210, 222, 0, 0, 24, 10, 1982, 672, 1049, 7, 3765, 38, 292, 12, 184, 271, 0, 68, 12, 46, 6750, 0, 695, 0, 83, 56, 299, 213, 0, 0, 7, 1, 194, 2266, 109] payload_rows_per_dst=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] dup_rows_per_dst=[1288, 2454, 1354, 2110, 3456, 3630, 1914, 512] combine_rows_per_src=[9382, 8352, 8848, 9966, 11562, 11364, 13944, 8168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277815296 dispatch_meta_bytes=339376 combine_payload_bytes=277815296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278154672 ingress_bytes=312418304 link_bytes=312418304 token_bytes=555969968 weight_movement_bytes=34603008 total_nvlink_bytes=590572976 weight_to_token_ratio=0.062239 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175604926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273440768 dispatch_meta_bytes=342672 combine_payload_bytes=273440768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273783440 ingress_bytes=342646784 link_bytes=342646784 token_bytes=547224208 weight_movement_bytes=69206016 total_nvlink_bytes=616430224 weight_to_token_ratio=0.126467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289352263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80806 payload_rows_remote=70146 dup_rows=17498 dup_rows_remote=14318 dedup_ratio=0.177999 router_tokens_per_expert=[947, 57, 25, 1, 351, 195, 3951, 0, 52, 1225, 1595, 2502, 5, 4577, 407, 2528, 2531, 51, 702, 2865, 109, 0, 629, 572, 1, 121, 4058, 507, 1852, 162, 456, 0, 0, 94, 72, 1605, 477, 1094, 26, 3552, 264, 94, 35, 64, 389, 0, 12, 2, 105, 4972, 0, 499, 0, 254, 62, 117, 282, 0, 0, 13, 12, 122, 1227, 673] payload_rows_per_dst=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] dup_rows_per_dst=[1004, 2904, 1136, 2812, 3180, 3984, 1328, 1150] combine_rows_per_src=[10050, 8668, 8720, 11502, 10660, 11946, 10690, 8570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78536 payload_rows_remote=69516 dup_rows=19768 dup_rows_remote=18560 dedup_ratio=0.201090 router_tokens_per_expert=[150, 0, 625, 402, 550, 0, 0, 0, 31, 4221, 22, 151, 3002, 5, 31, 9, 4699, 0, 1241, 0, 0, 5, 3387, 986, 384, 0, 0, 98, 0, 206, 254, 0, 130, 160, 988, 5311, 179, 1, 0, 256, 7208, 224, 364, 1143, 1339, 613, 1311, 969, 3, 3131, 184, 2, 0, 1208, 0, 5, 0, 36, 21, 734, 2076, 707, 340, 50] payload_rows_per_dst=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] dup_rows_per_dst=[1208, 10582, 2852, 938, 644, 1318, 1184, 1042] combine_rows_per_src=[9020, 14984, 12236, 15362, 2784, 8446, 8818, 6886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=282752 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258732160 ingress_bytes=293052416 link_bytes=293052416 token_bytes=517181568 weight_movement_bytes=34603008 total_nvlink_bytes=551784576 weight_to_token_ratio=0.066907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102732649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92004 payload_rows_remote=81312 dup_rows=6300 dup_rows_remote=6188 dedup_ratio=0.064087 router_tokens_per_expert=[53, 0, 0, 0, 0, 4, 7633, 2, 0, 6, 1, 9, 142, 1363, 5190, 54, 8173, 0, 1, 12, 254, 0, 1264, 4, 0, 0, 135, 0, 4, 0, 7093, 0, 0, 431, 1, 13, 3, 4, 1, 14, 186, 0, 0, 0, 365, 0, 1, 0, 293, 486, 0, 3, 0, 8093, 0, 3, 2, 0, 0, 851, 0, 512, 3, 6490] payload_rows_per_dst=[15374, 10692, 15260, 14194, 1854, 3820, 16200, 14610] dup_rows_per_dst=[10, 112, 3010, 0, 52, 10, 584, 2522] combine_rows_per_src=[15374, 10692, 15260, 14194, 1854, 3820, 16200, 14610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79310 payload_rows_remote=66752 dup_rows=18994 dup_rows_remote=17420 dedup_ratio=0.193217 router_tokens_per_expert=[979, 128, 30, 0, 389, 225, 2862, 0, 17, 2581, 1425, 2729, 0, 3652, 46, 3005, 842, 18, 837, 2347, 4, 4, 298, 733, 0, 239, 3973, 267, 2601, 294, 189, 4, 0, 35, 20, 1688, 207, 1826, 2, 4435, 11, 115, 24, 140, 155, 0, 11, 4, 0, 6023, 0, 653, 5, 30, 34, 321, 337, 0, 0, 3, 0, 130, 2129, 96] payload_rows_per_dst=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] dup_rows_per_dst=[1126, 4094, 774, 3330, 4310, 3404, 1574, 382] combine_rows_per_src=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287318016 dispatch_meta_bytes=337856 combine_payload_bytes=287318016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287655872 ingress_bytes=321921024 link_bytes=321921024 token_bytes=574973888 weight_movement_bytes=34603008 total_nvlink_bytes=609576896 weight_to_token_ratio=0.060182 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.211362897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=333240 combine_payload_bytes=287629312 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287962552 ingress_bytes=356835328 link_bytes=356835328 token_bytes=575591864 weight_movement_bytes=69206016 total_nvlink_bytes=644797880 weight_to_token_ratio=0.120235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342742612e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77510 payload_rows_remote=66758 dup_rows=20794 dup_rows_remote=18910 dedup_ratio=0.211528 router_tokens_per_expert=[1393, 44, 6, 0, 203, 208, 5157, 0, 7, 947, 1139, 1310, 4, 4392, 1038, 2583, 3854, 8, 650, 2133, 114, 0, 1284, 397, 0, 168, 4005, 136, 915, 119, 1531, 1, 0, 239, 36, 1242, 212, 1001, 8, 2058, 195, 148, 12, 60, 603, 0, 0, 2, 116, 5502, 0, 296, 0, 825, 17, 212, 196, 0, 0, 81, 2, 227, 906, 1210] payload_rows_per_dst=[11978, 6874, 8558, 10752, 9594, 11932, 7416, 10406] dup_rows_per_dst=[2044, 2016, 1728, 1884, 5034, 4058, 1484, 2546] combine_rows_per_src=[11978, 6874, 8558, 10752, 9594, 11932, 7416, 10406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80862 payload_rows_remote=66818 dup_rows=17442 dup_rows_remote=15346 dedup_ratio=0.177429 router_tokens_per_expert=[1100, 22, 8, 0, 80, 156, 6704, 0, 0, 468, 512, 414, 13, 4933, 1402, 1991, 6605, 96, 96, 1312, 203, 0, 1457, 86, 0, 9, 2096, 154, 422, 47, 2284, 2, 0, 438, 0, 699, 569, 154, 2, 1049, 394, 7, 0, 59, 1307, 0, 11, 5, 1008, 4302, 0, 422, 0, 2972, 2, 90, 55, 0, 0, 171, 0, 210, 289, 2265] payload_rows_per_dst=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] dup_rows_per_dst=[2096, 1510, 8390, 882, 1714, 1738, 920, 192] combine_rows_per_src=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77394 payload_rows_remote=67152 dup_rows=20910 dup_rows_remote=19008 dedup_ratio=0.212708 router_tokens_per_expert=[1059, 91, 98, 0, 140, 219, 4358, 1, 74, 1120, 1905, 2217, 3, 4425, 142, 4134, 2276, 246, 968, 1480, 123, 5, 334, 141, 0, 102, 2775, 734, 2009, 247, 308, 2, 17, 65, 2, 1293, 1127, 532, 3, 2273, 238, 46, 26, 148, 574, 0, 23, 297, 204, 6348, 0, 1091, 0, 143, 93, 574, 272, 0, 0, 14, 0, 66, 1690, 257] payload_rows_per_dst=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] dup_rows_per_dst=[1370, 5426, 1902, 942, 8016, 2468, 62, 724] combine_rows_per_src=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=284737536 dispatch_meta_bytes=352304 combine_payload_bytes=284737536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285089840 ingress_bytes=319340544 link_bytes=319340544 token_bytes=569827376 weight_movement_bytes=34603008 total_nvlink_bytes=604430384 weight_to_token_ratio=0.060725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201652758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79694 payload_rows_remote=66240 dup_rows=18610 dup_rows_remote=10994 dedup_ratio=0.189311 router_tokens_per_expert=[30, 215, 651, 7, 0, 2698, 0, 24, 415, 94, 0, 2103, 6373, 112, 69, 28, 27, 81, 3506, 0, 0, 353, 0, 0, 466, 18, 1807, 111, 42, 98, 499, 10, 634, 0, 0, 0, 0, 3632, 0, 6269, 2254, 1051, 42, 3480, 0, 271, 956, 187, 0, 72, 253, 8, 4, 1034, 217, 4699, 2, 101, 511, 0, 60, 2136, 1369, 73] payload_rows_per_dst=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] dup_rows_per_dst=[658, 4624, 104, 2114, 7616, 528, 1100, 1866] combine_rows_per_src=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333053952 dispatch_meta_bytes=350000 combine_payload_bytes=333053952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=333403952 ingress_bytes=436862976 link_bytes=436862976 token_bytes=666457904 weight_movement_bytes=103809024 total_nvlink_bytes=770266928 weight_to_token_ratio=0.155762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.643880208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79310 payload_rows_remote=66752 dup_rows=18994 dup_rows_remote=17420 dedup_ratio=0.193217 router_tokens_per_expert=[979, 128, 30, 0, 389, 225, 2862, 0, 17, 2581, 1425, 2729, 0, 3652, 46, 3005, 842, 18, 837, 2347, 4, 4, 298, 733, 0, 239, 3973, 267, 2601, 294, 189, 4, 0, 35, 20, 1688, 207, 1826, 2, 4435, 11, 115, 24, 140, 155, 0, 11, 4, 0, 6023, 0, 653, 5, 30, 34, 321, 337, 0, 0, 3, 0, 130, 2129, 96] payload_rows_per_dst=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] dup_rows_per_dst=[1126, 4094, 774, 3330, 4310, 3404, 1574, 382] combine_rows_per_src=[8100, 9502, 7708, 11804, 12116, 10830, 12558, 6692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273440768 dispatch_meta_bytes=342672 combine_payload_bytes=273440768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273783440 ingress_bytes=342646784 link_bytes=342646784 token_bytes=547224208 weight_movement_bytes=69206016 total_nvlink_bytes=616430224 weight_to_token_ratio=0.126467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289352263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273686528 dispatch_meta_bytes=328656 combine_payload_bytes=273686528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274015184 ingress_bytes=342892544 link_bytes=342892544 token_bytes=547701712 weight_movement_bytes=69206016 total_nvlink_bytes=616907728 weight_to_token_ratio=0.126357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290277038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=275054592 dispatch_meta_bytes=344640 combine_payload_bytes=275054592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275399232 ingress_bytes=344260608 link_bytes=344260608 token_bytes=550453824 weight_movement_bytes=69206016 total_nvlink_bytes=619659840 weight_to_token_ratio=0.125725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295424953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271319040 dispatch_meta_bytes=308936 combine_payload_bytes=271319040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271627976 ingress_bytes=340525056 link_bytes=340525056 token_bytes=542947016 weight_movement_bytes=69206016 total_nvlink_bytes=612153032 weight_to_token_ratio=0.127464 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281368371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82798 payload_rows_remote=67232 dup_rows=15506 dup_rows_remote=14808 dedup_ratio=0.157735 router_tokens_per_expert=[21, 981, 2325, 3, 0, 199, 219, 0, 0, 1005, 3, 1153, 2554, 7, 0, 0, 0, 7048, 104, 0, 0, 0, 2, 0, 85, 33, 217, 23, 4, 204, 3216, 178, 0, 2, 2, 0, 1, 80, 3, 407, 930, 7610, 11, 135, 932, 6476, 48, 149, 6, 0, 10, 0, 4170, 2, 47, 8025, 4, 133, 99, 0, 7, 137, 138, 4] payload_rows_per_dst=[6850, 7496, 14256, 7800, 550, 14118, 16162, 15566] dup_rows_per_dst=[646, 1948, 52, 390, 130, 2974, 8668, 698] combine_rows_per_src=[6850, 7496, 14256, 7800, 550, 14118, 16162, 15566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78288 payload_rows_remote=71346 dup_rows=20016 dup_rows_remote=19768 dedup_ratio=0.203613 router_tokens_per_expert=[21, 89, 416, 0, 0, 2759, 0, 30, 284, 0, 0, 2444, 6367, 79, 4, 20, 92, 21, 2571, 0, 0, 198, 0, 0, 268, 0, 3099, 80, 47, 53, 240, 5, 334, 0, 0, 0, 0, 4020, 0, 7093, 2665, 304, 7, 3871, 0, 59, 759, 105, 0, 244, 322, 2, 0, 466, 423, 3862, 1, 39, 545, 0, 45, 3489, 1231, 79] payload_rows_per_dst=[6112, 13496, 5714, 11696, 14674, 6942, 9794, 9860] dup_rows_per_dst=[518, 4900, 50, 3630, 8220, 248, 844, 1606] combine_rows_per_src=[6112, 13496, 5714, 11696, 14674, 6942, 9794, 9860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77394 payload_rows_remote=67152 dup_rows=20910 dup_rows_remote=19008 dedup_ratio=0.212708 router_tokens_per_expert=[1059, 91, 98, 0, 140, 219, 4358, 1, 74, 1120, 1905, 2217, 3, 4425, 142, 4134, 2276, 246, 968, 1480, 123, 5, 334, 141, 0, 102, 2775, 734, 2009, 247, 308, 2, 17, 65, 2, 1293, 1127, 532, 3, 2273, 238, 46, 26, 148, 574, 0, 23, 297, 204, 6348, 0, 1091, 0, 143, 93, 574, 272, 0, 0, 14, 0, 66, 1690, 257] payload_rows_per_dst=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] dup_rows_per_dst=[1370, 5426, 1902, 942, 8016, 2468, 62, 724] combine_rows_per_src=[10562, 11844, 10242, 5862, 15304, 11006, 4148, 8426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333053952 dispatch_meta_bytes=350000 combine_payload_bytes=333053952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=333403952 ingress_bytes=436862976 link_bytes=436862976 token_bytes=666457904 weight_movement_bytes=103809024 total_nvlink_bytes=770266928 weight_to_token_ratio=0.155762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.643880208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273416192 dispatch_meta_bytes=336688 combine_payload_bytes=273416192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273752880 ingress_bytes=377225216 link_bytes=377225216 token_bytes=547169072 weight_movement_bytes=103809024 total_nvlink_bytes=650978096 weight_to_token_ratio=0.189720 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.419468119e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80372 payload_rows_remote=70978 dup_rows=17932 dup_rows_remote=16610 dedup_ratio=0.182414 router_tokens_per_expert=[7, 379, 1375, 0, 0, 2184, 1, 22, 164, 135, 0, 2560, 6624, 70, 2, 11, 8, 915, 2484, 0, 0, 234, 0, 0, 210, 25, 1305, 110, 0, 119, 1283, 41, 336, 0, 0, 0, 0, 2527, 0, 4702, 3632, 2967, 46, 2265, 0, 1218, 448, 274, 0, 139, 176, 8, 80, 622, 250, 5662, 2, 128, 409, 0, 62, 1468, 1391, 72] payload_rows_per_dst=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] dup_rows_per_dst=[5490, 1212, 250, 1322, 5126, 1454, 774, 2304] combine_rows_per_src=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82798 payload_rows_remote=67232 dup_rows=15506 dup_rows_remote=14808 dedup_ratio=0.157735 router_tokens_per_expert=[21, 981, 2325, 3, 0, 199, 219, 0, 0, 1005, 3, 1153, 2554, 7, 0, 0, 0, 7048, 104, 0, 0, 0, 2, 0, 85, 33, 217, 23, 4, 204, 3216, 178, 0, 2, 2, 0, 1, 80, 3, 407, 930, 7610, 11, 135, 932, 6476, 48, 149, 6, 0, 10, 0, 4170, 2, 47, 8025, 4, 133, 99, 0, 7, 137, 138, 4] payload_rows_per_dst=[6850, 7496, 14256, 7800, 550, 14118, 16162, 15566] dup_rows_per_dst=[646, 1948, 52, 390, 130, 2974, 8668, 698] combine_rows_per_src=[6850, 7496, 14256, 7800, 550, 14118, 16162, 15566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292233216 dispatch_meta_bytes=364456 combine_payload_bytes=292233216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=292597672 ingress_bytes=430645248 link_bytes=430645248 token_bytes=584830888 weight_movement_bytes=138412032 total_nvlink_bytes=723242920 weight_to_token_ratio=0.236670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.620483398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=275054592 dispatch_meta_bytes=344640 combine_payload_bytes=275054592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275399232 ingress_bytes=344260608 link_bytes=344260608 token_bytes=550453824 weight_movement_bytes=69206016 total_nvlink_bytes=619659840 weight_to_token_ratio=0.125725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295424953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290725888 dispatch_meta_bytes=350352 combine_payload_bytes=290725888 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291076240 ingress_bytes=359931904 link_bytes=359931904 token_bytes=581802128 weight_movement_bytes=69206016 total_nvlink_bytes=651008144 weight_to_token_ratio=0.118951 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354394778e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80862 payload_rows_remote=66818 dup_rows=17442 dup_rows_remote=15346 dedup_ratio=0.177429 router_tokens_per_expert=[1100, 22, 8, 0, 80, 156, 6704, 0, 0, 468, 512, 414, 13, 4933, 1402, 1991, 6605, 96, 96, 1312, 203, 0, 1457, 86, 0, 9, 2096, 154, 422, 47, 2284, 2, 0, 438, 0, 699, 569, 154, 2, 1049, 394, 7, 0, 59, 1307, 0, 11, 5, 1008, 4302, 0, 422, 0, 2972, 2, 90, 55, 0, 0, 171, 0, 210, 289, 2265] payload_rows_per_dst=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] dup_rows_per_dst=[2096, 1510, 8390, 882, 1714, 1738, 920, 192] combine_rows_per_src=[14044, 8090, 15512, 8020, 9646, 11694, 8068, 5788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79694 payload_rows_remote=66240 dup_rows=18610 dup_rows_remote=10994 dedup_ratio=0.189311 router_tokens_per_expert=[30, 215, 651, 7, 0, 2698, 0, 24, 415, 94, 0, 2103, 6373, 112, 69, 28, 27, 81, 3506, 0, 0, 353, 0, 0, 466, 18, 1807, 111, 42, 98, 499, 10, 634, 0, 0, 0, 0, 3632, 0, 6269, 2254, 1051, 42, 3480, 0, 271, 956, 187, 0, 72, 253, 8, 4, 1034, 217, 4699, 2, 101, 511, 0, 60, 2136, 1369, 73] payload_rows_per_dst=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] dup_rows_per_dst=[658, 4624, 104, 2114, 7616, 528, 1100, 1866] combine_rows_per_src=[6592, 13764, 7830, 10948, 13454, 6892, 11474, 8740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=275382272 dispatch_meta_bytes=328160 combine_payload_bytes=275382272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275710432 ingress_bytes=309985280 link_bytes=309985280 token_bytes=551092704 weight_movement_bytes=34603008 total_nvlink_bytes=585695712 weight_to_token_ratio=0.062790 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166449653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75290 payload_rows_remote=70880 dup_rows=23014 dup_rows_remote=23000 dedup_ratio=0.234111 router_tokens_per_expert=[0, 726, 1211, 0, 0, 8, 402, 0, 1, 1468, 0, 726, 441, 2, 1, 0, 0, 7607, 17, 0, 0, 0, 0, 0, 1, 0, 10, 2, 0, 175, 3151, 0, 0, 0, 0, 0, 0, 12, 0, 14, 78, 7899, 0, 17, 3561, 7600, 9, 5, 5, 1, 0, 0, 5707, 0, 9, 8180, 0, 88, 1, 0, 1, 4, 10, 2] payload_rows_per_dst=[4854, 4410, 15800, 6838, 24, 15572, 11966, 15826] dup_rows_per_dst=[722, 14, 6664, 380, 0, 13690, 1360, 184] combine_rows_per_src=[4854, 4410, 15800, 6838, 24, 15572, 11966, 15826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79452 payload_rows_remote=71012 dup_rows=18852 dup_rows_remote=18230 dedup_ratio=0.191772 router_tokens_per_expert=[2, 28, 317, 0, 0, 3673, 0, 63, 673, 13, 0, 1313, 5676, 43, 10, 59, 41, 9, 2928, 0, 0, 258, 0, 0, 147, 5, 3388, 159, 0, 37, 307, 0, 398, 0, 0, 0, 0, 4364, 0, 7155, 2467, 87, 0, 5209, 0, 25, 1614, 134, 0, 281, 146, 7, 0, 831, 244, 3022, 1, 8, 517, 0, 44, 2188, 1049, 212] payload_rows_per_dst=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] dup_rows_per_dst=[432, 2828, 54, 4976, 8842, 422, 622, 676] combine_rows_per_src=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=273686528 dispatch_meta_bytes=328656 combine_payload_bytes=273686528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274015184 ingress_bytes=342892544 link_bytes=342892544 token_bytes=547701712 weight_movement_bytes=69206016 total_nvlink_bytes=616907728 weight_to_token_ratio=0.126357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290277038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271319040 dispatch_meta_bytes=308936 combine_payload_bytes=271319040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271627976 ingress_bytes=340525056 link_bytes=340525056 token_bytes=542947016 weight_movement_bytes=69206016 total_nvlink_bytes=612153032 weight_to_token_ratio=0.127464 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281368371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292233216 dispatch_meta_bytes=364456 combine_payload_bytes=292233216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=292597672 ingress_bytes=430645248 link_bytes=430645248 token_bytes=584830888 weight_movement_bytes=138412032 total_nvlink_bytes=723242920 weight_to_token_ratio=0.236670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.620483398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80372 payload_rows_remote=70978 dup_rows=17932 dup_rows_remote=16610 dedup_ratio=0.182414 router_tokens_per_expert=[7, 379, 1375, 0, 0, 2184, 1, 22, 164, 135, 0, 2560, 6624, 70, 2, 11, 8, 915, 2484, 0, 0, 234, 0, 0, 210, 25, 1305, 110, 0, 119, 1283, 41, 336, 0, 0, 0, 0, 2527, 0, 4702, 3632, 2967, 46, 2265, 0, 1218, 448, 274, 0, 139, 176, 8, 80, 622, 250, 5662, 2, 128, 409, 0, 62, 1468, 1391, 72] payload_rows_per_dst=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] dup_rows_per_dst=[5490, 1212, 250, 1322, 5126, 1454, 774, 2304] combine_rows_per_src=[13452, 6914, 7032, 9394, 10004, 9782, 13100, 10694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71884 payload_rows_remote=56334 dup_rows=26420 dup_rows_remote=18252 dedup_ratio=0.268758 router_tokens_per_expert=[13, 2206, 1658, 0, 1, 1159, 44, 12, 48, 212, 0, 3049, 6766, 78, 22, 6, 20, 3840, 1967, 0, 0, 24, 1, 0, 300, 63, 1325, 16, 9, 172, 569, 353, 122, 0, 0, 0, 0, 959, 0, 1979, 2661, 5688, 40, 688, 2, 2125, 152, 330, 0, 40, 197, 0, 239, 35, 90, 7207, 24, 738, 472, 0, 44, 844, 517, 26] payload_rows_per_dst=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] dup_rows_per_dst=[8168, 1308, 10964, 1748, 212, 2496, 14, 1510] combine_rows_per_src=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82754 payload_rows_remote=74910 dup_rows=15550 dup_rows_remote=15432 dedup_ratio=0.158183 router_tokens_per_expert=[1, 509, 499, 7, 0, 2410, 0, 11, 106, 47, 0, 3582, 6826, 7, 6, 58, 3, 83, 3830, 1, 0, 64, 0, 0, 441, 25, 2982, 196, 69, 87, 244, 2, 96, 0, 0, 0, 0, 3265, 0, 5670, 1797, 1057, 28, 2978, 0, 94, 716, 105, 0, 113, 190, 0, 0, 359, 760, 4977, 2, 84, 511, 0, 56, 3119, 977, 102] payload_rows_per_dst=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] dup_rows_per_dst=[5524, 1734, 118, 3182, 1656, 168, 1086, 2082] combine_rows_per_src=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79106 payload_rows_remote=72500 dup_rows=19198 dup_rows_remote=18900 dedup_ratio=0.195292 router_tokens_per_expert=[0, 0, 3, 412, 192, 3203, 18, 119, 5, 4, 136, 100, 2367, 150, 728, 1535, 6402, 163, 319, 888, 357, 1770, 185, 2, 233, 1192, 4251, 512, 66, 795, 0, 530, 1166, 125, 0, 2, 1746, 74, 339, 0, 219, 144, 0, 92, 81, 2108, 4, 210, 1923, 1300, 487, 184, 253, 2571, 6, 93, 3040, 3, 1546, 0, 1000, 178, 3620, 1] payload_rows_per_dst=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] dup_rows_per_dst=[4066, 1504, 666, 3536, 298, 4786, 2934, 1408] combine_rows_per_src=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290324480 dispatch_meta_bytes=375520 combine_payload_bytes=290324480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290700000 ingress_bytes=359530496 link_bytes=359530496 token_bytes=581024480 weight_movement_bytes=69206016 total_nvlink_bytes=650230496 weight_to_token_ratio=0.119110 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352884312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79452 payload_rows_remote=71012 dup_rows=18852 dup_rows_remote=18230 dedup_ratio=0.191772 router_tokens_per_expert=[2, 28, 317, 0, 0, 3673, 0, 63, 673, 13, 0, 1313, 5676, 43, 10, 59, 41, 9, 2928, 0, 0, 258, 0, 0, 147, 5, 3388, 159, 0, 37, 307, 0, 398, 0, 0, 0, 0, 4364, 0, 7155, 2467, 87, 0, 5209, 0, 25, 1614, 134, 0, 281, 146, 7, 0, 831, 244, 3022, 1, 8, 517, 0, 44, 2188, 1049, 212] payload_rows_per_dst=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] dup_rows_per_dst=[432, 2828, 54, 4976, 8842, 422, 622, 676] combine_rows_per_src=[7734, 12746, 6418, 13528, 14992, 8058, 8440, 7536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290725888 dispatch_meta_bytes=350352 combine_payload_bytes=290725888 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291076240 ingress_bytes=359931904 link_bytes=359931904 token_bytes=581802128 weight_movement_bytes=69206016 total_nvlink_bytes=651008144 weight_to_token_ratio=0.118951 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354394778e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=298344 combine_payload_bytes=230744064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231042408 ingress_bytes=265347072 link_bytes=265347072 token_bytes=461786472 weight_movement_bytes=34603008 total_nvlink_bytes=496389480 weight_to_token_ratio=0.074933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.984796697e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=306831360 dispatch_meta_bytes=361368 combine_payload_bytes=306831360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307192728 ingress_bytes=341434368 link_bytes=341434368 token_bytes=614024088 weight_movement_bytes=34603008 total_nvlink_bytes=648627096 weight_to_token_ratio=0.056354 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284790039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=365600 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297325600 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594285600 weight_movement_bytes=34603008 total_nvlink_bytes=628888608 weight_to_token_ratio=0.058226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73188 payload_rows_remote=59994 dup_rows=25116 dup_rows_remote=20574 dedup_ratio=0.255493 router_tokens_per_expert=[2, 29, 0, 11, 7, 7, 8, 1867, 2536, 0, 256, 25, 13, 1588, 5, 89, 286, 80, 40, 1301, 4, 2479, 27, 0, 106, 243, 8018, 2, 0, 8, 0, 26, 0, 498, 0, 0, 7936, 0, 2, 0, 10, 112, 1, 70, 4123, 17, 5, 20, 7673, 70, 468, 663, 24, 86, 14, 83, 346, 4012, 12, 2, 5, 1320, 445, 2072] payload_rows_per_dst=[4576, 7832, 7602, 1070, 15992, 8986, 13936, 13194] dup_rows_per_dst=[176, 1192, 260, 234, 16382, 302, 2028, 4542] combine_rows_per_src=[4576, 7832, 7602, 1070, 15992, 8986, 13936, 13194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79946 payload_rows_remote=64872 dup_rows=18358 dup_rows_remote=13520 dedup_ratio=0.186747 router_tokens_per_expert=[0, 0, 0, 588, 94, 3074, 19, 28, 0, 0, 87, 22, 2672, 22, 475, 1816, 7276, 304, 302, 775, 183, 1811, 285, 0, 159, 1228, 3167, 139, 81, 437, 0, 980, 1058, 7, 0, 0, 1327, 65, 156, 0, 69, 401, 0, 12, 19, 1993, 0, 186, 910, 3057, 317, 60, 224, 3180, 13, 48, 3143, 1, 1319, 0, 877, 9, 4677, 0] payload_rows_per_dst=[12282, 8990, 6662, 9998, 5138, 15074, 12044, 9758] dup_rows_per_dst=[4678, 1198, 658, 2384, 88, 4838, 3574, 940] combine_rows_per_src=[12282, 8990, 6662, 9998, 5138, 15074, 12044, 9758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82754 payload_rows_remote=74910 dup_rows=15550 dup_rows_remote=15432 dedup_ratio=0.158183 router_tokens_per_expert=[1, 509, 499, 7, 0, 2410, 0, 11, 106, 47, 0, 3582, 6826, 7, 6, 58, 3, 83, 3830, 1, 0, 64, 0, 0, 441, 25, 2982, 196, 69, 87, 244, 2, 96, 0, 0, 0, 0, 3265, 0, 5670, 1797, 1057, 28, 2978, 0, 94, 716, 105, 0, 113, 190, 0, 0, 359, 760, 4977, 2, 84, 511, 0, 56, 3119, 977, 102] payload_rows_per_dst=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] dup_rows_per_dst=[5524, 1734, 118, 3182, 1656, 168, 1086, 2082] combine_rows_per_src=[15002, 11430, 7844, 10866, 10854, 5312, 11712, 9734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290865152 dispatch_meta_bytes=356968 combine_payload_bytes=290865152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291222120 ingress_bytes=394674176 link_bytes=394674176 token_bytes=582087272 weight_movement_bytes=103809024 total_nvlink_bytes=685896296 weight_to_token_ratio=0.178339 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=290324480 dispatch_meta_bytes=375520 combine_payload_bytes=290324480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290700000 ingress_bytes=359530496 link_bytes=359530496 token_bytes=581024480 weight_movement_bytes=69206016 total_nvlink_bytes=650230496 weight_to_token_ratio=0.119110 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352884312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83172 payload_rows_remote=69380 dup_rows=15132 dup_rows_remote=12572 dedup_ratio=0.153931 router_tokens_per_expert=[0, 0, 0, 230, 51, 2217, 15, 431, 31, 0, 162, 244, 1508, 475, 308, 639, 4721, 149, 248, 1356, 402, 2833, 234, 0, 206, 857, 5734, 135, 16, 496, 0, 732, 777, 259, 0, 0, 3857, 70, 197, 0, 224, 48, 0, 78, 214, 1318, 13, 203, 3128, 605, 779, 670, 258, 1698, 9, 121, 3535, 110, 980, 0, 750, 875, 3709, 237] payload_rows_per_dst=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] dup_rows_per_dst=[3060, 584, 3152, 2560, 456, 424, 3020, 1876] combine_rows_per_src=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73188 payload_rows_remote=59994 dup_rows=25116 dup_rows_remote=20574 dedup_ratio=0.255493 router_tokens_per_expert=[2, 29, 0, 11, 7, 7, 8, 1867, 2536, 0, 256, 25, 13, 1588, 5, 89, 286, 80, 40, 1301, 4, 2479, 27, 0, 106, 243, 8018, 2, 0, 8, 0, 26, 0, 498, 0, 0, 7936, 0, 2, 0, 10, 112, 1, 70, 4123, 17, 5, 20, 7673, 70, 468, 663, 24, 86, 14, 83, 346, 4012, 12, 2, 5, 1320, 445, 2072] payload_rows_per_dst=[4576, 7832, 7602, 1070, 15992, 8986, 13936, 13194] dup_rows_per_dst=[176, 1192, 260, 234, 16382, 302, 2028, 4542] combine_rows_per_src=[4576, 7832, 7602, 1070, 15992, 8986, 13936, 13194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=313568 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266029280 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531744992 weight_movement_bytes=34603008 total_nvlink_bytes=566348000 weight_to_token_ratio=0.065074 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=306831360 dispatch_meta_bytes=361368 combine_payload_bytes=306831360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307192728 ingress_bytes=341434368 link_bytes=341434368 token_bytes=614024088 weight_movement_bytes=34603008 total_nvlink_bytes=648627096 weight_to_token_ratio=0.056354 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284790039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=284180480 dispatch_meta_bytes=327808 combine_payload_bytes=284180480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284508288 ingress_bytes=387989504 link_bytes=387989504 token_bytes=568688768 weight_movement_bytes=103809024 total_nvlink_bytes=672497792 weight_to_token_ratio=0.182541 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.459973268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71884 payload_rows_remote=56334 dup_rows=26420 dup_rows_remote=18252 dedup_ratio=0.268758 router_tokens_per_expert=[13, 2206, 1658, 0, 1, 1159, 44, 12, 48, 212, 0, 3049, 6766, 78, 22, 6, 20, 3840, 1967, 0, 0, 24, 1, 0, 300, 63, 1325, 16, 9, 172, 569, 353, 122, 0, 0, 0, 0, 959, 0, 1979, 2661, 5688, 40, 688, 2, 2125, 152, 330, 0, 40, 197, 0, 239, 35, 90, 7207, 24, 738, 472, 0, 44, 844, 517, 26] payload_rows_per_dst=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] dup_rows_per_dst=[8168, 1308, 10964, 1748, 212, 2496, 14, 1510] combine_rows_per_src=[15550, 9480, 15154, 9492, 1950, 9648, 1188, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79106 payload_rows_remote=72500 dup_rows=19198 dup_rows_remote=18900 dedup_ratio=0.195292 router_tokens_per_expert=[0, 0, 3, 412, 192, 3203, 18, 119, 5, 4, 136, 100, 2367, 150, 728, 1535, 6402, 163, 319, 888, 357, 1770, 185, 2, 233, 1192, 4251, 512, 66, 795, 0, 530, 1166, 125, 0, 2, 1746, 74, 339, 0, 219, 144, 0, 92, 81, 2108, 4, 210, 1923, 1300, 487, 184, 253, 2571, 6, 93, 3040, 3, 1546, 0, 1000, 178, 3620, 1] payload_rows_per_dst=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] dup_rows_per_dst=[4066, 1504, 666, 3536, 298, 4786, 2934, 1408] combine_rows_per_src=[11068, 8546, 6702, 11622, 6606, 13734, 10700, 10128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=245735424 dispatch_meta_bytes=322272 combine_payload_bytes=245735424 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=246057696 ingress_bytes=349544448 link_bytes=349544448 token_bytes=491793120 weight_movement_bytes=103809024 total_nvlink_bytes=595602144 weight_to_token_ratio=0.211083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315307617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=313568 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266029280 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531744992 weight_movement_bytes=34603008 total_nvlink_bytes=566348000 weight_to_token_ratio=0.065074 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81534 payload_rows_remote=66178 dup_rows=16770 dup_rows_remote=7300 dedup_ratio=0.170593 router_tokens_per_expert=[0, 1, 0, 0, 0, 6, 0, 1173, 4982, 0, 227, 15, 5, 571, 0, 0, 35, 0, 5, 173, 5, 914, 1, 0, 1, 4, 8155, 0, 4, 3, 0, 4, 2, 377, 0, 0, 8157, 2, 2, 0, 0, 5, 0, 1, 6685, 4, 0, 0, 7928, 3, 10, 463, 2, 12, 0, 0, 30, 5111, 2, 11, 1, 869, 41, 3150] payload_rows_per_dst=[2424, 15356, 2236, 16334, 16322, 13382, 1124, 14356] dup_rows_per_dst=[78, 9470, 30, 2474, 758, 8, 20, 3932] combine_rows_per_src=[2424, 15356, 2236, 16334, 16322, 13382, 1124, 14356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79626 payload_rows_remote=68464 dup_rows=18678 dup_rows_remote=16844 dedup_ratio=0.190002 router_tokens_per_expert=[0, 0, 5, 494, 133, 3401, 5, 11, 0, 0, 32, 51, 3271, 22, 567, 1828, 7224, 404, 432, 572, 105, 1183, 526, 0, 154, 761, 2282, 66, 13, 1395, 0, 1174, 1561, 25, 0, 0, 653, 96, 380, 2, 90, 109, 0, 20, 13, 3069, 6, 162, 257, 1451, 405, 27, 243, 4086, 5, 24, 3069, 0, 1601, 0, 1456, 42, 4189, 0] payload_rows_per_dst=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] dup_rows_per_dst=[5196, 1246, 656, 1896, 374, 6092, 1834, 1384] combine_rows_per_src=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=230744064 dispatch_meta_bytes=298344 combine_payload_bytes=230744064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231042408 ingress_bytes=265347072 link_bytes=265347072 token_bytes=461786472 weight_movement_bytes=34603008 total_nvlink_bytes=496389480 weight_to_token_ratio=0.074933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.984796697e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=365600 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297325600 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594285600 weight_movement_bytes=34603008 total_nvlink_bytes=628888608 weight_to_token_ratio=0.058226 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76586 payload_rows_remote=69018 dup_rows=21718 dup_rows_remote=20298 dedup_ratio=0.220927 router_tokens_per_expert=[43, 2199, 1, 258, 49, 879, 1456, 0, 4378, 371, 228, 678, 0, 67, 645, 341, 0, 0, 134, 99, 0, 3713, 962, 2, 0, 66, 2249, 793, 4, 153, 0, 0, 128, 592, 101, 333, 316, 3900, 125, 894, 652, 482, 3956, 337, 68, 54, 0, 2901, 66, 133, 0, 962, 3, 1917, 0, 1815, 0, 1462, 1885, 2678, 1, 3623, 0, 0] payload_rows_per_dst=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] dup_rows_per_dst=[896, 178, 662, 1340, 4834, 1420, 4292, 8096] combine_rows_per_src=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83172 payload_rows_remote=69380 dup_rows=15132 dup_rows_remote=12572 dedup_ratio=0.153931 router_tokens_per_expert=[0, 0, 0, 230, 51, 2217, 15, 431, 31, 0, 162, 244, 1508, 475, 308, 639, 4721, 149, 248, 1356, 402, 2833, 234, 0, 206, 857, 5734, 135, 16, 496, 0, 732, 777, 259, 0, 0, 3857, 70, 197, 0, 224, 48, 0, 78, 214, 1318, 13, 203, 3128, 605, 779, 670, 258, 1698, 9, 121, 3535, 110, 980, 0, 750, 875, 3709, 237] payload_rows_per_dst=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] dup_rows_per_dst=[3060, 584, 3152, 2560, 456, 424, 3020, 1876] combine_rows_per_src=[10246, 6150, 13872, 13792, 9864, 6634, 11516, 11098] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77838 payload_rows_remote=61756 dup_rows=20466 dup_rows_remote=13174 dedup_ratio=0.208191 router_tokens_per_expert=[0, 9, 1, 786, 316, 2386, 88, 141, 17, 0, 135, 50, 1272, 47, 290, 1738, 7318, 270, 913, 1025, 197, 1722, 242, 0, 127, 1087, 4028, 311, 119, 724, 0, 521, 506, 309, 0, 3, 1946, 358, 61, 1, 9, 394, 1, 206, 89, 1067, 0, 193, 2255, 2711, 415, 77, 248, 2750, 11, 56, 2891, 2, 1437, 0, 630, 49, 4594, 3] payload_rows_per_dst=[12296, 6698, 16082, 11450, 6052, 3660, 12682, 8918] dup_rows_per_dst=[4482, 400, 7292, 2384, 316, 258, 4364, 970] combine_rows_per_src=[12296, 6698, 16082, 11450, 6052, 3660, 12682, 8918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=271065088 dispatch_meta_bytes=293912 combine_payload_bytes=271065088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271359000 ingress_bytes=305668096 link_bytes=305668096 token_bytes=542424088 weight_movement_bytes=34603008 total_nvlink_bytes=577027096 weight_to_token_ratio=0.063793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150204437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78180 payload_rows_remote=67750 dup_rows=20124 dup_rows_remote=17310 dedup_ratio=0.204712 router_tokens_per_expert=[24, 2264, 0, 1668, 998, 2199, 884, 0, 4718, 37, 320, 416, 0, 27, 239, 1770, 0, 26, 57, 20, 0, 1800, 2067, 0, 0, 66, 2814, 1381, 8, 154, 0, 0, 882, 3025, 352, 104, 142, 1880, 63, 350, 237, 524, 5347, 344, 82, 80, 0, 1162, 215, 652, 0, 2082, 5, 1348, 0, 796, 1, 1330, 959, 1215, 2, 2016, 0, 0] payload_rows_per_dst=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] dup_rows_per_dst=[1996, 3006, 2374, 2814, 386, 2990, 2666, 3892] combine_rows_per_src=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=341232 combine_payload_bytes=280428544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=280769776 ingress_bytes=384237568 link_bytes=384237568 token_bytes=561198320 weight_movement_bytes=103809024 total_nvlink_bytes=665007344 weight_to_token_ratio=0.184977 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445855035e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77432 payload_rows_remote=75362 dup_rows=20872 dup_rows_remote=20534 dedup_ratio=0.212321 router_tokens_per_expert=[30, 346, 0, 7366, 7543, 6427, 12, 0, 935, 0, 0, 19, 0, 18, 11, 4557, 0, 21, 41, 1, 13, 18, 2661, 0, 3, 81, 125, 329, 0, 0, 19, 0, 2853, 7832, 185, 4, 3, 41, 50, 17, 24, 147, 5261, 417, 28, 11, 0, 225, 13, 419, 0, 660, 0, 20, 4, 93, 60, 88, 94, 5, 1, 20, 1, 0] payload_rows_per_dst=[3272, 10418, 6062, 14860, 15148, 13896, 11706, 2070] dup_rows_per_dst=[1586, 1694, 380, 8992, 5890, 758, 1234, 338] combine_rows_per_src=[3272, 10418, 6062, 14860, 15148, 13896, 11706, 2070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282697728 dispatch_meta_bytes=357264 combine_payload_bytes=282697728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283054992 ingress_bytes=351903744 link_bytes=351903744 token_bytes=565752720 weight_movement_bytes=69206016 total_nvlink_bytes=634958736 weight_to_token_ratio=0.122326 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324185458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252952576 dispatch_meta_bytes=299720 combine_payload_bytes=252952576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253252296 ingress_bytes=322158592 link_bytes=322158592 token_bytes=506204872 weight_movement_bytes=69206016 total_nvlink_bytes=575410888 weight_to_token_ratio=0.136715 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212256846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76610 payload_rows_remote=67030 dup_rows=21694 dup_rows_remote=18878 dedup_ratio=0.220683 router_tokens_per_expert=[0, 1, 3, 97, 66, 447, 101, 821, 489, 0, 191, 230, 495, 744, 35, 838, 2512, 120, 89, 2823, 220, 3494, 136, 0, 162, 550, 7115, 99, 65, 231, 0, 95, 210, 496, 0, 0, 6506, 31, 56, 0, 70, 72, 0, 356, 1538, 317, 0, 63, 6399, 1067, 677, 862, 227, 630, 2, 218, 1962, 638, 445, 0, 80, 386, 2700, 875] payload_rows_per_dst=[9580, 14692, 15038, 8278, 13596, 4712, 5992, 4722] dup_rows_per_dst=[2816, 4150, 11608, 498, 1002, 120, 1374, 126] combine_rows_per_src=[9580, 14692, 15038, 8278, 13596, 4712, 5992, 4722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77762 payload_rows_remote=69146 dup_rows=20542 dup_rows_remote=19678 dedup_ratio=0.208964 router_tokens_per_expert=[30, 2351, 1, 418, 301, 1154, 1143, 1, 5084, 237, 676, 682, 0, 53, 449, 1147, 0, 2, 35, 29, 0, 2729, 1081, 5, 0, 59, 2686, 1124, 9, 136, 0, 0, 277, 1412, 235, 152, 366, 2740, 142, 713, 289, 595, 4225, 250, 17, 123, 0, 2058, 200, 588, 0, 1596, 3, 1893, 11, 1132, 0, 1685, 1638, 1814, 0, 3376, 0, 0] payload_rows_per_dst=[7542, 13200, 9760, 8546, 8616, 5672, 14182, 10244] dup_rows_per_dst=[948, 3226, 826, 1790, 864, 992, 5114, 6782] combine_rows_per_src=[7542, 13200, 9760, 8546, 8616, 5672, 14182, 10244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277504000 dispatch_meta_bytes=340240 combine_payload_bytes=277504000 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277844240 ingress_bytes=346710016 link_bytes=346710016 token_bytes=555348240 weight_movement_bytes=69206016 total_nvlink_bytes=624554256 weight_to_token_ratio=0.124617 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304641878e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=308682752 dispatch_meta_bytes=383584 combine_payload_bytes=308682752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309066336 ingress_bytes=343285760 link_bytes=343285760 token_bytes=617749088 weight_movement_bytes=34603008 total_nvlink_bytes=652352096 weight_to_token_ratio=0.056015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291756678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12727685112 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6367743992 ingress_bytes=7744061440 link_bytes=7744061440 total_nvlink_bytes=14111805432 weight_to_token_ratio=0.108749 prefetch_share_of_ingress=0.093835 dedup_ratio=0.205659 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266968685e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=271065088 dispatch_meta_bytes=293912 combine_payload_bytes=271065088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271359000 ingress_bytes=305668096 link_bytes=305668096 token_bytes=542424088 weight_movement_bytes=34603008 total_nvlink_bytes=577027096 weight_to_token_ratio=0.063793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150204437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=284180480 dispatch_meta_bytes=327808 combine_payload_bytes=284180480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284508288 ingress_bytes=387989504 link_bytes=387989504 token_bytes=568688768 weight_movement_bytes=103809024 total_nvlink_bytes=672497792 weight_to_token_ratio=0.182541 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.459973268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79626 payload_rows_remote=68464 dup_rows=18678 dup_rows_remote=16844 dedup_ratio=0.190002 router_tokens_per_expert=[0, 0, 5, 494, 133, 3401, 5, 11, 0, 0, 32, 51, 3271, 22, 567, 1828, 7224, 404, 432, 572, 105, 1183, 526, 0, 154, 761, 2282, 66, 13, 1395, 0, 1174, 1561, 25, 0, 0, 653, 96, 380, 2, 90, 109, 0, 20, 13, 3069, 6, 162, 257, 1451, 405, 27, 243, 4086, 5, 24, 3069, 0, 1601, 0, 1456, 42, 4189, 0] payload_rows_per_dst=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] dup_rows_per_dst=[5196, 1246, 656, 1896, 374, 6092, 1834, 1384] combine_rows_per_src=[11280, 10296, 5788, 9794, 5060, 15294, 11162, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252952576 dispatch_meta_bytes=299720 combine_payload_bytes=252952576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=253252296 ingress_bytes=322158592 link_bytes=322158592 token_bytes=506204872 weight_movement_bytes=69206016 total_nvlink_bytes=575410888 weight_to_token_ratio=0.136715 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212256846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=274554880 dispatch_meta_bytes=343632 combine_payload_bytes=274554880 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274898512 ingress_bytes=343760896 link_bytes=343760896 token_bytes=549453392 weight_movement_bytes=69206016 total_nvlink_bytes=618659408 weight_to_token_ratio=0.125954 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293544577e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=283222016 dispatch_meta_bytes=355296 combine_payload_bytes=283222016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283577312 ingress_bytes=352428032 link_bytes=352428032 token_bytes=566799328 weight_movement_bytes=69206016 total_nvlink_bytes=636005344 weight_to_token_ratio=0.122100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326158312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77432 payload_rows_remote=75362 dup_rows=20872 dup_rows_remote=20534 dedup_ratio=0.212321 router_tokens_per_expert=[30, 346, 0, 7366, 7543, 6427, 12, 0, 935, 0, 0, 19, 0, 18, 11, 4557, 0, 21, 41, 1, 13, 18, 2661, 0, 3, 81, 125, 329, 0, 0, 19, 0, 2853, 7832, 185, 4, 3, 41, 50, 17, 24, 147, 5261, 417, 28, 11, 0, 225, 13, 419, 0, 660, 0, 20, 4, 93, 60, 88, 94, 5, 1, 20, 1, 0] payload_rows_per_dst=[3272, 10418, 6062, 14860, 15148, 13896, 11706, 2070] dup_rows_per_dst=[1586, 1694, 380, 8992, 5890, 758, 1234, 338] combine_rows_per_src=[3272, 10418, 6062, 14860, 15148, 13896, 11706, 2070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76586 payload_rows_remote=69018 dup_rows=21718 dup_rows_remote=20298 dedup_ratio=0.220927 router_tokens_per_expert=[43, 2199, 1, 258, 49, 879, 1456, 0, 4378, 371, 228, 678, 0, 67, 645, 341, 0, 0, 134, 99, 0, 3713, 962, 2, 0, 66, 2249, 793, 4, 153, 0, 0, 128, 592, 101, 333, 316, 3900, 125, 894, 652, 482, 3956, 337, 68, 54, 0, 2901, 66, 133, 0, 962, 3, 1917, 0, 1815, 0, 1462, 1885, 2678, 1, 3623, 0, 0] payload_rows_per_dst=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] dup_rows_per_dst=[896, 178, 662, 1340, 4834, 1420, 4292, 8096] combine_rows_per_src=[7116, 4482, 10342, 6948, 15516, 7568, 13412, 11202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=280428544 dispatch_meta_bytes=341232 combine_payload_bytes=280428544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=280769776 ingress_bytes=384237568 link_bytes=384237568 token_bytes=561198320 weight_movement_bytes=103809024 total_nvlink_bytes=665007344 weight_to_token_ratio=0.184977 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445855035e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12878820176 prefetch_bytes=865075200 reduce_bytes=588251136 weight_movement_bytes=1453326336 egress_bytes=6443340624 ingress_bytes=7888805888 link_bytes=7888805888 total_nvlink_bytes=14332146512 weight_to_token_ratio=0.112846 prefetch_share_of_ingress=0.109659 dedup_ratio=0.201599 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=17 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290649629e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=308682752 dispatch_meta_bytes=383584 combine_payload_bytes=308682752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309066336 ingress_bytes=343285760 link_bytes=343285760 token_bytes=617749088 weight_movement_bytes=34603008 total_nvlink_bytes=652352096 weight_to_token_ratio=0.056015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291756678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78298 payload_rows_remote=70782 dup_rows=20006 dup_rows_remote=19978 dedup_ratio=0.203512 router_tokens_per_expert=[0, 7, 0, 7852, 8150, 6954, 0, 0, 54, 0, 0, 10, 0, 1, 4, 3703, 0, 3, 0, 0, 0, 14, 1649, 0, 0, 80, 34, 427, 0, 0, 0, 0, 4821, 8170, 44, 0, 0, 7, 0, 0, 1, 18, 4820, 255, 0, 2, 0, 5, 1, 872, 0, 112, 0, 7, 0, 13, 1046, 4, 2, 4, 0, 6, 0, 0] payload_rows_per_dst=[16350, 7516, 16364, 14078, 9698, 10172, 2004, 2116] dup_rows_per_dst=[15668, 28, 3308, 912, 46, 30, 6, 8] combine_rows_per_src=[16350, 7516, 16364, 14078, 9698, 10172, 2004, 2116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12853885432 prefetch_bytes=692060160 reduce_bytes=899678208 weight_movement_bytes=1591738368 egress_bytes=6430841336 ingress_bytes=8014782464 link_bytes=8014782464 total_nvlink_bytes=14445623800 weight_to_token_ratio=0.123833 prefetch_share_of_ingress=0.086348 dedup_ratio=0.195642 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=26 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=27559 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.311260051e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76808 payload_rows_remote=63098 dup_rows=21496 dup_rows_remote=17342 dedup_ratio=0.218669 router_tokens_per_expert=[70, 1795, 0, 83, 9, 392, 1714, 0, 3554, 251, 550, 899, 0, 32, 769, 410, 0, 0, 36, 42, 0, 4218, 498, 0, 0, 49, 2539, 554, 21, 380, 0, 0, 158, 213, 117, 207, 90, 5198, 69, 758, 669, 506, 3116, 298, 60, 75, 0, 2373, 440, 312, 0, 790, 3, 3106, 1, 1164, 0, 1304, 2064, 2934, 0, 4262, 0, 0] payload_rows_per_dst=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] dup_rows_per_dst=[820, 418, 280, 1008, 1160, 1356, 4154, 12300] combine_rows_per_src=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12057787496 prefetch_bytes=657457152 reduce_bytes=865075200 weight_movement_bytes=1522532352 egress_bytes=6032571496 ingress_bytes=7547748352 link_bytes=7547748352 total_nvlink_bytes=13580319848 weight_to_token_ratio=0.126270 prefetch_share_of_ingress=0.087106 dedup_ratio=0.224767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=25 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234850844e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282697728 dispatch_meta_bytes=357264 combine_payload_bytes=282697728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283054992 ingress_bytes=351903744 link_bytes=351903744 token_bytes=565752720 weight_movement_bytes=69206016 total_nvlink_bytes=634958736 weight_to_token_ratio=0.122326 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324185458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78180 payload_rows_remote=67750 dup_rows=20124 dup_rows_remote=17310 dedup_ratio=0.204712 router_tokens_per_expert=[24, 2264, 0, 1668, 998, 2199, 884, 0, 4718, 37, 320, 416, 0, 27, 239, 1770, 0, 26, 57, 20, 0, 1800, 2067, 0, 0, 66, 2814, 1381, 8, 154, 0, 0, 882, 3025, 352, 104, 142, 1880, 63, 350, 237, 524, 5347, 344, 82, 80, 0, 1162, 215, 652, 0, 2082, 5, 1348, 0, 796, 1, 1330, 959, 1215, 2, 2016, 0, 0] payload_rows_per_dst=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] dup_rows_per_dst=[1996, 3006, 2374, 2814, 386, 2990, 2666, 3892] combine_rows_per_src=[9680, 12048, 11616, 10430, 7160, 8754, 11338, 7154] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75420 payload_rows_remote=65818 dup_rows=22884 dup_rows_remote=21700 dedup_ratio=0.232788 router_tokens_per_expert=[6, 2004, 0, 539, 283, 987, 422, 0, 6267, 44, 37, 319, 0, 107, 555, 1525, 0, 18, 54, 98, 0, 2205, 1605, 2, 0, 83, 1616, 513, 81, 335, 85, 0, 169, 1411, 600, 173, 392, 2975, 409, 853, 699, 1230, 4613, 509, 30, 174, 0, 3894, 8, 339, 0, 1369, 2, 1297, 52, 1391, 0, 1674, 1688, 1206, 5, 2200, 0, 0] payload_rows_per_dst=[5908, 13898, 9602, 6604, 9462, 12780, 8180, 8986] dup_rows_per_dst=[600, 3810, 1184, 796, 1680, 9518, 736, 4560] combine_rows_per_src=[5908, 13898, 9602, 6604, 9462, 12780, 8180, 8986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12878820176 prefetch_bytes=865075200 reduce_bytes=588251136 weight_movement_bytes=1453326336 egress_bytes=6443340624 ingress_bytes=7888805888 link_bytes=7888805888 total_nvlink_bytes=14332146512 weight_to_token_ratio=0.112846 prefetch_share_of_ingress=0.109659 dedup_ratio=0.201599 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=17 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290649629e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289923072 dispatch_meta_bytes=363040 combine_payload_bytes=289923072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290286112 ingress_bytes=393732096 link_bytes=393732096 token_bytes=580209184 weight_movement_bytes=103809024 total_nvlink_bytes=684018208 weight_to_token_ratio=0.178917 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481582179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=321760 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258771168 ingress_bytes=293052416 link_bytes=293052416 token_bytes=517220576 weight_movement_bytes=34603008 total_nvlink_bytes=551823584 weight_to_token_ratio=0.066902 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102732649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78724 payload_rows_remote=65572 dup_rows=19580 dup_rows_remote=13332 dedup_ratio=0.199178 router_tokens_per_expert=[8, 1075, 0, 4188, 4144, 3754, 285, 0, 5817, 20, 102, 67, 0, 73, 111, 3299, 0, 72, 2, 0, 0, 659, 1328, 1, 0, 82, 779, 1389, 6, 128, 24, 0, 1042, 5843, 705, 79, 61, 802, 84, 118, 92, 130, 5856, 723, 4, 44, 0, 1047, 12, 406, 0, 1623, 0, 314, 4, 426, 0, 684, 700, 249, 1, 690, 0, 0] payload_rows_per_dst=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] dup_rows_per_dst=[6248, 4926, 2298, 1796, 280, 2356, 220, 1456] combine_rows_per_src=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12057787496 prefetch_bytes=657457152 reduce_bytes=865075200 weight_movement_bytes=1522532352 egress_bytes=6032571496 ingress_bytes=7547748352 link_bytes=7547748352 total_nvlink_bytes=13580319848 weight_to_token_ratio=0.126270 prefetch_share_of_ingress=0.087106 dedup_ratio=0.224767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=25 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234850844e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12727685112 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6367743992 ingress_bytes=7744061440 link_bytes=7744061440 total_nvlink_bytes=14111805432 weight_to_token_ratio=0.108749 prefetch_share_of_ingress=0.093835 dedup_ratio=0.205659 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266968685e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=289923072 dispatch_meta_bytes=363040 combine_payload_bytes=289923072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290286112 ingress_bytes=393732096 link_bytes=393732096 token_bytes=580209184 weight_movement_bytes=103809024 total_nvlink_bytes=684018208 weight_to_token_ratio=0.178917 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481582179e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277504000 dispatch_meta_bytes=340240 combine_payload_bytes=277504000 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=277844240 ingress_bytes=346710016 link_bytes=346710016 token_bytes=555348240 weight_movement_bytes=69206016 total_nvlink_bytes=624554256 weight_to_token_ratio=0.124617 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304641878e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76808 payload_rows_remote=63098 dup_rows=21496 dup_rows_remote=17342 dedup_ratio=0.218669 router_tokens_per_expert=[70, 1795, 0, 83, 9, 392, 1714, 0, 3554, 251, 550, 899, 0, 32, 769, 410, 0, 0, 36, 42, 0, 4218, 498, 0, 0, 49, 2539, 554, 21, 380, 0, 0, 158, 213, 117, 207, 90, 5198, 69, 758, 669, 506, 3116, 298, 60, 75, 0, 2373, 440, 312, 0, 790, 3, 3106, 1, 1164, 0, 1304, 2064, 2934, 0, 4262, 0, 0] payload_rows_per_dst=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] dup_rows_per_dst=[820, 418, 280, 1008, 1160, 1356, 4154, 12300] combine_rows_per_src=[6522, 5404, 9734, 6862, 12062, 6606, 13710, 15908] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=269590528 dispatch_meta_bytes=350072 combine_payload_bytes=269590528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269940600 ingress_bytes=304193536 link_bytes=304193536 token_bytes=539531128 weight_movement_bytes=34603008 total_nvlink_bytes=574134136 weight_to_token_ratio=0.064135 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144655786e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=268582912 dispatch_meta_bytes=315616 combine_payload_bytes=268582912 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268898528 ingress_bytes=406994944 link_bytes=406994944 token_bytes=537481440 weight_movement_bytes=138412032 total_nvlink_bytes=675893472 weight_to_token_ratio=0.257520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.531489208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12870514128 prefetch_bytes=761266176 reduce_bytes=657457152 weight_movement_bytes=1418723328 egress_bytes=6439171536 ingress_bytes=7850065920 link_bytes=7850065920 total_nvlink_bytes=14289237456 weight_to_token_ratio=0.110231 prefetch_share_of_ingress=0.096976 dedup_ratio=0.199987 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=19 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.284311568e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78724 payload_rows_remote=65572 dup_rows=19580 dup_rows_remote=13332 dedup_ratio=0.199178 router_tokens_per_expert=[8, 1075, 0, 4188, 4144, 3754, 285, 0, 5817, 20, 102, 67, 0, 73, 111, 3299, 0, 72, 2, 0, 0, 659, 1328, 1, 0, 82, 779, 1389, 6, 128, 24, 0, 1042, 5843, 705, 79, 61, 802, 84, 118, 92, 130, 5856, 723, 4, 44, 0, 1047, 12, 406, 0, 1623, 0, 314, 4, 426, 0, 684, 700, 249, 1, 690, 0, 0] payload_rows_per_dst=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] dup_rows_per_dst=[6248, 4926, 2298, 1796, 280, 2356, 220, 1456] combine_rows_per_src=[13152, 14052, 13512, 10528, 5502, 13436, 5350, 3192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12492842888 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6250284936 ingress_bytes=7592075264 link_bytes=7592075264 total_nvlink_bytes=13842360200 weight_to_token_ratio=0.108023 prefetch_share_of_ingress=0.095713 dedup_ratio=0.215947 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.242102957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 2, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12853885432 prefetch_bytes=692060160 reduce_bytes=899678208 weight_movement_bytes=1591738368 egress_bytes=6430841336 ingress_bytes=8014782464 link_bytes=8014782464 total_nvlink_bytes=14445623800 weight_to_token_ratio=0.123833 prefetch_share_of_ingress=0.086348 dedup_ratio=0.195642 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=26 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=27559 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.311260051e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=258449408 dispatch_meta_bytes=321760 combine_payload_bytes=258449408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258771168 ingress_bytes=293052416 link_bytes=293052416 token_bytes=517220576 weight_movement_bytes=34603008 total_nvlink_bytes=551823584 weight_to_token_ratio=0.066902 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102732649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12583178440 prefetch_bytes=761266176 reduce_bytes=795869184 weight_movement_bytes=1557135360 egress_bytes=6295433416 ingress_bytes=7844880384 link_bytes=7844880384 total_nvlink_bytes=14140313800 weight_to_token_ratio=0.123747 prefetch_share_of_ingress=0.097040 dedup_ratio=0.207877 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=23 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.283463188e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=268582912 dispatch_meta_bytes=315616 combine_payload_bytes=268582912 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268898528 ingress_bytes=406994944 link_bytes=406994944 token_bytes=537481440 weight_movement_bytes=138412032 total_nvlink_bytes=675893472 weight_to_token_ratio=0.257520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.531489208e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11953454480 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=5980446096 ingress_bytes=7460937728 link_bytes=7460937728 total_nvlink_bytes=13441383824 weight_to_token_ratio=0.124477 prefetch_share_of_ingress=0.092758 dedup_ratio=0.229324 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27940 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220648175e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 16:55:24] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 791.1 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.091896E+01 | loss scale: 1.0 | grad norm: 452.732 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75604 payload_rows_remote=65262 dup_rows=22700 dup_rows_remote=18558 dedup_ratio=0.230916 router_tokens_per_expert=[1865, 601, 334, 1571, 260, 1116, 742, 91, 3355, 820, 1384, 319, 0, 2529, 587, 604, 652, 833, 714, 766, 139, 185, 252, 1019, 549, 670, 505, 598, 946, 249, 862, 724, 912, 562, 1681, 1058, 689, 429, 842, 542, 126, 363, 409, 147, 755, 1406, 608, 73, 138, 341, 550, 912, 859, 2308, 281, 280, 963, 526, 1641, 550, 213, 683, 1669, 795] payload_rows_per_dst=[9570, 6266, 10394, 8504, 10106, 10342, 9518, 10904] dup_rows_per_dst=[2608, 1162, 3784, 1702, 3324, 4142, 2802, 3176] combine_rows_per_src=[9570, 6266, 10394, 8504, 10106, 10342, 9518, 10904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73908 payload_rows_remote=61032 dup_rows=24396 dup_rows_remote=15132 dedup_ratio=0.248169 router_tokens_per_expert=[1884, 602, 375, 1549, 254, 1165, 790, 79, 3481, 693, 1212, 357, 3, 2505, 553, 665, 604, 894, 636, 707, 123, 230, 231, 898, 529, 643, 430, 577, 946, 231, 689, 700, 957, 519, 1601, 1138, 700, 408, 1013, 537, 150, 384, 416, 124, 807, 1455, 553, 101, 161, 343, 558, 951, 842, 2354, 262, 223, 1046, 605, 1558, 640, 206, 746, 1712, 847] payload_rows_per_dst=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] dup_rows_per_dst=[3064, 9264, 1582, 3232, 2212, 1006, 2068, 1968] combine_rows_per_src=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12492842888 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6250284936 ingress_bytes=7592075264 link_bytes=7592075264 total_nvlink_bytes=13842360200 weight_to_token_ratio=0.108023 prefetch_share_of_ingress=0.095713 dedup_ratio=0.215947 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.242102957e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-26 18:07:32] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 792.7 | throughput per GPU (TFLOP/s/GPU): 160.9 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.091896E+01 | loss scale: 1.0 | grad norm: 452.732 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75270 payload_rows_remote=65340 dup_rows=23034 dup_rows_remote=19742 dedup_ratio=0.234314 router_tokens_per_expert=[1805, 666, 398, 1528, 247, 1307, 744, 101, 3236, 748, 1222, 353, 4, 2458, 685, 573, 679, 869, 707, 698, 137, 199, 249, 868, 578, 756, 504, 520, 1021, 258, 749, 754, 929, 483, 1574, 1172, 614, 437, 889, 513, 170, 448, 501, 161, 802, 1389, 554, 69, 175, 408, 493, 998, 892, 2139, 245, 288, 1018, 521, 1706, 639, 176, 702, 1675, 751] payload_rows_per_dst=[10380, 6088, 9972, 10002, 9930, 10490, 9294, 9114] dup_rows_per_dst=[3212, 1082, 3756, 3376, 3292, 4170, 1982, 2164] combine_rows_per_src=[10380, 6088, 9972, 10002, 9930, 10490, 9294, 9114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=63244 dup_rows=23772 dup_rows_remote=20134 dedup_ratio=0.241821 router_tokens_per_expert=[1894, 649, 380, 1353, 214, 1336, 775, 112, 3344, 883, 1175, 306, 5, 2528, 512, 735, 663, 848, 654, 599, 160, 200, 246, 993, 487, 827, 528, 573, 959, 269, 731, 680, 960, 486, 1538, 1150, 680, 497, 918, 574, 170, 427, 431, 143, 787, 1415, 548, 85, 178, 389, 521, 901, 810, 2376, 288, 269, 1127, 599, 1640, 541, 177, 581, 1611, 717] payload_rows_per_dst=[7922, 6124, 10030, 8446, 9640, 10536, 10546, 11288] dup_rows_per_dst=[1716, 1108, 3752, 1662, 3026, 4164, 4706, 3638] combine_rows_per_src=[7922, 6124, 10030, 8446, 9640, 10536, 10546, 11288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74902 payload_rows_remote=65888 dup_rows=23402 dup_rows_remote=20632 dedup_ratio=0.238057 router_tokens_per_expert=[1808, 614, 338, 1512, 199, 1180, 714, 89, 3208, 792, 1214, 394, 2, 2439, 575, 724, 670, 969, 666, 697, 136, 163, 265, 992, 618, 644, 491, 447, 965, 293, 826, 932, 1035, 568, 1711, 1129, 613, 525, 1058, 494, 198, 406, 405, 166, 774, 1429, 577, 107, 142, 382, 496, 933, 804, 2089, 251, 331, 1004, 622, 1572, 597, 171, 610, 1567, 810] payload_rows_per_dst=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] dup_rows_per_dst=[2818, 6668, 2770, 3484, 2182, 1878, 1796, 1806] combine_rows_per_src=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73908 payload_rows_remote=61032 dup_rows=24396 dup_rows_remote=15132 dedup_ratio=0.248169 router_tokens_per_expert=[1884, 602, 375, 1549, 254, 1165, 790, 79, 3481, 693, 1212, 357, 3, 2505, 553, 665, 604, 894, 636, 707, 123, 230, 231, 898, 529, 643, 430, 577, 946, 231, 689, 700, 957, 519, 1601, 1138, 700, 408, 1013, 537, 150, 384, 416, 124, 807, 1455, 553, 101, 161, 343, 558, 951, 842, 2354, 262, 223, 1046, 605, 1558, 640, 206, 746, 1712, 847] payload_rows_per_dst=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] dup_rows_per_dst=[3064, 9264, 1582, 3232, 2212, 1006, 2068, 1968] combine_rows_per_src=[10332, 12876, 7064, 9682, 8332, 6974, 9320, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75604 payload_rows_remote=65262 dup_rows=22700 dup_rows_remote=18558 dedup_ratio=0.230916 router_tokens_per_expert=[1865, 601, 334, 1571, 260, 1116, 742, 91, 3355, 820, 1384, 319, 0, 2529, 587, 604, 652, 833, 714, 766, 139, 185, 252, 1019, 549, 670, 505, 598, 946, 249, 862, 724, 912, 562, 1681, 1058, 689, 429, 842, 542, 126, 363, 409, 147, 755, 1406, 608, 73, 138, 341, 550, 912, 859, 2308, 281, 280, 963, 526, 1641, 550, 213, 683, 1669, 795] payload_rows_per_dst=[9570, 6266, 10394, 8504, 10106, 10342, 9518, 10904] dup_rows_per_dst=[2608, 1162, 3784, 1702, 3324, 4142, 2802, 3176] combine_rows_per_src=[9570, 6266, 10394, 8504, 10106, 10342, 9518, 10904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75618 payload_rows_remote=65350 dup_rows=22686 dup_rows_remote=19240 dedup_ratio=0.230774 router_tokens_per_expert=[1906, 699, 343, 1391, 284, 1171, 767, 112, 3340, 729, 1207, 378, 2, 2309, 584, 734, 600, 874, 706, 685, 148, 220, 266, 903, 549, 745, 423, 429, 989, 260, 846, 936, 913, 542, 1748, 1122, 659, 441, 917, 595, 169, 423, 396, 156, 763, 1340, 528, 81, 201, 418, 489, 1039, 912, 2078, 267, 297, 995, 582, 1595, 538, 210, 695, 1680, 828] payload_rows_per_dst=[10138, 6508, 9748, 10268, 10126, 10430, 9286, 9114] dup_rows_per_dst=[3208, 1130, 3674, 3446, 3378, 3962, 2116, 1772] combine_rows_per_src=[10138, 6508, 9748, 10268, 10126, 10430, 9286, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=335280 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648432 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534961584 weight_movement_bytes=34603008 total_nvlink_bytes=569564592 weight_to_token_ratio=0.064683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64050 dup_rows=24008 dup_rows_remote=19378 dedup_ratio=0.244222 router_tokens_per_expert=[1933, 642, 343, 1449, 222, 1262, 676, 116, 3383, 660, 1144, 385, 8, 2566, 564, 629, 663, 933, 629, 628, 157, 205, 252, 906, 574, 729, 505, 509, 1015, 308, 798, 781, 977, 515, 1668, 1161, 718, 495, 887, 581, 160, 422, 435, 140, 760, 1458, 589, 91, 184, 402, 555, 938, 879, 1988, 269, 290, 942, 525, 1597, 582, 187, 697, 1630, 856] payload_rows_per_dst=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] dup_rows_per_dst=[1734, 932, 3848, 1780, 3688, 4326, 4630, 3070] combine_rows_per_src=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=63418 dup_rows=24356 dup_rows_remote=21060 dedup_ratio=0.247762 router_tokens_per_expert=[1827, 620, 368, 1343, 305, 1340, 778, 93, 3164, 781, 1325, 299, 12, 2462, 607, 620, 639, 839, 685, 708, 177, 208, 269, 918, 695, 812, 486, 549, 981, 253, 742, 837, 1089, 523, 1665, 1079, 615, 523, 912, 543, 126, 362, 370, 140, 719, 1404, 607, 114, 166, 396, 533, 836, 832, 2148, 273, 283, 1036, 607, 1550, 625, 212, 694, 1635, 793] payload_rows_per_dst=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] dup_rows_per_dst=[3296, 9184, 1554, 3648, 1908, 980, 1906, 1880] combine_rows_per_src=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=340328 combine_payload_bytes=267632640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267972968 ingress_bytes=371441664 link_bytes=371441664 token_bytes=535605608 weight_movement_bytes=103809024 total_nvlink_bytes=639414632 weight_to_token_ratio=0.193816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397705078e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259047424 dispatch_meta_bytes=333512 combine_payload_bytes=259047424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259380936 ingress_bytes=328253440 link_bytes=328253440 token_bytes=518428360 weight_movement_bytes=69206016 total_nvlink_bytes=587634376 weight_to_token_ratio=0.133492 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235191268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=346080 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270223328 ingress_bytes=304480256 link_bytes=304480256 token_bytes=540100576 weight_movement_bytes=34603008 total_nvlink_bytes=574703584 weight_to_token_ratio=0.064068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145734691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=249987072 dispatch_meta_bytes=304656 combine_payload_bytes=249987072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250291728 ingress_bytes=353796096 link_bytes=353796096 token_bytes=500278800 weight_movement_bytes=103809024 total_nvlink_bytes=604087824 weight_to_token_ratio=0.207502 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331306226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259047424 dispatch_meta_bytes=333512 combine_payload_bytes=259047424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259380936 ingress_bytes=328253440 link_bytes=328253440 token_bytes=518428360 weight_movement_bytes=69206016 total_nvlink_bytes=587634376 weight_to_token_ratio=0.133492 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235191268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75270 payload_rows_remote=65340 dup_rows=23034 dup_rows_remote=19742 dedup_ratio=0.234314 router_tokens_per_expert=[1805, 666, 398, 1528, 247, 1307, 744, 101, 3236, 748, 1222, 353, 4, 2458, 685, 573, 679, 869, 707, 698, 137, 199, 249, 868, 578, 756, 504, 520, 1021, 258, 749, 754, 929, 483, 1574, 1172, 614, 437, 889, 513, 170, 448, 501, 161, 802, 1389, 554, 69, 175, 408, 493, 998, 892, 2139, 245, 288, 1018, 521, 1706, 639, 176, 702, 1675, 751] payload_rows_per_dst=[10380, 6088, 9972, 10002, 9930, 10490, 9294, 9114] dup_rows_per_dst=[3212, 1082, 3756, 3376, 3292, 4170, 1982, 2164] combine_rows_per_src=[10380, 6088, 9972, 10002, 9930, 10490, 9294, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=335280 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648432 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534961584 weight_movement_bytes=34603008 total_nvlink_bytes=569564592 weight_to_token_ratio=0.064683 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267673600 dispatch_meta_bytes=338360 combine_payload_bytes=267673600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268011960 ingress_bytes=302276608 link_bytes=302276608 token_bytes=535685560 weight_movement_bytes=34603008 total_nvlink_bytes=570288568 weight_to_token_ratio=0.064596 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137442541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64050 dup_rows=24008 dup_rows_remote=19378 dedup_ratio=0.244222 router_tokens_per_expert=[1933, 642, 343, 1449, 222, 1262, 676, 116, 3383, 660, 1144, 385, 8, 2566, 564, 629, 663, 933, 629, 628, 157, 205, 252, 906, 574, 729, 505, 509, 1015, 308, 798, 781, 977, 515, 1668, 1161, 718, 495, 887, 581, 160, 422, 435, 140, 760, 1458, 589, 91, 184, 402, 555, 938, 879, 1988, 269, 290, 942, 525, 1597, 582, 187, 697, 1630, 856] payload_rows_per_dst=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] dup_rows_per_dst=[1734, 932, 3848, 1780, 3688, 4326, 4630, 3070] combine_rows_per_src=[7686, 5848, 10030, 8658, 10316, 10550, 10246, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259760128 dispatch_meta_bytes=337912 combine_payload_bytes=259760128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260098040 ingress_bytes=328966144 link_bytes=328966144 token_bytes=519858168 weight_movement_bytes=69206016 total_nvlink_bytes=589064184 weight_to_token_ratio=0.133125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237873116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=63418 dup_rows=24356 dup_rows_remote=21060 dedup_ratio=0.247762 router_tokens_per_expert=[1827, 620, 368, 1343, 305, 1340, 778, 93, 3164, 781, 1325, 299, 12, 2462, 607, 620, 639, 839, 685, 708, 177, 208, 269, 918, 695, 812, 486, 549, 981, 253, 742, 837, 1089, 523, 1665, 1079, 615, 523, 912, 543, 126, 362, 370, 140, 719, 1404, 607, 114, 166, 396, 533, 836, 832, 2148, 273, 283, 1036, 607, 1550, 625, 212, 694, 1635, 793] payload_rows_per_dst=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] dup_rows_per_dst=[3296, 9184, 1554, 3648, 1908, 980, 1906, 1880] combine_rows_per_src=[10530, 12686, 7332, 10332, 8182, 6704, 9028, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74902 payload_rows_remote=65888 dup_rows=23402 dup_rows_remote=20632 dedup_ratio=0.238057 router_tokens_per_expert=[1808, 614, 338, 1512, 199, 1180, 714, 89, 3208, 792, 1214, 394, 2, 2439, 575, 724, 670, 969, 666, 697, 136, 163, 265, 992, 618, 644, 491, 447, 965, 293, 826, 932, 1035, 568, 1711, 1129, 613, 525, 1058, 494, 198, 406, 405, 166, 774, 1429, 577, 107, 142, 382, 496, 933, 804, 2089, 251, 331, 1004, 622, 1572, 597, 171, 610, 1567, 810] payload_rows_per_dst=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] dup_rows_per_dst=[2818, 6668, 2770, 3484, 2182, 1878, 1796, 1806] combine_rows_per_src=[10090, 11378, 9014, 10082, 8662, 7650, 9060, 8966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262348800 dispatch_meta_bytes=333712 combine_payload_bytes=262348800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262682512 ingress_bytes=296951808 link_bytes=296951808 token_bytes=525031312 weight_movement_bytes=34603008 total_nvlink_bytes=559634320 weight_to_token_ratio=0.065907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117405747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=340328 combine_payload_bytes=267632640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267972968 ingress_bytes=371441664 link_bytes=371441664 token_bytes=535605608 weight_movement_bytes=103809024 total_nvlink_bytes=639414632 weight_to_token_ratio=0.193816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397705078e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79556 payload_rows_remote=66940 dup_rows=18748 dup_rows_remote=15148 dedup_ratio=0.190715 router_tokens_per_expert=[153, 497, 59, 140, 4661, 76, 1362, 443, 50, 510, 1207, 457, 436, 721, 71, 3314, 370, 922, 531, 530, 500, 1006, 420, 479, 396, 422, 799, 3897, 5, 82, 80, 548, 3, 0, 2, 8, 298, 0, 292, 539, 809, 505, 49, 712, 26, 1099, 2, 245, 6891, 250, 3796, 0, 11, 200, 437, 1319, 2813, 125, 709, 466, 356, 155, 1595, 296] payload_rows_per_dst=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] dup_rows_per_dst=[640, 1534, 3714, 2246, 1542, 3600, 2690, 2782] combine_rows_per_src=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81390 payload_rows_remote=67430 dup_rows=16914 dup_rows_remote=15286 dedup_ratio=0.172058 router_tokens_per_expert=[217, 490, 33, 130, 4586, 91, 1559, 319, 40, 524, 1666, 493, 548, 595, 55, 3404, 328, 964, 571, 880, 581, 854, 522, 448, 406, 354, 715, 3770, 0, 100, 131, 703, 1, 2, 14, 14, 387, 0, 258, 463, 755, 428, 173, 479, 16, 996, 1, 214, 6655, 295, 3367, 1, 18, 221, 423, 1331, 2733, 350, 883, 214, 439, 193, 1526, 225] payload_rows_per_dst=[5000, 11120, 8480, 10152, 13960, 12296, 9958, 10424] dup_rows_per_dst=[678, 3530, 1816, 2206, 1628, 3000, 1354, 2702] combine_rows_per_src=[5000, 11120, 8480, 10152, 13960, 12296, 9958, 10424] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71654 payload_rows_remote=61418 dup_rows=26650 dup_rows_remote=24038 dedup_ratio=0.271098 router_tokens_per_expert=[163, 420, 36, 127, 4397, 82, 1508, 302, 32, 485, 1527, 401, 501, 665, 51, 3496, 303, 960, 507, 632, 465, 1139, 541, 482, 455, 444, 812, 4033, 3, 98, 102, 615, 4, 3, 1, 4, 318, 1, 275, 499, 801, 479, 81, 592, 12, 967, 0, 247, 6956, 251, 3364, 0, 26, 205, 456, 1402, 2755, 392, 764, 189, 356, 152, 1620, 196] payload_rows_per_dst=[4708, 6374, 12206, 7828, 2202, 12112, 15988, 10236] dup_rows_per_dst=[568, 950, 4844, 1686, 8, 3040, 12942, 2612] combine_rows_per_src=[4708, 6374, 12206, 7828, 2202, 12112, 15988, 10236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75622 payload_rows_remote=67768 dup_rows=22682 dup_rows_remote=21268 dedup_ratio=0.230733 router_tokens_per_expert=[142, 559, 20, 148, 4882, 53, 1702, 334, 45, 504, 1394, 456, 528, 817, 46, 3284, 321, 978, 551, 460, 521, 885, 454, 464, 422, 425, 832, 3763, 2, 72, 97, 733, 2, 0, 5, 15, 365, 4, 234, 496, 781, 496, 58, 689, 22, 938, 3, 205, 6905, 226, 3654, 0, 12, 203, 404, 1361, 2590, 302, 687, 452, 404, 207, 1290, 248] payload_rows_per_dst=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] dup_rows_per_dst=[9582, 3288, 1414, 2404, 1752, 1474, 104, 2664] combine_rows_per_src=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76554 payload_rows_remote=65626 dup_rows=21750 dup_rows_remote=18708 dedup_ratio=0.221252 router_tokens_per_expert=[190, 452, 40, 138, 4732, 53, 1605, 374, 20, 480, 1493, 423, 502, 636, 31, 3400, 354, 1014, 488, 637, 483, 961, 470, 419, 399, 373, 725, 3861, 0, 84, 82, 661, 3, 3, 1, 4, 326, 5, 290, 542, 777, 503, 41, 674, 16, 855, 0, 212, 6927, 181, 3788, 0, 17, 205, 420, 1383, 2957, 307, 818, 178, 343, 133, 1415, 248] payload_rows_per_dst=[13548, 10928, 8100, 10276, 14400, 4738, 4318, 10246] dup_rows_per_dst=[9196, 3042, 1552, 2094, 1802, 1418, 94, 2552] combine_rows_per_src=[13548, 10928, 8100, 10276, 14400, 4738, 4318, 10246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259760128 dispatch_meta_bytes=337912 combine_payload_bytes=259760128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260098040 ingress_bytes=328966144 link_bytes=328966144 token_bytes=519858168 weight_movement_bytes=69206016 total_nvlink_bytes=589064184 weight_to_token_ratio=0.133125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237873116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79556 payload_rows_remote=66940 dup_rows=18748 dup_rows_remote=15148 dedup_ratio=0.190715 router_tokens_per_expert=[153, 497, 59, 140, 4661, 76, 1362, 443, 50, 510, 1207, 457, 436, 721, 71, 3314, 370, 922, 531, 530, 500, 1006, 420, 479, 396, 422, 799, 3897, 5, 82, 80, 548, 3, 0, 2, 8, 298, 0, 292, 539, 809, 505, 49, 712, 26, 1099, 2, 245, 6891, 250, 3796, 0, 11, 200, 437, 1319, 2813, 125, 709, 466, 356, 155, 1595, 296] payload_rows_per_dst=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] dup_rows_per_dst=[640, 1534, 3714, 2246, 1542, 3600, 2690, 2782] combine_rows_per_src=[4820, 6714, 11086, 10212, 12770, 12616, 11090, 10248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=346080 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270223328 ingress_bytes=304480256 link_bytes=304480256 token_bytes=540100576 weight_movement_bytes=34603008 total_nvlink_bytes=574703584 weight_to_token_ratio=0.064068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145734691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274186240 dispatch_meta_bytes=328352 combine_payload_bytes=274186240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274514592 ingress_bytes=308789248 link_bytes=308789248 token_bytes=548700832 weight_movement_bytes=34603008 total_nvlink_bytes=583303840 weight_to_token_ratio=0.063064 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161949081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80272 payload_rows_remote=69946 dup_rows=18032 dup_rows_remote=15650 dedup_ratio=0.183431 router_tokens_per_expert=[182, 424, 54, 163, 4885, 59, 1623, 422, 39, 707, 1438, 424, 503, 788, 27, 3446, 422, 973, 546, 663, 492, 912, 425, 389, 462, 365, 810, 3901, 0, 59, 94, 663, 7, 0, 2, 16, 335, 4, 208, 509, 789, 445, 85, 671, 19, 897, 0, 223, 6763, 228, 3052, 0, 29, 226, 412, 1386, 2697, 411, 793, 298, 384, 152, 1477, 274] payload_rows_per_dst=[7932, 10900, 7894, 10326, 14080, 12906, 6004, 10230] dup_rows_per_dst=[2152, 3844, 1750, 2382, 1608, 3122, 432, 2742] combine_rows_per_src=[7932, 10900, 7894, 10326, 14080, 12906, 6004, 10230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276193280 dispatch_meta_bytes=330864 combine_payload_bytes=276193280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276524144 ingress_bytes=310796288 link_bytes=310796288 token_bytes=552717424 weight_movement_bytes=34603008 total_nvlink_bytes=587320432 weight_to_token_ratio=0.062605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169501411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=251568128 dispatch_meta_bytes=341824 combine_payload_bytes=251568128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=251909952 ingress_bytes=320774144 link_bytes=320774144 token_bytes=503478080 weight_movement_bytes=69206016 total_nvlink_bytes=572684096 weight_to_token_ratio=0.137456 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207047279e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277577728 dispatch_meta_bytes=356144 combine_payload_bytes=277577728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933872 ingress_bytes=312180736 link_bytes=312180736 token_bytes=555511600 weight_movement_bytes=34603008 total_nvlink_bytes=590114608 weight_to_token_ratio=0.062290 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174710977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268804096 dispatch_meta_bytes=337336 combine_payload_bytes=268804096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269141432 ingress_bytes=338010112 link_bytes=338010112 token_bytes=537945528 weight_movement_bytes=69206016 total_nvlink_bytes=607151544 weight_to_token_ratio=0.128649 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271904839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274186240 dispatch_meta_bytes=328352 combine_payload_bytes=274186240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274514592 ingress_bytes=308789248 link_bytes=308789248 token_bytes=548700832 weight_movement_bytes=34603008 total_nvlink_bytes=583303840 weight_to_token_ratio=0.063064 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161949081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=58666 dup_rows=23870 dup_rows_remote=15010 dedup_ratio=0.242818 router_tokens_per_expert=[219, 463, 42, 153, 4754, 80, 1557, 341, 49, 523, 1352, 411, 502, 786, 74, 3210, 336, 914, 512, 880, 470, 927, 530, 474, 464, 351, 790, 3891, 0, 71, 106, 616, 1, 2, 4, 17, 319, 3, 265, 518, 818, 496, 167, 695, 17, 858, 1, 278, 6885, 220, 3091, 0, 21, 191, 504, 1402, 2625, 347, 802, 368, 406, 160, 1552, 271] payload_rows_per_dst=[4972, 6400, 11898, 10376, 2254, 12566, 15768, 10200] dup_rows_per_dst=[738, 994, 4608, 2202, 4, 3602, 8860, 2862] combine_rows_per_src=[4972, 6400, 11898, 10376, 2254, 12566, 15768, 10200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75366 payload_rows_remote=62750 dup_rows=22938 dup_rows_remote=15106 dedup_ratio=0.233337 router_tokens_per_expert=[216, 519, 32, 124, 4549, 63, 1359, 392, 58, 656, 1366, 373, 466, 683, 70, 3121, 406, 906, 512, 1044, 583, 1114, 468, 511, 455, 368, 743, 3841, 1, 83, 95, 590, 1, 0, 6, 5, 378, 0, 290, 500, 774, 461, 199, 503, 16, 1195, 0, 262, 6817, 251, 2970, 0, 25, 210, 424, 1287, 2931, 348, 778, 198, 383, 218, 1711, 244] payload_rows_per_dst=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] dup_rows_per_dst=[7832, 5708, 2318, 2898, 1748, 1426, 122, 886] combine_rows_per_src=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=286498816 dispatch_meta_bytes=342384 combine_payload_bytes=286498816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286841200 ingress_bytes=355704832 link_bytes=355704832 token_bytes=573340016 weight_movement_bytes=69206016 total_nvlink_bytes=642546032 weight_to_token_ratio=0.120707 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338488646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81390 payload_rows_remote=67430 dup_rows=16914 dup_rows_remote=15286 dedup_ratio=0.172058 router_tokens_per_expert=[217, 490, 33, 130, 4586, 91, 1559, 319, 40, 524, 1666, 493, 548, 595, 55, 3404, 328, 964, 571, 880, 581, 854, 522, 448, 406, 354, 715, 3770, 0, 100, 131, 703, 1, 2, 14, 14, 387, 0, 258, 463, 755, 428, 173, 479, 16, 996, 1, 214, 6655, 295, 3367, 1, 18, 221, 423, 1331, 2733, 350, 883, 214, 439, 193, 1526, 225] payload_rows_per_dst=[5000, 11120, 8480, 10152, 13960, 12296, 9958, 10424] dup_rows_per_dst=[678, 3530, 1816, 2206, 1628, 3000, 1354, 2702] combine_rows_per_src=[5000, 11120, 8480, 10152, 13960, 12296, 9958, 10424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=58666 dup_rows=23870 dup_rows_remote=15010 dedup_ratio=0.242818 router_tokens_per_expert=[219, 463, 42, 153, 4754, 80, 1557, 341, 49, 523, 1352, 411, 502, 786, 74, 3210, 336, 914, 512, 880, 470, 927, 530, 474, 464, 351, 790, 3891, 0, 71, 106, 616, 1, 2, 4, 17, 319, 3, 265, 518, 818, 496, 167, 695, 17, 858, 1, 278, 6885, 220, 3091, 0, 21, 191, 504, 1402, 2625, 347, 802, 368, 406, 160, 1552, 271] payload_rows_per_dst=[4972, 6400, 11898, 10376, 2254, 12566, 15768, 10200] dup_rows_per_dst=[738, 994, 4608, 2202, 4, 3602, 8860, 2862] combine_rows_per_src=[4972, 6400, 11898, 10376, 2254, 12566, 15768, 10200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=311424 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257335424 ingress_bytes=326230016 link_bytes=326230016 token_bytes=514359424 weight_movement_bytes=69206016 total_nvlink_bytes=583565440 weight_to_token_ratio=0.134548 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227577287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80364 payload_rows_remote=67260 dup_rows=17940 dup_rows_remote=14212 dedup_ratio=0.182495 router_tokens_per_expert=[0, 183, 61, 179, 0, 4, 408, 1945, 2, 2734, 156, 1638, 253, 339, 0, 0, 4555, 675, 5251, 102, 1, 0, 5236, 810, 204, 996, 2278, 2964, 2, 446, 238, 0, 32, 553, 205, 119, 4328, 431, 192, 83, 1145, 0, 0, 22, 415, 2200, 79, 0, 1050, 4, 476, 353, 2, 0, 259, 0, 3258, 152, 11, 5, 0, 1063, 0, 1055] payload_rows_per_dst=[6966, 8978, 9684, 10772, 9952, 13104, 11534, 9374] dup_rows_per_dst=[940, 1266, 1618, 3484, 1934, 3728, 3256, 1714] combine_rows_per_src=[6966, 8978, 9684, 10772, 9952, 13104, 11534, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75366 payload_rows_remote=62750 dup_rows=22938 dup_rows_remote=15106 dedup_ratio=0.233337 router_tokens_per_expert=[216, 519, 32, 124, 4549, 63, 1359, 392, 58, 656, 1366, 373, 466, 683, 70, 3121, 406, 906, 512, 1044, 583, 1114, 468, 511, 455, 368, 743, 3841, 1, 83, 95, 590, 1, 0, 6, 5, 378, 0, 290, 500, 774, 461, 199, 503, 16, 1195, 0, 262, 6817, 251, 2970, 0, 25, 210, 424, 1287, 2931, 348, 778, 198, 383, 218, 1711, 244] payload_rows_per_dst=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] dup_rows_per_dst=[7832, 5708, 2318, 2898, 1748, 1426, 122, 886] combine_rows_per_src=[12616, 12118, 8770, 10728, 14246, 5394, 4272, 7222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78938 payload_rows_remote=66170 dup_rows=19366 dup_rows_remote=14050 dedup_ratio=0.197001 router_tokens_per_expert=[0, 201, 46, 188, 0, 16, 371, 1914, 2, 2589, 112, 1701, 369, 352, 0, 2, 4477, 773, 5163, 130, 1, 1, 5339, 798, 213, 928, 2326, 3392, 3, 372, 254, 0, 40, 578, 156, 95, 4729, 363, 180, 104, 1037, 0, 1, 23, 414, 2202, 77, 0, 959, 2, 398, 307, 2, 0, 168, 0, 3067, 144, 10, 4, 1, 1020, 0, 1038] payload_rows_per_dst=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] dup_rows_per_dst=[208, 1080, 1760, 5210, 1930, 1212, 2650, 5316] combine_rows_per_src=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79854 payload_rows_remote=69834 dup_rows=18450 dup_rows_remote=16328 dedup_ratio=0.187683 router_tokens_per_expert=[0, 155, 50, 134, 0, 5, 409, 2016, 33, 2720, 132, 1798, 362, 343, 0, 0, 4475, 799, 5465, 86, 0, 0, 4828, 725, 223, 968, 2251, 3243, 0, 225, 222, 0, 43, 677, 208, 109, 4355, 386, 204, 89, 1048, 0, 0, 37, 446, 2175, 76, 0, 953, 5, 334, 415, 7, 0, 221, 0, 3258, 173, 10, 19, 0, 1177, 0, 1060] payload_rows_per_dst=[12334, 9364, 3088, 10860, 10020, 12854, 11762, 9572] dup_rows_per_dst=[2860, 1412, 132, 3404, 2122, 3660, 3038, 1822] combine_rows_per_src=[12334, 9364, 3088, 10860, 10020, 12854, 11762, 9572] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=240295936 dispatch_meta_bytes=294704 combine_payload_bytes=240295936 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=240590640 ingress_bytes=344104960 link_bytes=344104960 token_bytes=480886576 weight_movement_bytes=103809024 total_nvlink_bytes=584695600 weight_to_token_ratio=0.215870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294839262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75622 payload_rows_remote=67768 dup_rows=22682 dup_rows_remote=21268 dedup_ratio=0.230733 router_tokens_per_expert=[142, 559, 20, 148, 4882, 53, 1702, 334, 45, 504, 1394, 456, 528, 817, 46, 3284, 321, 978, 551, 460, 521, 885, 454, 464, 422, 425, 832, 3763, 2, 72, 97, 733, 2, 0, 5, 15, 365, 4, 234, 496, 781, 496, 58, 689, 22, 938, 3, 205, 6905, 226, 3654, 0, 12, 203, 404, 1361, 2590, 302, 687, 452, 404, 207, 1290, 248] payload_rows_per_dst=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] dup_rows_per_dst=[9582, 3288, 1414, 2404, 1752, 1474, 104, 2664] combine_rows_per_src=[13322, 10860, 7854, 10288, 14300, 4994, 4308, 9696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276193280 dispatch_meta_bytes=330864 combine_payload_bytes=276193280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276524144 ingress_bytes=310796288 link_bytes=310796288 token_bytes=552717424 weight_movement_bytes=34603008 total_nvlink_bytes=587320432 weight_to_token_ratio=0.062605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.169501411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78938 payload_rows_remote=66170 dup_rows=19366 dup_rows_remote=14050 dedup_ratio=0.197001 router_tokens_per_expert=[0, 201, 46, 188, 0, 16, 371, 1914, 2, 2589, 112, 1701, 369, 352, 0, 2, 4477, 773, 5163, 130, 1, 1, 5339, 798, 213, 928, 2326, 3392, 3, 372, 254, 0, 40, 578, 156, 95, 4729, 363, 180, 104, 1037, 0, 1, 23, 414, 2202, 77, 0, 959, 2, 398, 307, 2, 0, 168, 0, 3067, 144, 10, 4, 1, 1020, 0, 1038] payload_rows_per_dst=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] dup_rows_per_dst=[208, 1080, 1760, 5210, 1930, 1212, 2650, 5316] combine_rows_per_src=[5264, 9174, 9776, 12314, 10560, 7734, 11348, 12768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80364 payload_rows_remote=67260 dup_rows=17940 dup_rows_remote=14212 dedup_ratio=0.182495 router_tokens_per_expert=[0, 183, 61, 179, 0, 4, 408, 1945, 2, 2734, 156, 1638, 253, 339, 0, 0, 4555, 675, 5251, 102, 1, 0, 5236, 810, 204, 996, 2278, 2964, 2, 446, 238, 0, 32, 553, 205, 119, 4328, 431, 192, 83, 1145, 0, 0, 22, 415, 2200, 79, 0, 1050, 4, 476, 353, 2, 0, 259, 0, 3258, 152, 11, 5, 0, 1063, 0, 1055] payload_rows_per_dst=[6966, 8978, 9684, 10772, 9952, 13104, 11534, 9374] dup_rows_per_dst=[940, 1266, 1618, 3484, 1934, 3728, 3256, 1714] combine_rows_per_src=[6966, 8978, 9684, 10772, 9952, 13104, 11534, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80066 payload_rows_remote=70802 dup_rows=18238 dup_rows_remote=16928 dedup_ratio=0.185527 router_tokens_per_expert=[0, 164, 74, 155, 1, 4, 326, 2198, 40, 2706, 174, 1575, 312, 340, 0, 0, 4426, 1000, 5269, 123, 0, 0, 5189, 646, 272, 1043, 2311, 2495, 2, 274, 209, 0, 47, 620, 242, 125, 4438, 378, 143, 88, 986, 0, 0, 12, 441, 2295, 76, 1, 1017, 6, 412, 417, 2, 0, 186, 0, 3399, 185, 19, 6, 0, 1205, 1, 1077] payload_rows_per_dst=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] dup_rows_per_dst=[3298, 3108, 1310, 1408, 2916, 994, 3110, 2094] combine_rows_per_src=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77222 payload_rows_remote=65316 dup_rows=21082 dup_rows_remote=17028 dedup_ratio=0.214457 router_tokens_per_expert=[1, 132, 43, 156, 0, 1, 343, 1997, 5, 2544, 112, 1782, 212, 323, 0, 2, 4750, 824, 5270, 114, 0, 0, 5322, 692, 201, 882, 2158, 3000, 2, 408, 213, 0, 21, 595, 172, 95, 4597, 404, 205, 99, 834, 0, 0, 11, 448, 2083, 86, 0, 940, 5, 516, 426, 2, 1, 215, 1, 3873, 163, 4, 4, 0, 1013, 0, 850] payload_rows_per_dst=[12904, 11906, 13636, 6804, 10392, 6164, 5478, 9938] dup_rows_per_dst=[3086, 4054, 7388, 924, 1984, 760, 1010, 1876] combine_rows_per_src=[12904, 11906, 13636, 6804, 10392, 6164, 5478, 9938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275496960 dispatch_meta_bytes=325888 combine_payload_bytes=275496960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275822848 ingress_bytes=310099968 link_bytes=310099968 token_bytes=551319808 weight_movement_bytes=34603008 total_nvlink_bytes=585922816 weight_to_token_ratio=0.062764 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166881214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=257024000 dispatch_meta_bytes=311424 combine_payload_bytes=257024000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257335424 ingress_bytes=326230016 link_bytes=326230016 token_bytes=514359424 weight_movement_bytes=69206016 total_nvlink_bytes=583565440 weight_to_token_ratio=0.134548 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227577287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79516 payload_rows_remote=69112 dup_rows=18788 dup_rows_remote=15510 dedup_ratio=0.191121 router_tokens_per_expert=[0, 225, 72, 154, 0, 4, 351, 2086, 20, 2422, 168, 1803, 347, 376, 1, 1, 4545, 845, 5254, 143, 1, 0, 5121, 764, 216, 967, 2061, 2915, 1, 455, 226, 0, 48, 550, 195, 167, 4356, 415, 166, 79, 1011, 0, 0, 21, 424, 2302, 92, 0, 1095, 6, 451, 373, 1, 0, 238, 0, 3269, 154, 11, 12, 0, 1153, 0, 1019] payload_rows_per_dst=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] dup_rows_per_dst=[3126, 1338, 192, 3278, 1970, 3848, 3192, 1844] combine_rows_per_src=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286040064 dispatch_meta_bytes=344648 combine_payload_bytes=286040064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286384712 ingress_bytes=320643072 link_bytes=320643072 token_bytes=572424776 weight_movement_bytes=34603008 total_nvlink_bytes=607027784 weight_to_token_ratio=0.060450 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206554066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271032320 dispatch_meta_bytes=320880 combine_payload_bytes=271032320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271353200 ingress_bytes=305635328 link_bytes=305635328 token_bytes=542385520 weight_movement_bytes=34603008 total_nvlink_bytes=576988528 weight_to_token_ratio=0.063798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150081134e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79516 payload_rows_remote=69112 dup_rows=18788 dup_rows_remote=15510 dedup_ratio=0.191121 router_tokens_per_expert=[0, 225, 72, 154, 0, 4, 351, 2086, 20, 2422, 168, 1803, 347, 376, 1, 1, 4545, 845, 5254, 143, 1, 0, 5121, 764, 216, 967, 2061, 2915, 1, 455, 226, 0, 48, 550, 195, 167, 4356, 415, 166, 79, 1011, 0, 0, 21, 424, 2302, 92, 0, 1095, 6, 451, 373, 1, 0, 238, 0, 3269, 154, 11, 12, 0, 1153, 0, 1019] payload_rows_per_dst=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] dup_rows_per_dst=[3126, 1338, 192, 3278, 1970, 3848, 3192, 1844] combine_rows_per_src=[12900, 8938, 3314, 10404, 9982, 12942, 11644, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277577728 dispatch_meta_bytes=356144 combine_payload_bytes=277577728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933872 ingress_bytes=312180736 link_bytes=312180736 token_bytes=555511600 weight_movement_bytes=34603008 total_nvlink_bytes=590114608 weight_to_token_ratio=0.062290 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174710977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=329376 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267863712 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535398048 weight_movement_bytes=34603008 total_nvlink_bytes=570001056 weight_to_token_ratio=0.064630 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275496960 dispatch_meta_bytes=325888 combine_payload_bytes=275496960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275822848 ingress_bytes=310099968 link_bytes=310099968 token_bytes=551319808 weight_movement_bytes=34603008 total_nvlink_bytes=585922816 weight_to_token_ratio=0.062764 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166881214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290004992 dispatch_meta_bytes=350920 combine_payload_bytes=290004992 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=290355912 ingress_bytes=497623040 link_bytes=497623040 token_bytes=580360904 weight_movement_bytes=207618048 total_nvlink_bytes=787978952 weight_to_token_ratio=0.357740 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.872515438e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80062 payload_rows_remote=68558 dup_rows=18242 dup_rows_remote=15128 dedup_ratio=0.185567 router_tokens_per_expert=[1, 190, 55, 192, 0, 7, 359, 2062, 22, 2632, 150, 1563, 297, 337, 0, 1, 4519, 856, 5292, 169, 1, 0, 5187, 872, 259, 1022, 2284, 2682, 1, 315, 224, 0, 56, 578, 183, 140, 4268, 381, 180, 68, 1022, 0, 3, 28, 527, 2233, 91, 1, 916, 3, 422, 497, 8, 0, 169, 2, 3473, 156, 16, 5, 1, 1094, 1, 1079] payload_rows_per_dst=[5482, 8874, 11696, 10330, 9722, 12952, 11504, 9502] dup_rows_per_dst=[250, 1130, 2474, 3244, 1986, 3896, 3114, 2148] combine_rows_per_src=[5482, 8874, 11696, 10330, 9722, 12952, 11504, 9502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79854 payload_rows_remote=69834 dup_rows=18450 dup_rows_remote=16328 dedup_ratio=0.187683 router_tokens_per_expert=[0, 155, 50, 134, 0, 5, 409, 2016, 33, 2720, 132, 1798, 362, 343, 0, 0, 4475, 799, 5465, 86, 0, 0, 4828, 725, 223, 968, 2251, 3243, 0, 225, 222, 0, 43, 677, 208, 109, 4355, 386, 204, 89, 1048, 0, 0, 37, 446, 2175, 76, 0, 953, 5, 334, 415, 7, 0, 221, 0, 3258, 173, 10, 19, 0, 1177, 0, 1060] payload_rows_per_dst=[12334, 9364, 3088, 10860, 10020, 12854, 11762, 9572] dup_rows_per_dst=[2860, 1412, 132, 3404, 2122, 3660, 3038, 1822] combine_rows_per_src=[12334, 9364, 3088, 10860, 10020, 12854, 11762, 9572] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=329376 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267863712 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535398048 weight_movement_bytes=34603008 total_nvlink_bytes=570001056 weight_to_token_ratio=0.064630 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75944 payload_rows_remote=63416 dup_rows=22360 dup_rows_remote=19126 dedup_ratio=0.227458 router_tokens_per_expert=[0, 244, 41, 148, 0, 7, 366, 2075, 38, 2472, 121, 1735, 370, 313, 0, 0, 4662, 855, 5128, 143, 2, 0, 5000, 696, 240, 940, 2362, 2995, 1, 470, 223, 0, 44, 598, 203, 122, 4351, 386, 207, 107, 1125, 0, 0, 18, 425, 2110, 103, 1, 987, 5, 369, 458, 0, 0, 259, 0, 3221, 130, 11, 14, 0, 1221, 0, 1030] payload_rows_per_dst=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] dup_rows_per_dst=[3234, 4038, 8916, 1314, 2080, 848, 240, 1690] combine_rows_per_src=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=283082752 dispatch_meta_bytes=338488 combine_payload_bytes=283082752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283421240 ingress_bytes=352288768 link_bytes=352288768 token_bytes=566503992 weight_movement_bytes=69206016 total_nvlink_bytes=635710008 weight_to_token_ratio=0.122163 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.325634272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83002 payload_rows_remote=76238 dup_rows=15302 dup_rows_remote=14166 dedup_ratio=0.155660 router_tokens_per_expert=[0, 0, 4, 0, 2933, 839, 1005, 132, 357, 78, 313, 0, 255, 0, 6988, 4, 2611, 1323, 15, 1, 1514, 320, 0, 86, 1304, 769, 1203, 783, 560, 813, 452, 0, 0, 5, 46, 2, 1199, 17, 185, 1, 701, 5991, 94, 670, 299, 0, 0, 2186, 0, 0, 76, 522, 75, 2170, 0, 0, 1, 0, 749, 1, 2764, 5918, 2, 816] payload_rows_per_dst=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] dup_rows_per_dst=[1272, 1212, 2352, 3018, 1644, 1136, 3894, 774] combine_rows_per_src=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80062 payload_rows_remote=68558 dup_rows=18242 dup_rows_remote=15128 dedup_ratio=0.185567 router_tokens_per_expert=[1, 190, 55, 192, 0, 7, 359, 2062, 22, 2632, 150, 1563, 297, 337, 0, 1, 4519, 856, 5292, 169, 1, 0, 5187, 872, 259, 1022, 2284, 2682, 1, 315, 224, 0, 56, 578, 183, 140, 4268, 381, 180, 68, 1022, 0, 3, 28, 527, 2233, 91, 1, 916, 3, 422, 497, 8, 0, 169, 2, 3473, 156, 16, 5, 1, 1094, 1, 1079] payload_rows_per_dst=[5482, 8874, 11696, 10330, 9722, 12952, 11504, 9502] dup_rows_per_dst=[250, 1130, 2474, 3244, 1986, 3896, 3114, 2148] combine_rows_per_src=[5482, 8874, 11696, 10330, 9722, 12952, 11504, 9502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259751936 dispatch_meta_bytes=330168 combine_payload_bytes=259751936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260082104 ingress_bytes=294354944 link_bytes=294354944 token_bytes=519834040 weight_movement_bytes=34603008 total_nvlink_bytes=554437048 weight_to_token_ratio=0.066565 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107633957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280813568 dispatch_meta_bytes=334744 combine_payload_bytes=280813568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281148312 ingress_bytes=315416576 link_bytes=315416576 token_bytes=561961880 weight_movement_bytes=34603008 total_nvlink_bytes=596564888 weight_to_token_ratio=0.061575 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186887182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75944 payload_rows_remote=63416 dup_rows=22360 dup_rows_remote=19126 dedup_ratio=0.227458 router_tokens_per_expert=[0, 244, 41, 148, 0, 7, 366, 2075, 38, 2472, 121, 1735, 370, 313, 0, 0, 4662, 855, 5128, 143, 2, 0, 5000, 696, 240, 940, 2362, 2995, 1, 470, 223, 0, 44, 598, 203, 122, 4351, 386, 207, 107, 1125, 0, 0, 18, 425, 2110, 103, 1, 987, 5, 369, 458, 0, 0, 259, 0, 3221, 130, 11, 14, 0, 1221, 0, 1030] payload_rows_per_dst=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] dup_rows_per_dst=[3234, 4038, 8916, 1314, 2080, 848, 240, 1690] combine_rows_per_src=[12528, 12050, 14056, 7158, 9956, 6716, 3916, 9564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286040064 dispatch_meta_bytes=344648 combine_payload_bytes=286040064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286384712 ingress_bytes=320643072 link_bytes=320643072 token_bytes=572424776 weight_movement_bytes=34603008 total_nvlink_bytes=607027784 weight_to_token_ratio=0.060450 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206554066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=68968 dup_rows=16040 dup_rows_remote=14178 dedup_ratio=0.163167 router_tokens_per_expert=[0, 0, 3, 0, 2818, 723, 1106, 96, 437, 74, 354, 0, 252, 0, 6838, 0, 2487, 1367, 25, 0, 1552, 324, 0, 149, 1190, 928, 1212, 694, 484, 758, 483, 0, 0, 8, 10, 3, 1479, 14, 163, 0, 584, 5759, 118, 505, 239, 0, 0, 2293, 0, 0, 73, 537, 142, 2563, 0, 0, 2, 0, 715, 0, 2971, 5902, 0, 718] payload_rows_per_dst=[8178, 14418, 9512, 8700, 13296, 6542, 13584, 8034] dup_rows_per_dst=[1314, 1492, 2296, 2798, 1862, 936, 4564, 778] combine_rows_per_src=[8178, 14418, 9512, 8700, 13296, 6542, 13584, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83002 payload_rows_remote=76238 dup_rows=15302 dup_rows_remote=14166 dedup_ratio=0.155660 router_tokens_per_expert=[0, 0, 4, 0, 2933, 839, 1005, 132, 357, 78, 313, 0, 255, 0, 6988, 4, 2611, 1323, 15, 1, 1514, 320, 0, 86, 1304, 769, 1203, 783, 560, 813, 452, 0, 0, 5, 46, 2, 1199, 17, 185, 1, 701, 5991, 94, 670, 299, 0, 0, 2186, 0, 0, 76, 522, 75, 2170, 0, 0, 1, 0, 749, 1, 2764, 5918, 2, 816] payload_rows_per_dst=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] dup_rows_per_dst=[1272, 1212, 2352, 3018, 1644, 1136, 3894, 774] combine_rows_per_src=[8554, 14778, 9388, 8750, 13102, 6764, 13774, 7892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82404 payload_rows_remote=73130 dup_rows=15900 dup_rows_remote=13724 dedup_ratio=0.161743 router_tokens_per_expert=[0, 0, 0, 0, 3151, 780, 1136, 60, 414, 74, 396, 0, 248, 0, 7002, 1, 2436, 1450, 14, 1, 1403, 272, 0, 149, 1170, 975, 1145, 715, 427, 645, 491, 0, 0, 7, 59, 3, 1348, 30, 107, 0, 727, 5962, 104, 540, 227, 0, 0, 2460, 0, 0, 69, 487, 96, 2446, 0, 0, 0, 0, 667, 1, 2873, 5661, 0, 723] payload_rows_per_dst=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] dup_rows_per_dst=[1278, 1554, 2176, 2824, 1672, 5770, 54, 572] combine_rows_per_src=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70252 payload_rows_remote=54308 dup_rows=28052 dup_rows_remote=14186 dedup_ratio=0.285360 router_tokens_per_expert=[0, 0, 6, 3, 3055, 704, 1104, 68, 424, 78, 298, 0, 243, 0, 6988, 1, 2834, 1594, 23, 2, 1290, 284, 0, 108, 1160, 787, 1173, 808, 666, 759, 455, 1, 0, 10, 41, 6, 1325, 13, 144, 2, 586, 5993, 81, 552, 219, 3, 1, 2276, 2, 3, 72, 504, 113, 2346, 0, 0, 5, 0, 683, 0, 2698, 5850, 2, 706] payload_rows_per_dst=[8676, 5926, 11098, 11742, 3060, 6396, 7410, 15944] dup_rows_per_dst=[1204, 352, 5008, 5826, 22, 1040, 734, 13866] combine_rows_per_src=[8676, 5926, 11098, 11742, 3060, 6396, 7410, 15944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=312270848 dispatch_meta_bytes=361616 combine_payload_bytes=312270848 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=312632464 ingress_bytes=416079872 link_bytes=416079872 token_bytes=624903312 weight_movement_bytes=103809024 total_nvlink_bytes=728712336 weight_to_token_ratio=0.166120 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565675061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80066 payload_rows_remote=70802 dup_rows=18238 dup_rows_remote=16928 dedup_ratio=0.185527 router_tokens_per_expert=[0, 164, 74, 155, 1, 4, 326, 2198, 40, 2706, 174, 1575, 312, 340, 0, 0, 4426, 1000, 5269, 123, 0, 0, 5189, 646, 272, 1043, 2311, 2495, 2, 274, 209, 0, 47, 620, 242, 125, 4438, 378, 143, 88, 986, 0, 0, 12, 441, 2295, 76, 1, 1017, 6, 412, 417, 2, 0, 186, 0, 3399, 185, 19, 6, 0, 1205, 1, 1077] payload_rows_per_dst=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] dup_rows_per_dst=[3298, 3108, 1310, 1408, 2916, 994, 3110, 2094] combine_rows_per_src=[12924, 11594, 9264, 7396, 10798, 6892, 11508, 9690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80140 payload_rows_remote=71112 dup_rows=18164 dup_rows_remote=17544 dedup_ratio=0.184774 router_tokens_per_expert=[0, 0, 0, 2, 2789, 733, 1129, 119, 382, 86, 249, 1, 248, 0, 6883, 4, 2793, 1354, 9, 3, 1382, 279, 0, 134, 1183, 929, 1241, 724, 512, 731, 461, 0, 0, 6, 47, 0, 1427, 16, 156, 0, 929, 6017, 71, 648, 226, 0, 1, 2122, 1, 0, 86, 561, 51, 2626, 0, 2, 1, 1, 689, 2, 2860, 5544, 2, 700] payload_rows_per_dst=[11450, 9028, 9508, 8760, 12682, 14332, 6528, 7852] dup_rows_per_dst=[4152, 620, 2400, 2802, 1710, 5696, 126, 658] combine_rows_per_src=[11450, 9028, 9508, 8760, 12682, 14332, 6528, 7852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222445568 dispatch_meta_bytes=273976 combine_payload_bytes=222445568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222719544 ingress_bytes=291651584 link_bytes=291651584 token_bytes=445165112 weight_movement_bytes=69206016 total_nvlink_bytes=514371128 weight_to_token_ratio=0.155461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097461431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259751936 dispatch_meta_bytes=330168 combine_payload_bytes=259751936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260082104 ingress_bytes=294354944 link_bytes=294354944 token_bytes=519834040 weight_movement_bytes=34603008 total_nvlink_bytes=554437048 weight_to_token_ratio=0.066565 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107633957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82708 payload_rows_remote=73736 dup_rows=15596 dup_rows_remote=12566 dedup_ratio=0.158651 router_tokens_per_expert=[0, 0, 3, 0, 2905, 779, 1033, 117, 416, 82, 326, 0, 245, 0, 6969, 0, 2772, 1538, 15, 1, 1208, 266, 0, 184, 1211, 1037, 1234, 816, 517, 697, 489, 0, 0, 11, 5, 0, 1302, 20, 116, 1, 534, 5687, 103, 534, 259, 0, 0, 2202, 0, 2, 81, 516, 137, 2592, 0, 0, 0, 0, 675, 0, 2874, 5952, 0, 689] payload_rows_per_dst=[8478, 14700, 9662, 8972, 13130, 6686, 13188, 7892] dup_rows_per_dst=[1196, 1376, 2306, 3030, 1684, 1212, 4208, 584] combine_rows_per_src=[8478, 14700, 9662, 8972, 13130, 6686, 13188, 7892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290004992 dispatch_meta_bytes=350920 combine_payload_bytes=290004992 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=290355912 ingress_bytes=497623040 link_bytes=497623040 token_bytes=580360904 weight_movement_bytes=207618048 total_nvlink_bytes=787978952 weight_to_token_ratio=0.357740 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.872515438e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282492928 dispatch_meta_bytes=332584 combine_payload_bytes=282492928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282825512 ingress_bytes=317095936 link_bytes=317095936 token_bytes=565318440 weight_movement_bytes=34603008 total_nvlink_bytes=599921448 weight_to_token_ratio=0.061210 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193206479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=312270848 dispatch_meta_bytes=361616 combine_payload_bytes=312270848 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=312632464 ingress_bytes=416079872 link_bytes=416079872 token_bytes=624903312 weight_movement_bytes=103809024 total_nvlink_bytes=728712336 weight_to_token_ratio=0.166120 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565675061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=299540480 dispatch_meta_bytes=347416 combine_payload_bytes=299540480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299887896 ingress_bytes=334143488 link_bytes=334143488 token_bytes=599428376 weight_movement_bytes=34603008 total_nvlink_bytes=634031384 weight_to_token_ratio=0.057727 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257355045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=222445568 dispatch_meta_bytes=273976 combine_payload_bytes=222445568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=222719544 ingress_bytes=291651584 link_bytes=291651584 token_bytes=445165112 weight_movement_bytes=69206016 total_nvlink_bytes=514371128 weight_to_token_ratio=0.155461 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097461431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77210 payload_rows_remote=61816 dup_rows=21094 dup_rows_remote=12876 dedup_ratio=0.214579 router_tokens_per_expert=[0, 0, 2, 0, 2996, 796, 989, 122, 388, 81, 383, 0, 255, 0, 6901, 0, 2622, 1316, 34, 1, 1294, 311, 0, 138, 1200, 957, 1134, 686, 611, 685, 451, 0, 0, 9, 41, 1, 1332, 31, 164, 0, 631, 5803, 75, 608, 238, 0, 0, 2385, 0, 2, 86, 477, 92, 2546, 0, 0, 1, 0, 850, 0, 2957, 5753, 1, 716] payload_rows_per_dst=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] dup_rows_per_dst=[8218, 36, 2198, 2730, 1726, 5404, 88, 694] combine_rows_per_src=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=291274752 dispatch_meta_bytes=354624 combine_payload_bytes=291274752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=291629376 ingress_bytes=429686784 link_bytes=429686784 token_bytes=582904128 weight_movement_bytes=138412032 total_nvlink_bytes=721316160 weight_to_token_ratio=0.237452 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.616876776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80566 payload_rows_remote=66898 dup_rows=17738 dup_rows_remote=13714 dedup_ratio=0.180440 router_tokens_per_expert=[0, 1, 1, 1, 2954, 866, 1096, 63, 366, 90, 362, 0, 237, 1, 6869, 2, 2726, 1404, 13, 1, 1400, 278, 0, 124, 1217, 784, 1241, 790, 472, 712, 445, 0, 0, 11, 49, 11, 1340, 17, 147, 0, 718, 5791, 92, 584, 299, 0, 0, 2361, 1, 1, 90, 503, 91, 2367, 2, 0, 10, 3, 710, 0, 2815, 5880, 0, 743] payload_rows_per_dst=[8682, 14532, 9554, 8366, 6436, 6918, 13668, 12410] dup_rows_per_dst=[1282, 1322, 2338, 2956, 584, 1190, 4024, 4042] combine_rows_per_src=[8682, 14532, 9554, 8366, 6436, 6918, 13668, 12410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=68968 dup_rows=16040 dup_rows_remote=14178 dedup_ratio=0.163167 router_tokens_per_expert=[0, 0, 3, 0, 2818, 723, 1106, 96, 437, 74, 354, 0, 252, 0, 6838, 0, 2487, 1367, 25, 0, 1552, 324, 0, 149, 1190, 928, 1212, 694, 484, 758, 483, 0, 0, 8, 10, 3, 1479, 14, 163, 0, 584, 5759, 118, 505, 239, 0, 0, 2293, 0, 0, 73, 537, 142, 2563, 0, 0, 2, 0, 715, 0, 2971, 5902, 0, 718] payload_rows_per_dst=[8178, 14418, 9512, 8700, 13296, 6542, 13584, 8034] dup_rows_per_dst=[1314, 1492, 2296, 2798, 1862, 936, 4564, 778] combine_rows_per_src=[8178, 14418, 9512, 8700, 13296, 6542, 13584, 8034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=302022656 dispatch_meta_bytes=345208 combine_payload_bytes=302022656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302367864 ingress_bytes=336625664 link_bytes=336625664 token_bytes=604390520 weight_movement_bytes=34603008 total_nvlink_bytes=638993528 weight_to_token_ratio=0.057253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266695273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86582 payload_rows_remote=70726 dup_rows=11722 dup_rows_remote=7402 dedup_ratio=0.119242 router_tokens_per_expert=[0, 601, 3477, 0, 0, 34, 0, 63, 0, 0, 0, 91, 103, 0, 0, 0, 0, 3183, 0, 2686, 0, 216, 6780, 4, 2, 0, 308, 70, 0, 2095, 0, 0, 0, 0, 0, 521, 1880, 0, 3539, 260, 0, 2069, 0, 7817, 88, 0, 114, 0, 0, 406, 7199, 0, 0, 414, 4398, 476, 0, 0, 0, 0, 256, 0, 2, 0] payload_rows_per_dst=[9010, 13560, 9106, 4820, 10148, 15856, 9684, 14398] dup_rows_per_dst=[1054, 388, 1358, 130, 2252, 4320, 1704, 516] combine_rows_per_src=[9010, 13560, 9106, 4820, 10148, 15856, 9684, 14398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80566 payload_rows_remote=66898 dup_rows=17738 dup_rows_remote=13714 dedup_ratio=0.180440 router_tokens_per_expert=[0, 1, 1, 1, 2954, 866, 1096, 63, 366, 90, 362, 0, 237, 1, 6869, 2, 2726, 1404, 13, 1, 1400, 278, 0, 124, 1217, 784, 1241, 790, 472, 712, 445, 0, 0, 11, 49, 11, 1340, 17, 147, 0, 718, 5791, 92, 584, 299, 0, 0, 2361, 1, 1, 90, 503, 91, 2367, 2, 0, 10, 3, 710, 0, 2815, 5880, 0, 743] payload_rows_per_dst=[8682, 14532, 9554, 8366, 6436, 6918, 13668, 12410] dup_rows_per_dst=[1282, 1322, 2338, 2956, 584, 1190, 4024, 4042] combine_rows_per_src=[8682, 14532, 9554, 8366, 6436, 6918, 13668, 12410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=253198336 dispatch_meta_bytes=298768 combine_payload_bytes=253198336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253497104 ingress_bytes=287801344 link_bytes=287801344 token_bytes=506695440 weight_movement_bytes=34603008 total_nvlink_bytes=541298448 weight_to_token_ratio=0.068292 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.082973288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282492928 dispatch_meta_bytes=332584 combine_payload_bytes=282492928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282825512 ingress_bytes=317095936 link_bytes=317095936 token_bytes=565318440 weight_movement_bytes=34603008 total_nvlink_bytes=599921448 weight_to_token_ratio=0.061210 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193206479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77982 payload_rows_remote=63636 dup_rows=20322 dup_rows_remote=19854 dedup_ratio=0.206726 router_tokens_per_expert=[0, 533, 3167, 1, 2, 40, 2, 49, 0, 1, 0, 61, 75, 0, 0, 0, 0, 3027, 0, 3499, 0, 214, 6680, 12, 0, 0, 354, 65, 0, 2051, 1, 0, 2, 0, 0, 523, 1490, 3, 3841, 199, 0, 2348, 1, 7844, 52, 1, 132, 0, 1, 244, 7161, 1, 1, 424, 4332, 472, 0, 0, 0, 5, 238, 0, 2, 1] payload_rows_per_dst=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] dup_rows_per_dst=[584, 114, 8354, 120, 1710, 2862, 6110, 468] combine_rows_per_src=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86386 payload_rows_remote=76558 dup_rows=11918 dup_rows_remote=9692 dedup_ratio=0.121236 router_tokens_per_expert=[0, 574, 3491, 1, 0, 2, 0, 69, 0, 0, 0, 73, 110, 0, 0, 0, 0, 3097, 0, 3162, 1, 260, 6569, 9, 4, 0, 610, 23, 0, 1945, 0, 0, 0, 0, 0, 604, 1357, 0, 3784, 282, 0, 2166, 0, 7834, 68, 0, 114, 0, 0, 523, 7196, 0, 0, 415, 4124, 462, 0, 0, 0, 0, 222, 0, 1, 0] payload_rows_per_dst=[12240, 13138, 6546, 5118, 9828, 15842, 9282, 14392] dup_rows_per_dst=[2358, 366, 188, 46, 2226, 4522, 1766, 446] combine_rows_per_src=[12240, 13138, 6546, 5118, 9828, 15842, 9282, 14392] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274014208 dispatch_meta_bytes=322448 combine_payload_bytes=274014208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274336656 ingress_bytes=308617216 link_bytes=308617216 token_bytes=548350864 weight_movement_bytes=34603008 total_nvlink_bytes=582953872 weight_to_token_ratio=0.063104 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161301738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=289693696 dispatch_meta_bytes=312512 combine_payload_bytes=289693696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290006208 ingress_bytes=393502720 link_bytes=393502720 token_bytes=579699904 weight_movement_bytes=103809024 total_nvlink_bytes=683508928 weight_to_token_ratio=0.179074 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.480719056e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77210 payload_rows_remote=61816 dup_rows=21094 dup_rows_remote=12876 dedup_ratio=0.214579 router_tokens_per_expert=[0, 0, 2, 0, 2996, 796, 989, 122, 388, 81, 383, 0, 255, 0, 6901, 0, 2622, 1316, 34, 1, 1294, 311, 0, 138, 1200, 957, 1134, 686, 611, 685, 451, 0, 0, 9, 41, 1, 1332, 31, 164, 0, 631, 5803, 75, 608, 238, 0, 0, 2385, 0, 2, 86, 477, 92, 2546, 0, 0, 1, 0, 850, 0, 2957, 5753, 1, 716] payload_rows_per_dst=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] dup_rows_per_dst=[8218, 36, 2198, 2730, 1726, 5404, 88, 694] combine_rows_per_src=[15394, 2242, 9234, 8718, 12936, 14012, 6318, 8356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82404 payload_rows_remote=73130 dup_rows=15900 dup_rows_remote=13724 dedup_ratio=0.161743 router_tokens_per_expert=[0, 0, 0, 0, 3151, 780, 1136, 60, 414, 74, 396, 0, 248, 0, 7002, 1, 2436, 1450, 14, 1, 1403, 272, 0, 149, 1170, 975, 1145, 715, 427, 645, 491, 0, 0, 7, 59, 3, 1348, 30, 107, 0, 727, 5962, 104, 540, 227, 0, 0, 2460, 0, 0, 69, 487, 96, 2446, 0, 0, 0, 0, 667, 1, 2873, 5661, 0, 723] payload_rows_per_dst=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] dup_rows_per_dst=[1278, 1554, 2176, 2824, 1672, 5770, 54, 572] combine_rows_per_src=[8976, 14716, 9274, 8312, 12758, 14270, 6142, 7956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260653056 dispatch_meta_bytes=333960 combine_payload_bytes=260653056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260987016 ingress_bytes=295256064 link_bytes=295256064 token_bytes=521640072 weight_movement_bytes=34603008 total_nvlink_bytes=556243080 weight_to_token_ratio=0.066335 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111024799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86086 payload_rows_remote=78068 dup_rows=12218 dup_rows_remote=11428 dedup_ratio=0.124288 router_tokens_per_expert=[0, 529, 3386, 0, 0, 16, 0, 112, 0, 0, 0, 60, 72, 0, 0, 0, 0, 3116, 0, 3079, 2, 246, 6503, 7, 8, 0, 309, 76, 0, 1995, 0, 0, 0, 0, 0, 598, 1701, 1, 3908, 159, 0, 2196, 0, 7845, 60, 0, 151, 0, 0, 507, 7220, 0, 0, 382, 4228, 430, 0, 0, 0, 1, 248, 0, 1, 0] payload_rows_per_dst=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] dup_rows_per_dst=[2386, 258, 790, 4672, 2198, 120, 1296, 498] combine_rows_per_src=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86582 payload_rows_remote=70726 dup_rows=11722 dup_rows_remote=7402 dedup_ratio=0.119242 router_tokens_per_expert=[0, 601, 3477, 0, 0, 34, 0, 63, 0, 0, 0, 91, 103, 0, 0, 0, 0, 3183, 0, 2686, 0, 216, 6780, 4, 2, 0, 308, 70, 0, 2095, 0, 0, 0, 0, 0, 521, 1880, 0, 3539, 260, 0, 2069, 0, 7817, 88, 0, 114, 0, 0, 406, 7199, 0, 0, 414, 4398, 476, 0, 0, 0, 0, 256, 0, 2, 0] payload_rows_per_dst=[9010, 13560, 9106, 4820, 10148, 15856, 9684, 14398] dup_rows_per_dst=[1054, 388, 1358, 130, 2252, 4320, 1704, 516] combine_rows_per_src=[9010, 13560, 9106, 4820, 10148, 15856, 9684, 14398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77982 payload_rows_remote=63636 dup_rows=20322 dup_rows_remote=19854 dedup_ratio=0.206726 router_tokens_per_expert=[0, 533, 3167, 1, 2, 40, 2, 49, 0, 1, 0, 61, 75, 0, 0, 0, 0, 3027, 0, 3499, 0, 214, 6680, 12, 0, 0, 354, 65, 0, 2051, 1, 0, 2, 0, 0, 523, 1490, 3, 3841, 199, 0, 2348, 1, 7844, 52, 1, 132, 0, 1, 244, 7161, 1, 1, 424, 4332, 472, 0, 0, 0, 5, 238, 0, 2, 1] payload_rows_per_dst=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] dup_rows_per_dst=[584, 114, 8354, 120, 1710, 2862, 6110, 468] combine_rows_per_src=[7004, 4124, 14546, 4822, 8766, 11544, 12830, 14346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84922 payload_rows_remote=71706 dup_rows=13382 dup_rows_remote=12954 dedup_ratio=0.136129 router_tokens_per_expert=[0, 612, 3777, 0, 0, 4, 0, 106, 0, 1, 0, 88, 129, 0, 0, 0, 0, 2930, 0, 3330, 2, 199, 6604, 6, 1, 0, 274, 61, 0, 1905, 0, 0, 1, 0, 2, 517, 1603, 0, 3776, 263, 0, 2103, 1, 7817, 86, 0, 136, 0, 0, 227, 7292, 0, 0, 392, 4213, 446, 0, 0, 1, 0, 246, 0, 1, 0] payload_rows_per_dst=[12746, 13216, 12408, 15736, 10260, 4480, 7434, 8642] dup_rows_per_dst=[2912, 428, 2292, 4380, 2064, 172, 866, 268] combine_rows_per_src=[12746, 13216, 12408, 15736, 10260, 4480, 7434, 8642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86370 payload_rows_remote=73502 dup_rows=11934 dup_rows_remote=8182 dedup_ratio=0.121399 router_tokens_per_expert=[0, 546, 3109, 0, 0, 10, 0, 78, 0, 0, 0, 52, 73, 0, 0, 0, 0, 3330, 0, 3015, 0, 235, 6503, 6, 2, 0, 275, 96, 0, 2099, 0, 0, 0, 0, 0, 527, 1947, 0, 3889, 209, 0, 2059, 0, 7852, 52, 0, 140, 0, 0, 378, 7113, 0, 0, 414, 4435, 455, 0, 1, 0, 0, 247, 0, 5, 0] payload_rows_per_dst=[11474, 13024, 6984, 12868, 10688, 7344, 9752, 14236] dup_rows_per_dst=[2042, 232, 158, 3752, 2456, 1186, 1612, 496] combine_rows_per_src=[11474, 13024, 6984, 12868, 10688, 7344, 9752, 14236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=253198336 dispatch_meta_bytes=298768 combine_payload_bytes=253198336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253497104 ingress_bytes=287801344 link_bytes=287801344 token_bytes=506695440 weight_movement_bytes=34603008 total_nvlink_bytes=541298448 weight_to_token_ratio=0.068292 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.082973288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=313581568 dispatch_meta_bytes=345000 combine_payload_bytes=313581568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=313926568 ingress_bytes=348184576 link_bytes=348184576 token_bytes=627508136 weight_movement_bytes=34603008 total_nvlink_bytes=662111144 weight_to_token_ratio=0.055144 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310190528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=299540480 dispatch_meta_bytes=347416 combine_payload_bytes=299540480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299887896 ingress_bytes=334143488 link_bytes=334143488 token_bytes=599428376 weight_movement_bytes=34603008 total_nvlink_bytes=634031384 weight_to_token_ratio=0.057727 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257355045e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=357984 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320124512 ingress_bytes=423575552 link_bytes=423575552 token_bytes=639891040 weight_movement_bytes=103809024 total_nvlink_bytes=743700064 weight_to_token_ratio=0.162229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.593880702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=289693696 dispatch_meta_bytes=312512 combine_payload_bytes=289693696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290006208 ingress_bytes=393502720 link_bytes=393502720 token_bytes=579699904 weight_movement_bytes=103809024 total_nvlink_bytes=683508928 weight_to_token_ratio=0.179074 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.480719056e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260653056 dispatch_meta_bytes=333960 combine_payload_bytes=260653056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260987016 ingress_bytes=295256064 link_bytes=295256064 token_bytes=521640072 weight_movement_bytes=34603008 total_nvlink_bytes=556243080 weight_to_token_ratio=0.066335 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111024799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84636 payload_rows_remote=72406 dup_rows=13668 dup_rows_remote=11206 dedup_ratio=0.139038 router_tokens_per_expert=[0, 646, 3401, 0, 0, 3, 0, 96, 0, 0, 0, 61, 84, 0, 0, 0, 0, 2869, 0, 3200, 0, 216, 6539, 10, 5, 0, 562, 52, 0, 1851, 0, 0, 0, 0, 0, 648, 1487, 0, 3782, 291, 0, 2316, 0, 7860, 60, 0, 144, 0, 0, 256, 7301, 0, 0, 382, 4317, 457, 0, 0, 0, 0, 255, 0, 1, 0] payload_rows_per_dst=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] dup_rows_per_dst=[2462, 290, 2402, 4464, 2362, 120, 1568, 0] combine_rows_per_src=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=293707776 dispatch_meta_bytes=338640 combine_payload_bytes=293707776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294046416 ingress_bytes=328310784 link_bytes=328310784 token_bytes=587754192 weight_movement_bytes=34603008 total_nvlink_bytes=622357200 weight_to_token_ratio=0.058873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235407049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86386 payload_rows_remote=76558 dup_rows=11918 dup_rows_remote=9692 dedup_ratio=0.121236 router_tokens_per_expert=[0, 574, 3491, 1, 0, 2, 0, 69, 0, 0, 0, 73, 110, 0, 0, 0, 0, 3097, 0, 3162, 1, 260, 6569, 9, 4, 0, 610, 23, 0, 1945, 0, 0, 0, 0, 0, 604, 1357, 0, 3784, 282, 0, 2166, 0, 7834, 68, 0, 114, 0, 0, 523, 7196, 0, 0, 415, 4124, 462, 0, 0, 0, 0, 222, 0, 1, 0] payload_rows_per_dst=[12240, 13138, 6546, 5118, 9828, 15842, 9282, 14392] dup_rows_per_dst=[2358, 366, 188, 46, 2226, 4522, 1766, 446] combine_rows_per_src=[12240, 13138, 6546, 5118, 9828, 15842, 9282, 14392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85798 payload_rows_remote=76118 dup_rows=12506 dup_rows_remote=10742 dedup_ratio=0.127218 router_tokens_per_expert=[0, 599, 3242, 5, 0, 13, 2, 111, 1, 2, 1, 51, 70, 0, 2, 0, 0, 3176, 0, 2957, 0, 225, 6411, 8, 6, 0, 523, 106, 0, 2055, 1, 2, 1, 0, 3, 514, 1621, 0, 4070, 169, 3, 2110, 0, 7851, 49, 0, 147, 1, 1, 487, 7081, 7, 0, 406, 4374, 447, 0, 3, 3, 0, 234, 0, 0, 1] payload_rows_per_dst=[7320, 12842, 10408, 5180, 10340, 15846, 9680, 14182] dup_rows_per_dst=[624, 234, 2324, 206, 2416, 4476, 1764, 462] combine_rows_per_src=[7320, 12842, 10408, 5180, 10340, 15846, 9680, 14182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83192 payload_rows_remote=70574 dup_rows=15112 dup_rows_remote=13856 dedup_ratio=0.153727 router_tokens_per_expert=[81, 7379, 76, 0, 6567, 2201, 3754, 0, 0, 0, 0, 1034, 341, 0, 0, 3003, 337, 689, 542, 71, 48, 701, 5762, 0, 0, 1156, 336, 0, 0, 0, 139, 0, 928, 0, 0, 0, 0, 0, 137, 1923, 5876, 0, 0, 0, 973, 1, 63, 24, 0, 0, 2342, 120, 0, 0, 0, 702, 0, 0, 76, 0, 111, 1444, 0, 215] payload_rows_per_dst=[4716, 7110, 14140, 3246, 11112, 12618, 15980, 14270] dup_rows_per_dst=[0, 1646, 2160, 16, 2372, 1256, 5106, 2556] combine_rows_per_src=[4716, 7110, 14140, 3246, 11112, 12618, 15980, 14270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=301064192 dispatch_meta_bytes=326736 combine_payload_bytes=301064192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301390928 ingress_bytes=335667200 link_bytes=335667200 token_bytes=602455120 weight_movement_bytes=34603008 total_nvlink_bytes=637058128 weight_to_token_ratio=0.057437 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85798 payload_rows_remote=76118 dup_rows=12506 dup_rows_remote=10742 dedup_ratio=0.127218 router_tokens_per_expert=[0, 599, 3242, 5, 0, 13, 2, 111, 1, 2, 1, 51, 70, 0, 2, 0, 0, 3176, 0, 2957, 0, 225, 6411, 8, 6, 0, 523, 106, 0, 2055, 1, 2, 1, 0, 3, 514, 1621, 0, 4070, 169, 3, 2110, 0, 7851, 49, 0, 147, 1, 1, 487, 7081, 7, 0, 406, 4374, 447, 0, 3, 3, 0, 234, 0, 0, 1] payload_rows_per_dst=[7320, 12842, 10408, 5180, 10340, 15846, 9680, 14182] dup_rows_per_dst=[624, 234, 2324, 206, 2416, 4476, 1764, 462] combine_rows_per_src=[7320, 12842, 10408, 5180, 10340, 15846, 9680, 14182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=334448 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296909424 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593484400 weight_movement_bytes=34603008 total_nvlink_bytes=628087408 weight_to_token_ratio=0.058305 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83142 payload_rows_remote=68338 dup_rows=15162 dup_rows_remote=12426 dedup_ratio=0.154236 router_tokens_per_expert=[70, 7503, 76, 0, 6894, 1927, 3657, 0, 0, 0, 1, 1021, 220, 0, 0, 3149, 352, 694, 525, 53, 65, 749, 5799, 0, 2, 1300, 350, 2, 1, 0, 183, 0, 689, 0, 0, 3, 1, 1, 213, 1491, 6327, 0, 1, 0, 920, 10, 52, 26, 0, 0, 2297, 14, 0, 0, 1, 637, 0, 0, 75, 0, 80, 1515, 1, 205] payload_rows_per_dst=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] dup_rows_per_dst=[2, 1480, 2210, 42, 1594, 2192, 4906, 2736] combine_rows_per_src=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=313581568 dispatch_meta_bytes=345000 combine_payload_bytes=313581568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=313926568 ingress_bytes=348184576 link_bytes=348184576 token_bytes=627508136 weight_movement_bytes=34603008 total_nvlink_bytes=662111144 weight_to_token_ratio=0.055144 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310190528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84328 payload_rows_remote=73682 dup_rows=13976 dup_rows_remote=11852 dedup_ratio=0.142171 router_tokens_per_expert=[72, 7386, 72, 0, 6738, 2163, 3771, 0, 0, 0, 0, 886, 246, 0, 0, 2896, 347, 624, 553, 67, 70, 775, 5690, 0, 0, 1335, 346, 0, 0, 0, 194, 0, 782, 0, 0, 0, 0, 0, 209, 1623, 6001, 0, 0, 0, 1005, 0, 72, 27, 0, 0, 2388, 28, 0, 0, 0, 718, 0, 0, 72, 0, 118, 1634, 0, 244] payload_rows_per_dst=[4614, 6720, 14028, 12670, 10646, 12828, 8224, 14598] dup_rows_per_dst=[0, 1336, 2224, 2614, 2124, 1382, 1282, 3014] combine_rows_per_src=[4614, 6720, 14028, 12670, 10646, 12828, 8224, 14598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=311779328 dispatch_meta_bytes=347440 combine_payload_bytes=311779328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312126768 ingress_bytes=415588352 link_bytes=415588352 token_bytes=623906096 weight_movement_bytes=103809024 total_nvlink_bytes=727715120 weight_to_token_ratio=0.166386 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.563825511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289071104 dispatch_meta_bytes=337720 combine_payload_bytes=289071104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289408824 ingress_bytes=358277120 link_bytes=358277120 token_bytes=578479928 weight_movement_bytes=69206016 total_nvlink_bytes=647685944 weight_to_token_ratio=0.119634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348167959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84636 payload_rows_remote=72406 dup_rows=13668 dup_rows_remote=11206 dedup_ratio=0.139038 router_tokens_per_expert=[0, 646, 3401, 0, 0, 3, 0, 96, 0, 0, 0, 61, 84, 0, 0, 0, 0, 2869, 0, 3200, 0, 216, 6539, 10, 5, 0, 562, 52, 0, 1851, 0, 0, 0, 0, 0, 648, 1487, 0, 3782, 291, 0, 2316, 0, 7860, 60, 0, 144, 0, 0, 256, 7301, 0, 0, 382, 4317, 457, 0, 0, 0, 0, 255, 0, 1, 0] payload_rows_per_dst=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] dup_rows_per_dst=[2462, 290, 2402, 4464, 2362, 120, 1568, 0] combine_rows_per_src=[12230, 13078, 12422, 16196, 10054, 4920, 15224, 512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86086 payload_rows_remote=78068 dup_rows=12218 dup_rows_remote=11428 dedup_ratio=0.124288 router_tokens_per_expert=[0, 529, 3386, 0, 0, 16, 0, 112, 0, 0, 0, 60, 72, 0, 0, 0, 0, 3116, 0, 3079, 2, 246, 6503, 7, 8, 0, 309, 76, 0, 1995, 0, 0, 0, 0, 0, 598, 1701, 1, 3908, 159, 0, 2196, 0, 7845, 60, 0, 151, 0, 0, 507, 7220, 0, 0, 382, 4228, 430, 0, 0, 0, 1, 248, 0, 1, 0] payload_rows_per_dst=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] dup_rows_per_dst=[2386, 258, 790, 4672, 2198, 120, 1296, 498] combine_rows_per_src=[11858, 13012, 8018, 15794, 10536, 4694, 7732, 14442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279912448 dispatch_meta_bytes=323056 combine_payload_bytes=279912448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280235504 ingress_bytes=314515456 link_bytes=314515456 token_bytes=560147952 weight_movement_bytes=34603008 total_nvlink_bytes=594750960 weight_to_token_ratio=0.061775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183496340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82044 payload_rows_remote=74634 dup_rows=16260 dup_rows_remote=15042 dedup_ratio=0.165405 router_tokens_per_expert=[70, 7477, 70, 0, 6612, 2153, 3407, 0, 0, 0, 0, 896, 330, 0, 2, 3057, 337, 678, 521, 57, 12, 819, 5712, 0, 0, 1071, 335, 0, 0, 0, 174, 0, 837, 0, 0, 0, 0, 0, 232, 1677, 6277, 0, 0, 0, 836, 0, 71, 37, 0, 0, 2728, 15, 0, 0, 0, 628, 0, 0, 71, 0, 90, 1672, 0, 191] payload_rows_per_dst=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] dup_rows_per_dst=[3588, 4710, 1218, 2864, 2086, 1224, 132, 438] combine_rows_per_src=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83192 payload_rows_remote=70574 dup_rows=15112 dup_rows_remote=13856 dedup_ratio=0.153727 router_tokens_per_expert=[81, 7379, 76, 0, 6567, 2201, 3754, 0, 0, 0, 0, 1034, 341, 0, 0, 3003, 337, 689, 542, 71, 48, 701, 5762, 0, 0, 1156, 336, 0, 0, 0, 139, 0, 928, 0, 0, 0, 0, 0, 137, 1923, 5876, 0, 0, 0, 973, 1, 63, 24, 0, 0, 2342, 120, 0, 0, 0, 702, 0, 0, 76, 0, 111, 1444, 0, 215] payload_rows_per_dst=[4716, 7110, 14140, 3246, 11112, 12618, 15980, 14270] dup_rows_per_dst=[0, 1646, 2160, 16, 2372, 1256, 5106, 2556] combine_rows_per_src=[4716, 7110, 14140, 3246, 11112, 12618, 15980, 14270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83142 payload_rows_remote=68338 dup_rows=15162 dup_rows_remote=12426 dedup_ratio=0.154236 router_tokens_per_expert=[70, 7503, 76, 0, 6894, 1927, 3657, 0, 0, 0, 1, 1021, 220, 0, 0, 3149, 352, 694, 525, 53, 65, 749, 5799, 0, 2, 1300, 350, 2, 1, 0, 183, 0, 689, 0, 0, 3, 1, 1, 213, 1491, 6327, 0, 1, 0, 920, 10, 52, 26, 0, 0, 2297, 14, 0, 0, 1, 637, 0, 0, 75, 0, 80, 1515, 1, 205] payload_rows_per_dst=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] dup_rows_per_dst=[2, 1480, 2210, 42, 1594, 2192, 4906, 2736] combine_rows_per_src=[4144, 7302, 13398, 3634, 10516, 13346, 15998, 14804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76504 payload_rows_remote=62732 dup_rows=21800 dup_rows_remote=15354 dedup_ratio=0.221761 router_tokens_per_expert=[88, 7445, 87, 0, 6693, 2546, 3556, 0, 0, 2, 0, 1069, 248, 0, 0, 3221, 321, 715, 540, 44, 74, 713, 5569, 0, 2, 1247, 319, 0, 0, 0, 185, 0, 603, 1, 2, 0, 1, 2, 244, 1472, 6155, 0, 0, 2, 865, 1, 82, 35, 0, 1, 2128, 5, 0, 0, 1, 706, 0, 0, 86, 0, 134, 1720, 3, 219] payload_rows_per_dst=[14930, 13772, 3966, 15298, 10148, 8958, 5606, 3826] dup_rows_per_dst=[8350, 6446, 848, 3098, 1614, 870, 76, 498] combine_rows_per_src=[14930, 13772, 3966, 15298, 10148, 8958, 5606, 3826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85570 payload_rows_remote=70308 dup_rows=12734 dup_rows_remote=9520 dedup_ratio=0.129537 router_tokens_per_expert=[85, 7462, 81, 1, 6477, 2291, 3860, 0, 0, 0, 0, 1041, 324, 0, 0, 2863, 372, 677, 577, 82, 170, 586, 5515, 0, 0, 1212, 372, 0, 0, 0, 192, 0, 601, 0, 0, 0, 0, 0, 222, 1667, 6121, 0, 0, 0, 928, 0, 79, 37, 0, 0, 2564, 2, 0, 0, 0, 666, 0, 0, 81, 0, 84, 1625, 0, 235] payload_rows_per_dst=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] dup_rows_per_dst=[0, 1442, 2018, 3214, 1746, 1372, 144, 2798] combine_rows_per_src=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=334448 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296909424 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593484400 weight_movement_bytes=34603008 total_nvlink_bytes=628087408 weight_to_token_ratio=0.058305 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=301801472 dispatch_meta_bytes=342136 combine_payload_bytes=301801472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302143608 ingress_bytes=336404480 link_bytes=336404480 token_bytes=603945080 weight_movement_bytes=34603008 total_nvlink_bytes=638548088 weight_to_token_ratio=0.057295 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91418 payload_rows_remote=83504 dup_rows=6886 dup_rows_remote=5734 dedup_ratio=0.070048 router_tokens_per_expert=[364, 0, 22, 0, 0, 0, 0, 450, 0, 0, 3903, 0, 2546, 7854, 0, 3760, 0, 0, 0, 303, 0, 0, 476, 97, 0, 6, 76, 0, 0, 76, 0, 0, 0, 0, 5160, 0, 0, 0, 0, 0, 0, 7455, 0, 0, 0, 120, 811, 619, 0, 0, 0, 92, 3979, 2816, 49, 0, 2, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9128, 10686, 9704, 15708, 10722, 7914, 11324, 16232] dup_rows_per_dst=[64, 994, 992, 316, 816, 1152, 2548, 4] combine_rows_per_src=[9128, 10686, 9704, 15708, 10722, 7914, 11324, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=357984 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320124512 ingress_bytes=423575552 link_bytes=423575552 token_bytes=639891040 weight_movement_bytes=103809024 total_nvlink_bytes=743700064 weight_to_token_ratio=0.162229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.593880702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84328 payload_rows_remote=73682 dup_rows=13976 dup_rows_remote=11852 dedup_ratio=0.142171 router_tokens_per_expert=[72, 7386, 72, 0, 6738, 2163, 3771, 0, 0, 0, 0, 886, 246, 0, 0, 2896, 347, 624, 553, 67, 70, 775, 5690, 0, 0, 1335, 346, 0, 0, 0, 194, 0, 782, 0, 0, 0, 0, 0, 209, 1623, 6001, 0, 0, 0, 1005, 0, 72, 27, 0, 0, 2388, 28, 0, 0, 0, 718, 0, 0, 72, 0, 118, 1634, 0, 244] payload_rows_per_dst=[4614, 6720, 14028, 12670, 10646, 12828, 8224, 14598] dup_rows_per_dst=[0, 1336, 2224, 2614, 2124, 1382, 1282, 3014] combine_rows_per_src=[4614, 6720, 14028, 12670, 10646, 12828, 8224, 14598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=305700864 dispatch_meta_bytes=358704 combine_payload_bytes=305700864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306059568 ingress_bytes=374906880 link_bytes=374906880 token_bytes=611760432 weight_movement_bytes=69206016 total_nvlink_bytes=680966448 weight_to_token_ratio=0.113126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410744407e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=289071104 dispatch_meta_bytes=337720 combine_payload_bytes=289071104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289408824 ingress_bytes=358277120 link_bytes=358277120 token_bytes=578479928 weight_movement_bytes=69206016 total_nvlink_bytes=647685944 weight_to_token_ratio=0.119634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348167959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85570 payload_rows_remote=70308 dup_rows=12734 dup_rows_remote=9520 dedup_ratio=0.129537 router_tokens_per_expert=[85, 7462, 81, 1, 6477, 2291, 3860, 0, 0, 0, 0, 1041, 324, 0, 0, 2863, 372, 677, 577, 82, 170, 586, 5515, 0, 0, 1212, 372, 0, 0, 0, 192, 0, 601, 0, 0, 0, 0, 0, 222, 1667, 6121, 0, 0, 0, 928, 0, 79, 37, 0, 0, 2564, 2, 0, 0, 0, 666, 0, 0, 81, 0, 84, 1625, 0, 235] payload_rows_per_dst=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] dup_rows_per_dst=[0, 1442, 2018, 3214, 1746, 1372, 144, 2798] combine_rows_per_src=[4926, 7014, 13940, 15262, 10954, 12958, 6320, 14196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67844 payload_rows_remote=52062 dup_rows=30460 dup_rows_remote=11862 dedup_ratio=0.309855 router_tokens_per_expert=[80, 7324, 78, 0, 6922, 1970, 3962, 0, 0, 0, 0, 991, 248, 0, 0, 3046, 309, 669, 469, 83, 133, 682, 5386, 0, 0, 1417, 306, 0, 0, 0, 164, 0, 740, 0, 0, 0, 0, 0, 266, 1671, 6110, 0, 0, 0, 834, 0, 78, 30, 0, 0, 2405, 74, 0, 0, 0, 696, 0, 0, 78, 0, 91, 1626, 0, 214] payload_rows_per_dst=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] dup_rows_per_dst=[18598, 5810, 816, 3078, 1536, 2, 176, 444] combine_rows_per_src=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279912448 dispatch_meta_bytes=323056 combine_payload_bytes=279912448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280235504 ingress_bytes=314515456 link_bytes=314515456 token_bytes=560147952 weight_movement_bytes=34603008 total_nvlink_bytes=594750960 weight_to_token_ratio=0.061775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183496340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256950272 dispatch_meta_bytes=312344 combine_payload_bytes=256950272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257262616 ingress_bytes=291553280 link_bytes=291553280 token_bytes=514212888 weight_movement_bytes=34603008 total_nvlink_bytes=548815896 weight_to_token_ratio=0.067293 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097091521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82898 payload_rows_remote=66882 dup_rows=15406 dup_rows_remote=9852 dedup_ratio=0.156718 router_tokens_per_expert=[83, 7466, 85, 0, 6375, 2048, 3675, 0, 0, 0, 3, 979, 282, 0, 0, 2976, 351, 722, 436, 86, 73, 745, 5614, 0, 0, 1345, 350, 0, 0, 1, 187, 2, 677, 22, 0, 0, 1, 0, 237, 1817, 6039, 0, 0, 1, 929, 6, 88, 36, 0, 1, 2697, 22, 0, 0, 2, 597, 1, 0, 92, 0, 85, 1711, 4, 203] payload_rows_per_dst=[4432, 7022, 13928, 3708, 10880, 12864, 16016, 14048] dup_rows_per_dst=[0, 1458, 2126, 62, 1978, 1334, 5554, 2894] combine_rows_per_src=[4432, 7022, 13928, 3708, 10880, 12864, 16016, 14048] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=287981568 dispatch_meta_bytes=319312 combine_payload_bytes=287981568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288300880 ingress_bytes=322584576 link_bytes=322584576 token_bytes=576282448 weight_movement_bytes=34603008 total_nvlink_bytes=610885456 weight_to_token_ratio=0.060045 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213859789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=213245952 dispatch_meta_bytes=255696 combine_payload_bytes=213245952 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=213501648 ingress_bytes=386260992 link_bytes=386260992 token_bytes=426747600 weight_movement_bytes=173015040 total_nvlink_bytes=599762640 weight_to_token_ratio=0.405427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453469016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85492 payload_rows_remote=69272 dup_rows=12812 dup_rows_remote=7054 dedup_ratio=0.130330 router_tokens_per_expert=[363, 0, 46, 0, 0, 2, 0, 421, 0, 0, 4161, 1, 2521, 7824, 1, 3720, 0, 0, 0, 230, 0, 2, 483, 118, 1, 39, 73, 0, 1, 71, 1, 0, 3, 0, 5142, 0, 2, 1, 0, 0, 27, 7584, 0, 0, 0, 122, 751, 627, 0, 0, 4, 57, 4133, 2508, 0, 0, 3, 2, 0, 0, 0, 0, 8107, 0] payload_rows_per_dst=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] dup_rows_per_dst=[80, 0, 152, 366, 3114, 2480, 862, 5758] combine_rows_per_src=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342032384 dispatch_meta_bytes=356952 combine_payload_bytes=342032384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342389336 ingress_bytes=411238400 link_bytes=411238400 token_bytes=684421720 weight_movement_bytes=69206016 total_nvlink_bytes=753627736 weight_to_token_ratio=0.101116 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.547456992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=301801472 dispatch_meta_bytes=342136 combine_payload_bytes=301801472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302143608 ingress_bytes=336404480 link_bytes=336404480 token_bytes=603945080 weight_movement_bytes=34603008 total_nvlink_bytes=638548088 weight_to_token_ratio=0.057295 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92370 payload_rows_remote=81770 dup_rows=5934 dup_rows_remote=5934 dedup_ratio=0.060364 router_tokens_per_expert=[380, 0, 16, 0, 0, 0, 0, 475, 0, 0, 3705, 0, 2751, 7844, 0, 4055, 0, 0, 0, 296, 0, 0, 459, 99, 0, 62, 72, 0, 0, 72, 0, 0, 0, 0, 5300, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 120, 765, 592, 0, 0, 0, 74, 3599, 2768, 26, 0, 1, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] dup_rows_per_dst=[72, 1380, 1532, 412, 0, 290, 2246, 2] combine_rows_per_src=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273948672 dispatch_meta_bytes=306936 combine_payload_bytes=273948672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274255608 ingress_bytes=308551680 link_bytes=308551680 token_bytes=548204280 weight_movement_bytes=34603008 total_nvlink_bytes=582807288 weight_to_token_ratio=0.063121 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161055131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67844 payload_rows_remote=52062 dup_rows=30460 dup_rows_remote=11862 dedup_ratio=0.309855 router_tokens_per_expert=[80, 7324, 78, 0, 6922, 1970, 3962, 0, 0, 0, 0, 991, 248, 0, 0, 3046, 309, 669, 469, 83, 133, 682, 5386, 0, 0, 1417, 306, 0, 0, 0, 164, 0, 740, 0, 0, 0, 0, 0, 266, 1671, 6110, 0, 0, 0, 834, 0, 78, 30, 0, 0, 2405, 74, 0, 0, 0, 696, 0, 0, 78, 0, 91, 1626, 0, 214] payload_rows_per_dst=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] dup_rows_per_dst=[18598, 5810, 816, 3078, 1536, 2, 176, 444] combine_rows_per_src=[15782, 13532, 3874, 14248, 8778, 1882, 6174, 3574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283738112 dispatch_meta_bytes=305304 combine_payload_bytes=283738112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284043416 ingress_bytes=352944128 link_bytes=352944128 token_bytes=567781528 weight_movement_bytes=69206016 total_nvlink_bytes=636987544 weight_to_token_ratio=0.121888 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328100339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82044 payload_rows_remote=74634 dup_rows=16260 dup_rows_remote=15042 dedup_ratio=0.165405 router_tokens_per_expert=[70, 7477, 70, 0, 6612, 2153, 3407, 0, 0, 0, 0, 896, 330, 0, 2, 3057, 337, 678, 521, 57, 12, 819, 5712, 0, 0, 1071, 335, 0, 0, 0, 174, 0, 837, 0, 0, 0, 0, 0, 232, 1677, 6277, 0, 0, 0, 836, 0, 71, 37, 0, 0, 2728, 15, 0, 0, 0, 628, 0, 0, 71, 0, 90, 1672, 0, 191] payload_rows_per_dst=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] dup_rows_per_dst=[3588, 4710, 1218, 2864, 2086, 1224, 132, 438] combine_rows_per_src=[14222, 11504, 7410, 15250, 10220, 13218, 6610, 3610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89922 payload_rows_remote=74454 dup_rows=8382 dup_rows_remote=6814 dedup_ratio=0.085266 router_tokens_per_expert=[340, 0, 19, 0, 0, 0, 0, 446, 0, 0, 3584, 0, 2751, 7852, 0, 3535, 0, 0, 0, 304, 0, 0, 461, 141, 0, 17, 71, 0, 0, 70, 0, 0, 0, 0, 5525, 0, 0, 0, 0, 0, 1, 7612, 0, 0, 0, 128, 727, 585, 0, 0, 0, 61, 3840, 2887, 74, 0, 0, 0, 0, 0, 0, 0, 8121, 0] payload_rows_per_dst=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] dup_rows_per_dst=[2970, 0, 1568, 314, 0, 1012, 2518, 0] combine_rows_per_src=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91418 payload_rows_remote=83504 dup_rows=6886 dup_rows_remote=5734 dedup_ratio=0.070048 router_tokens_per_expert=[364, 0, 22, 0, 0, 0, 0, 450, 0, 0, 3903, 0, 2546, 7854, 0, 3760, 0, 0, 0, 303, 0, 0, 476, 97, 0, 6, 76, 0, 0, 76, 0, 0, 0, 0, 5160, 0, 0, 0, 0, 0, 0, 7455, 0, 0, 0, 120, 811, 619, 0, 0, 0, 92, 3979, 2816, 49, 0, 2, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9128, 10686, 9704, 15708, 10722, 7914, 11324, 16232] dup_rows_per_dst=[64, 994, 992, 316, 816, 1152, 2548, 4] combine_rows_per_src=[9128, 10686, 9704, 15708, 10722, 7914, 11324, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=334929920 dispatch_meta_bytes=350816 combine_payload_bytes=334929920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335280736 ingress_bytes=369532928 link_bytes=369532928 token_bytes=670210656 weight_movement_bytes=34603008 total_nvlink_bytes=704813664 weight_to_token_ratio=0.051630 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390522658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85492 payload_rows_remote=69272 dup_rows=12812 dup_rows_remote=7054 dedup_ratio=0.130330 router_tokens_per_expert=[363, 0, 46, 0, 0, 2, 0, 421, 0, 0, 4161, 1, 2521, 7824, 1, 3720, 0, 0, 0, 230, 0, 2, 483, 118, 1, 39, 73, 0, 1, 71, 1, 0, 3, 0, 5142, 0, 2, 1, 0, 0, 27, 7584, 0, 0, 0, 122, 751, 627, 0, 0, 4, 57, 4133, 2508, 0, 0, 3, 2, 0, 0, 0, 0, 8107, 0] payload_rows_per_dst=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] dup_rows_per_dst=[80, 0, 152, 366, 3114, 2480, 862, 5758] combine_rows_per_src=[9024, 8098, 1514, 15654, 12224, 15742, 7016, 16220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83510 payload_rows_remote=67856 dup_rows=14794 dup_rows_remote=10148 dedup_ratio=0.150492 router_tokens_per_expert=[401, 0, 33, 0, 0, 0, 2, 466, 0, 1, 3374, 0, 2845, 7869, 0, 3680, 0, 2, 1, 361, 0, 1, 468, 117, 0, 39, 88, 0, 1, 86, 0, 0, 0, 0, 5617, 0, 2, 2, 0, 0, 0, 7508, 0, 0, 0, 133, 860, 728, 0, 1, 0, 94, 3915, 2283, 65, 1, 1, 1, 1, 0, 1, 1, 8103, 0] payload_rows_per_dst=[12928, 15654, 15290, 1378, 11238, 16378, 10638, 6] dup_rows_per_dst=[2984, 4646, 1626, 180, 4, 3270, 2080, 4] combine_rows_per_src=[12928, 15654, 15290, 1378, 11238, 16378, 10638, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92292 payload_rows_remote=76660 dup_rows=6012 dup_rows_remote=5564 dedup_ratio=0.061157 router_tokens_per_expert=[376, 0, 54, 0, 0, 0, 0, 463, 0, 0, 3453, 0, 2534, 7816, 0, 4118, 0, 0, 0, 329, 0, 0, 518, 103, 0, 61, 82, 0, 0, 81, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 1, 7545, 0, 0, 0, 127, 817, 600, 0, 0, 1, 53, 3845, 2379, 52, 0, 2, 0, 0, 0, 0, 0, 8111, 0] payload_rows_per_dst=[10216, 10230, 15296, 15632, 11262, 2770, 10664, 16222] dup_rows_per_dst=[514, 1036, 1694, 448, 0, 320, 1996, 4] combine_rows_per_src=[10216, 10230, 15296, 15632, 11262, 2770, 10664, 16222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=213245952 dispatch_meta_bytes=255696 combine_payload_bytes=213245952 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=213501648 ingress_bytes=386260992 link_bytes=386260992 token_bytes=426747600 weight_movement_bytes=173015040 total_nvlink_bytes=599762640 weight_to_token_ratio=0.405427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453469016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87948 payload_rows_remote=73518 dup_rows=10356 dup_rows_remote=10356 dedup_ratio=0.105347 router_tokens_per_expert=[255, 0, 0, 0, 255, 0, 0, 6, 0, 710, 1914, 1832, 0, 0, 345, 2, 753, 2161, 0, 0, 7774, 0, 0, 1894, 0, 0, 0, 7145, 0, 338, 0, 0, 0, 0, 0, 0, 1646, 497, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 78, 7215, 6184, 1, 0, 0, 0, 0, 0, 0, 7771, 0] payload_rows_per_dst=[12322, 8250, 9900, 10148, 4852, 14430, 12426, 15620] dup_rows_per_dst=[916, 1356, 3058, 452, 176, 0, 110, 4288] combine_rows_per_src=[12322, 8250, 9900, 10148, 4852, 14430, 12426, 15620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=325072 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305288656 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610252240 weight_movement_bytes=34603008 total_nvlink_bytes=644855248 weight_to_token_ratio=0.056703 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=305700864 dispatch_meta_bytes=358704 combine_payload_bytes=305700864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=306059568 ingress_bytes=374906880 link_bytes=374906880 token_bytes=611760432 weight_movement_bytes=69206016 total_nvlink_bytes=680966448 weight_to_token_ratio=0.113126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410744407e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92370 payload_rows_remote=81770 dup_rows=5934 dup_rows_remote=5934 dedup_ratio=0.060364 router_tokens_per_expert=[380, 0, 16, 0, 0, 0, 0, 475, 0, 0, 3705, 0, 2751, 7844, 0, 4055, 0, 0, 0, 296, 0, 0, 459, 99, 0, 62, 72, 0, 0, 72, 0, 0, 0, 0, 5300, 0, 0, 0, 0, 0, 0, 7505, 0, 0, 0, 120, 765, 592, 0, 0, 0, 74, 3599, 2768, 26, 0, 1, 0, 0, 0, 0, 0, 8116, 0] payload_rows_per_dst=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] dup_rows_per_dst=[72, 1380, 1532, 412, 0, 290, 2246, 2] combine_rows_per_src=[9780, 11532, 15186, 15688, 10600, 2664, 10688, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89790 payload_rows_remote=79058 dup_rows=8514 dup_rows_remote=6520 dedup_ratio=0.086609 router_tokens_per_expert=[342, 0, 28, 0, 4, 0, 0, 460, 0, 1, 3648, 0, 2532, 7833, 1, 4015, 1, 4, 0, 290, 0, 3, 491, 140, 3, 47, 81, 2, 0, 82, 0, 1, 1, 1, 5570, 0, 1, 0, 3, 0, 12, 7541, 1, 0, 0, 134, 753, 660, 0, 0, 1, 48, 3606, 2688, 16, 4, 1, 0, 6, 0, 1, 4, 8089, 2] payload_rows_per_dst=[9648, 7300, 1680, 15676, 12834, 15726, 10732, 16194] dup_rows_per_dst=[50, 0, 178, 422, 3382, 2476, 1994, 12] combine_rows_per_src=[9648, 7300, 1680, 15676, 12834, 15726, 10732, 16194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342032384 dispatch_meta_bytes=356952 combine_payload_bytes=342032384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342389336 ingress_bytes=411238400 link_bytes=411238400 token_bytes=684421720 weight_movement_bytes=69206016 total_nvlink_bytes=753627736 weight_to_token_ratio=0.101116 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.547456992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92292 payload_rows_remote=76660 dup_rows=6012 dup_rows_remote=5564 dedup_ratio=0.061157 router_tokens_per_expert=[376, 0, 54, 0, 0, 0, 0, 463, 0, 0, 3453, 0, 2534, 7816, 0, 4118, 0, 0, 0, 329, 0, 0, 518, 103, 0, 61, 82, 0, 0, 81, 0, 0, 0, 0, 5631, 0, 0, 0, 0, 0, 1, 7545, 0, 0, 0, 127, 817, 600, 0, 0, 1, 53, 3845, 2379, 52, 0, 2, 0, 0, 0, 0, 0, 8111, 0] payload_rows_per_dst=[10216, 10230, 15296, 15632, 11262, 2770, 10664, 16222] dup_rows_per_dst=[514, 1036, 1694, 448, 0, 320, 1996, 4] combine_rows_per_src=[10216, 10230, 15296, 15632, 11262, 2770, 10664, 16222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83152 payload_rows_remote=69166 dup_rows=15152 dup_rows_remote=11170 dedup_ratio=0.154134 router_tokens_per_expert=[337, 0, 39, 0, 0, 0, 0, 400, 0, 0, 4047, 0, 2741, 7882, 0, 4161, 0, 0, 0, 250, 0, 0, 435, 118, 0, 46, 78, 0, 0, 78, 0, 0, 0, 0, 5529, 0, 0, 0, 0, 0, 69, 7409, 0, 0, 0, 117, 726, 685, 0, 0, 0, 62, 3646, 2176, 12, 0, 0, 0, 0, 0, 0, 0, 8109, 0] payload_rows_per_dst=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] dup_rows_per_dst=[3982, 4870, 1420, 156, 0, 3028, 1696, 0] combine_rows_per_src=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283738112 dispatch_meta_bytes=305304 combine_payload_bytes=283738112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284043416 ingress_bytes=352944128 link_bytes=352944128 token_bytes=567781528 weight_movement_bytes=69206016 total_nvlink_bytes=636987544 weight_to_token_ratio=0.121888 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328100339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277938176 dispatch_meta_bytes=312016 combine_payload_bytes=277938176 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=278250192 ingress_bytes=450953216 link_bytes=450953216 token_bytes=556188368 weight_movement_bytes=173015040 total_nvlink_bytes=729203408 weight_to_token_ratio=0.311073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.696900647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89790 payload_rows_remote=79058 dup_rows=8514 dup_rows_remote=6520 dedup_ratio=0.086609 router_tokens_per_expert=[342, 0, 28, 0, 4, 0, 0, 460, 0, 1, 3648, 0, 2532, 7833, 1, 4015, 1, 4, 0, 290, 0, 3, 491, 140, 3, 47, 81, 2, 0, 82, 0, 1, 1, 1, 5570, 0, 1, 0, 3, 0, 12, 7541, 1, 0, 0, 134, 753, 660, 0, 0, 1, 48, 3606, 2688, 16, 4, 1, 0, 6, 0, 1, 4, 8089, 2] payload_rows_per_dst=[9648, 7300, 1680, 15676, 12834, 15726, 10732, 16194] dup_rows_per_dst=[50, 0, 178, 422, 3382, 2476, 1994, 12] combine_rows_per_src=[9648, 7300, 1680, 15676, 12834, 15726, 10732, 16194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=313999360 dispatch_meta_bytes=328896 combine_payload_bytes=313999360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314328256 ingress_bytes=348602368 link_bytes=348602368 token_bytes=628327616 weight_movement_bytes=34603008 total_nvlink_bytes=662930624 weight_to_token_ratio=0.055072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311762646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93446 payload_rows_remote=88896 dup_rows=4858 dup_rows_remote=4680 dedup_ratio=0.049418 router_tokens_per_expert=[252, 0, 0, 0, 250, 0, 0, 3, 0, 713, 2204, 2121, 0, 0, 406, 5, 711, 1985, 0, 0, 7791, 0, 0, 1557, 0, 2, 0, 7120, 0, 348, 46, 0, 0, 0, 0, 0, 1494, 491, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 7303, 6098, 5, 0, 0, 0, 0, 0, 0, 7795, 0] payload_rows_per_dst=[15582, 9170, 7398, 14330, 4550, 14606, 12220, 15590] dup_rows_per_dst=[1010, 1728, 1108, 702, 178, 0, 132, 0] combine_rows_per_src=[15582, 9170, 7398, 14330, 4550, 14606, 12220, 15590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283303936 dispatch_meta_bytes=321344 combine_payload_bytes=283303936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283625280 ingress_bytes=352509952 link_bytes=352509952 token_bytes=566929216 weight_movement_bytes=69206016 total_nvlink_bytes=636135232 weight_to_token_ratio=0.122072 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326466570e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=323821568 dispatch_meta_bytes=342312 combine_payload_bytes=323821568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324163880 ingress_bytes=393027584 link_bytes=393027584 token_bytes=647985448 weight_movement_bytes=69206016 total_nvlink_bytes=717191464 weight_to_token_ratio=0.106802 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478931157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89610 payload_rows_remote=81646 dup_rows=8694 dup_rows_remote=7236 dedup_ratio=0.088440 router_tokens_per_expert=[248, 6, 0, 0, 248, 0, 0, 7, 0, 940, 1930, 1795, 0, 0, 335, 0, 653, 2285, 0, 0, 7872, 0, 0, 1773, 0, 0, 0, 7225, 1, 335, 0, 0, 0, 0, 1, 0, 1617, 429, 295, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 75, 7291, 5974, 3, 0, 0, 0, 0, 0, 0, 7811, 0] payload_rows_per_dst=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] dup_rows_per_dst=[1018, 1370, 1458, 670, 4042, 2, 134, 0] combine_rows_per_src=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87948 payload_rows_remote=73518 dup_rows=10356 dup_rows_remote=10356 dedup_ratio=0.105347 router_tokens_per_expert=[255, 0, 0, 0, 255, 0, 0, 6, 0, 710, 1914, 1832, 0, 0, 345, 2, 753, 2161, 0, 0, 7774, 0, 0, 1894, 0, 0, 0, 7145, 0, 338, 0, 0, 0, 0, 0, 0, 1646, 497, 371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 78, 7215, 6184, 1, 0, 0, 0, 0, 0, 0, 7771, 0] payload_rows_per_dst=[12322, 8250, 9900, 10148, 4852, 14430, 12426, 15620] dup_rows_per_dst=[916, 1356, 3058, 452, 176, 0, 110, 4288] combine_rows_per_src=[12322, 8250, 9900, 10148, 4852, 14430, 12426, 15620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364118016 dispatch_meta_bytes=374304 combine_payload_bytes=364118016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=364492320 ingress_bytes=398721024 link_bytes=398721024 token_bytes=728610336 weight_movement_bytes=34603008 total_nvlink_bytes=763213344 weight_to_token_ratio=0.047492 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500355114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67452 payload_rows_remote=51574 dup_rows=30852 dup_rows_remote=16616 dedup_ratio=0.313843 router_tokens_per_expert=[243, 5, 0, 0, 243, 0, 0, 4, 1, 954, 1762, 2196, 2, 4, 370, 0, 580, 1981, 0, 0, 7854, 2, 0, 1624, 0, 0, 0, 7251, 1, 350, 12, 2, 1, 1, 0, 0, 1672, 418, 290, 0, 2, 4, 0, 0, 0, 8, 0, 2, 1, 3, 0, 0, 75, 7261, 6165, 2, 0, 0, 0, 0, 0, 1, 7805, 0] payload_rows_per_dst=[504, 9084, 9894, 11274, 4672, 28, 16118, 15878] dup_rows_per_dst=[486, 1494, 3166, 478, 92, 4, 10896, 14236] combine_rows_per_src=[504, 9084, 9894, 11274, 4672, 28, 16118, 15878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301129728 dispatch_meta_bytes=335496 combine_payload_bytes=301129728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301465224 ingress_bytes=335732736 link_bytes=335732736 token_bytes=602594952 weight_movement_bytes=34603008 total_nvlink_bytes=637197960 weight_to_token_ratio=0.057423 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=334929920 dispatch_meta_bytes=350816 combine_payload_bytes=334929920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335280736 ingress_bytes=369532928 link_bytes=369532928 token_bytes=670210656 weight_movement_bytes=34603008 total_nvlink_bytes=704813664 weight_to_token_ratio=0.051630 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390522658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86088 payload_rows_remote=75154 dup_rows=12216 dup_rows_remote=8604 dedup_ratio=0.124268 router_tokens_per_expert=[248, 1, 3, 0, 249, 0, 0, 2, 0, 577, 2228, 1948, 2, 1, 320, 4, 712, 2204, 0, 0, 7745, 0, 0, 1669, 0, 0, 0, 7157, 1, 316, 4, 0, 0, 0, 0, 0, 1663, 506, 352, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 1, 94, 7224, 6156, 1, 0, 0, 0, 1, 0, 0, 7755, 1] payload_rows_per_dst=[15496, 10934, 9316, 14322, 16180, 14454, 5382, 4] dup_rows_per_dst=[1000, 3612, 2520, 634, 4372, 2, 76, 0] combine_rows_per_src=[15496, 10934, 9316, 14322, 16180, 14454, 5382, 4] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=323821568 dispatch_meta_bytes=342312 combine_payload_bytes=323821568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324163880 ingress_bytes=393027584 link_bytes=393027584 token_bytes=647985448 weight_movement_bytes=69206016 total_nvlink_bytes=717191464 weight_to_token_ratio=0.106802 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478931157e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83152 payload_rows_remote=69166 dup_rows=15152 dup_rows_remote=11170 dedup_ratio=0.154134 router_tokens_per_expert=[337, 0, 39, 0, 0, 0, 0, 400, 0, 0, 4047, 0, 2741, 7882, 0, 4161, 0, 0, 0, 250, 0, 0, 435, 118, 0, 46, 78, 0, 0, 78, 0, 0, 0, 0, 5529, 0, 0, 0, 0, 0, 69, 7409, 0, 0, 0, 117, 726, 685, 0, 0, 0, 62, 3646, 2176, 12, 0, 0, 0, 0, 0, 0, 0, 8109, 0] payload_rows_per_dst=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] dup_rows_per_dst=[3982, 4870, 1420, 156, 0, 3028, 1696, 0] combine_rows_per_src=[13986, 16376, 15004, 248, 11058, 16384, 10096, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=334422016 dispatch_meta_bytes=355528 combine_payload_bytes=334422016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=334777544 ingress_bytes=438231040 link_bytes=438231040 token_bytes=669199560 weight_movement_bytes=103809024 total_nvlink_bytes=773008584 weight_to_token_ratio=0.155124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.649028123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=301129728 dispatch_meta_bytes=335496 combine_payload_bytes=301129728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301465224 ingress_bytes=335732736 link_bytes=335732736 token_bytes=602594952 weight_movement_bytes=34603008 total_nvlink_bytes=637197960 weight_to_token_ratio=0.057423 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=211247104 dispatch_meta_bytes=272760 combine_payload_bytes=211247104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211519864 ingress_bytes=280453120 link_bytes=280453120 token_bytes=422766968 weight_movement_bytes=69206016 total_nvlink_bytes=491972984 weight_to_token_ratio=0.163698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055322512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89922 payload_rows_remote=74454 dup_rows=8382 dup_rows_remote=6814 dedup_ratio=0.085266 router_tokens_per_expert=[340, 0, 19, 0, 0, 0, 0, 446, 0, 0, 3584, 0, 2751, 7852, 0, 3535, 0, 0, 0, 304, 0, 0, 461, 141, 0, 17, 71, 0, 0, 70, 0, 0, 0, 0, 5525, 0, 0, 0, 0, 0, 1, 7612, 0, 0, 0, 128, 727, 585, 0, 0, 0, 61, 3840, 2887, 74, 0, 0, 0, 0, 0, 0, 0, 8121, 0] payload_rows_per_dst=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] dup_rows_per_dst=[2970, 0, 1568, 314, 0, 1012, 2518, 0] combine_rows_per_src=[12878, 5502, 15468, 15706, 11050, 6698, 11206, 11414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86088 payload_rows_remote=75154 dup_rows=12216 dup_rows_remote=8604 dedup_ratio=0.124268 router_tokens_per_expert=[248, 1, 3, 0, 249, 0, 0, 2, 0, 577, 2228, 1948, 2, 1, 320, 4, 712, 2204, 0, 0, 7745, 0, 0, 1669, 0, 0, 0, 7157, 1, 316, 4, 0, 0, 0, 0, 0, 1663, 506, 352, 0, 0, 1, 1, 0, 0, 0, 0, 2, 1, 2, 0, 1, 94, 7224, 6156, 1, 0, 0, 0, 1, 0, 0, 7755, 1] payload_rows_per_dst=[15496, 10934, 9316, 14322, 16180, 14454, 5382, 4] dup_rows_per_dst=[1000, 3612, 2520, 634, 4372, 2, 76, 0] combine_rows_per_src=[15496, 10934, 9316, 14322, 16180, 14454, 5382, 4] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90256 payload_rows_remote=75804 dup_rows=8048 dup_rows_remote=7302 dedup_ratio=0.081868 router_tokens_per_expert=[245, 1, 0, 0, 245, 0, 0, 3, 0, 806, 1796, 1956, 0, 0, 394, 0, 670, 2310, 0, 0, 7807, 0, 0, 1560, 0, 1, 0, 7198, 0, 370, 28, 2, 0, 0, 2, 0, 1673, 430, 302, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 2, 84, 7260, 6183, 9, 0, 0, 0, 0, 0, 0, 7812, 0] payload_rows_per_dst=[15614, 8530, 7788, 14452, 14278, 14524, 12388, 2682] dup_rows_per_dst=[988, 1374, 1292, 746, 3478, 2, 168, 0] combine_rows_per_src=[15614, 8530, 7788, 14452, 14278, 14524, 12388, 2682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283303936 dispatch_meta_bytes=321344 combine_payload_bytes=283303936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283625280 ingress_bytes=352509952 link_bytes=352509952 token_bytes=566929216 weight_movement_bytes=69206016 total_nvlink_bytes=636135232 weight_to_token_ratio=0.122072 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326466570e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93216 payload_rows_remote=78854 dup_rows=5088 dup_rows_remote=5088 dedup_ratio=0.051758 router_tokens_per_expert=[7193, 0, 0, 3296, 125, 0, 219, 0, 0, 0, 0, 1, 0, 0, 0, 8069, 0, 6250, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 0, 0, 74, 0, 0, 1902, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7181, 214, 0, 0, 0, 0, 657, 0, 0, 55, 0, 0, 924, 340, 224, 404, 7923] payload_rows_per_dst=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] dup_rows_per_dst=[28, 2, 0, 0, 0, 0, 1462, 3596] combine_rows_per_src=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=325072 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305288656 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610252240 weight_movement_bytes=34603008 total_nvlink_bytes=644855248 weight_to_token_ratio=0.056703 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93446 payload_rows_remote=88896 dup_rows=4858 dup_rows_remote=4680 dedup_ratio=0.049418 router_tokens_per_expert=[252, 0, 0, 0, 250, 0, 0, 3, 0, 713, 2204, 2121, 0, 0, 406, 5, 711, 1985, 0, 0, 7791, 0, 0, 1557, 0, 2, 0, 7120, 0, 348, 46, 0, 0, 0, 0, 0, 1494, 491, 379, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 7303, 6098, 5, 0, 0, 0, 0, 0, 0, 7795, 0] payload_rows_per_dst=[15582, 9170, 7398, 14330, 4550, 14606, 12220, 15590] dup_rows_per_dst=[1010, 1728, 1108, 702, 178, 0, 132, 0] combine_rows_per_src=[15582, 9170, 7398, 14330, 4550, 14606, 12220, 15590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83610 payload_rows_remote=67984 dup_rows=14694 dup_rows_remote=13658 dedup_ratio=0.149475 router_tokens_per_expert=[255, 1, 0, 0, 255, 0, 0, 7, 0, 742, 2061, 1983, 0, 0, 315, 0, 654, 1929, 0, 0, 7813, 0, 0, 1763, 0, 1, 0, 7146, 1, 307, 28, 0, 0, 0, 1, 0, 1573, 472, 321, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 80, 7338, 6291, 12, 0, 0, 0, 0, 0, 0, 7799, 0] payload_rows_per_dst=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] dup_rows_per_dst=[1036, 7706, 1232, 614, 4104, 2, 0, 0] combine_rows_per_src=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=307830784 dispatch_meta_bytes=335032 combine_payload_bytes=307830784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308165816 ingress_bytes=342433792 link_bytes=342433792 token_bytes=615996600 weight_movement_bytes=34603008 total_nvlink_bytes=650599608 weight_to_token_ratio=0.056174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288550791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70782 payload_rows_remote=57470 dup_rows=27522 dup_rows_remote=24740 dedup_ratio=0.279968 router_tokens_per_expert=[236, 1, 0, 2, 236, 0, 1, 5, 0, 765, 1705, 2168, 0, 0, 354, 2, 679, 1972, 4, 0, 7864, 0, 0, 1856, 1, 0, 1, 7198, 5, 332, 0, 0, 1, 0, 2, 4, 1603, 418, 391, 0, 6, 0, 0, 1, 2, 0, 0, 0, 3, 6, 0, 4, 104, 7231, 6181, 3, 0, 4, 5, 1, 0, 1, 7793, 1] payload_rows_per_dst=[488, 8568, 16186, 676, 4670, 10986, 13312, 15896] dup_rows_per_dst=[474, 1420, 8564, 2, 168, 2, 2782, 14110] combine_rows_per_src=[488, 8568, 16186, 676, 4670, 10986, 13312, 15896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=310493184 dispatch_meta_bytes=332424 combine_payload_bytes=310493184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=310825608 ingress_bytes=379699200 link_bytes=379699200 token_bytes=621318792 weight_movement_bytes=69206016 total_nvlink_bytes=690524808 weight_to_token_ratio=0.111386 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428777521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72530 payload_rows_remote=56166 dup_rows=25774 dup_rows_remote=9488 dedup_ratio=0.262187 router_tokens_per_expert=[7328, 0, 1, 3513, 20, 0, 227, 0, 0, 0, 1, 2, 0, 0, 9, 7955, 0, 6377, 0, 0, 0, 0, 3, 5, 0, 3896, 0, 0, 0, 69, 0, 0, 2087, 0, 0, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 3, 4, 7236, 280, 3, 0, 0, 0, 518, 0, 0, 63, 2, 0, 799, 324, 153, 340, 7919] payload_rows_per_dst=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] dup_rows_per_dst=[138, 2, 8, 7930, 0, 14, 1396, 16286] combine_rows_per_src=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=322985984 dispatch_meta_bytes=335768 combine_payload_bytes=322985984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=323321752 ingress_bytes=426795008 link_bytes=426795008 token_bytes=646307736 weight_movement_bytes=103809024 total_nvlink_bytes=750116760 weight_to_token_ratio=0.160619 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605995255e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=364118016 dispatch_meta_bytes=374304 combine_payload_bytes=364118016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=364492320 ingress_bytes=398721024 link_bytes=398721024 token_bytes=728610336 weight_movement_bytes=34603008 total_nvlink_bytes=763213344 weight_to_token_ratio=0.047492 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500355114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=235397120 dispatch_meta_bytes=328840 combine_payload_bytes=235397120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=235725960 ingress_bytes=339206144 link_bytes=339206144 token_bytes=471123080 weight_movement_bytes=103809024 total_nvlink_bytes=574932104 weight_to_token_ratio=0.220344 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276405412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83610 payload_rows_remote=67984 dup_rows=14694 dup_rows_remote=13658 dedup_ratio=0.149475 router_tokens_per_expert=[255, 1, 0, 0, 255, 0, 0, 7, 0, 742, 2061, 1983, 0, 0, 315, 0, 654, 1929, 0, 0, 7813, 0, 0, 1763, 0, 1, 0, 7146, 1, 307, 28, 0, 0, 0, 1, 0, 1573, 472, 321, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 80, 7338, 6291, 12, 0, 0, 0, 0, 0, 0, 7799, 0] payload_rows_per_dst=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] dup_rows_per_dst=[1036, 7706, 1232, 614, 4104, 2, 0, 0] combine_rows_per_src=[15626, 15078, 7460, 14352, 16228, 14676, 190, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230055936 dispatch_meta_bytes=262616 combine_payload_bytes=230055936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=230318552 ingress_bytes=299261952 link_bytes=299261952 token_bytes=460374488 weight_movement_bytes=69206016 total_nvlink_bytes=529580504 weight_to_token_ratio=0.150325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89610 payload_rows_remote=81646 dup_rows=8694 dup_rows_remote=7236 dedup_ratio=0.088440 router_tokens_per_expert=[248, 6, 0, 0, 248, 0, 0, 7, 0, 940, 1930, 1795, 0, 0, 335, 0, 653, 2285, 0, 0, 7872, 0, 0, 1773, 0, 0, 0, 7225, 1, 335, 0, 0, 0, 0, 1, 0, 1617, 429, 295, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 75, 7291, 5974, 3, 0, 0, 0, 0, 0, 0, 7811, 0] payload_rows_per_dst=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] dup_rows_per_dst=[1018, 1370, 1458, 670, 4042, 2, 134, 0] combine_rows_per_src=[15744, 8630, 7964, 14452, 16264, 14586, 11970, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=326568 combine_payload_bytes=278462464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278789032 ingress_bytes=313065472 link_bytes=313065472 token_bytes=557251496 weight_movement_bytes=34603008 total_nvlink_bytes=591854504 weight_to_token_ratio=0.062096 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178040167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89076 payload_rows_remote=72788 dup_rows=9228 dup_rows_remote=9226 dedup_ratio=0.093872 router_tokens_per_expert=[7203, 0, 3, 3150, 51, 3, 194, 0, 0, 1, 0, 0, 2, 0, 4, 8138, 0, 6272, 0, 0, 1, 0, 2, 0, 0, 4340, 0, 0, 0, 85, 1, 1, 1658, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 7202, 243, 0, 0, 0, 0, 940, 0, 0, 57, 2, 0, 704, 334, 199, 468, 7890] payload_rows_per_dst=[15646, 16288, 12548, 8854, 16040, 14408, 2364, 2928] dup_rows_per_dst=[5562, 2, 2, 0, 3060, 0, 2, 600] combine_rows_per_src=[15646, 16288, 12548, 8854, 16040, 14408, 2364, 2928] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92884 payload_rows_remote=84972 dup_rows=5420 dup_rows_remote=5420 dedup_ratio=0.055135 router_tokens_per_expert=[7365, 0, 0, 3805, 103, 0, 205, 0, 0, 0, 0, 0, 0, 0, 3, 8131, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 79, 0, 0, 1623, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7264, 197, 0, 1, 0, 0, 753, 0, 0, 67, 0, 0, 721, 289, 210, 419, 7923] payload_rows_per_dst=[8108, 16262, 12232, 7912, 15990, 14528, 14956, 2896] dup_rows_per_dst=[118, 6, 0, 0, 3104, 0, 1676, 516] combine_rows_per_src=[8108, 16262, 12232, 7912, 15990, 14528, 14956, 2896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95266 payload_rows_remote=80230 dup_rows=3038 dup_rows_remote=2782 dedup_ratio=0.030904 router_tokens_per_expert=[0, 7512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 2, 0, 0, 0, 8192, 0, 0, 224, 8192, 0, 274, 0, 0, 0, 0, 0, 2312, 0, 0, 76, 0, 0, 5391, 7518, 0, 76, 0, 0, 0, 52, 0, 0, 0, 8033, 0, 0, 650, 0, 0, 0, 0, 0, 1, 0, 103, 19, 0, 0, 0] payload_rows_per_dst=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] dup_rows_per_dst=[0, 516, 210, 510, 0, 256, 1300, 246] combine_rows_per_src=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=310493184 dispatch_meta_bytes=332424 combine_payload_bytes=310493184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=310825608 ingress_bytes=379699200 link_bytes=379699200 token_bytes=621318792 weight_movement_bytes=69206016 total_nvlink_bytes=690524808 weight_to_token_ratio=0.111386 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.428777521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93336 payload_rows_remote=84764 dup_rows=4968 dup_rows_remote=3534 dedup_ratio=0.050537 router_tokens_per_expert=[7336, 0, 0, 3604, 47, 0, 208, 0, 0, 0, 0, 0, 0, 0, 4, 7948, 0, 6173, 0, 0, 0, 0, 0, 0, 0, 4118, 0, 0, 0, 72, 0, 0, 2074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7276, 277, 0, 0, 0, 0, 595, 0, 0, 78, 0, 0, 655, 338, 139, 337, 7873] payload_rows_per_dst=[7600, 15896, 12346, 8380, 8572, 14552, 14870, 11120] dup_rows_per_dst=[118, 8, 0, 0, 1434, 0, 1546, 1862] combine_rows_per_src=[7600, 15896, 12346, 8380, 8572, 14552, 14870, 11120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=334422016 dispatch_meta_bytes=355528 combine_payload_bytes=334422016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=334777544 ingress_bytes=438231040 link_bytes=438231040 token_bytes=669199560 weight_movement_bytes=103809024 total_nvlink_bytes=773008584 weight_to_token_ratio=0.155124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.649028123e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298139648 dispatch_meta_bytes=328056 combine_payload_bytes=298139648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=298467704 ingress_bytes=367345664 link_bytes=367345664 token_bytes=596607352 weight_movement_bytes=69206016 total_nvlink_bytes=665813368 weight_to_token_ratio=0.115999 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382292160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=348045312 dispatch_meta_bytes=361568 combine_payload_bytes=348045312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348406880 ingress_bytes=382648320 link_bytes=382648320 token_bytes=696452192 weight_movement_bytes=34603008 total_nvlink_bytes=731055200 weight_to_token_ratio=0.049685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439874822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84440 payload_rows_remote=69782 dup_rows=13864 dup_rows_remote=11710 dedup_ratio=0.141032 router_tokens_per_expert=[7203, 0, 0, 3567, 54, 2, 200, 0, 2, 0, 4, 2, 9, 0, 4, 7892, 0, 6026, 0, 0, 1, 3, 9, 1, 1, 4141, 0, 5, 0, 89, 1, 0, 1800, 2, 2, 0, 0, 1, 1, 7, 2, 1, 0, 6, 0, 2, 0, 7255, 346, 0, 0, 0, 0, 856, 1, 0, 52, 1, 0, 727, 291, 203, 442, 7938] payload_rows_per_dst=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] dup_rows_per_dst=[84, 18, 4, 8406, 8, 4, 2154, 3186] combine_rows_per_src=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72840 payload_rows_remote=56460 dup_rows=25464 dup_rows_remote=25194 dedup_ratio=0.259033 router_tokens_per_expert=[0, 7451, 0, 0, 0, 6, 0, 0, 0, 2, 1, 1, 1, 555, 0, 0, 0, 0, 0, 8186, 0, 0, 160, 8182, 0, 253, 0, 0, 0, 0, 0, 2501, 1, 0, 81, 0, 1, 5195, 7560, 0, 71, 0, 3, 3, 60, 0, 0, 1, 8051, 2, 0, 682, 0, 2, 0, 1, 0, 1, 0, 75, 56, 1, 1, 5] payload_rows_per_dst=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] dup_rows_per_dst=[0, 6, 12758, 486, 38, 270, 11636, 270] combine_rows_per_src=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=332048 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328954128 ingress_bytes=363225088 link_bytes=363225088 token_bytes=657576208 weight_movement_bytes=34603008 total_nvlink_bytes=692179216 weight_to_token_ratio=0.052622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366786764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=278462464 dispatch_meta_bytes=326568 combine_payload_bytes=278462464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278789032 ingress_bytes=313065472 link_bytes=313065472 token_bytes=557251496 weight_movement_bytes=34603008 total_nvlink_bytes=591854504 weight_to_token_ratio=0.062096 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178040167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=235397120 dispatch_meta_bytes=328840 combine_payload_bytes=235397120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=235725960 ingress_bytes=339206144 link_bytes=339206144 token_bytes=471123080 weight_movement_bytes=103809024 total_nvlink_bytes=574932104 weight_to_token_ratio=0.220344 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276405412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91554 payload_rows_remote=79034 dup_rows=6750 dup_rows_remote=6750 dedup_ratio=0.068665 router_tokens_per_expert=[7347, 0, 0, 3685, 73, 0, 175, 0, 0, 0, 0, 1, 0, 0, 0, 8171, 0, 6260, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 67, 0, 0, 1502, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7377, 38, 0, 0, 0, 0, 992, 0, 0, 71, 0, 0, 656, 261, 145, 442, 7912] payload_rows_per_dst=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] dup_rows_per_dst=[2362, 2, 0, 0, 2870, 0, 1092, 424] combine_rows_per_src=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93216 payload_rows_remote=78854 dup_rows=5088 dup_rows_remote=5088 dedup_ratio=0.051758 router_tokens_per_expert=[7193, 0, 0, 3296, 125, 0, 219, 0, 0, 0, 0, 1, 0, 0, 0, 8069, 0, 6250, 0, 0, 0, 0, 0, 0, 0, 4100, 0, 0, 0, 74, 0, 0, 1902, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7181, 214, 0, 0, 0, 0, 657, 0, 0, 55, 0, 0, 924, 340, 224, 404, 7923] payload_rows_per_dst=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] dup_rows_per_dst=[28, 2, 0, 0, 0, 0, 1462, 3596] combine_rows_per_src=[7252, 16138, 12500, 8348, 3806, 14362, 14666, 16144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=347193344 dispatch_meta_bytes=353192 combine_payload_bytes=347193344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=347546536 ingress_bytes=381796352 link_bytes=381796352 token_bytes=694739880 weight_movement_bytes=34603008 total_nvlink_bytes=729342888 weight_to_token_ratio=0.049807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436668935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72530 payload_rows_remote=56166 dup_rows=25774 dup_rows_remote=9488 dedup_ratio=0.262187 router_tokens_per_expert=[7328, 0, 1, 3513, 20, 0, 227, 0, 0, 0, 1, 2, 0, 0, 9, 7955, 0, 6377, 0, 0, 0, 0, 3, 5, 0, 3896, 0, 0, 0, 69, 0, 0, 2087, 0, 0, 1, 0, 0, 4, 0, 0, 2, 0, 0, 8, 3, 4, 7236, 280, 3, 0, 0, 0, 518, 0, 0, 63, 2, 0, 799, 324, 153, 340, 7919] payload_rows_per_dst=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] dup_rows_per_dst=[138, 2, 8, 7930, 0, 14, 1396, 16286] combine_rows_per_src=[7384, 22, 12762, 15910, 4184, 1042, 14862, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89076 payload_rows_remote=72788 dup_rows=9228 dup_rows_remote=9226 dedup_ratio=0.093872 router_tokens_per_expert=[7203, 0, 3, 3150, 51, 3, 194, 0, 0, 1, 0, 0, 2, 0, 4, 8138, 0, 6272, 0, 0, 1, 0, 2, 0, 0, 4340, 0, 0, 0, 85, 1, 1, 1658, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 7202, 243, 0, 0, 0, 0, 940, 0, 0, 57, 2, 0, 704, 334, 199, 468, 7890] payload_rows_per_dst=[15646, 16288, 12548, 8854, 16040, 14408, 2364, 2928] dup_rows_per_dst=[5562, 2, 2, 0, 3060, 0, 2, 600] combine_rows_per_src=[15646, 16288, 12548, 8854, 16040, 14408, 2364, 2928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323723264 dispatch_meta_bytes=343136 combine_payload_bytes=323723264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324066400 ingress_bytes=392929280 link_bytes=392929280 token_bytes=647789664 weight_movement_bytes=69206016 total_nvlink_bytes=716995680 weight_to_token_ratio=0.106834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478561247e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=322985984 dispatch_meta_bytes=335768 combine_payload_bytes=322985984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=323321752 ingress_bytes=426795008 link_bytes=426795008 token_bytes=646307736 weight_movement_bytes=103809024 total_nvlink_bytes=750116760 weight_to_token_ratio=0.160619 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.605995255e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92884 payload_rows_remote=84972 dup_rows=5420 dup_rows_remote=5420 dedup_ratio=0.055135 router_tokens_per_expert=[7365, 0, 0, 3805, 103, 0, 205, 0, 0, 0, 0, 0, 0, 0, 3, 8131, 0, 6116, 0, 0, 0, 0, 0, 0, 0, 3877, 0, 0, 0, 79, 0, 0, 1623, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7264, 197, 0, 1, 0, 0, 753, 0, 0, 67, 0, 0, 721, 289, 210, 419, 7923] payload_rows_per_dst=[8108, 16262, 12232, 7912, 15990, 14528, 14956, 2896] dup_rows_per_dst=[118, 6, 0, 0, 3104, 0, 1676, 516] combine_rows_per_src=[8108, 16262, 12232, 7912, 15990, 14528, 14956, 2896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=230055936 dispatch_meta_bytes=262616 combine_payload_bytes=230055936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=230318552 ingress_bytes=299261952 link_bytes=299261952 token_bytes=460374488 weight_movement_bytes=69206016 total_nvlink_bytes=529580504 weight_to_token_ratio=0.150325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82740 payload_rows_remote=67036 dup_rows=15564 dup_rows_remote=5616 dedup_ratio=0.158325 router_tokens_per_expert=[7178, 0, 0, 3438, 41, 0, 193, 0, 0, 0, 0, 0, 0, 0, 0, 8016, 0, 5941, 0, 0, 0, 0, 0, 0, 0, 4298, 0, 0, 0, 77, 0, 0, 1886, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 260, 0, 0, 0, 0, 802, 0, 0, 57, 0, 0, 792, 287, 175, 456, 7940] payload_rows_per_dst=[15704, 16086, 10908, 8750, 16048, 10112, 2124, 3008] dup_rows_per_dst=[9948, 1086, 398, 0, 3606, 0, 0, 526] combine_rows_per_src=[15704, 16086, 10908, 8750, 16048, 10112, 2124, 3008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=285827072 dispatch_meta_bytes=325968 combine_payload_bytes=285827072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286153040 ingress_bytes=320430080 link_bytes=320430080 token_bytes=571980112 weight_movement_bytes=34603008 total_nvlink_bytes=606583120 weight_to_token_ratio=0.060497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205752594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231260160 dispatch_meta_bytes=326616 combine_payload_bytes=231260160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231586776 ingress_bytes=265863168 link_bytes=265863168 token_bytes=462846936 weight_movement_bytes=34603008 total_nvlink_bytes=497449944 weight_to_token_ratio=0.074761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000421697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84440 payload_rows_remote=69782 dup_rows=13864 dup_rows_remote=11710 dedup_ratio=0.141032 router_tokens_per_expert=[7203, 0, 0, 3567, 54, 2, 200, 0, 2, 0, 4, 2, 9, 0, 4, 7892, 0, 6026, 0, 0, 1, 3, 9, 1, 1, 4141, 0, 5, 0, 89, 1, 0, 1800, 2, 2, 0, 0, 1, 1, 7, 2, 1, 0, 6, 0, 2, 0, 7255, 346, 0, 0, 0, 0, 856, 1, 0, 52, 1, 0, 727, 291, 203, 442, 7938] payload_rows_per_dst=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] dup_rows_per_dst=[84, 18, 4, 8406, 8, 4, 2154, 3186] combine_rows_per_src=[7562, 144, 12076, 15732, 3618, 14528, 14658, 16122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298139648 dispatch_meta_bytes=328056 combine_payload_bytes=298139648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=298467704 ingress_bytes=367345664 link_bytes=367345664 token_bytes=596607352 weight_movement_bytes=69206016 total_nvlink_bytes=665813368 weight_to_token_ratio=0.115999 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.382292160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=84182 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[0, 7501, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 8192, 0, 0, 175, 8186, 0, 260, 0, 6, 0, 0, 0, 2404, 0, 0, 72, 0, 0, 5144, 7613, 0, 72, 0, 0, 0, 90, 0, 0, 0, 8001, 0, 0, 723, 0, 0, 0, 0, 0, 1, 0, 110, 23, 0, 0, 0] payload_rows_per_dst=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] dup_rows_per_dst=[4, 1148, 0, 504, 0, 324, 1442, 268] combine_rows_per_src=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=348045312 dispatch_meta_bytes=361568 combine_payload_bytes=348045312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=348406880 ingress_bytes=382648320 link_bytes=382648320 token_bytes=696452192 weight_movement_bytes=34603008 total_nvlink_bytes=731055200 weight_to_token_ratio=0.049685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439874822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274579456 dispatch_meta_bytes=290608 combine_payload_bytes=274579456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274870064 ingress_bytes=343785472 link_bytes=343785472 token_bytes=549449520 weight_movement_bytes=69206016 total_nvlink_bytes=618655536 weight_to_token_ratio=0.125955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293637054e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91554 payload_rows_remote=79034 dup_rows=6750 dup_rows_remote=6750 dedup_ratio=0.068665 router_tokens_per_expert=[7347, 0, 0, 3685, 73, 0, 175, 0, 0, 0, 0, 1, 0, 0, 0, 8171, 0, 6260, 0, 0, 0, 0, 0, 0, 0, 3974, 0, 0, 0, 67, 0, 0, 1502, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 7377, 38, 0, 0, 0, 0, 992, 0, 0, 71, 0, 0, 656, 261, 145, 442, 7912] payload_rows_per_dst=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] dup_rows_per_dst=[2362, 2, 0, 0, 2870, 0, 1092, 424] combine_rows_per_src=[10690, 16342, 12520, 8082, 15964, 14754, 10476, 2726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93140 payload_rows_remote=83340 dup_rows=5164 dup_rows_remote=2524 dedup_ratio=0.052531 router_tokens_per_expert=[0, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 8192, 0, 0, 216, 8192, 0, 254, 0, 0, 0, 0, 0, 2327, 0, 0, 81, 0, 0, 5199, 7584, 0, 81, 0, 0, 0, 97, 0, 0, 0, 8047, 0, 0, 740, 0, 0, 0, 0, 0, 2, 0, 72, 31, 0, 0, 0] payload_rows_per_dst=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] dup_rows_per_dst=[0, 1170, 0, 2640, 0, 356, 788, 210] combine_rows_per_src=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=285827072 dispatch_meta_bytes=325968 combine_payload_bytes=285827072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286153040 ingress_bytes=320430080 link_bytes=320430080 token_bytes=571980112 weight_movement_bytes=34603008 total_nvlink_bytes=606583120 weight_to_token_ratio=0.060497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205752594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=88650 dup_rows=7202 dup_rows_remote=7166 dedup_ratio=0.073263 router_tokens_per_expert=[0, 7481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 8192, 0, 0, 165, 8190, 0, 268, 0, 0, 0, 0, 0, 2474, 0, 0, 71, 0, 0, 5071, 7556, 0, 70, 0, 0, 0, 101, 0, 0, 0, 8044, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 72, 46, 0, 0, 0] payload_rows_per_dst=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] dup_rows_per_dst=[8, 1152, 36, 5472, 0, 342, 0, 192] combine_rows_per_src=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=344809472 dispatch_meta_bytes=351488 combine_payload_bytes=344809472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=345160960 ingress_bytes=448618496 link_bytes=448618496 token_bytes=689970432 weight_movement_bytes=103809024 total_nvlink_bytes=793779456 weight_to_token_ratio=0.150454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688115284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95266 payload_rows_remote=80230 dup_rows=3038 dup_rows_remote=2782 dedup_ratio=0.030904 router_tokens_per_expert=[0, 7512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 2, 0, 0, 0, 8192, 0, 0, 224, 8192, 0, 274, 0, 0, 0, 0, 0, 2312, 0, 0, 76, 0, 0, 5391, 7518, 0, 76, 0, 0, 0, 52, 0, 0, 0, 8033, 0, 0, 650, 0, 0, 0, 0, 0, 1, 0, 103, 19, 0, 0, 0] payload_rows_per_dst=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] dup_rows_per_dst=[0, 516, 210, 510, 0, 256, 1300, 246] combine_rows_per_src=[15024, 8154, 9006, 4662, 10934, 15036, 16066, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72840 payload_rows_remote=56460 dup_rows=25464 dup_rows_remote=25194 dedup_ratio=0.259033 router_tokens_per_expert=[0, 7451, 0, 0, 0, 6, 0, 0, 0, 2, 1, 1, 1, 555, 0, 0, 0, 0, 0, 8186, 0, 0, 160, 8182, 0, 253, 0, 0, 0, 0, 0, 2501, 1, 0, 81, 0, 1, 5195, 7560, 0, 71, 0, 3, 3, 60, 0, 0, 1, 8051, 2, 0, 682, 0, 2, 0, 1, 0, 1, 0, 75, 56, 1, 1, 5] payload_rows_per_dst=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] dup_rows_per_dst=[0, 6, 12758, 486, 38, 270, 11636, 270] combine_rows_per_src=[12, 1114, 16376, 5022, 2580, 15126, 16230, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84384 payload_rows_remote=68006 dup_rows=13920 dup_rows_remote=12894 dedup_ratio=0.141602 router_tokens_per_expert=[1, 7375, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 513, 0, 0, 0, 0, 0, 8188, 0, 2, 224, 8184, 0, 259, 1, 1, 0, 0, 0, 2470, 0, 0, 86, 2, 0, 5434, 7516, 0, 90, 0, 1, 0, 70, 0, 0, 2, 8007, 0, 0, 519, 0, 3, 0, 1, 0, 2, 0, 128, 68, 0, 0, 0] payload_rows_per_dst=[14752, 16378, 16380, 16018, 14292, 5124, 1044, 396] dup_rows_per_dst=[0, 1026, 448, 5458, 6842, 144, 2, 0] combine_rows_per_src=[14752, 16378, 16380, 16018, 14292, 5124, 1044, 396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91284 payload_rows_remote=83256 dup_rows=7020 dup_rows_remote=7020 dedup_ratio=0.071411 router_tokens_per_expert=[0, 0, 0, 459, 0, 486, 0, 0, 7018, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4125, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 178, 0, 0, 257, 0, 7824, 8192, 8112, 0, 0, 0, 0, 0, 3678, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] dup_rows_per_dst=[434, 0, 5778, 324, 50, 0, 0, 434] combine_rows_per_src=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=363110400 dispatch_meta_bytes=383264 combine_payload_bytes=363110400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=363493664 ingress_bytes=466919424 link_bytes=466919424 token_bytes=726604064 weight_movement_bytes=103809024 total_nvlink_bytes=830413088 weight_to_token_ratio=0.142869 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.756980202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=84182 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[0, 7501, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 8192, 0, 0, 175, 8186, 0, 260, 0, 6, 0, 0, 0, 2404, 0, 0, 72, 0, 0, 5144, 7613, 0, 72, 0, 0, 0, 90, 0, 0, 0, 8001, 0, 0, 723, 0, 0, 0, 0, 0, 1, 0, 110, 23, 0, 0, 0] payload_rows_per_dst=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] dup_rows_per_dst=[4, 1148, 0, 504, 0, 324, 1442, 268] combine_rows_per_src=[15002, 16378, 350, 4836, 10432, 15226, 16006, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323723264 dispatch_meta_bytes=343136 combine_payload_bytes=323723264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324066400 ingress_bytes=392929280 link_bytes=392929280 token_bytes=647789664 weight_movement_bytes=69206016 total_nvlink_bytes=716995680 weight_to_token_ratio=0.106834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478561247e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93140 payload_rows_remote=83340 dup_rows=5164 dup_rows_remote=2524 dedup_ratio=0.052531 router_tokens_per_expert=[0, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 8192, 0, 0, 216, 8192, 0, 254, 0, 0, 0, 0, 0, 2327, 0, 0, 81, 0, 0, 5199, 7584, 0, 81, 0, 0, 0, 97, 0, 0, 0, 8047, 0, 0, 740, 0, 0, 0, 0, 0, 2, 0, 72, 31, 0, 0, 0] payload_rows_per_dst=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] dup_rows_per_dst=[0, 1170, 0, 2640, 0, 356, 788, 210] combine_rows_per_src=[14904, 16384, 432, 9800, 10560, 15168, 9508, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=332048 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328954128 ingress_bytes=363225088 link_bytes=363225088 token_bytes=657576208 weight_movement_bytes=34603008 total_nvlink_bytes=692179216 weight_to_token_ratio=0.052622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366786764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81718 payload_rows_remote=66744 dup_rows=16586 dup_rows_remote=16582 dedup_ratio=0.168722 router_tokens_per_expert=[0, 7487, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 8192, 0, 0, 298, 8186, 0, 286, 0, 0, 0, 0, 0, 2337, 0, 0, 78, 0, 0, 5188, 7568, 0, 78, 0, 0, 0, 79, 0, 0, 0, 8050, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 84, 55, 0, 0, 0] payload_rows_per_dst=[14974, 16384, 16384, 16110, 15874, 314, 1400, 278] dup_rows_per_dst=[4, 956, 596, 5236, 9794, 0, 0, 0] combine_rows_per_src=[14974, 16384, 16384, 16110, 15874, 314, 1400, 278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=231260160 dispatch_meta_bytes=326616 combine_payload_bytes=231260160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231586776 ingress_bytes=265863168 link_bytes=265863168 token_bytes=462846936 weight_movement_bytes=34603008 total_nvlink_bytes=497449944 weight_to_token_ratio=0.074761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000421697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85430 payload_rows_remote=69270 dup_rows=12874 dup_rows_remote=7690 dedup_ratio=0.130961 router_tokens_per_expert=[0, 7415, 0, 0, 0, 0, 0, 0, 0, 10, 7, 0, 0, 559, 0, 1, 0, 0, 0, 8181, 0, 1, 181, 8170, 0, 258, 2, 2, 0, 0, 0, 2463, 0, 0, 88, 3, 1, 5144, 7542, 4, 81, 2, 8, 7, 109, 3, 0, 1, 8058, 0, 0, 705, 0, 0, 0, 6, 0, 1, 0, 70, 61, 0, 7, 1] payload_rows_per_dst=[8658, 1140, 16344, 4974, 6670, 15106, 16160, 16378] dup_rows_per_dst=[0, 14, 6532, 476, 4, 400, 5184, 264] combine_rows_per_src=[8658, 1140, 16344, 4974, 6670, 15106, 16160, 16378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=341360640 dispatch_meta_bytes=343456 combine_payload_bytes=341360640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341704096 ingress_bytes=375963648 link_bytes=375963648 token_bytes=683064736 weight_movement_bytes=34603008 total_nvlink_bytes=717667744 weight_to_token_ratio=0.050658 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.414720940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=341016576 dispatch_meta_bytes=361104 combine_payload_bytes=341016576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341377680 ingress_bytes=375619584 link_bytes=375619584 token_bytes=682394256 weight_movement_bytes=34603008 total_nvlink_bytes=716997264 weight_to_token_ratio=0.050708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413426255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=323600 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278876176 ingress_bytes=313155584 link_bytes=313155584 token_bytes=557428752 weight_movement_bytes=34603008 total_nvlink_bytes=592031760 weight_to_token_ratio=0.062076 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178379251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69656 payload_rows_remote=53276 dup_rows=28648 dup_rows_remote=28458 dedup_ratio=0.291423 router_tokens_per_expert=[0, 0, 0, 709, 0, 527, 0, 3, 7053, 0, 0, 0, 0, 0, 2, 73, 0, 0, 4220, 0, 5, 3, 0, 0, 133, 0, 0, 0, 0, 137, 0, 0, 224, 0, 7885, 8188, 8108, 0, 2, 1, 0, 0, 3250, 343, 1, 0, 6, 0, 3, 0, 3, 0, 0, 8174, 0, 2, 94, 2, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] dup_rows_per_dst=[32, 0, 1566, 0, 23102, 2, 3756, 190] combine_rows_per_src=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81718 payload_rows_remote=66744 dup_rows=16586 dup_rows_remote=16582 dedup_ratio=0.168722 router_tokens_per_expert=[0, 7487, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 484, 0, 0, 0, 0, 0, 8192, 0, 0, 298, 8186, 0, 286, 0, 0, 0, 0, 0, 2337, 0, 0, 78, 0, 0, 5188, 7568, 0, 78, 0, 0, 0, 79, 0, 0, 0, 8050, 0, 0, 700, 0, 0, 0, 0, 0, 0, 0, 84, 55, 0, 0, 0] payload_rows_per_dst=[14974, 16384, 16384, 16110, 15874, 314, 1400, 278] dup_rows_per_dst=[4, 956, 596, 5236, 9794, 0, 0, 0] combine_rows_per_src=[14974, 16384, 16384, 16110, 15874, 314, 1400, 278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=344809472 dispatch_meta_bytes=351488 combine_payload_bytes=344809472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=345160960 ingress_bytes=448618496 link_bytes=448618496 token_bytes=689970432 weight_movement_bytes=103809024 total_nvlink_bytes=793779456 weight_to_token_ratio=0.150454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688115284e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283729920 dispatch_meta_bytes=307840 combine_payload_bytes=283729920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284037760 ingress_bytes=352935936 link_bytes=352935936 token_bytes=567767680 weight_movement_bytes=69206016 total_nvlink_bytes=636973696 weight_to_token_ratio=0.121891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328069513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=218218496 dispatch_meta_bytes=326936 combine_payload_bytes=218218496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=218545432 ingress_bytes=252821504 link_bytes=252821504 token_bytes=436763928 weight_movement_bytes=34603008 total_nvlink_bytes=471366936 weight_to_token_ratio=0.079226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.513469658e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95822 payload_rows_remote=95314 dup_rows=2482 dup_rows_remote=2482 dedup_ratio=0.025248 router_tokens_per_expert=[0, 0, 0, 514, 0, 484, 0, 0, 7135, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4063, 0, 2, 0, 0, 0, 50, 0, 0, 0, 0, 170, 0, 0, 254, 0, 7870, 8192, 8116, 0, 0, 0, 0, 0, 3536, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] dup_rows_per_dst=[1726, 0, 0, 440, 0, 0, 0, 316] combine_rows_per_src=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=341360640 dispatch_meta_bytes=343456 combine_payload_bytes=341360640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341704096 ingress_bytes=375963648 link_bytes=375963648 token_bytes=683064736 weight_movement_bytes=34603008 total_nvlink_bytes=717667744 weight_to_token_ratio=0.050658 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.414720940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=273383424 dispatch_meta_bytes=333304 combine_payload_bytes=273383424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=273716728 ingress_bytes=342589440 link_bytes=342589440 token_bytes=547100152 weight_movement_bytes=69206016 total_nvlink_bytes=616306168 weight_to_token_ratio=0.126496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289136482e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=88650 dup_rows=7202 dup_rows_remote=7166 dedup_ratio=0.073263 router_tokens_per_expert=[0, 7481, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 578, 0, 0, 0, 0, 0, 8192, 0, 0, 165, 8190, 0, 268, 0, 0, 0, 0, 0, 2474, 0, 0, 71, 0, 0, 5071, 7556, 0, 70, 0, 0, 0, 101, 0, 0, 0, 8044, 0, 0, 769, 0, 0, 0, 0, 0, 0, 0, 72, 46, 0, 0, 0] payload_rows_per_dst=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] dup_rows_per_dst=[8, 1152, 36, 5472, 0, 342, 0, 192] combine_rows_per_src=[14962, 16384, 2452, 16100, 10284, 15112, 1538, 14270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95990 payload_rows_remote=79786 dup_rows=2314 dup_rows_remote=1776 dedup_ratio=0.023539 router_tokens_per_expert=[0, 0, 0, 437, 0, 514, 0, 0, 7015, 0, 0, 0, 0, 0, 0, 81, 0, 0, 3976, 0, 4, 0, 0, 0, 140, 0, 0, 0, 0, 129, 0, 0, 233, 0, 7856, 8192, 8102, 0, 0, 0, 0, 0, 3846, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] dup_rows_per_dst=[1644, 0, 2, 538, 0, 2, 0, 128] combine_rows_per_src=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89968 payload_rows_remote=73584 dup_rows=8336 dup_rows_remote=8336 dedup_ratio=0.084798 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8057, 0, 0, 1, 0, 0, 0, 3217, 0, 0, 76, 39, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 72, 201, 0, 25, 0, 6014, 0, 3, 0, 980, 2277, 0, 0] payload_rows_per_dst=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] dup_rows_per_dst=[492, 0, 0, 50, 8, 0, 4052, 3734] combine_rows_per_src=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=283729920 dispatch_meta_bytes=307840 combine_payload_bytes=283729920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=284037760 ingress_bytes=352935936 link_bytes=352935936 token_bytes=567767680 weight_movement_bytes=69206016 total_nvlink_bytes=636973696 weight_to_token_ratio=0.121891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328069513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=390406144 dispatch_meta_bytes=391184 combine_payload_bytes=390406144 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=390797328 ingress_bytes=563421184 link_bytes=563421184 token_bytes=781203472 weight_movement_bytes=173015040 total_nvlink_bytes=954218512 weight_to_token_ratio=0.221472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.120108556e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94432 payload_rows_remote=85768 dup_rows=3872 dup_rows_remote=3872 dedup_ratio=0.039388 router_tokens_per_expert=[0, 0, 0, 423, 0, 478, 0, 0, 7047, 0, 1, 0, 0, 0, 0, 70, 0, 0, 4332, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 156, 0, 0, 214, 0, 7845, 8192, 8114, 0, 0, 0, 0, 0, 3581, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] dup_rows_per_dst=[1528, 2, 0, 354, 68, 1636, 0, 284] combine_rows_per_src=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91284 payload_rows_remote=83256 dup_rows=7020 dup_rows_remote=7020 dedup_ratio=0.071411 router_tokens_per_expert=[0, 0, 0, 459, 0, 486, 0, 0, 7018, 0, 0, 0, 0, 0, 0, 76, 0, 0, 4125, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 178, 0, 0, 257, 0, 7824, 8192, 8112, 0, 0, 0, 0, 0, 3678, 336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] dup_rows_per_dst=[434, 0, 5778, 324, 50, 0, 0, 434] combine_rows_per_src=[5468, 14188, 14112, 13552, 3170, 8028, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69656 payload_rows_remote=53276 dup_rows=28648 dup_rows_remote=28458 dedup_ratio=0.291423 router_tokens_per_expert=[0, 0, 0, 709, 0, 527, 0, 3, 7053, 0, 0, 0, 0, 0, 2, 73, 0, 0, 4220, 0, 5, 3, 0, 0, 133, 0, 0, 0, 0, 137, 0, 0, 224, 0, 7885, 8188, 8108, 0, 2, 1, 0, 0, 3250, 343, 1, 0, 6, 0, 3, 0, 3, 0, 0, 8174, 0, 2, 94, 2, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] dup_rows_per_dst=[32, 0, 1566, 0, 23102, 2, 3756, 190] combine_rows_per_src=[2446, 150, 10214, 540, 16374, 7198, 16354, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87982 payload_rows_remote=73744 dup_rows=10322 dup_rows_remote=10318 dedup_ratio=0.105001 router_tokens_per_expert=[0, 0, 0, 518, 2, 463, 0, 3, 7029, 1, 0, 2, 0, 2, 1, 86, 0, 0, 4302, 0, 3, 0, 0, 0, 74, 1, 0, 0, 1, 185, 2, 0, 300, 0, 7843, 8189, 8095, 1, 0, 1, 0, 0, 3412, 320, 1, 0, 0, 2, 0, 0, 3, 0, 0, 8189, 1, 0, 118, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15934, 14238, 8610, 16196, 16380, 16378, 6, 240] dup_rows_per_dst=[1724, 4, 0, 520, 602, 7470, 2, 0] combine_rows_per_src=[15934, 14238, 8610, 16196, 16380, 16378, 6, 240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=351305728 dispatch_meta_bytes=358560 combine_payload_bytes=351305728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=351664288 ingress_bytes=385908736 link_bytes=385908736 token_bytes=702970016 weight_movement_bytes=34603008 total_nvlink_bytes=737573024 weight_to_token_ratio=0.049224 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.452143505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95822 payload_rows_remote=95314 dup_rows=2482 dup_rows_remote=2482 dedup_ratio=0.025248 router_tokens_per_expert=[0, 0, 0, 514, 0, 484, 0, 0, 7135, 0, 0, 0, 0, 0, 0, 72, 0, 0, 4063, 0, 2, 0, 0, 0, 50, 0, 0, 0, 0, 170, 0, 0, 254, 0, 7870, 8192, 8116, 0, 0, 0, 0, 0, 3536, 344, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 158, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] dup_rows_per_dst=[1726, 0, 0, 440, 0, 0, 0, 316] combine_rows_per_src=[16010, 14414, 8130, 16232, 508, 7760, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95990 payload_rows_remote=79786 dup_rows=2314 dup_rows_remote=1776 dedup_ratio=0.023539 router_tokens_per_expert=[0, 0, 0, 437, 0, 514, 0, 0, 7015, 0, 0, 0, 0, 0, 0, 81, 0, 0, 3976, 0, 4, 0, 0, 0, 140, 0, 0, 0, 0, 129, 0, 0, 233, 0, 7856, 8192, 8102, 0, 0, 0, 0, 0, 3846, 372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] dup_rows_per_dst=[1644, 0, 2, 538, 0, 2, 0, 128] combine_rows_per_src=[15970, 14192, 7958, 16204, 466, 8434, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=341016576 dispatch_meta_bytes=361104 combine_payload_bytes=341016576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=341377680 ingress_bytes=375619584 link_bytes=375619584 token_bytes=682394256 weight_movement_bytes=34603008 total_nvlink_bytes=716997264 weight_to_token_ratio=0.050708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.413426255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=218218496 dispatch_meta_bytes=326936 combine_payload_bytes=218218496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=218545432 ingress_bytes=252821504 link_bytes=252821504 token_bytes=436763928 weight_movement_bytes=34603008 total_nvlink_bytes=471366936 weight_to_token_ratio=0.079226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.513469658e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88280 payload_rows_remote=72362 dup_rows=10024 dup_rows_remote=8452 dedup_ratio=0.101969 router_tokens_per_expert=[0, 0, 0, 446, 0, 452, 0, 0, 7071, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4362, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 164, 0, 0, 239, 0, 7847, 8192, 8113, 0, 0, 0, 0, 0, 3499, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15918, 14298, 8724, 16226, 16384, 16378, 0, 352] dup_rows_per_dst=[1572, 0, 0, 364, 478, 7610, 0, 0] combine_rows_per_src=[15918, 14298, 8724, 16226, 16384, 16378, 0, 352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87866 payload_rows_remote=71494 dup_rows=10438 dup_rows_remote=10428 dedup_ratio=0.106181 router_tokens_per_expert=[0, 0, 0, 452, 8, 499, 0, 8, 6998, 1, 0, 7, 0, 1, 4, 83, 0, 0, 4391, 0, 5, 5, 0, 0, 136, 1, 0, 0, 0, 152, 2, 0, 240, 0, 7872, 8180, 8088, 0, 0, 1, 0, 0, 3385, 352, 1, 5, 1, 1, 2, 0, 6, 0, 2, 8175, 0, 6, 72, 8, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[1922, 12570, 16212, 582, 16340, 7486, 16372, 16382] dup_rows_per_dst=[12, 14, 8334, 0, 1922, 4, 10, 142] combine_rows_per_src=[1922, 12570, 16212, 582, 16340, 7486, 16372, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=363110400 dispatch_meta_bytes=383264 combine_payload_bytes=363110400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=363493664 ingress_bytes=466919424 link_bytes=466919424 token_bytes=726604064 weight_movement_bytes=103809024 total_nvlink_bytes=830413088 weight_to_token_ratio=0.142869 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.756980202e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=326803456 dispatch_meta_bytes=326248 combine_payload_bytes=326803456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327129704 ingress_bytes=361406464 link_bytes=361406464 token_bytes=653933160 weight_movement_bytes=34603008 total_nvlink_bytes=688536168 weight_to_token_ratio=0.052915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359943428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301400064 dispatch_meta_bytes=327680 combine_payload_bytes=301400064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=301727744 ingress_bytes=370606080 link_bytes=370606080 token_bytes=603127808 weight_movement_bytes=69206016 total_nvlink_bytes=672333824 weight_to_token_ratio=0.114745 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394560843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302055424 dispatch_meta_bytes=336248 combine_payload_bytes=302055424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=302391672 ingress_bytes=371261440 link_bytes=371261440 token_bytes=604447096 weight_movement_bytes=69206016 total_nvlink_bytes=673653112 weight_to_token_ratio=0.114495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397026910e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68266 payload_rows_remote=53128 dup_rows=30038 dup_rows_remote=23714 dedup_ratio=0.305562 router_tokens_per_expert=[0, 0, 186, 0, 6, 0, 0, 3, 1, 0, 0, 5, 0, 3, 3, 0, 1, 2, 0, 0, 1, 8185, 3, 0, 1, 1, 0, 8186, 8116, 0, 1, 3, 0, 0, 0, 3089, 0, 2, 76, 27, 8185, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3419, 71, 214, 0, 26, 0, 5799, 0, 5, 0, 1071, 2455, 2, 0] payload_rows_per_dst=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] dup_rows_per_dst=[0, 12, 4, 16238, 4, 0, 7456, 6324] combine_rows_per_src=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93546 payload_rows_remote=84200 dup_rows=4758 dup_rows_remote=2938 dedup_ratio=0.048401 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8096, 0, 0, 0, 0, 0, 0, 3213, 0, 0, 72, 11, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 72, 231, 0, 47, 0, 6084, 0, 0, 0, 1064, 2193, 0, 0] payload_rows_per_dst=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] dup_rows_per_dst=[618, 0, 0, 0, 1820, 0, 0, 2320] combine_rows_per_src=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88280 payload_rows_remote=72362 dup_rows=10024 dup_rows_remote=8452 dedup_ratio=0.101969 router_tokens_per_expert=[0, 0, 0, 446, 0, 452, 0, 0, 7071, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4362, 0, 0, 0, 0, 0, 17, 1, 0, 0, 0, 164, 0, 0, 239, 0, 7847, 8192, 8113, 0, 0, 0, 0, 0, 3499, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15918, 14298, 8724, 16226, 16384, 16378, 0, 352] dup_rows_per_dst=[1572, 0, 0, 364, 478, 7610, 0, 0] combine_rows_per_src=[15918, 14298, 8724, 16226, 16384, 16378, 0, 352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=390406144 dispatch_meta_bytes=391184 combine_payload_bytes=390406144 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=390797328 ingress_bytes=563421184 link_bytes=563421184 token_bytes=781203472 weight_movement_bytes=173015040 total_nvlink_bytes=954218512 weight_to_token_ratio=0.221472 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.120108556e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292839424 dispatch_meta_bytes=327688 combine_payload_bytes=292839424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293167112 ingress_bytes=362045440 link_bytes=362045440 token_bytes=586006536 weight_movement_bytes=69206016 total_nvlink_bytes=655212552 weight_to_token_ratio=0.118098 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362347844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217612288 dispatch_meta_bytes=307368 combine_payload_bytes=217612288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217919656 ingress_bytes=321421312 link_bytes=321421312 token_bytes=435531944 weight_movement_bytes=103809024 total_nvlink_bytes=539340968 weight_to_token_ratio=0.238350 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209482521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=326803456 dispatch_meta_bytes=326248 combine_payload_bytes=326803456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327129704 ingress_bytes=361406464 link_bytes=361406464 token_bytes=653933160 weight_movement_bytes=34603008 total_nvlink_bytes=688536168 weight_to_token_ratio=0.052915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359943428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292839424 dispatch_meta_bytes=327688 combine_payload_bytes=292839424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293167112 ingress_bytes=362045440 link_bytes=362045440 token_bytes=586006536 weight_movement_bytes=69206016 total_nvlink_bytes=655212552 weight_to_token_ratio=0.118098 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362347844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296394752 dispatch_meta_bytes=323256 combine_payload_bytes=296394752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296718008 ingress_bytes=330997760 link_bytes=330997760 token_bytes=593112760 weight_movement_bytes=34603008 total_nvlink_bytes=627715768 weight_to_token_ratio=0.058341 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245517923e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94432 payload_rows_remote=85768 dup_rows=3872 dup_rows_remote=3872 dedup_ratio=0.039388 router_tokens_per_expert=[0, 0, 0, 423, 0, 478, 0, 0, 7047, 0, 1, 0, 0, 0, 0, 70, 0, 0, 4332, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 156, 0, 0, 214, 0, 7845, 8192, 8114, 0, 0, 0, 0, 0, 3581, 335, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 152, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] dup_rows_per_dst=[1528, 2, 0, 354, 68, 1636, 0, 284] combine_rows_per_src=[15964, 14234, 8664, 16232, 2460, 9918, 12656, 14304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92836 payload_rows_remote=92832 dup_rows=5468 dup_rows_remote=5468 dedup_ratio=0.055623 router_tokens_per_expert=[0, 0, 253, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8078, 0, 0, 2, 0, 0, 0, 3201, 0, 0, 81, 21, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 81, 225, 0, 48, 0, 6026, 0, 5, 0, 959, 2246, 0, 0] payload_rows_per_dst=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] dup_rows_per_dst=[526, 0, 0, 0, 4836, 0, 4, 102] combine_rows_per_src=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96164 payload_rows_remote=81582 dup_rows=2140 dup_rows_remote=2140 dedup_ratio=0.021769 router_tokens_per_expert=[0, 0, 0, 0, 3491, 0, 0, 0, 0, 7909, 0, 0, 8106, 4, 218, 142, 162, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 255, 0, 0, 8192, 0, 1, 155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7290, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 78, 0, 0, 7746, 0] payload_rows_per_dst=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] dup_rows_per_dst=[0, 0, 570, 510, 158, 0, 0, 902] combine_rows_per_src=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92418 payload_rows_remote=76034 dup_rows=5886 dup_rows_remote=5886 dedup_ratio=0.059875 router_tokens_per_expert=[0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8129, 0, 0, 1, 0, 0, 0, 3146, 0, 0, 70, 57, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 70, 177, 0, 22, 0, 6186, 0, 0, 0, 1121, 2319, 0, 0] payload_rows_per_dst=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] dup_rows_per_dst=[618, 0, 0, 0, 5132, 0, 4, 132] combine_rows_per_src=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=344883200 dispatch_meta_bytes=348552 combine_payload_bytes=344883200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345231752 ingress_bytes=379486208 link_bytes=379486208 token_bytes=690114952 weight_movement_bytes=34603008 total_nvlink_bytes=724717960 weight_to_token_ratio=0.050141 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.427976050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89968 payload_rows_remote=73584 dup_rows=8336 dup_rows_remote=8336 dedup_ratio=0.084798 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8057, 0, 0, 1, 0, 0, 0, 3217, 0, 0, 76, 39, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3305, 72, 201, 0, 25, 0, 6014, 0, 3, 0, 980, 2277, 0, 0] payload_rows_per_dst=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] dup_rows_per_dst=[492, 0, 0, 50, 8, 0, 4052, 3734] combine_rows_per_src=[13142, 16336, 7044, 3098, 6656, 16384, 12494, 14814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68266 payload_rows_remote=53128 dup_rows=30038 dup_rows_remote=23714 dedup_ratio=0.305562 router_tokens_per_expert=[0, 0, 186, 0, 6, 0, 0, 3, 1, 0, 0, 5, 0, 3, 3, 0, 1, 2, 0, 0, 1, 8185, 3, 0, 1, 1, 0, 8186, 8116, 0, 1, 3, 0, 0, 0, 3089, 0, 2, 76, 27, 8185, 3, 0, 0, 0, 0, 0, 0, 1, 0, 3419, 71, 214, 0, 26, 0, 5799, 0, 5, 0, 1071, 2455, 2, 0] payload_rows_per_dst=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] dup_rows_per_dst=[0, 12, 4, 16238, 4, 0, 7456, 6324] combine_rows_per_src=[390, 12, 10, 16378, 6384, 13578, 16376, 15138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91674 payload_rows_remote=75292 dup_rows=6630 dup_rows_remote=6626 dedup_ratio=0.067444 router_tokens_per_expert=[0, 0, 228, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 2, 8187, 1, 0, 0, 0, 0, 8189, 8061, 0, 0, 0, 1, 1, 0, 3977, 0, 2, 88, 45, 8186, 3, 1, 0, 0, 0, 2, 0, 2, 0, 2739, 83, 193, 0, 18, 2, 6152, 0, 0, 0, 1065, 1918, 0, 0] payload_rows_per_dst=[16128, 16382, 16380, 13556, 14484, 2822, 6074, 5848] dup_rows_per_dst=[454, 4, 0, 0, 6048, 6, 0, 118] combine_rows_per_src=[16128, 16382, 16380, 13556, 14484, 2822, 6074, 5848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311435264 dispatch_meta_bytes=327680 combine_payload_bytes=311435264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311762944 ingress_bytes=380641280 link_bytes=380641280 token_bytes=623198208 weight_movement_bytes=69206016 total_nvlink_bytes=692404224 weight_to_token_ratio=0.111050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432322493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301400064 dispatch_meta_bytes=327680 combine_payload_bytes=301400064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=301727744 ingress_bytes=370606080 link_bytes=370606080 token_bytes=603127808 weight_movement_bytes=69206016 total_nvlink_bytes=672333824 weight_to_token_ratio=0.114745 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394560843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93546 payload_rows_remote=84200 dup_rows=4758 dup_rows_remote=2938 dedup_ratio=0.048401 router_tokens_per_expert=[0, 0, 308, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8096, 0, 0, 0, 0, 0, 0, 3213, 0, 0, 72, 11, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3184, 72, 231, 0, 47, 0, 6084, 0, 0, 0, 1064, 2193, 0, 0] payload_rows_per_dst=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] dup_rows_per_dst=[618, 0, 0, 0, 1820, 0, 0, 2320] combine_rows_per_src=[16192, 16384, 16384, 0, 9346, 16384, 7068, 11788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=351305728 dispatch_meta_bytes=358560 combine_payload_bytes=351305728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=351664288 ingress_bytes=385908736 link_bytes=385908736 token_bytes=702970016 weight_movement_bytes=34603008 total_nvlink_bytes=737573024 weight_to_token_ratio=0.049224 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.452143505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92836 payload_rows_remote=92832 dup_rows=5468 dup_rows_remote=5468 dedup_ratio=0.055623 router_tokens_per_expert=[0, 0, 253, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8078, 0, 0, 2, 0, 0, 0, 3201, 0, 0, 81, 21, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3340, 81, 225, 0, 48, 0, 6026, 0, 5, 0, 959, 2246, 0, 0] payload_rows_per_dst=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] dup_rows_per_dst=[526, 0, 0, 0, 4836, 0, 4, 102] combine_rows_per_src=[16156, 16384, 16384, 4, 13822, 16384, 7384, 6318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=217612288 dispatch_meta_bytes=307368 combine_payload_bytes=217612288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217919656 ingress_bytes=321421312 link_bytes=321421312 token_bytes=435531944 weight_movement_bytes=103809024 total_nvlink_bytes=539340968 weight_to_token_ratio=0.238350 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209482521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71144 payload_rows_remote=54772 dup_rows=27160 dup_rows_remote=19926 dedup_ratio=0.276286 router_tokens_per_expert=[1, 0, 277, 8, 3, 0, 0, 4, 4, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 3, 8178, 4, 1, 0, 0, 2, 8181, 8109, 1, 0, 4, 1, 0, 3, 3123, 1, 8, 93, 49, 8157, 2, 13, 0, 0, 3, 6, 0, 1, 0, 3315, 81, 174, 0, 48, 6, 6196, 0, 3, 1, 948, 2132, 0, 0] payload_rows_per_dst=[584, 22, 16, 16362, 6544, 16340, 16372, 14904] dup_rows_per_dst=[2, 2, 0, 16232, 12, 22, 7234, 3656] combine_rows_per_src=[584, 22, 16, 16362, 6544, 16340, 16372, 14904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=380239872 dispatch_meta_bytes=393200 combine_payload_bytes=380239872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380633072 ingress_bytes=484048896 link_bytes=484048896 token_bytes=760872944 weight_movement_bytes=103809024 total_nvlink_bytes=864681968 weight_to_token_ratio=0.136434 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821437027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88734 payload_rows_remote=73228 dup_rows=9570 dup_rows_remote=8934 dedup_ratio=0.097351 router_tokens_per_expert=[3, 0, 0, 0, 4027, 0, 0, 1, 2, 7908, 0, 0, 8106, 8, 228, 113, 181, 0, 4, 2, 1, 113, 2, 1, 0, 3, 3, 243, 0, 1, 8185, 0, 2, 81, 1, 5, 2, 0, 0, 1, 0, 1, 0, 1, 2, 0, 7205, 2, 4640, 0, 0, 2, 1, 0, 0, 0, 3, 0, 388, 28, 0, 0, 7652, 0] payload_rows_per_dst=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] dup_rows_per_dst=[2, 6, 584, 490, 28, 2, 7822, 636] combine_rows_per_src=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=334159872 dispatch_meta_bytes=334888 combine_payload_bytes=334159872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=334494760 ingress_bytes=403365888 link_bytes=403365888 token_bytes=668654632 weight_movement_bytes=69206016 total_nvlink_bytes=737860648 weight_to_token_ratio=0.103500 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517833363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308396032 dispatch_meta_bytes=327672 combine_payload_bytes=308396032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308723704 ingress_bytes=342999040 link_bytes=342999040 token_bytes=617119736 weight_movement_bytes=34603008 total_nvlink_bytes=651722744 weight_to_token_ratio=0.056072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290677774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87198 payload_rows_remote=71056 dup_rows=11106 dup_rows_remote=10508 dedup_ratio=0.112976 router_tokens_per_expert=[0, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8071, 0, 0, 0, 0, 0, 0, 3083, 0, 0, 78, 18, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3333, 78, 216, 0, 31, 0, 6189, 0, 1, 0, 925, 2254, 0, 0] payload_rows_per_dst=[16142, 16384, 16384, 16384, 8310, 0, 7316, 6278] dup_rows_per_dst=[598, 0, 9314, 0, 1112, 0, 0, 82] combine_rows_per_src=[16142, 16384, 16384, 16384, 8310, 0, 7316, 6278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=344883200 dispatch_meta_bytes=348552 combine_payload_bytes=344883200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345231752 ingress_bytes=379486208 link_bytes=379486208 token_bytes=690114952 weight_movement_bytes=34603008 total_nvlink_bytes=724717960 weight_to_token_ratio=0.050141 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.427976050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308396032 dispatch_meta_bytes=327672 combine_payload_bytes=308396032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308723704 ingress_bytes=342999040 link_bytes=342999040 token_bytes=617119736 weight_movement_bytes=34603008 total_nvlink_bytes=651722744 weight_to_token_ratio=0.056072 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290677774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96368 payload_rows_remote=79984 dup_rows=1936 dup_rows_remote=1744 dedup_ratio=0.019694 router_tokens_per_expert=[0, 0, 0, 0, 3418, 0, 0, 0, 0, 7911, 0, 0, 8120, 5, 208, 169, 187, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 252, 0, 0, 8192, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7309, 0, 4888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 103, 0, 0, 7692, 0] payload_rows_per_dst=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] dup_rows_per_dst=[0, 0, 542, 504, 192, 0, 0, 698] combine_rows_per_src=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=380239872 dispatch_meta_bytes=393200 combine_payload_bytes=380239872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=380633072 ingress_bytes=484048896 link_bytes=484048896 token_bytes=760872944 weight_movement_bytes=103809024 total_nvlink_bytes=864681968 weight_to_token_ratio=0.136434 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.821437027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=224346112 dispatch_meta_bytes=298792 combine_payload_bytes=224346112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=224644904 ingress_bytes=258949120 link_bytes=258949120 token_bytes=448991016 weight_movement_bytes=34603008 total_nvlink_bytes=483594024 weight_to_token_ratio=0.077068 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.744046914e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299941888 dispatch_meta_bytes=328648 combine_payload_bytes=299941888 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=300270536 ingress_bytes=403750912 link_bytes=403750912 token_bytes=600212424 weight_movement_bytes=103809024 total_nvlink_bytes=704021448 weight_to_token_ratio=0.172954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519282177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291045376 dispatch_meta_bytes=326256 combine_payload_bytes=291045376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291371632 ingress_bytes=325648384 link_bytes=325648384 token_bytes=582417008 weight_movement_bytes=34603008 total_nvlink_bytes=617020016 weight_to_token_ratio=0.059413 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225388652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92418 payload_rows_remote=76034 dup_rows=5886 dup_rows_remote=5886 dedup_ratio=0.059875 router_tokens_per_expert=[0, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8129, 0, 0, 1, 0, 0, 0, 3146, 0, 0, 70, 57, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 70, 177, 0, 22, 0, 6186, 0, 0, 0, 1121, 2319, 0, 0] payload_rows_per_dst=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] dup_rows_per_dst=[618, 0, 0, 0, 5132, 0, 4, 132] combine_rows_per_src=[16262, 16382, 16384, 2, 13786, 16384, 6470, 6748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96512 payload_rows_remote=80128 dup_rows=1792 dup_rows_remote=1302 dedup_ratio=0.018229 router_tokens_per_expert=[0, 0, 0, 0, 3457, 0, 0, 0, 0, 7910, 0, 0, 8108, 5, 205, 162, 168, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 245, 0, 0, 8192, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 5001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 67, 0, 0, 7726, 0] payload_rows_per_dst=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] dup_rows_per_dst=[0, 0, 592, 490, 34, 0, 0, 676] combine_rows_per_src=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84016 payload_rows_remote=68704 dup_rows=14288 dup_rows_remote=6070 dedup_ratio=0.145345 router_tokens_per_expert=[0, 0, 0, 63, 0, 0, 0, 8192, 3019, 0, 0, 0, 0, 0, 11, 0, 472, 6268, 0, 0, 4415, 0, 0, 4305, 4636, 0, 0, 695, 0, 0, 0, 0, 0, 654, 0, 0, 8, 0, 0, 0, 0, 0, 6851, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 0, 62, 0, 0, 0, 0, 336, 0, 0] payload_rows_per_dst=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] dup_rows_per_dst=[80, 0, 3868, 742, 540, 8218, 718, 122] combine_rows_per_src=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91110 payload_rows_remote=77376 dup_rows=7194 dup_rows_remote=6858 dedup_ratio=0.073181 router_tokens_per_expert=[0, 0, 0, 0, 3387, 0, 0, 0, 0, 7905, 0, 0, 8120, 11, 175, 152, 162, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 248, 0, 0, 8192, 0, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 68, 0, 0, 7758, 0] payload_rows_per_dst=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] dup_rows_per_dst=[3470, 916, 336, 236, 12, 1590, 0, 634] combine_rows_per_src=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=327614464 dispatch_meta_bytes=326912 combine_payload_bytes=327614464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327941376 ingress_bytes=362217472 link_bytes=362217472 token_bytes=655555840 weight_movement_bytes=34603008 total_nvlink_bytes=690158848 weight_to_token_ratio=0.052784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362995186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96164 payload_rows_remote=81582 dup_rows=2140 dup_rows_remote=2140 dedup_ratio=0.021769 router_tokens_per_expert=[0, 0, 0, 0, 3491, 0, 0, 0, 0, 7909, 0, 0, 8106, 4, 218, 142, 162, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 255, 0, 0, 8192, 0, 1, 155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7290, 0, 4672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 485, 78, 0, 0, 7746, 0] payload_rows_per_dst=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] dup_rows_per_dst=[0, 0, 570, 510, 158, 0, 0, 902] combine_rows_per_src=[6982, 728, 16062, 16384, 16366, 14582, 9344, 15716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88734 payload_rows_remote=73228 dup_rows=9570 dup_rows_remote=8934 dedup_ratio=0.097351 router_tokens_per_expert=[3, 0, 0, 0, 4027, 0, 0, 1, 2, 7908, 0, 0, 8106, 8, 228, 113, 181, 0, 4, 2, 1, 113, 2, 1, 0, 3, 3, 243, 0, 1, 8185, 0, 2, 81, 1, 5, 2, 0, 0, 1, 0, 1, 0, 1, 2, 0, 7205, 2, 4640, 0, 0, 2, 1, 0, 0, 0, 3, 0, 388, 28, 0, 0, 7652, 0] payload_rows_per_dst=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] dup_rows_per_dst=[2, 6, 584, 490, 28, 2, 7822, 636] combine_rows_per_src=[8060, 696, 15840, 16380, 16368, 10, 15874, 15506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=316932096 dispatch_meta_bytes=336936 combine_payload_bytes=316932096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317269032 ingress_bytes=351535104 link_bytes=351535104 token_bytes=634201128 weight_movement_bytes=34603008 total_nvlink_bytes=668804136 weight_to_token_ratio=0.054562 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322798295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63862 payload_rows_remote=47484 dup_rows=34442 dup_rows_remote=8030 dedup_ratio=0.350362 router_tokens_per_expert=[0, 0, 1, 0, 3392, 0, 0, 0, 1, 7900, 0, 0, 8097, 13, 197, 158, 179, 0, 1, 0, 0, 385, 0, 0, 0, 3, 1, 248, 1, 0, 8188, 0, 1, 95, 2, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 7312, 4, 4634, 3, 0, 0, 1, 0, 0, 0, 2, 0, 468, 126, 1, 0, 7733, 2] payload_rows_per_dst=[16378, 16378, 1130, 4860, 198, 14700, 9276, 942] dup_rows_per_dst=[6784, 26412, 0, 136, 2, 852, 0, 256] combine_rows_per_src=[16378, 16378, 1130, 4860, 198, 14700, 9276, 942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96368 payload_rows_remote=79984 dup_rows=1936 dup_rows_remote=1744 dedup_ratio=0.019694 router_tokens_per_expert=[0, 0, 0, 0, 3418, 0, 0, 0, 0, 7911, 0, 0, 8120, 5, 208, 169, 187, 0, 0, 0, 0, 156, 0, 0, 0, 0, 0, 252, 0, 0, 8192, 0, 0, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7309, 0, 4888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 374, 103, 0, 0, 7692, 0] payload_rows_per_dst=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] dup_rows_per_dst=[0, 0, 542, 504, 192, 0, 0, 698] combine_rows_per_src=[6836, 764, 15966, 16384, 16384, 14618, 9776, 15640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311435264 dispatch_meta_bytes=327680 combine_payload_bytes=311435264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311762944 ingress_bytes=380641280 link_bytes=380641280 token_bytes=623198208 weight_movement_bytes=69206016 total_nvlink_bytes=692404224 weight_to_token_ratio=0.111050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432322493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92964 payload_rows_remote=80778 dup_rows=5340 dup_rows_remote=1816 dedup_ratio=0.054321 router_tokens_per_expert=[7, 1, 5, 3, 3697, 0, 1, 0, 5, 7907, 0, 0, 8089, 17, 200, 142, 174, 0, 3, 0, 2, 117, 1, 0, 0, 4, 0, 236, 5, 0, 8182, 0, 3, 124, 6, 0, 8, 4, 0, 0, 1, 0, 0, 1, 0, 1, 7195, 6, 4858, 1, 0, 1, 0, 0, 0, 7, 2, 0, 359, 64, 0, 4, 7707, 2] payload_rows_per_dst=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] dup_rows_per_dst=[12, 8, 552, 474, 104, 6, 3524, 660] combine_rows_per_src=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96512 payload_rows_remote=80128 dup_rows=1792 dup_rows_remote=1302 dedup_ratio=0.018229 router_tokens_per_expert=[0, 0, 0, 0, 3457, 0, 0, 0, 0, 7910, 0, 0, 8108, 5, 205, 162, 168, 0, 0, 0, 0, 264, 0, 0, 0, 0, 0, 245, 0, 0, 8192, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7187, 0, 5001, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 67, 0, 0, 7726, 0] payload_rows_per_dst=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] dup_rows_per_dst=[0, 0, 592, 490, 34, 0, 0, 676] combine_rows_per_src=[6914, 744, 16092, 16384, 16378, 14374, 10002, 15624] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281411584 dispatch_meta_bytes=299096 combine_payload_bytes=281411584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=281710680 ingress_bytes=385220608 link_bytes=385220608 token_bytes=563122264 weight_movement_bytes=103809024 total_nvlink_bytes=666931288 weight_to_token_ratio=0.184345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=328204288 dispatch_meta_bytes=325720 combine_payload_bytes=328204288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=328530008 ingress_bytes=397410304 link_bytes=397410304 token_bytes=656734296 weight_movement_bytes=69206016 total_nvlink_bytes=725940312 weight_to_token_ratio=0.105379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.495422980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70296 payload_rows_remote=57178 dup_rows=28008 dup_rows_remote=27904 dedup_ratio=0.284912 router_tokens_per_expert=[1, 1, 0, 97, 0, 2, 0, 8185, 3019, 0, 0, 3, 3, 3, 12, 0, 499, 6209, 4, 0, 4845, 0, 0, 3996, 4386, 0, 0, 747, 0, 0, 0, 0, 0, 653, 1, 2, 10, 0, 0, 2, 6, 0, 6830, 8139, 0, 0, 0, 6, 0, 0, 0, 0, 1087, 0, 2, 0, 51, 0, 0, 0, 0, 351, 0, 0] payload_rows_per_dst=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] dup_rows_per_dst=[2, 10, 292, 8424, 2, 13582, 5592, 104] combine_rows_per_src=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=334159872 dispatch_meta_bytes=334888 combine_payload_bytes=334159872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=334494760 ingress_bytes=403365888 link_bytes=403365888 token_bytes=668654632 weight_movement_bytes=69206016 total_nvlink_bytes=737860648 weight_to_token_ratio=0.103500 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517833363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299941888 dispatch_meta_bytes=328648 combine_payload_bytes=299941888 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=300270536 ingress_bytes=403750912 link_bytes=403750912 token_bytes=600212424 weight_movement_bytes=103809024 total_nvlink_bytes=704021448 weight_to_token_ratio=0.172954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519282177e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59264 payload_rows_remote=42880 dup_rows=39040 dup_rows_remote=31740 dedup_ratio=0.397135 router_tokens_per_expert=[0, 0, 0, 0, 3650, 0, 0, 0, 0, 7881, 0, 0, 8112, 5, 193, 140, 168, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 255, 0, 0, 8192, 0, 2, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7229, 0, 5130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 78, 0, 0, 7653, 0] payload_rows_per_dst=[16384, 16384, 448, 510, 164, 14458, 10260, 656] dup_rows_per_dst=[7300, 31584, 0, 0, 0, 0, 0, 156] combine_rows_per_src=[16384, 16384, 448, 510, 164, 14458, 10260, 656] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92964 payload_rows_remote=80778 dup_rows=5340 dup_rows_remote=1816 dedup_ratio=0.054321 router_tokens_per_expert=[7, 1, 5, 3, 3697, 0, 1, 0, 5, 7907, 0, 0, 8089, 17, 200, 142, 174, 0, 3, 0, 2, 117, 1, 0, 0, 4, 0, 236, 5, 0, 8182, 0, 3, 124, 6, 0, 8, 4, 0, 0, 1, 0, 0, 1, 0, 1, 7195, 6, 4858, 1, 0, 1, 0, 0, 0, 7, 2, 0, 359, 64, 0, 4, 7707, 2] payload_rows_per_dst=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] dup_rows_per_dst=[12, 8, 552, 474, 104, 6, 3524, 660] combine_rows_per_src=[7416, 720, 15856, 16380, 16364, 8426, 12186, 15616] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=194494464 dispatch_meta_bytes=222056 combine_payload_bytes=194494464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=194716520 ingress_bytes=298303488 link_bytes=298303488 token_bytes=389210984 weight_movement_bytes=103809024 total_nvlink_bytes=493020008 weight_to_token_ratio=0.266717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122492010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=327614464 dispatch_meta_bytes=326912 combine_payload_bytes=327614464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327941376 ingress_bytes=362217472 link_bytes=362217472 token_bytes=655555840 weight_movement_bytes=34603008 total_nvlink_bytes=690158848 weight_to_token_ratio=0.052784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362995186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95132 payload_rows_remote=78894 dup_rows=3172 dup_rows_remote=1808 dedup_ratio=0.032267 router_tokens_per_expert=[0, 0, 0, 91, 0, 0, 0, 8192, 2935, 0, 0, 0, 0, 0, 15, 0, 563, 6305, 0, 0, 4103, 0, 0, 4331, 4901, 0, 0, 627, 0, 0, 0, 0, 0, 666, 0, 0, 16, 0, 0, 0, 0, 0, 6968, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 901, 0, 0, 0, 73, 0, 0, 0, 0, 346, 0, 0] payload_rows_per_dst=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] dup_rows_per_dst=[182, 0, 372, 626, 1364, 0, 484, 144] combine_rows_per_src=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=330866688 dispatch_meta_bytes=330376 combine_payload_bytes=330866688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331197064 ingress_bytes=365469696 link_bytes=365469696 token_bytes=662063752 weight_movement_bytes=34603008 total_nvlink_bytes=696666760 weight_to_token_ratio=0.052265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375233043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=340328 combine_payload_bytes=234201088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=234541416 ingress_bytes=268804096 link_bytes=268804096 token_bytes=468742504 weight_movement_bytes=34603008 total_nvlink_bytes=503345512 weight_to_token_ratio=0.073821 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.011488173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=328204288 dispatch_meta_bytes=325720 combine_payload_bytes=328204288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=328530008 ingress_bytes=397410304 link_bytes=397410304 token_bytes=656734296 weight_movement_bytes=69206016 total_nvlink_bytes=725940312 weight_to_token_ratio=0.105379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.495422980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=175636480 dispatch_meta_bytes=298480 combine_payload_bytes=175636480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=175934960 ingress_bytes=210239488 link_bytes=210239488 token_bytes=351571440 weight_movement_bytes=34603008 total_nvlink_bytes=386174448 weight_to_token_ratio=0.098424 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.911142677e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91110 payload_rows_remote=77376 dup_rows=7194 dup_rows_remote=6858 dedup_ratio=0.073181 router_tokens_per_expert=[0, 0, 0, 0, 3387, 0, 0, 0, 0, 7905, 0, 0, 8120, 11, 175, 152, 162, 0, 0, 0, 0, 153, 0, 0, 0, 0, 0, 248, 0, 0, 8192, 0, 2, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7254, 0, 5120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 68, 0, 0, 7758, 0] payload_rows_per_dst=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] dup_rows_per_dst=[3470, 916, 336, 236, 12, 1590, 0, 634] combine_rows_per_src=[11766, 2964, 13734, 8182, 15552, 14686, 10240, 13986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92610 payload_rows_remote=79082 dup_rows=5694 dup_rows_remote=1418 dedup_ratio=0.057922 router_tokens_per_expert=[0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 6061, 0, 0, 0, 0, 7875, 193, 47, 0, 0, 0, 0, 0, 0, 0, 6763, 0, 0, 8, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 5976, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 0, 0, 0, 457, 0, 0, 0] payload_rows_per_dst=[10772, 12122, 16104, 16, 16028, 13528, 10514, 13526] dup_rows_per_dst=[378, 0, 126, 0, 0, 4276, 0, 914] combine_rows_per_src=[10772, 12122, 16104, 16, 16028, 13528, 10514, 13526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88950 payload_rows_remote=79200 dup_rows=9354 dup_rows_remote=8818 dedup_ratio=0.095154 router_tokens_per_expert=[0, 0, 0, 70, 0, 0, 0, 8192, 3210, 0, 0, 0, 0, 0, 3, 0, 511, 5961, 0, 0, 4273, 0, 0, 4540, 4540, 0, 0, 603, 0, 0, 0, 0, 0, 681, 0, 0, 11, 0, 0, 0, 0, 0, 6912, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 78, 0, 0, 0, 0, 372, 0, 0] payload_rows_per_dst=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] dup_rows_per_dst=[140, 5950, 318, 536, 1384, 0, 870, 156] combine_rows_per_src=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=330866688 dispatch_meta_bytes=330376 combine_payload_bytes=330866688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331197064 ingress_bytes=365469696 link_bytes=365469696 token_bytes=662063752 weight_movement_bytes=34603008 total_nvlink_bytes=696666760 weight_to_token_ratio=0.052265 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375233043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88628 payload_rows_remote=79568 dup_rows=9676 dup_rows_remote=9416 dedup_ratio=0.098429 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 8192, 3336, 0, 0, 0, 0, 0, 11, 0, 455, 6234, 0, 0, 4205, 0, 0, 4255, 4475, 0, 0, 750, 0, 0, 0, 0, 0, 655, 0, 0, 6, 0, 0, 0, 0, 0, 6992, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 66, 0, 0, 0, 0, 339, 0, 0] payload_rows_per_dst=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] dup_rows_per_dst=[160, 6650, 260, 732, 1322, 0, 412, 140] combine_rows_per_src=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=323149824 dispatch_meta_bytes=322808 combine_payload_bytes=323149824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323472632 ingress_bytes=357752832 link_bytes=357752832 token_bytes=646622456 weight_movement_bytes=34603008 total_nvlink_bytes=681225464 weight_to_token_ratio=0.053513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346195105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84016 payload_rows_remote=68704 dup_rows=14288 dup_rows_remote=6070 dedup_ratio=0.145345 router_tokens_per_expert=[0, 0, 0, 63, 0, 0, 0, 8192, 3019, 0, 0, 0, 0, 0, 11, 0, 472, 6268, 0, 0, 4415, 0, 0, 4305, 4636, 0, 0, 695, 0, 0, 0, 0, 0, 654, 0, 0, 8, 0, 0, 0, 0, 0, 6851, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 1058, 0, 0, 0, 62, 0, 0, 0, 0, 336, 0, 0] payload_rows_per_dst=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] dup_rows_per_dst=[80, 0, 3868, 742, 540, 8218, 718, 122] combine_rows_per_src=[9930, 6060, 12406, 9920, 7170, 15312, 10008, 13210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70296 payload_rows_remote=57178 dup_rows=28008 dup_rows_remote=27904 dedup_ratio=0.284912 router_tokens_per_expert=[1, 1, 0, 97, 0, 2, 0, 8185, 3019, 0, 0, 3, 3, 3, 12, 0, 499, 6209, 4, 0, 4845, 0, 0, 3996, 4386, 0, 0, 747, 0, 0, 0, 0, 0, 653, 1, 2, 10, 0, 0, 2, 6, 0, 6830, 8139, 0, 0, 0, 6, 0, 0, 0, 0, 1087, 0, 2, 0, 51, 0, 0, 0, 0, 351, 0, 0] payload_rows_per_dst=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] dup_rows_per_dst=[2, 10, 292, 8424, 2, 13582, 5592, 104] combine_rows_per_src=[200, 6070, 4186, 14740, 1334, 16380, 14268, 13118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=325910528 dispatch_meta_bytes=355936 combine_payload_bytes=325910528 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326266464 ingress_bytes=464322560 link_bytes=464322560 token_bytes=652176992 weight_movement_bytes=138412032 total_nvlink_bytes=790589024 weight_to_token_ratio=0.212231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747208412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89672 payload_rows_remote=75824 dup_rows=8632 dup_rows_remote=3374 dedup_ratio=0.087809 router_tokens_per_expert=[2, 0, 0, 66, 0, 2, 0, 8187, 2657, 0, 0, 2, 0, 3, 1, 0, 483, 6320, 4, 0, 4257, 0, 0, 4408, 4640, 0, 0, 854, 0, 0, 0, 0, 0, 652, 1, 0, 4, 0, 0, 0, 5, 1, 6890, 8101, 0, 0, 0, 3, 3, 0, 0, 0, 1202, 1, 2, 0, 81, 0, 0, 0, 0, 320, 0, 0] payload_rows_per_dst=[16380, 13848, 9274, 10150, 16206, 16, 10520, 13278] dup_rows_per_dst=[134, 5258, 214, 838, 1310, 2, 712, 164] combine_rows_per_src=[16380, 13848, 9274, 10150, 16206, 16, 10520, 13278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95132 payload_rows_remote=78894 dup_rows=3172 dup_rows_remote=1808 dedup_ratio=0.032267 router_tokens_per_expert=[0, 0, 0, 91, 0, 0, 0, 8192, 2935, 0, 0, 0, 0, 0, 15, 0, 563, 6305, 0, 0, 4103, 0, 0, 4331, 4901, 0, 0, 627, 0, 0, 0, 0, 0, 666, 0, 0, 16, 0, 0, 0, 0, 0, 6968, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 901, 0, 0, 0, 73, 0, 0, 0, 0, 346, 0, 0] payload_rows_per_dst=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] dup_rows_per_dst=[182, 0, 372, 626, 1364, 0, 484, 144] combine_rows_per_src=[16384, 5900, 8960, 10430, 16238, 13936, 9980, 13304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=316932096 dispatch_meta_bytes=336936 combine_payload_bytes=316932096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317269032 ingress_bytes=351535104 link_bytes=351535104 token_bytes=634201128 weight_movement_bytes=34603008 total_nvlink_bytes=668804136 weight_to_token_ratio=0.054562 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322798295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75112 payload_rows_remote=63670 dup_rows=23192 dup_rows_remote=21274 dedup_ratio=0.235921 router_tokens_per_expert=[1, 1, 0, 63, 1, 1, 0, 8178, 2988, 0, 7, 2, 7, 10, 6, 1, 496, 6312, 11, 0, 4234, 0, 5, 4242, 4663, 0, 0, 766, 2, 0, 0, 0, 0, 660, 1, 0, 15, 0, 1, 0, 4, 4, 6858, 8127, 1, 0, 0, 7, 1, 0, 0, 1, 1010, 4, 4, 1, 86, 0, 11, 0, 0, 359, 0, 0] payload_rows_per_dst=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] dup_rows_per_dst=[0, 16, 6634, 798, 2, 13632, 1918, 192] combine_rows_per_src=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88950 payload_rows_remote=79200 dup_rows=9354 dup_rows_remote=8818 dedup_ratio=0.095154 router_tokens_per_expert=[0, 0, 0, 70, 0, 0, 0, 8192, 3210, 0, 0, 0, 0, 0, 3, 0, 511, 5961, 0, 0, 4273, 0, 0, 4540, 4540, 0, 0, 603, 0, 0, 0, 0, 0, 681, 0, 0, 11, 0, 0, 0, 0, 0, 6912, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 1055, 0, 0, 0, 78, 0, 0, 0, 0, 372, 0, 0] payload_rows_per_dst=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] dup_rows_per_dst=[140, 5950, 318, 536, 1384, 0, 870, 156] combine_rows_per_src=[16384, 13444, 9250, 9750, 16280, 856, 10320, 12666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=281411584 dispatch_meta_bytes=299096 combine_payload_bytes=281411584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=281710680 ingress_bytes=385220608 link_bytes=385220608 token_bytes=563122264 weight_movement_bytes=103809024 total_nvlink_bytes=666931288 weight_to_token_ratio=0.184345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=322000 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324241872 ingress_bytes=427728896 link_bytes=427728896 token_bytes=648161744 weight_movement_bytes=103809024 total_nvlink_bytes=751970768 weight_to_token_ratio=0.160159 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609509401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=61388 dup_rows=23738 dup_rows_remote=22932 dedup_ratio=0.241475 router_tokens_per_expert=[0, 2, 5, 352, 9, 0, 0, 0, 4, 0, 0, 6321, 0, 0, 3, 0, 7968, 123, 42, 0, 1, 0, 3, 0, 3, 0, 6587, 0, 0, 18, 0, 8006, 5, 0, 0, 0, 1, 5, 0, 0, 5851, 0, 0, 8184, 0, 1, 0, 0, 1, 4, 0, 0, 0, 0, 5248, 0, 0, 2, 0, 1, 402, 0, 0, 0] payload_rows_per_dst=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] dup_rows_per_dst=[4, 6, 0, 0, 10, 14774, 8138, 806] combine_rows_per_src=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=324403200 dispatch_meta_bytes=352072 combine_payload_bytes=324403200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324755272 ingress_bytes=359006208 link_bytes=359006208 token_bytes=649158472 weight_movement_bytes=34603008 total_nvlink_bytes=683761480 weight_to_token_ratio=0.053304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350911458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82466 payload_rows_remote=66082 dup_rows=15838 dup_rows_remote=10412 dedup_ratio=0.161112 router_tokens_per_expert=[0, 0, 0, 82, 0, 0, 0, 8192, 3215, 0, 0, 0, 0, 0, 3, 0, 461, 6169, 0, 0, 4014, 0, 0, 4586, 4758, 0, 0, 660, 0, 0, 0, 0, 0, 627, 0, 0, 9, 0, 0, 0, 0, 0, 6935, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 926, 0, 0, 0, 77, 0, 0, 0, 0, 306, 0, 0] payload_rows_per_dst=[16384, 11132, 12872, 10210, 16264, 0, 10428, 5176] dup_rows_per_dst=[5426, 3912, 3958, 626, 1272, 0, 596, 48] combine_rows_per_src=[16384, 11132, 12872, 10210, 16264, 0, 10428, 5176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=234201088 dispatch_meta_bytes=340328 combine_payload_bytes=234201088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=234541416 ingress_bytes=268804096 link_bytes=268804096 token_bytes=468742504 weight_movement_bytes=34603008 total_nvlink_bytes=503345512 weight_to_token_ratio=0.073821 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.011488173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75112 payload_rows_remote=63670 dup_rows=23192 dup_rows_remote=21274 dedup_ratio=0.235921 router_tokens_per_expert=[1, 1, 0, 63, 1, 1, 0, 8178, 2988, 0, 7, 2, 7, 10, 6, 1, 496, 6312, 11, 0, 4234, 0, 5, 4242, 4663, 0, 0, 766, 2, 0, 0, 0, 0, 660, 1, 0, 15, 0, 1, 0, 4, 4, 6858, 8127, 1, 0, 0, 7, 1, 0, 0, 1, 1010, 4, 4, 1, 86, 0, 11, 0, 0, 359, 0, 0] payload_rows_per_dst=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] dup_rows_per_dst=[0, 16, 6634, 798, 2, 13632, 1918, 192] combine_rows_per_src=[134, 6026, 16380, 10064, 1352, 16370, 11442, 13344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96550 payload_rows_remote=80454 dup_rows=1754 dup_rows_remote=1754 dedup_ratio=0.017843 router_tokens_per_expert=[0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 7903, 157, 61, 0, 0, 0, 0, 0, 0, 0, 6679, 0, 0, 8, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 5854, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 498, 0, 0, 0] payload_rows_per_dst=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] dup_rows_per_dst=[620, 0, 122, 16, 0, 0, 0, 996] combine_rows_per_src=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=310575104 dispatch_meta_bytes=316792 combine_payload_bytes=310575104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310891896 ingress_bytes=345178112 link_bytes=345178112 token_bytes=621467000 weight_movement_bytes=34603008 total_nvlink_bytes=656070008 weight_to_token_ratio=0.055680 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298877446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=323149824 dispatch_meta_bytes=322808 combine_payload_bytes=323149824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323472632 ingress_bytes=357752832 link_bytes=357752832 token_bytes=646622456 weight_movement_bytes=34603008 total_nvlink_bytes=681225464 weight_to_token_ratio=0.053513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346195105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260792320 dispatch_meta_bytes=339776 combine_payload_bytes=260792320 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=261132096 ingress_bytes=329998336 link_bytes=329998336 token_bytes=521924416 weight_movement_bytes=69206016 total_nvlink_bytes=591130432 weight_to_token_ratio=0.132598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241757171e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=251445248 dispatch_meta_bytes=337280 combine_payload_bytes=251445248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251782528 ingress_bytes=286048256 link_bytes=286048256 token_bytes=503227776 weight_movement_bytes=34603008 total_nvlink_bytes=537830784 weight_to_token_ratio=0.068762 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076376559e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=324403200 dispatch_meta_bytes=352072 combine_payload_bytes=324403200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324755272 ingress_bytes=359006208 link_bytes=359006208 token_bytes=649158472 weight_movement_bytes=34603008 total_nvlink_bytes=683761480 weight_to_token_ratio=0.053304 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350911458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=270671872 dispatch_meta_bytes=305976 combine_payload_bytes=270671872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270977848 ingress_bytes=374480896 link_bytes=374480896 token_bytes=541649720 weight_movement_bytes=103809024 total_nvlink_bytes=645458744 weight_to_token_ratio=0.191653 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409141463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88628 payload_rows_remote=79568 dup_rows=9676 dup_rows_remote=9416 dedup_ratio=0.098429 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 8192, 3336, 0, 0, 0, 0, 0, 11, 0, 455, 6234, 0, 0, 4205, 0, 0, 4255, 4475, 0, 0, 750, 0, 0, 0, 0, 0, 655, 0, 0, 6, 0, 0, 0, 0, 0, 6992, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 969, 0, 0, 0, 66, 0, 0, 0, 0, 339, 0, 0] payload_rows_per_dst=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] dup_rows_per_dst=[160, 6650, 260, 732, 1322, 0, 412, 140] combine_rows_per_src=[16384, 14028, 9060, 9718, 16264, 0, 10036, 13138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96260 payload_rows_remote=95114 dup_rows=2044 dup_rows_remote=1502 dedup_ratio=0.020793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 7972, 8192, 5239, 0, 0, 0, 0, 0, 303, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] dup_rows_per_dst=[0, 150, 0, 264, 482, 542, 606, 0] combine_rows_per_src=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96546 payload_rows_remote=84908 dup_rows=1758 dup_rows_remote=1752 dedup_ratio=0.017883 router_tokens_per_expert=[0, 0, 0, 370, 0, 0, 0, 0, 2, 0, 0, 6083, 0, 0, 0, 0, 7916, 192, 17, 0, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 3, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 5819, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5426, 0, 0, 0, 0, 0, 428, 0, 0, 0] payload_rows_per_dst=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] dup_rows_per_dst=[740, 2, 156, 6, 0, 0, 0, 854] combine_rows_per_src=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=325910528 dispatch_meta_bytes=355936 combine_payload_bytes=325910528 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326266464 ingress_bytes=464322560 link_bytes=464322560 token_bytes=652176992 weight_movement_bytes=138412032 total_nvlink_bytes=790589024 weight_to_token_ratio=0.212231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747208412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96550 payload_rows_remote=80454 dup_rows=1754 dup_rows_remote=1754 dedup_ratio=0.017843 router_tokens_per_expert=[0, 0, 0, 310, 0, 0, 0, 0, 0, 0, 0, 6096, 0, 0, 0, 0, 7903, 157, 61, 0, 0, 0, 0, 0, 0, 0, 6679, 0, 0, 8, 0, 8048, 0, 0, 0, 0, 0, 0, 0, 0, 5854, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5346, 0, 0, 0, 0, 0, 498, 0, 0, 0] payload_rows_per_dst=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] dup_rows_per_dst=[620, 0, 122, 16, 0, 0, 0, 996] combine_rows_per_src=[16384, 12192, 16120, 5680, 16096, 6028, 10692, 13358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260792320 dispatch_meta_bytes=339776 combine_payload_bytes=260792320 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=261132096 ingress_bytes=329998336 link_bytes=329998336 token_bytes=521924416 weight_movement_bytes=69206016 total_nvlink_bytes=591130432 weight_to_token_ratio=0.132598 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241757171e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96802 payload_rows_remote=80720 dup_rows=1502 dup_rows_remote=1372 dedup_ratio=0.015279 router_tokens_per_expert=[0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 6313, 0, 0, 0, 0, 7974, 125, 7, 0, 0, 0, 0, 0, 0, 0, 6717, 0, 0, 2, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 5700, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 0, 356, 0, 0, 0] payload_rows_per_dst=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] dup_rows_per_dst=[656, 0, 130, 4, 0, 0, 0, 712] combine_rows_per_src=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328832 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329868416 ingress_bytes=364142592 link_bytes=364142592 token_bytes=659408000 weight_movement_bytes=34603008 total_nvlink_bytes=694011008 weight_to_token_ratio=0.052476 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370239258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92610 payload_rows_remote=79082 dup_rows=5694 dup_rows_remote=1418 dedup_ratio=0.057922 router_tokens_per_expert=[0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 6061, 0, 0, 0, 0, 7875, 193, 47, 0, 0, 0, 0, 0, 0, 0, 6763, 0, 0, 8, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 5976, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 0, 0, 0, 457, 0, 0, 0] payload_rows_per_dst=[10772, 12122, 16104, 16, 16028, 13528, 10514, 13526] dup_rows_per_dst=[378, 0, 126, 0, 0, 4276, 0, 914] combine_rows_per_src=[10772, 12122, 16104, 16, 16028, 13528, 10514, 13526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=330629120 dispatch_meta_bytes=328368 combine_payload_bytes=330629120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=330957488 ingress_bytes=434438144 link_bytes=434438144 token_bytes=661586608 weight_movement_bytes=103809024 total_nvlink_bytes=765395632 weight_to_token_ratio=0.156909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634755761e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=61388 dup_rows=23738 dup_rows_remote=22932 dedup_ratio=0.241475 router_tokens_per_expert=[0, 2, 5, 352, 9, 0, 0, 0, 4, 0, 0, 6321, 0, 0, 3, 0, 7968, 123, 42, 0, 1, 0, 3, 0, 3, 0, 6587, 0, 0, 18, 0, 8006, 5, 0, 0, 0, 1, 5, 0, 0, 5851, 0, 0, 8184, 0, 1, 0, 0, 1, 4, 0, 0, 0, 0, 5248, 0, 0, 2, 0, 1, 402, 0, 0, 0] payload_rows_per_dst=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] dup_rows_per_dst=[4, 6, 0, 0, 10, 14774, 8138, 806] combine_rows_per_src=[732, 12650, 338, 42, 16024, 16376, 15226, 13178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83972 payload_rows_remote=71450 dup_rows=14332 dup_rows_remote=14330 dedup_ratio=0.145793 router_tokens_per_expert=[0, 0, 4, 395, 5, 0, 3, 0, 1, 1, 0, 6259, 1, 0, 0, 0, 7831, 205, 97, 0, 2, 0, 2, 0, 1, 0, 6782, 0, 0, 3, 0, 7994, 3, 0, 0, 0, 0, 0, 0, 0, 5842, 0, 0, 8186, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 5082, 0, 0, 4, 0, 0, 442, 1, 0, 3] payload_rows_per_dst=[16380, 12522, 16100, 15860, 1832, 4, 10166, 11108] dup_rows_per_dst=[806, 2, 174, 12614, 0, 0, 0, 736] combine_rows_per_src=[16380, 12522, 16100, 15860, 1832, 4, 10166, 11108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=389586944 dispatch_meta_bytes=386464 combine_payload_bytes=389586944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=389973408 ingress_bytes=562601984 link_bytes=562601984 token_bytes=779560352 weight_movement_bytes=173015040 total_nvlink_bytes=952575392 weight_to_token_ratio=0.221939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.117025973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65644 payload_rows_remote=49414 dup_rows=32660 dup_rows_remote=32594 dedup_ratio=0.332235 router_tokens_per_expert=[1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 606, 2, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 127, 2, 5, 3, 3, 0, 4, 0, 0, 450, 0, 6, 0, 7958, 8186, 5537, 0, 0, 2, 3, 0, 254, 0, 8185, 0, 0, 0, 8110, 3, 4, 0, 1, 0, 0] payload_rows_per_dst=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] dup_rows_per_dst=[0, 152, 0, 0, 14, 31918, 510, 66] combine_rows_per_src=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96546 payload_rows_remote=84908 dup_rows=1758 dup_rows_remote=1752 dedup_ratio=0.017883 router_tokens_per_expert=[0, 0, 0, 370, 0, 0, 0, 0, 2, 0, 0, 6083, 0, 0, 0, 0, 7916, 192, 17, 0, 0, 0, 0, 0, 0, 0, 6650, 0, 0, 3, 0, 8054, 0, 0, 0, 0, 0, 0, 0, 0, 5819, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5426, 0, 0, 0, 0, 0, 428, 0, 0, 0] payload_rows_per_dst=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] dup_rows_per_dst=[740, 2, 156, 6, 0, 0, 0, 854] combine_rows_per_src=[16384, 12168, 16094, 11638, 16108, 0, 10852, 13302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=322000 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324241872 ingress_bytes=427728896 link_bytes=427728896 token_bytes=648161744 weight_movement_bytes=103809024 total_nvlink_bytes=751970768 weight_to_token_ratio=0.160159 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.609509401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70322 payload_rows_remote=59512 dup_rows=27982 dup_rows_remote=27980 dedup_ratio=0.284648 router_tokens_per_expert=[0, 0, 15, 466, 3, 0, 2, 4, 0, 0, 0, 5996, 1, 11, 0, 1, 7954, 151, 23, 11, 0, 0, 7, 1, 3, 0, 6713, 0, 0, 2, 1, 8014, 8, 0, 0, 0, 0, 2, 0, 0, 5702, 0, 0, 8167, 0, 1, 0, 0, 3, 0, 1, 0, 0, 3, 5399, 0, 0, 10, 0, 0, 472, 0, 1, 4] payload_rows_per_dst=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] dup_rows_per_dst=[12, 2, 48, 0, 2, 26972, 2, 944] combine_rows_per_src=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347783168 dispatch_meta_bytes=346640 combine_payload_bytes=347783168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348129808 ingress_bytes=451592192 link_bytes=451592192 token_bytes=695912976 weight_movement_bytes=103809024 total_nvlink_bytes=799722000 weight_to_token_ratio=0.149170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699305062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72686 payload_rows_remote=56302 dup_rows=25618 dup_rows_remote=24918 dedup_ratio=0.260600 router_tokens_per_expert=[0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 5917, 0, 0, 0, 0, 7886, 200, 16, 0, 0, 0, 0, 0, 2, 0, 6586, 0, 0, 1, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 5830, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 0, 490, 0, 0, 0] payload_rows_per_dst=[16384, 11864, 15888, 16384, 0, 0, 11186, 980] dup_rows_per_dst=[700, 132, 154, 24632, 0, 0, 0, 0] combine_rows_per_src=[16384, 11864, 15888, 16384, 0, 0, 11186, 980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328832 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=329868416 ingress_bytes=364142592 link_bytes=364142592 token_bytes=659408000 weight_movement_bytes=34603008 total_nvlink_bytes=694011008 weight_to_token_ratio=0.052476 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370239258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70322 payload_rows_remote=59512 dup_rows=27982 dup_rows_remote=27980 dedup_ratio=0.284648 router_tokens_per_expert=[0, 0, 15, 466, 3, 0, 2, 4, 0, 0, 0, 5996, 1, 11, 0, 1, 7954, 151, 23, 11, 0, 0, 7, 1, 3, 0, 6713, 0, 0, 2, 1, 8014, 8, 0, 0, 0, 0, 2, 0, 0, 5702, 0, 0, 8167, 0, 1, 0, 0, 3, 0, 1, 0, 0, 3, 5399, 0, 0, 10, 0, 0, 472, 0, 1, 4] payload_rows_per_dst=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] dup_rows_per_dst=[12, 2, 48, 0, 2, 26972, 2, 944] combine_rows_per_src=[968, 12016, 668, 12, 16046, 16346, 10810, 13456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=251445248 dispatch_meta_bytes=337280 combine_payload_bytes=251445248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251782528 ingress_bytes=286048256 link_bytes=286048256 token_bytes=503227776 weight_movement_bytes=34603008 total_nvlink_bytes=537830784 weight_to_token_ratio=0.068762 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.076376559e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=79772 dup_rows=2148 dup_rows_remote=1506 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 7984, 8192, 5382, 0, 0, 0, 0, 0, 298, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] dup_rows_per_dst=[0, 144, 658, 238, 642, 0, 466, 0] combine_rows_per_src=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292659200 dispatch_meta_bytes=343120 combine_payload_bytes=292659200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293002320 ingress_bytes=327262208 link_bytes=327262208 token_bytes=585661520 weight_movement_bytes=34603008 total_nvlink_bytes=620264528 weight_to_token_ratio=0.059084 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231461342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347783168 dispatch_meta_bytes=346640 combine_payload_bytes=347783168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348129808 ingress_bytes=451592192 link_bytes=451592192 token_bytes=695912976 weight_movement_bytes=103809024 total_nvlink_bytes=799722000 weight_to_token_ratio=0.149170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699305062e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=202399744 dispatch_meta_bytes=328032 combine_payload_bytes=202399744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=202727776 ingress_bytes=271605760 link_bytes=271605760 token_bytes=405127520 weight_movement_bytes=69206016 total_nvlink_bytes=474333536 weight_to_token_ratio=0.170825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.022030609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=243761152 dispatch_meta_bytes=349968 combine_payload_bytes=243761152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244111120 ingress_bytes=278364160 link_bytes=278364160 token_bytes=487872272 weight_movement_bytes=34603008 total_nvlink_bytes=522475280 weight_to_token_ratio=0.070926 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.047461924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=230612992 dispatch_meta_bytes=324880 combine_payload_bytes=230612992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=230937872 ingress_bytes=265216000 link_bytes=265216000 token_bytes=461550864 weight_movement_bytes=34603008 total_nvlink_bytes=496153872 weight_to_token_ratio=0.074971 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.979864564e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96802 payload_rows_remote=80720 dup_rows=1502 dup_rows_remote=1372 dedup_ratio=0.015279 router_tokens_per_expert=[0, 0, 0, 328, 0, 0, 0, 0, 0, 0, 0, 6313, 0, 0, 0, 0, 7974, 125, 7, 0, 0, 0, 0, 0, 0, 0, 6717, 0, 0, 2, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 5700, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5359, 0, 0, 0, 0, 0, 356, 0, 0, 0] payload_rows_per_dst=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] dup_rows_per_dst=[656, 0, 130, 4, 0, 0, 0, 712] combine_rows_per_src=[16384, 12626, 16082, 11400, 16158, 0, 10718, 13434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97042 payload_rows_remote=95864 dup_rows=1262 dup_rows_remote=1232 dedup_ratio=0.012838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 0, 0, 0, 2773, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6296, 8192, 0, 0] payload_rows_per_dst=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] dup_rows_per_dst=[0, 0, 0, 0, 1198, 30, 0, 34] combine_rows_per_src=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93420 payload_rows_remote=77288 dup_rows=4884 dup_rows_remote=4586 dedup_ratio=0.049683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 7986, 8192, 4970, 0, 0, 0, 0, 0, 275, 0, 8192, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11368, 16384, 16132, 16382, 0, 550, 16220] dup_rows_per_dst=[0, 162, 3704, 298, 720, 0, 0, 0] combine_rows_per_src=[16384, 11368, 16384, 16132, 16382, 0, 550, 16220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=330629120 dispatch_meta_bytes=328368 combine_payload_bytes=330629120 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=330957488 ingress_bytes=434438144 link_bytes=434438144 token_bytes=661586608 weight_movement_bytes=103809024 total_nvlink_bytes=765395632 weight_to_token_ratio=0.156909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634755761e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96156 payload_rows_remote=79772 dup_rows=2148 dup_rows_remote=1506 dedup_ratio=0.021851 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 1715, 0, 0, 0, 0, 0, 0, 0, 0, 8048, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 529, 0, 0, 0, 7984, 8192, 5382, 0, 0, 0, 0, 0, 298, 0, 8192, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] dup_rows_per_dst=[0, 144, 658, 238, 642, 0, 466, 0] combine_rows_per_src=[16384, 11756, 6040, 16106, 16384, 0, 13246, 16240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93598 payload_rows_remote=77214 dup_rows=4706 dup_rows_remote=1038 dedup_ratio=0.047872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 8017, 8191, 5200, 0, 0, 0, 0, 0, 277, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] dup_rows_per_dst=[0, 144, 3668, 264, 630, 0, 0, 0] combine_rows_per_src=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=326746112 dispatch_meta_bytes=325112 combine_payload_bytes=326746112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327071224 ingress_bytes=361349120 link_bytes=361349120 token_bytes=653817336 weight_movement_bytes=34603008 total_nvlink_bytes=688420344 weight_to_token_ratio=0.052925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359727648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96260 payload_rows_remote=95114 dup_rows=2044 dup_rows_remote=1502 dedup_ratio=0.020793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 525, 0, 0, 0, 0, 0, 1952, 0, 0, 0, 0, 0, 0, 0, 0, 8041, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 475, 0, 0, 0, 7972, 8192, 5239, 0, 0, 0, 0, 0, 303, 0, 8192, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] dup_rows_per_dst=[0, 150, 0, 264, 482, 542, 606, 0] combine_rows_per_src=[16370, 11378, 3904, 15574, 15272, 1146, 16384, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316268544 dispatch_meta_bytes=313008 combine_payload_bytes=316268544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316581552 ingress_bytes=350871552 link_bytes=350871552 token_bytes=632850096 weight_movement_bytes=34603008 total_nvlink_bytes=667453104 weight_to_token_ratio=0.054678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65644 payload_rows_remote=49414 dup_rows=32660 dup_rows_remote=32594 dedup_ratio=0.332235 router_tokens_per_expert=[1, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 606, 2, 0, 0, 0, 0, 1667, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 127, 2, 5, 3, 3, 0, 4, 0, 0, 450, 0, 6, 0, 7958, 8186, 5537, 0, 0, 2, 3, 0, 254, 0, 8185, 0, 0, 0, 8110, 3, 4, 0, 1, 0, 0] payload_rows_per_dst=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] dup_rows_per_dst=[0, 152, 0, 0, 14, 31918, 510, 66] combine_rows_per_src=[6, 12138, 3334, 258, 916, 16384, 16378, 16230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93094 payload_rows_remote=81460 dup_rows=5210 dup_rows_remote=5036 dedup_ratio=0.052999 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 603, 0, 0, 0, 0, 2, 1924, 0, 0, 0, 1, 0, 3, 3, 0, 8013, 0, 0, 0, 166, 0, 4, 6, 2, 0, 2, 0, 0, 380, 0, 0, 0, 7996, 8184, 5299, 0, 0, 2, 0, 0, 272, 0, 8185, 0, 0, 0, 8096, 1, 4, 0, 0, 0, 0] payload_rows_per_dst=[16372, 11634, 16384, 16094, 16384, 0, 15338, 888] dup_rows_per_dst=[0, 174, 3846, 270, 396, 0, 522, 2] combine_rows_per_src=[16372, 11634, 16384, 16094, 16384, 0, 15338, 888] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=392658944 dispatch_meta_bytes=388384 combine_payload_bytes=392658944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=393047328 ingress_bytes=496467968 link_bytes=496467968 token_bytes=785706272 weight_movement_bytes=103809024 total_nvlink_bytes=889515296 weight_to_token_ratio=0.132122 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.868168995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50760 payload_rows_remote=34384 dup_rows=47544 dup_rows_remote=33058 dedup_ratio=0.483643 router_tokens_per_expert=[1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 7939, 0, 6, 0, 12, 0, 0, 0, 6, 0, 0, 3748, 1, 1, 1, 0, 2572, 0, 2, 0, 0, 8185, 8170, 0, 5, 5, 0, 4247, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 6055, 8186, 0, 0] payload_rows_per_dst=[10, 4, 14, 34, 9444, 16382, 8496, 16376] dup_rows_per_dst=[0, 0, 0, 2, 830, 32226, 0, 14486] combine_rows_per_src=[10, 4, 14, 34, 9444, 16382, 8496, 16376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93420 payload_rows_remote=77288 dup_rows=4884 dup_rows_remote=4586 dedup_ratio=0.049683 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 795, 0, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 7986, 8192, 4970, 0, 0, 0, 0, 0, 275, 0, 8192, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11368, 16384, 16132, 16382, 0, 550, 16220] dup_rows_per_dst=[0, 162, 3704, 298, 720, 0, 0, 0] combine_rows_per_src=[16384, 11368, 16384, 16132, 16382, 0, 550, 16220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=389586944 dispatch_meta_bytes=386464 combine_payload_bytes=389586944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=389973408 ingress_bytes=562601984 link_bytes=562601984 token_bytes=779560352 weight_movement_bytes=173015040 total_nvlink_bytes=952575392 weight_to_token_ratio=0.221939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.117025973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65518 payload_rows_remote=49166 dup_rows=32786 dup_rows_remote=32172 dedup_ratio=0.333516 router_tokens_per_expert=[3, 2, 0, 0, 2, 2, 1, 1, 0, 0, 1, 738, 3, 1, 0, 0, 7, 2018, 0, 0, 0, 0, 0, 9, 7, 0, 8004, 1, 0, 2, 183, 0, 13, 10, 2, 0, 8, 1, 0, 491, 0, 7, 1, 7967, 8177, 4911, 0, 1, 2, 0, 2, 309, 0, 8170, 0, 0, 0, 8077, 4, 12, 0, 1, 0, 1] payload_rows_per_dst=[14, 11132, 4062, 384, 1026, 16370, 16352, 16178] dup_rows_per_dst=[8, 176, 6, 2, 24, 31944, 614, 12] combine_rows_per_src=[14, 11132, 4062, 384, 1026, 16370, 16352, 16178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316571648 dispatch_meta_bytes=327496 combine_payload_bytes=316571648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316899144 ingress_bytes=385777664 link_bytes=385777664 token_bytes=633470792 weight_movement_bytes=69206016 total_nvlink_bytes=702676808 weight_to_token_ratio=0.109249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451650292e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90330 payload_rows_remote=73946 dup_rows=7974 dup_rows_remote=7974 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 698, 0, 0, 0, 0, 0, 1773, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 123, 0, 0, 0, 0, 0, 0, 0, 0, 492, 0, 0, 0, 7998, 8192, 5256, 0, 0, 0, 0, 0, 278, 0, 8192, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 13052, 16384, 16092, 12190, 0, 16228, 0] dup_rows_per_dst=[0, 3188, 3546, 226, 458, 0, 556, 0] combine_rows_per_src=[16384, 13052, 16384, 16092, 12190, 0, 16228, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=326746112 dispatch_meta_bytes=325112 combine_payload_bytes=326746112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327071224 ingress_bytes=361349120 link_bytes=361349120 token_bytes=653817336 weight_movement_bytes=34603008 total_nvlink_bytes=688420344 weight_to_token_ratio=0.052925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359727648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=202399744 dispatch_meta_bytes=328032 combine_payload_bytes=202399744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=202727776 ingress_bytes=271605760 link_bytes=271605760 token_bytes=405127520 weight_movement_bytes=69206016 total_nvlink_bytes=474333536 weight_to_token_ratio=0.170825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.022030609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97038 payload_rows_remote=85122 dup_rows=1266 dup_rows_remote=0 dedup_ratio=0.012878 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 2521, 0, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5990, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] dup_rows_per_dst=[0, 0, 0, 0, 1266, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=333660160 dispatch_meta_bytes=345984 combine_payload_bytes=333660160 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=334006144 ingress_bytes=402866176 link_bytes=402866176 token_bytes=667666304 weight_movement_bytes=69206016 total_nvlink_bytes=736872320 weight_to_token_ratio=0.103654 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515952987e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316571648 dispatch_meta_bytes=327496 combine_payload_bytes=316571648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316899144 ingress_bytes=385777664 link_bytes=385777664 token_bytes=633470792 weight_movement_bytes=69206016 total_nvlink_bytes=702676808 weight_to_token_ratio=0.109249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.451650292e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=140836864 dispatch_meta_bytes=269768 combine_payload_bytes=140836864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=141106632 ingress_bytes=244645888 link_bytes=244645888 token_bytes=281943496 weight_movement_bytes=103809024 total_nvlink_bytes=385752520 weight_to_token_ratio=0.368191 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.205827809e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93410 payload_rows_remote=77560 dup_rows=4894 dup_rows_remote=4894 dedup_ratio=0.049784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 2795, 0, 0, 0, 0, 8192, 8185, 0, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6321, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] dup_rows_per_dst=[0, 0, 0, 0, 1204, 0, 3690, 0] combine_rows_per_src=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=201383936 dispatch_meta_bytes=325352 combine_payload_bytes=201383936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201709288 ingress_bytes=270589952 link_bytes=270589952 token_bytes=403093224 weight_movement_bytes=69206016 total_nvlink_bytes=472299240 weight_to_token_ratio=0.171687 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.018208205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=302882816 dispatch_meta_bytes=327680 combine_payload_bytes=302882816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303210496 ingress_bytes=337485824 link_bytes=337485824 token_bytes=606093312 weight_movement_bytes=34603008 total_nvlink_bytes=640696320 weight_to_token_ratio=0.057092 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269931986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93598 payload_rows_remote=77214 dup_rows=4706 dup_rows_remote=1038 dedup_ratio=0.047872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 490, 0, 0, 0, 8017, 8191, 5200, 0, 0, 0, 0, 0, 277, 0, 8192, 0, 0, 0, 8118, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] dup_rows_per_dst=[0, 144, 3668, 264, 630, 0, 0, 0] combine_rows_per_src=[16382, 11556, 16384, 16102, 16384, 0, 554, 16236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94706 payload_rows_remote=78322 dup_rows=3598 dup_rows_remote=1436 dedup_ratio=0.036601 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7393, 7848, 0, 2705, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 4184, 0, 0, 0, 0, 0, 0, 1071, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 340, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] dup_rows_per_dst=[0, 0, 0, 2, 662, 2162, 0, 772] combine_rows_per_src=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=316268544 dispatch_meta_bytes=313008 combine_payload_bytes=316268544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316581552 ingress_bytes=350871552 link_bytes=350871552 token_bytes=632850096 weight_movement_bytes=34603008 total_nvlink_bytes=667453104 weight_to_token_ratio=0.054678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320301403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97138 payload_rows_remote=80756 dup_rows=1166 dup_rows_remote=1166 dedup_ratio=0.011861 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 2565, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15822, 16382, 12106, 0, 7736, 12324] dup_rows_per_dst=[0, 0, 0, 0, 1166, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15822, 16382, 12106, 0, 7736, 12324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320806912 dispatch_meta_bytes=319032 combine_payload_bytes=320806912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321125944 ingress_bytes=355409920 link_bytes=355409920 token_bytes=641932856 weight_movement_bytes=34603008 total_nvlink_bytes=676535864 weight_to_token_ratio=0.053904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337378916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97038 payload_rows_remote=85122 dup_rows=1266 dup_rows_remote=0 dedup_ratio=0.012878 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4070, 0, 0, 0, 0, 2521, 0, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5990, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] dup_rows_per_dst=[0, 0, 0, 0, 1266, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15880, 16376, 11916, 0, 8118, 11980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=348659712 dispatch_meta_bytes=340488 combine_payload_bytes=348659712 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=349000200 ingress_bytes=417865728 link_bytes=417865728 token_bytes=697659912 weight_movement_bytes=69206016 total_nvlink_bytes=766865928 weight_to_token_ratio=0.099197 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.572395094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97042 payload_rows_remote=95864 dup_rows=1262 dup_rows_remote=1232 dedup_ratio=0.012838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7945, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 0, 0, 0, 0, 2773, 0, 0, 0, 0, 8192, 8178, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6296, 8192, 0, 0] payload_rows_per_dst=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] dup_rows_per_dst=[0, 0, 0, 0, 1198, 30, 0, 34] combine_rows_per_src=[16368, 16340, 14712, 16342, 11604, 1178, 7896, 12602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317685760 dispatch_meta_bytes=329816 combine_payload_bytes=317685760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318015576 ingress_bytes=386891776 link_bytes=386891776 token_bytes=635701336 weight_movement_bytes=69206016 total_nvlink_bytes=704907352 weight_to_token_ratio=0.108866 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455842606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=50760 payload_rows_remote=34384 dup_rows=47544 dup_rows_remote=33058 dedup_ratio=0.483643 router_tokens_per_expert=[1, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 7939, 0, 6, 0, 12, 0, 0, 0, 6, 0, 0, 3748, 1, 1, 1, 0, 2572, 0, 2, 0, 0, 8185, 8170, 0, 5, 5, 0, 4247, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 6055, 8186, 0, 0] payload_rows_per_dst=[10, 4, 14, 34, 9444, 16382, 8496, 16376] dup_rows_per_dst=[0, 0, 0, 2, 830, 32226, 0, 14486] combine_rows_per_src=[10, 4, 14, 34, 9444, 16382, 8496, 16376] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92376 payload_rows_remote=75992 dup_rows=5928 dup_rows_remote=5926 dedup_ratio=0.060303 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 2, 7927, 0, 2, 0, 5, 0, 0, 0, 4, 0, 0, 3838, 0, 0, 0, 0, 2670, 0, 0, 0, 0, 8187, 8183, 0, 4, 1, 0, 3828, 2, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 6300, 8188, 0, 0] payload_rows_per_dst=[16376, 16384, 15860, 16376, 11750, 8, 15622, 0] dup_rows_per_dst=[0, 2, 2, 8, 1266, 2, 4648, 0] combine_rows_per_src=[16376, 16384, 15860, 16376, 11750, 8, 15622, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87514 payload_rows_remote=71568 dup_rows=10790 dup_rows_remote=10186 dedup_ratio=0.109762 router_tokens_per_expert=[8186, 0, 5, 0, 6, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 7525, 7909, 1, 3049, 0, 8169, 1, 4, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 1, 4, 0, 932, 0, 3748, 2, 0, 0, 0, 6, 4, 1010, 0, 30, 0, 0, 1, 0, 0, 1, 0, 0, 0, 74, 282, 0, 2, 6, 0, 2] payload_rows_per_dst=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] dup_rows_per_dst=[10, 0, 0, 4, 8084, 2088, 0, 604] combine_rows_per_src=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97138 payload_rows_remote=80756 dup_rows=1166 dup_rows_remote=1166 dedup_ratio=0.011861 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 2565, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 3868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6162, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15822, 16382, 12106, 0, 7736, 12324] dup_rows_per_dst=[0, 0, 0, 0, 1166, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15822, 16382, 12106, 0, 7736, 12324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=392658944 dispatch_meta_bytes=388384 combine_payload_bytes=392658944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=393047328 ingress_bytes=496467968 link_bytes=496467968 token_bytes=785706272 weight_movement_bytes=103809024 total_nvlink_bytes=889515296 weight_to_token_ratio=0.132122 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.868168995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=140836864 dispatch_meta_bytes=269768 combine_payload_bytes=140836864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=141106632 ingress_bytes=244645888 link_bytes=244645888 token_bytes=281943496 weight_movement_bytes=103809024 total_nvlink_bytes=385752520 weight_to_token_ratio=0.368191 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.205827809e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=52738 payload_rows_remote=45042 dup_rows=45566 dup_rows_remote=45520 dedup_ratio=0.463521 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 5, 15, 0, 0, 0, 2, 0, 9, 7874, 1, 0, 0, 8, 0, 0, 0, 16, 0, 0, 3996, 1, 0, 0, 0, 2762, 1, 0, 0, 0, 8170, 8168, 3, 16, 8, 0, 3835, 25, 0, 0, 0, 0, 10, 1, 2, 0, 0, 0, 6038, 8178, 0, 1] payload_rows_per_dst=[14, 30, 24, 48, 12184, 16382, 7696, 16360] dup_rows_per_dst=[0, 10, 0, 0, 1336, 32096, 46, 12078] combine_rows_per_src=[14, 30, 24, 48, 12184, 16382, 7696, 16360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330776576 dispatch_meta_bytes=327688 combine_payload_bytes=330776576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331104264 ingress_bytes=365379584 link_bytes=365379584 token_bytes=661880840 weight_movement_bytes=34603008 total_nvlink_bytes=696483848 weight_to_token_ratio=0.052280 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374893959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89186 payload_rows_remote=72802 dup_rows=9118 dup_rows_remote=9118 dedup_ratio=0.092753 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4183, 0, 0, 0, 0, 2491, 0, 0, 0, 0, 8192, 8183, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5907, 8191, 0, 0] payload_rows_per_dst=[16384, 16382, 16094, 16366, 8360, 0, 15600, 0] dup_rows_per_dst=[0, 0, 4046, 0, 694, 0, 4378, 0] combine_rows_per_src=[16384, 16382, 16094, 16366, 8360, 0, 15600, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=348659712 dispatch_meta_bytes=340488 combine_payload_bytes=348659712 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=349000200 ingress_bytes=417865728 link_bytes=417865728 token_bytes=697659912 weight_movement_bytes=69206016 total_nvlink_bytes=766865928 weight_to_token_ratio=0.099197 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.572395094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95004 payload_rows_remote=85542 dup_rows=3300 dup_rows_remote=2644 dedup_ratio=0.033569 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7332, 7910, 0, 2925, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4221, 0, 0, 0, 0, 0, 0, 919, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 350, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] dup_rows_per_dst=[0, 0, 0, 0, 656, 1866, 0, 778] combine_rows_per_src=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=311263232 dispatch_meta_bytes=327672 combine_payload_bytes=311263232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311590904 ingress_bytes=345866240 link_bytes=345866240 token_bytes=622854136 weight_movement_bytes=34603008 total_nvlink_bytes=657457144 weight_to_token_ratio=0.055556 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301466817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=330776576 dispatch_meta_bytes=327688 combine_payload_bytes=330776576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331104264 ingress_bytes=365379584 link_bytes=365379584 token_bytes=661880840 weight_movement_bytes=34603008 total_nvlink_bytes=696483848 weight_to_token_ratio=0.052280 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.374893959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293142528 dispatch_meta_bytes=327016 combine_payload_bytes=293142528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293469544 ingress_bytes=327745536 link_bytes=327745536 token_bytes=586612072 weight_movement_bytes=34603008 total_nvlink_bytes=621215080 weight_to_token_ratio=0.058988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233280066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=184492032 dispatch_meta_bytes=362248 combine_payload_bytes=184492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184854280 ingress_bytes=219095040 link_bytes=219095040 token_bytes=369346312 weight_movement_bytes=34603008 total_nvlink_bytes=403949320 weight_to_token_ratio=0.093687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.244369969e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92046 payload_rows_remote=88078 dup_rows=6258 dup_rows_remote=6258 dedup_ratio=0.063660 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7320, 7957, 0, 2516, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 880, 0, 4628, 0, 0, 0, 0, 0, 0, 886, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 313, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] dup_rows_per_dst=[5030, 0, 0, 2, 678, 4, 0, 544] combine_rows_per_src=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=298196992 dispatch_meta_bytes=327680 combine_payload_bytes=298196992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298524672 ingress_bytes=332800000 link_bytes=332800000 token_bytes=596721664 weight_movement_bytes=34603008 total_nvlink_bytes=631324672 weight_to_token_ratio=0.057989 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252299607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93410 payload_rows_remote=77560 dup_rows=4894 dup_rows_remote=4894 dedup_ratio=0.049784 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3797, 0, 0, 0, 0, 2795, 0, 0, 0, 0, 8192, 8185, 0, 0, 0, 0, 3745, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6321, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] dup_rows_per_dst=[0, 0, 0, 0, 1204, 0, 3690, 0] combine_rows_per_src=[16384, 16384, 15850, 16370, 11980, 0, 14378, 2064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87126 payload_rows_remote=84764 dup_rows=11178 dup_rows_remote=11178 dedup_ratio=0.113708 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 8192, 0, 0, 76, 0, 0] payload_rows_per_dst=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] dup_rows_per_dst=[0, 0, 0, 42, 0, 0, 260, 10876] combine_rows_per_src=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=184492032 dispatch_meta_bytes=362248 combine_payload_bytes=184492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184854280 ingress_bytes=219095040 link_bytes=219095040 token_bytes=369346312 weight_movement_bytes=34603008 total_nvlink_bytes=403949320 weight_to_token_ratio=0.093687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.244369969e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=350380032 dispatch_meta_bytes=352744 combine_payload_bytes=350380032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=350732776 ingress_bytes=384983040 link_bytes=384983040 token_bytes=701112808 weight_movement_bytes=34603008 total_nvlink_bytes=735715816 weight_to_token_ratio=0.049354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94706 payload_rows_remote=78322 dup_rows=3598 dup_rows_remote=1436 dedup_ratio=0.036601 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7393, 7848, 0, 2705, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 895, 0, 4184, 0, 0, 0, 0, 0, 0, 1071, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 129, 340, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] dup_rows_per_dst=[0, 0, 0, 2, 662, 2162, 0, 772] combine_rows_per_src=[0, 14786, 5410, 16384, 9496, 16384, 16384, 15862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=360767488 dispatch_meta_bytes=377344 combine_payload_bytes=360767488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=361144832 ingress_bytes=499179520 link_bytes=499179520 token_bytes=721912320 weight_movement_bytes=138412032 total_nvlink_bytes=860324352 weight_to_token_ratio=0.191730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.878372347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87514 payload_rows_remote=71568 dup_rows=10790 dup_rows_remote=10186 dedup_ratio=0.109762 router_tokens_per_expert=[8186, 0, 5, 0, 6, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 7525, 7909, 1, 3049, 0, 8169, 1, 4, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 1, 4, 0, 932, 0, 3748, 2, 0, 0, 0, 6, 4, 1010, 0, 30, 0, 0, 1, 0, 0, 1, 0, 0, 0, 74, 282, 0, 2, 6, 0, 2] payload_rows_per_dst=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] dup_rows_per_dst=[10, 0, 0, 4, 8084, 2088, 0, 604] combine_rows_per_src=[12, 0, 6110, 16380, 16340, 16384, 16342, 15946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94404 payload_rows_remote=78020 dup_rows=3900 dup_rows_remote=3900 dedup_ratio=0.039673 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 7404, 7900, 0, 2444, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 857, 0, 4297, 0, 0, 0, 0, 0, 0, 1094, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 429, 0, 0, 0, 0, 0] payload_rows_per_dst=[908, 14808, 3980, 16384, 9652, 16384, 16384, 15904] dup_rows_per_dst=[0, 0, 0, 0, 656, 2264, 0, 980] combine_rows_per_src=[908, 14808, 3980, 16384, 9652, 16384, 16384, 15904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317685760 dispatch_meta_bytes=329816 combine_payload_bytes=317685760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318015576 ingress_bytes=386891776 link_bytes=386891776 token_bytes=635701336 weight_movement_bytes=69206016 total_nvlink_bytes=704907352 weight_to_token_ratio=0.108866 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455842606e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347193344 dispatch_meta_bytes=383768 combine_payload_bytes=347193344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=347577112 ingress_bytes=381796352 link_bytes=381796352 token_bytes=694770456 weight_movement_bytes=34603008 total_nvlink_bytes=729373464 weight_to_token_ratio=0.049805 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436668935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95004 payload_rows_remote=85542 dup_rows=3300 dup_rows_remote=2644 dedup_ratio=0.033569 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7332, 7910, 0, 2925, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 838, 0, 4221, 0, 0, 0, 0, 0, 0, 919, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 350, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] dup_rows_per_dst=[0, 0, 0, 0, 656, 1866, 0, 778] combine_rows_per_src=[0, 14664, 5850, 16384, 9462, 16384, 16384, 15876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59706 payload_rows_remote=45144 dup_rows=38598 dup_rows_remote=38598 dedup_ratio=0.392639 router_tokens_per_expert=[8187, 0, 3, 0, 4, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1, 7280, 7792, 1, 3036, 0, 8170, 0, 3, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 2, 0, 0, 841, 0, 4100, 0, 1, 2, 0, 3, 4, 887, 0, 17, 0, 0, 1, 0, 2, 3, 0, 0, 2, 105, 510, 0, 1, 4, 0, 0] payload_rows_per_dst=[16384, 14562, 16378, 6, 9208, 1820, 106, 1242] dup_rows_per_dst=[22450, 0, 15460, 0, 678, 8, 0, 2] combine_rows_per_src=[16384, 14562, 16378, 6, 9208, 1820, 106, 1242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65284 payload_rows_remote=48900 dup_rows=33020 dup_rows_remote=16502 dedup_ratio=0.335897 router_tokens_per_expert=[0, 0, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 5, 0, 6917, 2, 0, 0, 8183, 0, 0, 3, 3, 0, 8011, 4, 0, 0, 0, 8184, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 2, 1290, 0, 0, 6, 0, 0, 0, 0, 0, 75, 2, 0, 8186, 0, 6, 71, 0, 0] payload_rows_per_dst=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] dup_rows_per_dst=[0, 0, 0, 16364, 0, 0, 138, 16518] combine_rows_per_src=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92670 payload_rows_remote=76348 dup_rows=5634 dup_rows_remote=5634 dedup_ratio=0.057312 router_tokens_per_expert=[8174, 0, 14, 0, 15, 0, 0, 8177, 0, 0, 0, 0, 0, 7, 0, 7348, 7850, 4, 2964, 0, 8145, 0, 0, 0, 0, 0, 2, 0, 0, 12, 0, 0, 0, 4, 0, 0, 919, 0, 3929, 0, 2, 1, 0, 9, 34, 1003, 0, 41, 0, 0, 0, 2, 0, 14, 0, 0, 7, 75, 381, 0, 1, 14, 0, 4] payload_rows_per_dst=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] dup_rows_per_dst=[22, 8, 1124, 4, 1540, 2158, 0, 778] combine_rows_per_src=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=319569920 dispatch_meta_bytes=327680 combine_payload_bytes=319569920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319897600 ingress_bytes=354172928 link_bytes=354172928 token_bytes=639467520 weight_movement_bytes=34603008 total_nvlink_bytes=674070528 weight_to_token_ratio=0.054112 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332724215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=320806912 dispatch_meta_bytes=319032 combine_payload_bytes=320806912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321125944 ingress_bytes=355409920 link_bytes=355409920 token_bytes=641932856 weight_movement_bytes=34603008 total_nvlink_bytes=676535864 weight_to_token_ratio=0.053904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.337378916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61094 payload_rows_remote=44710 dup_rows=37210 dup_rows_remote=16486 dedup_ratio=0.378520 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 7389, 7906, 0, 2170, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 951, 0, 4580, 0, 0, 0, 0, 0, 0, 1065, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 119, 378, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 14778, 16384, 0, 10388, 2166, 0, 994] dup_rows_per_dst=[20724, 0, 15812, 0, 674, 0, 0, 0] combine_rows_per_src=[16384, 14778, 16384, 0, 10388, 2166, 0, 994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=350380032 dispatch_meta_bytes=352744 combine_payload_bytes=350380032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=350732776 ingress_bytes=384983040 link_bytes=384983040 token_bytes=701112808 weight_movement_bytes=34603008 total_nvlink_bytes=735715816 weight_to_token_ratio=0.049354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=184909824 dispatch_meta_bytes=334968 combine_payload_bytes=184909824 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=185244792 ingress_bytes=254115840 link_bytes=254115840 token_bytes=370154616 weight_movement_bytes=69206016 total_nvlink_bytes=439360632 weight_to_token_ratio=0.186965 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.562174479e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293142528 dispatch_meta_bytes=327016 combine_payload_bytes=293142528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293469544 ingress_bytes=327745536 link_bytes=327745536 token_bytes=586612072 weight_movement_bytes=34603008 total_nvlink_bytes=621215080 weight_to_token_ratio=0.058988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233280066e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92670 payload_rows_remote=76348 dup_rows=5634 dup_rows_remote=5634 dedup_ratio=0.057312 router_tokens_per_expert=[8174, 0, 14, 0, 15, 0, 0, 8177, 0, 0, 0, 0, 0, 7, 0, 7348, 7850, 4, 2964, 0, 8145, 0, 0, 0, 0, 0, 2, 0, 0, 12, 0, 0, 0, 4, 0, 0, 919, 0, 3929, 0, 2, 1, 0, 9, 34, 1003, 0, 41, 0, 0, 0, 2, 0, 14, 0, 0, 7, 75, 381, 0, 1, 14, 0, 4] payload_rows_per_dst=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] dup_rows_per_dst=[22, 8, 1124, 4, 1540, 2158, 0, 778] combine_rows_per_src=[36, 10058, 7828, 16372, 9792, 16376, 16322, 15886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=319569920 dispatch_meta_bytes=327680 combine_payload_bytes=319569920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319897600 ingress_bytes=354172928 link_bytes=354172928 token_bytes=639467520 weight_movement_bytes=34603008 total_nvlink_bytes=674070528 weight_to_token_ratio=0.054112 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332724215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96030 payload_rows_remote=79646 dup_rows=2274 dup_rows_remote=2274 dedup_ratio=0.023132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 8192, 0, 0, 72, 0, 0] payload_rows_per_dst=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] dup_rows_per_dst=[0, 0, 0, 0, 0, 2042, 192, 40] combine_rows_per_src=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=200294400 dispatch_meta_bytes=261608 combine_payload_bytes=200294400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200556008 ingress_bytes=269500416 link_bytes=269500416 token_bytes=400850408 weight_movement_bytes=69206016 total_nvlink_bytes=470056424 weight_to_token_ratio=0.172648 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.014108369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=312721408 dispatch_meta_bytes=327928 combine_payload_bytes=312721408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313049336 ingress_bytes=347324416 link_bytes=347324416 token_bytes=625770744 weight_movement_bytes=34603008 total_nvlink_bytes=660373752 weight_to_token_ratio=0.055297 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306953815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=184909824 dispatch_meta_bytes=334968 combine_payload_bytes=184909824 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=185244792 ingress_bytes=254115840 link_bytes=254115840 token_bytes=370154616 weight_movement_bytes=69206016 total_nvlink_bytes=439360632 weight_to_token_ratio=0.186965 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.562174479e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95904 payload_rows_remote=79520 dup_rows=2400 dup_rows_remote=2400 dedup_ratio=0.024414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 8192, 0, 0, 70, 0, 0] payload_rows_per_dst=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] dup_rows_per_dst=[0, 0, 0, 0, 0, 2280, 120, 0] combine_rows_per_src=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=183132160 dispatch_meta_bytes=244784 combine_payload_bytes=183132160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=183376944 ingress_bytes=286941184 link_bytes=286941184 token_bytes=366509104 weight_movement_bytes=103809024 total_nvlink_bytes=470318128 weight_to_token_ratio=0.283237 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.079736575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92046 payload_rows_remote=88078 dup_rows=6258 dup_rows_remote=6258 dedup_ratio=0.063660 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7320, 7957, 0, 2516, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 880, 0, 4628, 0, 0, 0, 0, 0, 0, 886, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 313, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] dup_rows_per_dst=[5030, 0, 0, 2, 678, 4, 0, 544] combine_rows_per_src=[16344, 14640, 3968, 16382, 10338, 1856, 16382, 12136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=81300 dup_rows=1910 dup_rows_remote=196 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8105, 1206, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 1037, 0, 7492, 0, 6210, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 6351, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 607, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] dup_rows_per_dst=[0, 0, 14, 182, 0, 1714, 0, 0] combine_rows_per_src=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326230016 dispatch_meta_bytes=327680 combine_payload_bytes=326230016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326557696 ingress_bytes=395436032 link_bytes=395436032 token_bytes=652787712 weight_movement_bytes=69206016 total_nvlink_bytes=721993728 weight_to_token_ratio=0.106016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487993953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=312721408 dispatch_meta_bytes=327928 combine_payload_bytes=312721408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313049336 ingress_bytes=347324416 link_bytes=347324416 token_bytes=625770744 weight_movement_bytes=34603008 total_nvlink_bytes=660373752 weight_to_token_ratio=0.055297 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306953815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87126 payload_rows_remote=84764 dup_rows=11178 dup_rows_remote=11178 dedup_ratio=0.113708 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7032, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7965, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 8192, 0, 0, 76, 0, 0] payload_rows_per_dst=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] dup_rows_per_dst=[0, 0, 0, 42, 0, 0, 260, 10876] combine_rows_per_src=[16342, 14064, 14572, 1812, 5660, 2362, 15930, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325713920 dispatch_meta_bytes=327680 combine_payload_bytes=325713920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326041600 ingress_bytes=394919936 link_bytes=394919936 token_bytes=651755520 weight_movement_bytes=69206016 total_nvlink_bytes=720961536 weight_to_token_ratio=0.106184 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486051926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95458 payload_rows_remote=95458 dup_rows=2846 dup_rows_remote=2846 dedup_ratio=0.028951 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6918, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 7962, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1308, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 0, 8192, 0, 0, 81, 0, 0] payload_rows_per_dst=[16384, 13836, 16384, 0, 16384, 16384, 15924, 162] dup_rows_per_dst=[0, 0, 2, 0, 0, 2616, 228, 0] combine_rows_per_src=[16384, 13836, 16384, 0, 16384, 16384, 15924, 162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333004800 dispatch_meta_bytes=325984 combine_payload_bytes=333004800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333330784 ingress_bytes=367607808 link_bytes=367607808 token_bytes=666335584 weight_movement_bytes=34603008 total_nvlink_bytes=700938592 weight_to_token_ratio=0.051930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383278587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=360767488 dispatch_meta_bytes=377344 combine_payload_bytes=360767488 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=361144832 ingress_bytes=499179520 link_bytes=499179520 token_bytes=721912320 weight_movement_bytes=138412032 total_nvlink_bytes=860324352 weight_to_token_ratio=0.191730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.878372347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96030 payload_rows_remote=79646 dup_rows=2274 dup_rows_remote=2274 dedup_ratio=0.023132 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6768, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1438, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 8192, 0, 0, 72, 0, 0] payload_rows_per_dst=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] dup_rows_per_dst=[0, 0, 0, 0, 0, 2042, 192, 40] combine_rows_per_src=[16384, 13536, 16384, 0, 16384, 12760, 16020, 4562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65284 payload_rows_remote=48900 dup_rows=33020 dup_rows_remote=16502 dedup_ratio=0.335897 router_tokens_per_expert=[0, 0, 4, 0, 1, 0, 0, 1, 0, 0, 0, 0, 5, 0, 6917, 2, 0, 0, 8183, 0, 0, 3, 3, 0, 8011, 4, 0, 0, 0, 8184, 0, 0, 0, 0, 8186, 5, 0, 0, 0, 2, 0, 0, 0, 3, 0, 2, 1290, 0, 0, 6, 0, 0, 0, 0, 0, 75, 2, 0, 8186, 0, 6, 71, 0, 0] payload_rows_per_dst=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] dup_rows_per_dst=[0, 0, 0, 16364, 0, 0, 138, 16518] combine_rows_per_src=[12, 1398, 1796, 16378, 10680, 2590, 16046, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347193344 dispatch_meta_bytes=383768 combine_payload_bytes=347193344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=347577112 ingress_bytes=381796352 link_bytes=381796352 token_bytes=694770456 weight_movement_bytes=34603008 total_nvlink_bytes=729373464 weight_to_token_ratio=0.049805 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436668935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95168 payload_rows_remote=81836 dup_rows=3136 dup_rows_remote=3136 dedup_ratio=0.031901 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6662, 0, 2, 0, 8186, 0, 0, 0, 2, 0, 7943, 1, 0, 0, 0, 8187, 0, 0, 0, 0, 8187, 5, 0, 0, 1, 0, 0, 0, 1, 2, 0, 0, 1564, 0, 1, 3, 0, 0, 0, 1, 0, 118, 1, 0, 8187, 0, 5, 86, 0, 0] payload_rows_per_dst=[16380, 13332, 16380, 14978, 16384, 16378, 1154, 182] dup_rows_per_dst=[0, 0, 0, 0, 2, 3130, 2, 2] combine_rows_per_src=[16380, 13332, 16380, 14978, 16384, 16378, 1154, 182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92504 payload_rows_remote=76292 dup_rows=5800 dup_rows_remote=5792 dedup_ratio=0.059001 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 8101, 1052, 2, 0, 1, 1, 6, 0, 1235, 0, 6, 1008, 0, 7449, 0, 6262, 0, 141, 0, 0, 5, 0, 2, 0, 0, 0, 6538, 8177, 0, 0, 2, 0, 0, 0, 1, 3, 523, 436, 3, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4] payload_rows_per_dst=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] dup_rows_per_dst=[0, 2106, 16, 116, 1752, 1798, 4, 8] combine_rows_per_src=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65306 payload_rows_remote=49396 dup_rows=32998 dup_rows_remote=32878 dedup_ratio=0.335673 router_tokens_per_expert=[0, 0, 14, 0, 3, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7045, 2, 0, 0, 8171, 0, 0, 0, 9, 0, 7940, 14, 0, 0, 0, 8176, 0, 0, 0, 2, 8174, 17, 0, 0, 2, 0, 3, 0, 0, 15, 1, 1, 1211, 0, 0, 6, 1, 0, 0, 0, 0, 68, 2, 0, 8175, 0, 17, 81, 0, 0] payload_rows_per_dst=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] dup_rows_per_dst=[0, 6, 0, 16344, 4, 8, 120, 16516] combine_rows_per_src=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=390995968 dispatch_meta_bytes=393216 combine_payload_bytes=390995968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=391389184 ingress_bytes=494804992 link_bytes=494804992 token_bytes=782385152 weight_movement_bytes=103809024 total_nvlink_bytes=886194176 weight_to_token_ratio=0.132683 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.861911350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95772 payload_rows_remote=79388 dup_rows=2532 dup_rows_remote=2532 dedup_ratio=0.025757 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6914, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8004, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1266, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 8192, 0, 0, 78, 0, 0] payload_rows_per_dst=[16384, 13828, 16384, 16008, 16384, 16384, 244, 156] dup_rows_per_dst=[0, 0, 0, 0, 0, 2532, 0, 0] combine_rows_per_src=[16384, 13828, 16384, 16008, 16384, 16384, 244, 156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326230016 dispatch_meta_bytes=327680 combine_payload_bytes=326230016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326557696 ingress_bytes=395436032 link_bytes=395436032 token_bytes=652787712 weight_movement_bytes=69206016 total_nvlink_bytes=721993728 weight_to_token_ratio=0.106016 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487993953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=335200256 dispatch_meta_bytes=339888 combine_payload_bytes=335200256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335540144 ingress_bytes=404406272 link_bytes=404406272 token_bytes=670740400 weight_movement_bytes=69206016 total_nvlink_bytes=739946416 weight_to_token_ratio=0.103179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=390995968 dispatch_meta_bytes=393216 combine_payload_bytes=390995968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=391389184 ingress_bytes=494804992 link_bytes=494804992 token_bytes=782385152 weight_movement_bytes=103809024 total_nvlink_bytes=886194176 weight_to_token_ratio=0.132683 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.861911350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=200294400 dispatch_meta_bytes=261608 combine_payload_bytes=200294400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200556008 ingress_bytes=269500416 link_bytes=269500416 token_bytes=400850408 weight_movement_bytes=69206016 total_nvlink_bytes=470056424 weight_to_token_ratio=0.172648 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.014108369e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=83596 dup_rows=1910 dup_rows_remote=1910 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8033, 1123, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 1065, 0, 7739, 0, 6076, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 6399, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 493, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] dup_rows_per_dst=[0, 0, 20, 142, 0, 1746, 0, 2] combine_rows_per_src=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65306 payload_rows_remote=49396 dup_rows=32998 dup_rows_remote=32878 dedup_ratio=0.335673 router_tokens_per_expert=[0, 0, 14, 0, 3, 0, 0, 0, 1, 0, 0, 0, 1, 0, 7045, 2, 0, 0, 8171, 0, 0, 0, 9, 0, 7940, 14, 0, 0, 0, 8176, 0, 0, 0, 2, 8174, 17, 0, 0, 2, 0, 3, 0, 0, 15, 1, 1, 1211, 0, 0, 6, 1, 0, 0, 0, 0, 68, 2, 0, 8175, 0, 17, 81, 0, 0] payload_rows_per_dst=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] dup_rows_per_dst=[0, 6, 0, 16344, 4, 8, 120, 16516] combine_rows_per_src=[34, 14092, 18, 16378, 38, 2454, 15910, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312492032 dispatch_meta_bytes=328336 combine_payload_bytes=312492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312820368 ingress_bytes=347095040 link_bytes=347095040 token_bytes=625312400 weight_movement_bytes=34603008 total_nvlink_bytes=659915408 weight_to_token_ratio=0.055337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306090692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=202326016 dispatch_meta_bytes=329096 combine_payload_bytes=202326016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202655112 ingress_bytes=236929024 link_bytes=236929024 token_bytes=404981128 weight_movement_bytes=34603008 total_nvlink_bytes=439584136 weight_to_token_ratio=0.085444 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.915448430e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=335200256 dispatch_meta_bytes=339888 combine_payload_bytes=335200256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335540144 ingress_bytes=404406272 link_bytes=404406272 token_bytes=670740400 weight_movement_bytes=69206016 total_nvlink_bytes=739946416 weight_to_token_ratio=0.103179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88214 payload_rows_remote=72242 dup_rows=10090 dup_rows_remote=1118 dedup_ratio=0.102641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8108, 999, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 1059, 0, 7517, 0, 6297, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 6405, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 510, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0] payload_rows_per_dst=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] dup_rows_per_dst=[0, 0, 8972, 150, 0, 948, 0, 20] combine_rows_per_src=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325173248 dispatch_meta_bytes=327680 combine_payload_bytes=325173248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325500928 ingress_bytes=359776256 link_bytes=359776256 token_bytes=650674176 weight_movement_bytes=34603008 total_nvlink_bytes=685277184 weight_to_token_ratio=0.053180 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353809087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=69342 dup_rows=12578 dup_rows_remote=12448 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1170, 531, 0, 0, 0, 0, 0, 0, 1229, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 65, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 1336, 0, 90] payload_rows_per_dst=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] dup_rows_per_dst=[0, 1062, 8070, 0, 0, 130, 0, 3316] combine_rows_per_src=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95904 payload_rows_remote=79520 dup_rows=2400 dup_rows_remote=2400 dedup_ratio=0.024414 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7106, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8008, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 8192, 0, 0, 70, 0, 0] payload_rows_per_dst=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] dup_rows_per_dst=[0, 0, 0, 0, 0, 2280, 120, 0] combine_rows_per_src=[16384, 14212, 16384, 0, 16384, 16384, 16016, 140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342409216 dispatch_meta_bytes=342024 combine_payload_bytes=342409216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=342751240 ingress_bytes=480821248 link_bytes=480821248 token_bytes=685160456 weight_movement_bytes=138412032 total_nvlink_bytes=823572488 weight_to_token_ratio=0.202014 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.809291647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=202326016 dispatch_meta_bytes=329096 combine_payload_bytes=202326016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202655112 ingress_bytes=236929024 link_bytes=236929024 token_bytes=404981128 weight_movement_bytes=34603008 total_nvlink_bytes=439584136 weight_to_token_ratio=0.085444 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.915448430e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=81300 dup_rows=1910 dup_rows_remote=196 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8105, 1206, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 1037, 0, 7492, 0, 6210, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 6351, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 607, 305, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] dup_rows_per_dst=[0, 0, 14, 182, 0, 1714, 0, 0] combine_rows_per_src=[16384, 2412, 4554, 12654, 12702, 15094, 16384, 16210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=293440 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296196672 ingress_bytes=365109248 link_bytes=365109248 token_bytes=592099904 weight_movement_bytes=69206016 total_nvlink_bytes=661305920 weight_to_token_ratio=0.116882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373876707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95988 payload_rows_remote=83722 dup_rows=2316 dup_rows_remote=2154 dedup_ratio=0.023560 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8072, 1233, 0, 0, 0, 0, 0, 0, 1278, 0, 0, 984, 0, 7513, 0, 6019, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 6441, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 549, 484, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16380, 2466, 5636, 12266, 11634, 15078, 16384, 16144] dup_rows_per_dst=[0, 0, 136, 162, 0, 2014, 0, 4] combine_rows_per_src=[16380, 2466, 5636, 12266, 11634, 15078, 16384, 16144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284024832 dispatch_meta_bytes=327160 combine_payload_bytes=284024832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284351992 ingress_bytes=353230848 link_bytes=353230848 token_bytes=568376824 weight_movement_bytes=69206016 total_nvlink_bytes=637582840 weight_to_token_ratio=0.121761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329179244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325713920 dispatch_meta_bytes=327680 combine_payload_bytes=325713920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326041600 ingress_bytes=394919936 link_bytes=394919936 token_bytes=651755520 weight_movement_bytes=69206016 total_nvlink_bytes=720961536 weight_to_token_ratio=0.106184 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486051926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92504 payload_rows_remote=76292 dup_rows=5800 dup_rows_remote=5792 dedup_ratio=0.059001 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 8183, 8101, 1052, 2, 0, 1, 1, 6, 0, 1235, 0, 6, 1008, 0, 7449, 0, 6262, 0, 141, 0, 0, 5, 0, 2, 0, 0, 0, 6538, 8177, 0, 0, 2, 0, 0, 0, 1, 3, 523, 436, 3, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 4, 4] payload_rows_per_dst=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] dup_rows_per_dst=[0, 2106, 16, 116, 1752, 1798, 4, 8] combine_rows_per_src=[2, 16372, 4494, 10538, 13490, 15030, 16366, 16212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=333004800 dispatch_meta_bytes=325984 combine_payload_bytes=333004800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=333330784 ingress_bytes=367607808 link_bytes=367607808 token_bytes=666335584 weight_movement_bytes=34603008 total_nvlink_bytes=700938592 weight_to_token_ratio=0.051930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.383278587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96394 payload_rows_remote=83596 dup_rows=1910 dup_rows_remote=1910 dedup_ratio=0.019430 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8033, 1123, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 1065, 0, 7739, 0, 6076, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 6399, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 493, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] dup_rows_per_dst=[0, 0, 20, 142, 0, 1746, 0, 2] combine_rows_per_src=[16382, 2246, 4682, 12344, 12798, 15492, 16382, 16068] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79418 payload_rows_remote=64226 dup_rows=18886 dup_rows_remote=12918 dedup_ratio=0.192118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8090, 1222, 3, 0, 3, 0, 4, 1, 1525, 0, 1, 1085, 0, 7392, 0, 5864, 0, 203, 0, 0, 3, 0, 0, 0, 0, 0, 6386, 8184, 0, 0, 0, 0, 0, 0, 2, 0, 634, 351, 2, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 1, 1, 3] payload_rows_per_dst=[16372, 15192, 16356, 11970, 16330, 1970, 48, 1180] dup_rows_per_dst=[0, 5968, 12736, 170, 0, 4, 4, 4] combine_rows_per_src=[16372, 15192, 16356, 11970, 16330, 1970, 48, 1180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77000 payload_rows_remote=60632 dup_rows=21304 dup_rows_remote=17550 dedup_ratio=0.216715 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 895, 491, 5, 1, 0, 0, 0, 1, 1242, 0, 5, 3897, 0, 0, 0, 0, 0, 0, 3, 4158, 0, 0, 0, 0, 4, 7793, 3, 1, 0, 0, 8184, 1, 8184, 0, 2, 0, 8183, 8, 6, 87, 0, 2, 3719, 0, 0, 4, 0, 0, 2, 0, 0, 608, 0, 0, 1553, 0, 107] payload_rows_per_dst=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] dup_rows_per_dst=[0, 980, 0, 0, 16372, 190, 8, 3754] combine_rows_per_src=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93702 payload_rows_remote=77360 dup_rows=4602 dup_rows_remote=4596 dedup_ratio=0.046814 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 10, 0, 0, 8169, 8080, 1174, 2, 0, 9, 0, 16, 7, 1292, 0, 12, 986, 0, 7408, 2, 5963, 0, 141, 0, 0, 15, 0, 0, 0, 0, 0, 6607, 8154, 0, 0, 4, 0, 0, 0, 1, 2, 550, 497, 4, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 20, 8] payload_rows_per_dst=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] dup_rows_per_dst=[0, 2346, 34, 166, 0, 2008, 6, 42] combine_rows_per_src=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342925312 dispatch_meta_bytes=343504 combine_payload_bytes=342925312 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=343268816 ingress_bytes=377528320 link_bytes=377528320 token_bytes=686194128 weight_movement_bytes=34603008 total_nvlink_bytes=720797136 weight_to_token_ratio=0.050427 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.420608675e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79720 payload_rows_remote=63346 dup_rows=18584 dup_rows_remote=18584 dedup_ratio=0.189046 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8097, 990, 0, 0, 0, 0, 0, 0, 1439, 0, 0, 1035, 0, 7571, 0, 6132, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 6336, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 561, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0] payload_rows_per_dst=[16374, 16228, 16288, 12500, 16384, 1932, 0, 14] dup_rows_per_dst=[0, 14618, 3802, 164, 0, 0, 0, 0] combine_rows_per_src=[16374, 16228, 16288, 12500, 16384, 1932, 0, 14] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342925312 dispatch_meta_bytes=343504 combine_payload_bytes=342925312 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=343268816 ingress_bytes=377528320 link_bytes=377528320 token_bytes=686194128 weight_movement_bytes=34603008 total_nvlink_bytes=720797136 weight_to_token_ratio=0.050427 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.420608675e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=342409216 dispatch_meta_bytes=342024 combine_payload_bytes=342409216 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=342751240 ingress_bytes=480821248 link_bytes=480821248 token_bytes=685160456 weight_movement_bytes=138412032 total_nvlink_bytes=823572488 weight_to_token_ratio=0.202014 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.809291647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312492032 dispatch_meta_bytes=328336 combine_payload_bytes=312492032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312820368 ingress_bytes=347095040 link_bytes=347095040 token_bytes=625312400 weight_movement_bytes=34603008 total_nvlink_bytes=659915408 weight_to_token_ratio=0.055337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306090692e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85640 payload_rows_remote=84724 dup_rows=12664 dup_rows_remote=12664 dedup_ratio=0.128825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1257, 449, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 4183, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 92, 0, 0, 3658, 0, 0, 0, 5, 0, 0, 0, 0, 528, 0, 0, 1476, 0, 85] payload_rows_per_dst=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] dup_rows_per_dst=[2356, 1756, 4844, 0, 0, 184, 0, 3524] combine_rows_per_src=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93702 payload_rows_remote=77360 dup_rows=4602 dup_rows_remote=4596 dedup_ratio=0.046814 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 10, 0, 0, 8169, 8080, 1174, 2, 0, 9, 0, 16, 7, 1292, 0, 12, 986, 0, 7408, 2, 5963, 0, 141, 0, 0, 15, 0, 0, 0, 0, 0, 6607, 8154, 0, 0, 4, 0, 0, 0, 1, 2, 550, 497, 4, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 1, 0, 1, 20, 8] payload_rows_per_dst=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] dup_rows_per_dst=[0, 2346, 34, 166, 0, 2008, 6, 42] combine_rows_per_src=[32, 16352, 4592, 12076, 13214, 14916, 16342, 16178] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85488 payload_rows_remote=77142 dup_rows=12816 dup_rows_remote=12816 dedup_ratio=0.130371 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1093, 501, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 94, 0, 0, 3452, 0, 0, 0, 1, 0, 0, 0, 0, 650, 0, 0, 1377, 0, 83] payload_rows_per_dst=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] dup_rows_per_dst=[2186, 3458, 0, 48, 0, 184, 6906, 34] combine_rows_per_src=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=263069696 dispatch_meta_bytes=308576 combine_payload_bytes=263069696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263378272 ingress_bytes=366878720 link_bytes=366878720 token_bytes=526447968 weight_movement_bytes=103809024 total_nvlink_bytes=630256992 weight_to_token_ratio=0.197188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380535087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316866560 dispatch_meta_bytes=327824 combine_payload_bytes=316866560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317194384 ingress_bytes=351469568 link_bytes=351469568 token_bytes=634060944 weight_movement_bytes=34603008 total_nvlink_bytes=668663952 weight_to_token_ratio=0.054574 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=248348672 dispatch_meta_bytes=312728 combine_payload_bytes=248348672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248661400 ingress_bytes=282951680 link_bytes=282951680 token_bytes=497010072 weight_movement_bytes=34603008 total_nvlink_bytes=531613080 weight_to_token_ratio=0.069622 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064724392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97820 payload_rows_remote=81436 dup_rows=484 dup_rows_remote=22 dedup_ratio=0.004924 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 8192, 0, 0, 0, 75, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 462, 0, 22] combine_rows_per_src=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=259465216 dispatch_meta_bytes=327720 combine_payload_bytes=259465216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=259792936 ingress_bytes=294068224 link_bytes=294068224 token_bytes=519258152 weight_movement_bytes=34603008 total_nvlink_bytes=553861160 weight_to_token_ratio=0.066639 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106555052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88214 payload_rows_remote=72242 dup_rows=10090 dup_rows_remote=1118 dedup_ratio=0.102641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8108, 999, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 1059, 0, 7517, 0, 6297, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 6405, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 510, 405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0] payload_rows_per_dst=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] dup_rows_per_dst=[0, 0, 8972, 150, 0, 948, 0, 20] combine_rows_per_src=[16378, 1998, 15972, 12704, 0, 8560, 16384, 16218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333561856 dispatch_meta_bytes=325832 combine_payload_bytes=333561856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333887688 ingress_bytes=402767872 link_bytes=402767872 token_bytes=667449544 weight_movement_bytes=69206016 total_nvlink_bytes=736655560 weight_to_token_ratio=0.103687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515583077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=347029504 dispatch_meta_bytes=389552 combine_payload_bytes=347029504 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=347419056 ingress_bytes=554647552 link_bytes=554647552 token_bytes=694448560 weight_movement_bytes=207618048 total_nvlink_bytes=902066608 weight_to_token_ratio=0.298968 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.087094085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316866560 dispatch_meta_bytes=327824 combine_payload_bytes=316866560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317194384 ingress_bytes=351469568 link_bytes=351469568 token_bytes=634060944 weight_movement_bytes=34603008 total_nvlink_bytes=668663952 weight_to_token_ratio=0.054574 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=69342 dup_rows=12578 dup_rows_remote=12448 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1170, 531, 0, 0, 0, 0, 0, 0, 1229, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4126, 0, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 65, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 651, 0, 0, 1336, 0, 90] payload_rows_per_dst=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] dup_rows_per_dst=[0, 1062, 8070, 0, 0, 130, 0, 3316] combine_rows_per_src=[2340, 2458, 16384, 8252, 16384, 16384, 7140, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=315973632 dispatch_meta_bytes=359832 combine_payload_bytes=315973632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316333464 ingress_bytes=350576640 link_bytes=350576640 token_bytes=632307096 weight_movement_bytes=34603008 total_nvlink_bytes=666910104 weight_to_token_ratio=0.054725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319191673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86024 payload_rows_remote=77624 dup_rows=12280 dup_rows_remote=12280 dedup_ratio=0.124919 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1074, 447, 0, 0, 0, 0, 0, 1, 1117, 0, 0, 4113, 0, 0, 0, 0, 0, 0, 0, 4200, 0, 0, 0, 0, 0, 7829, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 8192, 0, 0, 122, 0, 0, 3528, 0, 0, 0, 0, 0, 0, 0, 0, 545, 0, 0, 1511, 0, 90] payload_rows_per_dst=[16384, 16384, 8226, 8400, 0, 16384, 14812, 5434] dup_rows_per_dst=[2146, 3130, 0, 0, 0, 244, 6424, 336] combine_rows_per_src=[16384, 16384, 8226, 8400, 0, 16384, 14812, 5434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=295903232 dispatch_meta_bytes=293440 combine_payload_bytes=295903232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296196672 ingress_bytes=365109248 link_bytes=365109248 token_bytes=592099904 weight_movement_bytes=69206016 total_nvlink_bytes=661305920 weight_to_token_ratio=0.116882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373876707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77000 payload_rows_remote=60632 dup_rows=21304 dup_rows_remote=17550 dedup_ratio=0.216715 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 895, 491, 5, 1, 0, 0, 0, 1, 1242, 0, 5, 3897, 0, 0, 0, 0, 0, 0, 3, 4158, 0, 0, 0, 0, 4, 7793, 3, 1, 0, 0, 8184, 1, 8184, 0, 2, 0, 8183, 8, 6, 87, 0, 2, 3719, 0, 0, 4, 0, 0, 2, 0, 0, 608, 0, 0, 1553, 0, 107] payload_rows_per_dst=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] dup_rows_per_dst=[0, 980, 0, 0, 16372, 190, 8, 3754] combine_rows_per_src=[1796, 2500, 7804, 8330, 16374, 16382, 7446, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=284024832 dispatch_meta_bytes=327160 combine_payload_bytes=284024832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=284351992 ingress_bytes=353230848 link_bytes=353230848 token_bytes=568376824 weight_movement_bytes=69206016 total_nvlink_bytes=637582840 weight_to_token_ratio=0.121761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329179244e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85640 payload_rows_remote=84724 dup_rows=12664 dup_rows_remote=12664 dedup_ratio=0.128825 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1257, 449, 0, 0, 0, 0, 0, 0, 1054, 0, 0, 4183, 0, 0, 0, 0, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 92, 0, 0, 3658, 0, 0, 0, 5, 0, 0, 0, 0, 528, 0, 0, 1476, 0, 85] payload_rows_per_dst=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] dup_rows_per_dst=[2356, 1756, 4844, 0, 0, 184, 0, 3524] combine_rows_per_src=[15626, 8022, 13134, 7848, 916, 16384, 7326, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65172 payload_rows_remote=48790 dup_rows=33132 dup_rows_remote=16750 dedup_ratio=0.337036 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8184, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 8184, 0, 1, 0, 4, 0, 0, 0, 0, 184, 0, 0, 8, 8185, 0, 0, 7, 79, 0, 0, 22, 7899, 0, 0, 0, 6, 0, 4, 8184, 0, 0, 4] payload_rows_per_dst=[2, 12, 16364, 6, 10, 16384, 16012, 16382] dup_rows_per_dst=[0, 0, 2, 0, 0, 16738, 10, 16382] combine_rows_per_src=[2, 12, 16364, 6, 10, 16384, 16012, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79152 payload_rows_remote=62770 dup_rows=19152 dup_rows_remote=15884 dedup_ratio=0.194824 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 1, 1004, 546, 5, 0, 0, 0, 0, 0, 1088, 0, 1, 4062, 0, 0, 0, 2, 0, 0, 6, 4169, 0, 0, 0, 0, 1, 7757, 3, 0, 0, 0, 8187, 0, 8186, 0, 3, 1, 8186, 4, 3, 55, 0, 0, 3778, 0, 0, 3, 1, 0, 2, 0, 2, 619, 0, 0, 1367, 0, 108] payload_rows_per_dst=[16380, 16382, 8130, 16376, 4642, 118, 13016, 4108] dup_rows_per_dst=[2008, 3268, 0, 8348, 0, 14, 5430, 84] combine_rows_per_src=[16380, 16382, 8130, 16376, 4642, 118, 13016, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=72908 dup_rows=18350 dup_rows_remote=18328 dedup_ratio=0.186666 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 4, 1127, 531, 16, 0, 0, 0, 0, 0, 1050, 0, 5, 4239, 0, 0, 0, 5, 0, 2, 18, 3908, 0, 0, 0, 0, 7, 7752, 14, 0, 0, 0, 8173, 0, 8172, 0, 13, 0, 8168, 15, 12, 49, 0, 1, 3509, 0, 1, 12, 0, 0, 11, 0, 2, 698, 0, 0, 1536, 0, 100] payload_rows_per_dst=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] dup_rows_per_dst=[0, 1052, 2994, 6, 10304, 144, 22, 3828] combine_rows_per_src=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=317947904 dispatch_meta_bytes=359616 combine_payload_bytes=317947904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318307520 ingress_bytes=352550912 link_bytes=352550912 token_bytes=636255424 weight_movement_bytes=34603008 total_nvlink_bytes=670858432 weight_to_token_ratio=0.054385 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326620699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85072 payload_rows_remote=68688 dup_rows=13232 dup_rows_remote=11184 dedup_ratio=0.134603 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1024, 457, 0, 0, 0, 0, 0, 0, 1014, 0, 0, 4407, 0, 0, 0, 0, 0, 0, 0, 4104, 0, 0, 0, 0, 0, 7823, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 80, 0, 0, 3559, 0, 0, 0, 0, 0, 0, 0, 0, 572, 0, 0, 1451, 0, 85] payload_rows_per_dst=[16384, 16384, 8814, 16384, 15646, 160, 7118, 4182] dup_rows_per_dst=[2048, 2942, 0, 8208, 0, 0, 0, 34] combine_rows_per_src=[16384, 16384, 8814, 16384, 15646, 160, 7118, 4182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=317947904 dispatch_meta_bytes=359616 combine_payload_bytes=317947904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318307520 ingress_bytes=352550912 link_bytes=352550912 token_bytes=636255424 weight_movement_bytes=34603008 total_nvlink_bytes=670858432 weight_to_token_ratio=0.054385 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326620699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=199843840 dispatch_meta_bytes=262160 combine_payload_bytes=199843840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200106000 ingress_bytes=269049856 link_bytes=269049856 token_bytes=399949840 weight_movement_bytes=69206016 total_nvlink_bytes=469155856 weight_to_token_ratio=0.173037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.012412948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97978 payload_rows_remote=94854 dup_rows=326 dup_rows_remote=326 dedup_ratio=0.003316 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 8192, 0, 0, 0, 72, 0, 0, 62, 7848, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 326, 0, 0] combine_rows_per_src=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=347029504 dispatch_meta_bytes=389552 combine_payload_bytes=347029504 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=347419056 ingress_bytes=554647552 link_bytes=554647552 token_bytes=694448560 weight_movement_bytes=207618048 total_nvlink_bytes=902066608 weight_to_token_ratio=0.298968 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=2.087094085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=248348672 dispatch_meta_bytes=312728 combine_payload_bytes=248348672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=248661400 ingress_bytes=282951680 link_bytes=282951680 token_bytes=497010072 weight_movement_bytes=34603008 total_nvlink_bytes=531613080 weight_to_token_ratio=0.069622 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.064724392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=314616 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257420536 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514526456 weight_movement_bytes=34603008 total_nvlink_bytes=549129464 weight_to_token_ratio=0.067252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=298631168 dispatch_meta_bytes=364944 combine_payload_bytes=298631168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298996112 ingress_bytes=333234176 link_bytes=333234176 token_bytes=597627280 weight_movement_bytes=34603008 total_nvlink_bytes=632230288 weight_to_token_ratio=0.057901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253933377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88172 payload_rows_remote=71788 dup_rows=10132 dup_rows_remote=7150 dedup_ratio=0.103068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 347, 0, 1032, 1488, 0, 0, 0, 3472, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 520, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] dup_rows_per_dst=[494, 162, 0, 0, 0, 2982, 6494, 0] combine_rows_per_src=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=72908 dup_rows=18350 dup_rows_remote=18328 dedup_ratio=0.186666 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 4, 1127, 531, 16, 0, 0, 0, 0, 0, 1050, 0, 5, 4239, 0, 0, 0, 5, 0, 2, 18, 3908, 0, 0, 0, 0, 7, 7752, 14, 0, 0, 0, 8173, 0, 8172, 0, 13, 0, 8168, 15, 12, 49, 0, 1, 3509, 0, 1, 12, 0, 0, 11, 0, 2, 698, 0, 0, 1536, 0, 100] payload_rows_per_dst=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] dup_rows_per_dst=[0, 1052, 2994, 6, 10304, 144, 22, 3828] combine_rows_per_src=[2266, 2142, 11558, 7864, 16360, 16370, 7046, 16348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281346048 dispatch_meta_bytes=319488 combine_payload_bytes=281346048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281665536 ingress_bytes=315949056 link_bytes=315949056 token_bytes=563011584 weight_movement_bytes=34603008 total_nvlink_bytes=597614592 weight_to_token_ratio=0.061461 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188890862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85488 payload_rows_remote=77142 dup_rows=12816 dup_rows_remote=12816 dedup_ratio=0.130371 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1093, 501, 0, 0, 0, 0, 0, 0, 1228, 0, 0, 4173, 0, 0, 0, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 0, 7805, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 8192, 0, 0, 94, 0, 0, 3452, 0, 0, 0, 1, 0, 0, 0, 0, 650, 0, 0, 1377, 0, 83] payload_rows_per_dst=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] dup_rows_per_dst=[2186, 3458, 0, 48, 0, 184, 6906, 34] combine_rows_per_src=[16384, 16384, 8346, 8240, 1306, 15032, 15610, 4186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=294043648 dispatch_meta_bytes=315752 combine_payload_bytes=294043648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294359400 ingress_bytes=328646656 link_bytes=328646656 token_bytes=588403048 weight_movement_bytes=34603008 total_nvlink_bytes=623006056 weight_to_token_ratio=0.058808 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236670908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=388521984 dispatch_meta_bytes=380720 combine_payload_bytes=388521984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388902704 ingress_bytes=457728000 link_bytes=457728000 token_bytes=777424688 weight_movement_bytes=69206016 total_nvlink_bytes=846630704 weight_to_token_ratio=0.089020 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722393614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97820 payload_rows_remote=81436 dup_rows=484 dup_rows_remote=22 dedup_ratio=0.004924 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 223, 0, 0, 0, 8192, 0, 0, 0, 75, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 462, 0, 22] combine_rows_per_src=[16368, 0, 16384, 16362, 0, 16384, 15938, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335872000 ingress_bytes=404750336 link_bytes=404750336 token_bytes=671416320 weight_movement_bytes=69206016 total_nvlink_bytes=740622336 weight_to_token_ratio=0.103075 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523042929e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=298631168 dispatch_meta_bytes=364944 combine_payload_bytes=298631168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298996112 ingress_bytes=333234176 link_bytes=333234176 token_bytes=597627280 weight_movement_bytes=34603008 total_nvlink_bytes=632230288 weight_to_token_ratio=0.057901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253933377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 8192, 0, 0, 0, 81, 0, 0, 17, 7905, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 378, 16006, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 378, 16006, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=315973632 dispatch_meta_bytes=359832 combine_payload_bytes=315973632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316333464 ingress_bytes=350576640 link_bytes=350576640 token_bytes=632307096 weight_movement_bytes=34603008 total_nvlink_bytes=666910104 weight_to_token_ratio=0.054725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319191673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=333561856 dispatch_meta_bytes=325832 combine_payload_bytes=333561856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333887688 ingress_bytes=402767872 link_bytes=402767872 token_bytes=667449544 weight_movement_bytes=69206016 total_nvlink_bytes=736655560 weight_to_token_ratio=0.103687 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515583077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65172 payload_rows_remote=48790 dup_rows=33132 dup_rows_remote=16750 dedup_ratio=0.337036 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8184, 0, 0, 0, 8185, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 8184, 0, 1, 0, 4, 0, 0, 0, 0, 184, 0, 0, 8, 8185, 0, 0, 7, 79, 0, 0, 22, 7899, 0, 0, 0, 6, 0, 4, 8184, 0, 0, 4] payload_rows_per_dst=[2, 12, 16364, 6, 10, 16384, 16012, 16382] dup_rows_per_dst=[0, 0, 2, 0, 0, 16738, 10, 16382] combine_rows_per_src=[2, 12, 16364, 6, 10, 16384, 16012, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14566961624 prefetch_bytes=622854144 reduce_bytes=934281216 weight_movement_bytes=1557135360 egress_bytes=7287370200 ingress_bytes=8836726784 link_bytes=8836726784 total_nvlink_bytes=16124096984 weight_to_token_ratio=0.106895 prefetch_share_of_ingress=0.070485 dedup_ratio=0.091799 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=27 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.445734412e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73926 payload_rows_remote=57576 dup_rows=24378 dup_rows_remote=21788 dedup_ratio=0.247986 router_tokens_per_expert=[0, 0, 3, 7, 6, 0, 0, 372, 0, 824, 1331, 0, 0, 0, 3201, 0, 0, 4, 0, 6, 5, 8184, 3, 0, 2, 2, 0, 0, 8038, 0, 4, 0, 4, 6, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 2193, 0, 4, 0, 434, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 7, 0, 0, 8171, 0, 0] payload_rows_per_dst=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] dup_rows_per_dst=[16, 172, 12, 6, 8, 4390, 17184, 2590] combine_rows_per_src=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97978 payload_rows_remote=94854 dup_rows=326 dup_rows_remote=326 dedup_ratio=0.003316 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 0, 0, 8192, 0, 0, 0, 72, 0, 0, 62, 7848, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] dup_rows_per_dst=[0, 0, 0, 0, 0, 326, 0, 0] combine_rows_per_src=[16384, 14, 16384, 16384, 3124, 13354, 15964, 16370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68220 payload_rows_remote=51836 dup_rows=30084 dup_rows_remote=13762 dedup_ratio=0.306030 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8186, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 3, 0, 2, 0, 0, 211, 0, 0, 6, 8186, 0, 0, 4, 89, 0, 0, 11, 7876, 0, 0, 0, 3, 0, 4, 8187, 0, 0, 4] payload_rows_per_dst=[16376, 16384, 50, 16376, 16376, 434, 2214, 10] dup_rows_per_dst=[0, 16322, 0, 0, 13746, 0, 4, 12] combine_rows_per_src=[16376, 16384, 50, 16376, 16376, 434, 2214, 10] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65184 payload_rows_remote=49228 dup_rows=33120 dup_rows_remote=33118 dedup_ratio=0.336914 router_tokens_per_expert=[5, 0, 0, 4, 0, 0, 2, 0, 17, 0, 0, 0, 8169, 0, 0, 1, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4, 0, 0, 0, 8166, 5, 0, 1, 13, 0, 0, 0, 0, 208, 0, 1, 18, 8173, 0, 0, 21, 91, 0, 0, 74, 7793, 0, 0, 0, 14, 0, 13, 8173, 0, 0, 2] payload_rows_per_dst=[20, 36, 16348, 24, 36, 16382, 15956, 16382] dup_rows_per_dst=[2, 0, 4, 0, 2, 16750, 2, 16360] combine_rows_per_src=[20, 36, 16348, 24, 36, 16382, 15956, 16382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335872000 ingress_bytes=370147328 link_bytes=370147328 token_bytes=671416320 weight_movement_bytes=34603008 total_nvlink_bytes=706019328 weight_to_token_ratio=0.051537 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66062 payload_rows_remote=49678 dup_rows=32242 dup_rows_remote=32242 dedup_ratio=0.327983 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 184, 0, 0, 0, 8192, 0, 0, 0, 78, 0, 0, 1, 7929, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 0, 16384, 16384, 368, 158, 0] dup_rows_per_dst=[0, 16384, 0, 0, 15858, 0, 0, 0] combine_rows_per_src=[16384, 16384, 0, 16384, 16384, 368, 158, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=235831296 dispatch_meta_bytes=317456 combine_payload_bytes=235831296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236148752 ingress_bytes=305037312 link_bytes=305037312 token_bytes=471980048 weight_movement_bytes=69206016 total_nvlink_bytes=541186064 weight_to_token_ratio=0.146629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147830848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=388521984 dispatch_meta_bytes=380720 combine_payload_bytes=388521984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388902704 ingress_bytes=457728000 link_bytes=457728000 token_bytes=777424688 weight_movement_bytes=69206016 total_nvlink_bytes=846630704 weight_to_token_ratio=0.089020 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722393614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93778 payload_rows_remote=79570 dup_rows=4526 dup_rows_remote=4526 dedup_ratio=0.046041 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 351, 0, 1061, 1793, 0, 0, 0, 2801, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 553, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0] payload_rows_per_dst=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] dup_rows_per_dst=[702, 150, 0, 0, 0, 3562, 112, 0] combine_rows_per_src=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335872000 ingress_bytes=370147328 link_bytes=370147328 token_bytes=671416320 weight_movement_bytes=34603008 total_nvlink_bytes=706019328 weight_to_token_ratio=0.051537 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=780 dedup_ratio=0.031921 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 315, 0, 1160, 1269, 0, 0, 0, 3078, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 616, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] dup_rows_per_dst=[630, 150, 2358, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=199843840 dispatch_meta_bytes=262160 combine_payload_bytes=199843840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=200106000 ingress_bytes=269049856 link_bytes=269049856 token_bytes=399949840 weight_movement_bytes=69206016 total_nvlink_bytes=469155856 weight_to_token_ratio=0.173037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.012412948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=203481088 dispatch_meta_bytes=327680 combine_payload_bytes=203481088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203808768 ingress_bytes=238084096 link_bytes=238084096 token_bytes=407289856 weight_movement_bytes=34603008 total_nvlink_bytes=441892864 weight_to_token_ratio=0.084959 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.958912859e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65184 payload_rows_remote=49228 dup_rows=33120 dup_rows_remote=33118 dedup_ratio=0.336914 router_tokens_per_expert=[5, 0, 0, 4, 0, 0, 2, 0, 17, 0, 0, 0, 8169, 0, 0, 1, 8173, 3, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 4, 0, 0, 0, 8166, 5, 0, 1, 13, 0, 0, 0, 0, 208, 0, 1, 18, 8173, 0, 0, 21, 91, 0, 0, 74, 7793, 0, 0, 0, 14, 0, 13, 8173, 0, 0, 2] payload_rows_per_dst=[20, 36, 16348, 24, 36, 16382, 15956, 16382] dup_rows_per_dst=[2, 0, 4, 0, 2, 16750, 2, 16360] combine_rows_per_src=[20, 36, 16348, 24, 36, 16382, 15956, 16382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=212320256 dispatch_meta_bytes=262392 combine_payload_bytes=212320256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=212582648 ingress_bytes=281526272 link_bytes=281526272 token_bytes=424902904 weight_movement_bytes=69206016 total_nvlink_bytes=494108920 weight_to_token_ratio=0.162875 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.059360697e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=325918720 dispatch_meta_bytes=336384 combine_payload_bytes=325918720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326255104 ingress_bytes=395124736 link_bytes=395124736 token_bytes=652173824 weight_movement_bytes=69206016 total_nvlink_bytes=721379840 weight_to_token_ratio=0.106116 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486822571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=318248 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323009320 ingress_bytes=391897088 link_bytes=391897088 token_bytes=645700392 weight_movement_bytes=69206016 total_nvlink_bytes=714906408 weight_to_token_ratio=0.107180 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474677192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88172 payload_rows_remote=71788 dup_rows=10132 dup_rows_remote=7150 dedup_ratio=0.103068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 347, 0, 1032, 1488, 0, 0, 0, 3472, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 1491, 0, 0, 0, 520, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] dup_rows_per_dst=[494, 162, 0, 0, 0, 2982, 6494, 0] combine_rows_per_src=[11130, 11822, 0, 16078, 0, 16384, 16374, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=201637888 dispatch_meta_bytes=329384 combine_payload_bytes=201637888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=201967272 ingress_bytes=270843904 link_bytes=270843904 token_bytes=403605160 weight_movement_bytes=69206016 total_nvlink_bytes=472811176 weight_to_token_ratio=0.171470 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.019163806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=203481088 dispatch_meta_bytes=327680 combine_payload_bytes=203481088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203808768 ingress_bytes=238084096 link_bytes=238084096 token_bytes=407289856 weight_movement_bytes=34603008 total_nvlink_bytes=441892864 weight_to_token_ratio=0.084959 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.958912859e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=98304 payload_rows_remote=81920 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 8192, 0, 0, 0, 71, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 344, 16040, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11118279448 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=5562710808 ingress_bytes=6905085952 link_bytes=6905085952 total_nvlink_bytes=12467796760 weight_to_token_ratio=0.121378 prefetch_share_of_ingress=0.105236 dedup_ratio=0.246288 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.129707936e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=15188540784 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=7598259568 ingress_bytes=9078210560 link_bytes=9078210560 total_nvlink_bytes=16676470128 weight_to_token_ratio=0.097964 prefetch_share_of_ingress=0.076233 dedup_ratio=0.071782 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.026630 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.485242412e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96888 payload_rows_remote=80832 dup_rows=1416 dup_rows_remote=1416 dedup_ratio=0.014404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 334, 0, 911, 1637, 0, 0, 0, 2922, 0, 0, 0, 0, 19, 0, 8192, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2055, 0, 0, 0, 499, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0] payload_rows_per_dst=[16384, 10774, 13872, 16056, 16384, 6078, 998, 16342] dup_rows_per_dst=[668, 166, 36, 0, 0, 546, 0, 0] combine_rows_per_src=[16384, 10774, 13872, 16056, 16384, 6078, 998, 16342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14638774392 prefetch_bytes=657457152 reduce_bytes=1072693248 weight_movement_bytes=1730150400 egress_bytes=7323326584 ingress_bytes=9045598208 link_bytes=9045598208 total_nvlink_bytes=16368924792 weight_to_token_ratio=0.118190 prefetch_share_of_ingress=0.072683 dedup_ratio=0.093835 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=31 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.479906862e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=294043648 dispatch_meta_bytes=315752 combine_payload_bytes=294043648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294359400 ingress_bytes=328646656 link_bytes=328646656 token_bytes=588403048 weight_movement_bytes=34603008 total_nvlink_bytes=623006056 weight_to_token_ratio=0.058808 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236670908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73926 payload_rows_remote=57576 dup_rows=24378 dup_rows_remote=21788 dedup_ratio=0.247986 router_tokens_per_expert=[0, 0, 3, 7, 6, 0, 0, 372, 0, 824, 1331, 0, 0, 0, 3201, 0, 0, 4, 0, 6, 5, 8184, 3, 0, 2, 2, 0, 0, 8038, 0, 4, 0, 4, 6, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 2193, 0, 4, 0, 434, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 7, 0, 0, 8171, 0, 0] payload_rows_per_dst=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] dup_rows_per_dst=[16, 172, 12, 6, 8, 4390, 17184, 2590] combine_rows_per_src=[760, 10540, 24, 10, 13504, 16372, 16366, 16350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=335544320 dispatch_meta_bytes=327680 combine_payload_bytes=335544320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=335872000 ingress_bytes=404750336 link_bytes=404750336 token_bytes=671416320 weight_movement_bytes=69206016 total_nvlink_bytes=740622336 weight_to_token_ratio=0.103075 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523042929e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93778 payload_rows_remote=79570 dup_rows=4526 dup_rows_remote=4526 dedup_ratio=0.046041 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 351, 0, 1061, 1793, 0, 0, 0, 2801, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 8047, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1781, 0, 0, 0, 553, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0] payload_rows_per_dst=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] dup_rows_per_dst=[702, 150, 0, 0, 0, 3562, 112, 0] combine_rows_per_src=[16384, 11160, 18, 16094, 14208, 16384, 3170, 16360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81278 payload_rows_remote=69996 dup_rows=17026 dup_rows_remote=16848 dedup_ratio=0.173197 router_tokens_per_expert=[2, 0, 2, 2, 5, 0, 0, 238, 0, 954, 1120, 0, 0, 0, 3656, 0, 0, 1, 0, 7, 6, 8185, 2, 0, 1, 3, 0, 0, 8014, 0, 3, 0, 4, 4, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 1731, 0, 2, 0, 676, 1, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 8163, 3, 0] payload_rows_per_dst=[16384, 11282, 16382, 16034, 16372, 3464, 1354, 6] dup_rows_per_dst=[484, 178, 16346, 8, 8, 2, 0, 0] combine_rows_per_src=[16384, 11282, 16382, 16034, 16372, 3464, 1354, 6] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331087872 dispatch_meta_bytes=328992 combine_payload_bytes=331087872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=331416864 ingress_bytes=434896896 link_bytes=434896896 token_bytes=662504736 weight_movement_bytes=103809024 total_nvlink_bytes=766313760 weight_to_token_ratio=0.156692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.636482008e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76278 payload_rows_remote=59936 dup_rows=22026 dup_rows_remote=4424 dedup_ratio=0.224060 router_tokens_per_expert=[2, 0, 9, 10, 19, 0, 0, 394, 0, 1100, 1215, 0, 0, 0, 2972, 0, 0, 12, 0, 22, 18, 8167, 2, 0, 2, 6, 0, 1, 7999, 0, 10, 0, 13, 15, 0, 0, 0, 8133, 5, 0, 0, 0, 0, 0, 2055, 0, 6, 0, 675, 0, 0, 4, 0, 8160, 0, 0, 1, 0, 3, 0, 0, 8117, 5, 0] payload_rows_per_dst=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] dup_rows_per_dst=[34, 208, 26, 20, 26, 4110, 17602, 0] combine_rows_per_src=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=58668 dup_rows=23252 dup_rows_remote=22620 dedup_ratio=0.236532 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 316, 0, 1135, 1630, 0, 0, 0, 2788, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1943, 0, 0, 0, 538, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0] payload_rows_per_dst=[16384, 14070, 16384, 6868, 16384, 3886, 1076, 0] dup_rows_per_dst=[632, 6240, 16380, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 14070, 16384, 6868, 16384, 3886, 1076, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331087872 dispatch_meta_bytes=328992 combine_payload_bytes=331087872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=331416864 ingress_bytes=434896896 link_bytes=434896896 token_bytes=662504736 weight_movement_bytes=103809024 total_nvlink_bytes=766313760 weight_to_token_ratio=0.156692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.636482008e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14566961624 prefetch_bytes=622854144 reduce_bytes=934281216 weight_movement_bytes=1557135360 egress_bytes=7287370200 ingress_bytes=8836726784 link_bytes=8836726784 total_nvlink_bytes=16124096984 weight_to_token_ratio=0.106895 prefetch_share_of_ingress=0.070485 dedup_ratio=0.091799 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=27 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.445734412e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=235831296 dispatch_meta_bytes=317456 combine_payload_bytes=235831296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=236148752 ingress_bytes=305037312 link_bytes=305037312 token_bytes=471980048 weight_movement_bytes=69206016 total_nvlink_bytes=541186064 weight_to_token_ratio=0.146629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147830848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76278 payload_rows_remote=59936 dup_rows=22026 dup_rows_remote=4424 dedup_ratio=0.224060 router_tokens_per_expert=[2, 0, 9, 10, 19, 0, 0, 394, 0, 1100, 1215, 0, 0, 0, 2972, 0, 0, 12, 0, 22, 18, 8167, 2, 0, 2, 6, 0, 1, 7999, 0, 10, 0, 13, 15, 0, 0, 0, 8133, 5, 0, 0, 0, 0, 0, 2055, 0, 6, 0, 675, 0, 0, 4, 0, 8160, 0, 0, 1, 0, 3, 0, 0, 8117, 5, 0] payload_rows_per_dst=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] dup_rows_per_dst=[34, 208, 26, 20, 26, 4110, 17602, 0] combine_rows_per_src=[834, 10366, 82, 11028, 5028, 16346, 16342, 16252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=325918720 dispatch_meta_bytes=336384 combine_payload_bytes=325918720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326255104 ingress_bytes=395124736 link_bytes=395124736 token_bytes=652173824 weight_movement_bytes=69206016 total_nvlink_bytes=721379840 weight_to_token_ratio=0.106116 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486822571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286703616 dispatch_meta_bytes=347376 combine_payload_bytes=286703616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287050992 ingress_bytes=321306624 link_bytes=321306624 token_bytes=573754608 weight_movement_bytes=34603008 total_nvlink_bytes=608357616 weight_to_token_ratio=0.060310 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209050959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=240304128 dispatch_meta_bytes=325152 combine_payload_bytes=240304128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240629280 ingress_bytes=274907136 link_bytes=274907136 token_bytes=480933408 weight_movement_bytes=34603008 total_nvlink_bytes=515536416 weight_to_token_ratio=0.071950 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.034453421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14878007752 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=7442932168 ingress_bytes=8680783872 link_bytes=8680783872 total_nvlink_bytes=16123716040 weight_to_token_ratio=0.083728 prefetch_share_of_ingress=0.079723 dedup_ratio=0.076917 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=21717 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.420221341e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11118279448 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=5562710808 ingress_bytes=6905085952 link_bytes=6905085952 total_nvlink_bytes=12467796760 weight_to_token_ratio=0.121378 prefetch_share_of_ingress=0.105236 dedup_ratio=0.246288 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=22352 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.129707936e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=245497856 dispatch_meta_bytes=257440 combine_payload_bytes=245497856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245755296 ingress_bytes=314703872 link_bytes=314703872 token_bytes=491253152 weight_movement_bytes=69206016 total_nvlink_bytes=560459168 weight_to_token_ratio=0.140876 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184205335e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=240304128 dispatch_meta_bytes=325152 combine_payload_bytes=240304128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240629280 ingress_bytes=274907136 link_bytes=274907136 token_bytes=480933408 weight_movement_bytes=34603008 total_nvlink_bytes=515536416 weight_to_token_ratio=0.071950 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.034453421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=780 dedup_ratio=0.031921 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 315, 0, 1160, 1269, 0, 0, 0, 3078, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 0, 616, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0] payload_rows_per_dst=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] dup_rows_per_dst=[630, 150, 2358, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 10864, 16384, 16102, 16382, 3798, 1232, 14020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=15188540784 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=7598259568 ingress_bytes=9078210560 link_bytes=9078210560 total_nvlink_bytes=16676470128 weight_to_token_ratio=0.097964 prefetch_share_of_ingress=0.076233 dedup_ratio=0.071782 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.026630 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.485242412e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12773668384 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6390535712 ingress_bytes=7871062016 link_bytes=7871062016 total_nvlink_bytes=14261597728 weight_to_token_ratio=0.116484 prefetch_share_of_ingress=0.087925 dedup_ratio=0.169443 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.287746639e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12183793264 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=6095654512 ingress_bytes=7333847040 link_bytes=7333847040 total_nvlink_bytes=13429501552 weight_to_token_ratio=0.102243 prefetch_share_of_ingress=0.108520 dedup_ratio=0.205104 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.199855478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12022502504 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=6014874728 ingress_bytes=7253336064 link_bytes=7253336064 total_nvlink_bytes=13268210792 weight_to_token_ratio=0.103615 prefetch_share_of_ingress=0.109725 dedup_ratio=0.195773 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.186683464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=318248 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323009320 ingress_bytes=391897088 link_bytes=391897088 token_bytes=645700392 weight_movement_bytes=69206016 total_nvlink_bytes=714906408 weight_to_token_ratio=0.107180 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474677192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14638774392 prefetch_bytes=657457152 reduce_bytes=1072693248 weight_movement_bytes=1730150400 egress_bytes=7323326584 ingress_bytes=9045598208 link_bytes=9045598208 total_nvlink_bytes=16368924792 weight_to_token_ratio=0.118190 prefetch_share_of_ingress=0.072683 dedup_ratio=0.093835 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=31 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.479906862e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:55:25] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 793.6 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.375167E+00 | loss scale: 1.0 | grad norm: 4.992 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65582 dup_rows=24062 dup_rows_remote=20102 dedup_ratio=0.244771 router_tokens_per_expert=[1440, 578, 487, 1359, 463, 1487, 1319, 139, 2172, 1046, 1298, 330, 10, 2502, 712, 742, 709, 865, 824, 689, 110, 211, 261, 1006, 343, 724, 743, 529, 999, 159, 767, 574, 1027, 656, 1778, 1044, 573, 243, 775, 648, 245, 457, 610, 200, 764, 820, 496, 165, 321, 419, 502, 662, 797, 1963, 307, 435, 1864, 514, 1537, 664, 161, 673, 1504, 731] payload_rows_per_dst=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] dup_rows_per_dst=[4074, 3960, 2704, 3570, 2602, 3446, 1978, 1728] combine_rows_per_src=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75124 payload_rows_remote=65414 dup_rows=23180 dup_rows_remote=20334 dedup_ratio=0.235799 router_tokens_per_expert=[1514, 604, 499, 1248, 447, 1531, 1160, 110, 2285, 732, 1276, 262, 4, 2685, 752, 812, 672, 925, 840, 712, 86, 191, 222, 1116, 301, 758, 663, 447, 940, 153, 724, 466, 1024, 636, 1838, 1069, 708, 204, 824, 696, 188, 413, 558, 162, 744, 816, 550, 119, 306, 389, 542, 647, 885, 2155, 356, 438, 2178, 411, 1609, 618, 143, 678, 1387, 724] payload_rows_per_dst=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] dup_rows_per_dst=[2674, 3598, 2846, 3432, 3544, 3374, 2130, 1582] combine_rows_per_src=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75614 payload_rows_remote=66450 dup_rows=22690 dup_rows_remote=19244 dedup_ratio=0.230815 router_tokens_per_expert=[1607, 614, 539, 1281, 463, 1400, 1036, 216, 2278, 715, 1215, 245, 8, 2532, 742, 832, 674, 910, 733, 709, 135, 240, 296, 1062, 354, 735, 716, 496, 996, 123, 664, 460, 951, 758, 1754, 1169, 824, 242, 774, 637, 237, 517, 611, 185, 787, 780, 532, 124, 316, 424, 589, 652, 802, 2048, 271, 472, 1923, 486, 1641, 691, 164, 673, 1380, 712] payload_rows_per_dst=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] dup_rows_per_dst=[2860, 3506, 2548, 3386, 2514, 3446, 2626, 1804] combine_rows_per_src=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74782 payload_rows_remote=64672 dup_rows=23522 dup_rows_remote=19876 dedup_ratio=0.239278 router_tokens_per_expert=[1556, 617, 563, 1238, 444, 1366, 1186, 153, 2146, 968, 1363, 228, 12, 2648, 698, 715, 758, 951, 785, 805, 134, 257, 263, 1029, 367, 690, 766, 475, 946, 217, 745, 551, 1103, 668, 1751, 1131, 587, 210, 783, 645, 255, 463, 610, 182, 695, 939, 565, 138, 349, 421, 514, 666, 810, 1906, 274, 487, 1958, 431, 1545, 576, 206, 642, 1322, 680] payload_rows_per_dst=[8468, 8606, 10092, 9848, 10110, 9448, 8884, 9326] dup_rows_per_dst=[2666, 3654, 2984, 3582, 3646, 3542, 1970, 1478] combine_rows_per_src=[8468, 8606, 10092, 9848, 10110, 9448, 8884, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12183793264 prefetch_bytes=795869184 reduce_bytes=449839104 weight_movement_bytes=1245708288 egress_bytes=6095654512 ingress_bytes=7333847040 link_bytes=7333847040 total_nvlink_bytes=13429501552 weight_to_token_ratio=0.102243 prefetch_share_of_ingress=0.108520 dedup_ratio=0.205104 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=13 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.199855478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none + [2026-08-26 18:07:32] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 793.3 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.375167E+00 | loss scale: 1.0 | grad norm: 4.992 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75064 payload_rows_remote=63456 dup_rows=23240 dup_rows_remote=18818 dedup_ratio=0.236410 router_tokens_per_expert=[1526, 637, 579, 1256, 413, 1421, 1022, 134, 2302, 967, 1358, 265, 4, 2658, 662, 844, 694, 953, 816, 681, 153, 209, 247, 1110, 310, 693, 795, 455, 1022, 197, 636, 399, 998, 622, 1779, 1079, 613, 197, 854, 683, 199, 440, 575, 173, 675, 834, 537, 155, 327, 445, 591, 649, 811, 2105, 326, 472, 2051, 474, 1530, 614, 151, 632, 1425, 718] payload_rows_per_dst=[8184, 7536, 9894, 7460, 9332, 9274, 11776, 11608] dup_rows_per_dst=[2740, 2618, 2884, 1554, 2570, 3218, 3234, 4422] combine_rows_per_src=[8184, 7536, 9894, 7460, 9332, 9274, 11776, 11608] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74944 payload_rows_remote=65340 dup_rows=23360 dup_rows_remote=20086 dedup_ratio=0.237630 router_tokens_per_expert=[1603, 702, 553, 1309, 430, 1536, 1172, 132, 2277, 862, 1176, 268, 3, 2558, 692, 829, 691, 897, 737, 748, 103, 205, 275, 1102, 341, 715, 717, 414, 954, 178, 712, 497, 1033, 644, 1701, 1170, 690, 224, 854, 611, 210, 438, 590, 179, 707, 804, 507, 122, 307, 416, 572, 726, 770, 1997, 331, 474, 1911, 437, 1749, 618, 185, 738, 1375, 674] payload_rows_per_dst=[8684, 8574, 10002, 9604, 10216, 8964, 9060, 9840] dup_rows_per_dst=[2984, 3640, 2720, 3274, 3638, 3266, 2126, 1712] combine_rows_per_src=[8684, 8574, 10002, 9604, 10216, 8964, 9060, 9840] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=63490 dup_rows=23312 dup_rows_remote=20022 dedup_ratio=0.237142 router_tokens_per_expert=[1581, 661, 583, 1255, 475, 1500, 1206, 167, 2125, 865, 1201, 239, 10, 2692, 730, 754, 680, 852, 757, 711, 95, 237, 286, 992, 307, 730, 668, 531, 982, 155, 629, 470, 1052, 663, 1699, 1113, 623, 200, 870, 635, 195, 528, 636, 130, 768, 868, 554, 173, 326, 363, 541, 778, 824, 2113, 326, 426, 1993, 487, 1579, 602, 203, 717, 1373, 668] payload_rows_per_dst=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] dup_rows_per_dst=[3060, 3464, 2704, 2202, 1842, 3602, 3290, 3148] combine_rows_per_src=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75124 payload_rows_remote=65414 dup_rows=23180 dup_rows_remote=20334 dedup_ratio=0.235799 router_tokens_per_expert=[1514, 604, 499, 1248, 447, 1531, 1160, 110, 2285, 732, 1276, 262, 4, 2685, 752, 812, 672, 925, 840, 712, 86, 191, 222, 1116, 301, 758, 663, 447, 940, 153, 724, 466, 1024, 636, 1838, 1069, 708, 204, 824, 696, 188, 413, 558, 162, 744, 816, 550, 119, 306, 389, 542, 647, 885, 2155, 356, 438, 2178, 411, 1609, 618, 143, 678, 1387, 724] payload_rows_per_dst=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] dup_rows_per_dst=[2674, 3598, 2846, 3432, 3544, 3374, 2130, 1582] combine_rows_per_src=[8524, 8648, 9710, 9828, 10454, 9096, 9306, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75614 payload_rows_remote=66450 dup_rows=22690 dup_rows_remote=19244 dedup_ratio=0.230815 router_tokens_per_expert=[1607, 614, 539, 1281, 463, 1400, 1036, 216, 2278, 715, 1215, 245, 8, 2532, 742, 832, 674, 910, 733, 709, 135, 240, 296, 1062, 354, 735, 716, 496, 996, 123, 664, 460, 951, 758, 1754, 1169, 824, 242, 774, 637, 237, 517, 611, 185, 787, 780, 532, 124, 316, 424, 589, 652, 802, 2048, 271, 472, 1923, 486, 1641, 691, 164, 673, 1380, 712] payload_rows_per_dst=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] dup_rows_per_dst=[2860, 3506, 2548, 3386, 2514, 3446, 2626, 1804] combine_rows_per_src=[8238, 8564, 10184, 9548, 9830, 9164, 10396, 9690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65582 dup_rows=24062 dup_rows_remote=20102 dedup_ratio=0.244771 router_tokens_per_expert=[1440, 578, 487, 1359, 463, 1487, 1319, 139, 2172, 1046, 1298, 330, 10, 2502, 712, 742, 709, 865, 824, 689, 110, 211, 261, 1006, 343, 724, 743, 529, 999, 159, 767, 574, 1027, 656, 1778, 1044, 573, 243, 775, 648, 245, 457, 610, 200, 764, 820, 496, 165, 321, 419, 502, 662, 797, 1963, 307, 435, 1864, 514, 1537, 664, 161, 673, 1504, 731] payload_rows_per_dst=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] dup_rows_per_dst=[4074, 3960, 2704, 3570, 2602, 3446, 1978, 1728] combine_rows_per_src=[9570, 8660, 9526, 9834, 8906, 9072, 8834, 9840] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268623872 dispatch_meta_bytes=342736 combine_payload_bytes=268623872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=268966608 ingress_bytes=372432896 link_bytes=372432896 token_bytes=537590480 weight_movement_bytes=103809024 total_nvlink_bytes=641399504 weight_to_token_ratio=0.193101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401435004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=342776 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272521976 ingress_bytes=306782208 link_bytes=306782208 token_bytes=544701176 weight_movement_bytes=34603008 total_nvlink_bytes=579304184 weight_to_token_ratio=0.063527 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154396751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=342992 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268278736 ingress_bytes=302538752 link_bytes=302538752 token_bytes=536214480 weight_movement_bytes=34603008 total_nvlink_bytes=570817488 weight_to_token_ratio=0.064532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138428967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=338192 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265234704 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530131216 weight_movement_bytes=103809024 total_nvlink_bytes=633940240 weight_to_token_ratio=0.195818 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=63490 dup_rows=23312 dup_rows_remote=20022 dedup_ratio=0.237142 router_tokens_per_expert=[1581, 661, 583, 1255, 475, 1500, 1206, 167, 2125, 865, 1201, 239, 10, 2692, 730, 754, 680, 852, 757, 711, 95, 237, 286, 992, 307, 730, 668, 531, 982, 155, 629, 470, 1052, 663, 1699, 1113, 623, 200, 870, 635, 195, 528, 636, 130, 768, 868, 554, 173, 326, 363, 541, 778, 824, 2113, 326, 426, 1993, 487, 1579, 602, 203, 717, 1373, 668] payload_rows_per_dst=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] dup_rows_per_dst=[3060, 3464, 2704, 2202, 1842, 3602, 3290, 3148] combine_rows_per_src=[8634, 8384, 9678, 8132, 8470, 9486, 11502, 10706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74782 payload_rows_remote=64672 dup_rows=23522 dup_rows_remote=19876 dedup_ratio=0.239278 router_tokens_per_expert=[1556, 617, 563, 1238, 444, 1366, 1186, 153, 2146, 968, 1363, 228, 12, 2648, 698, 715, 758, 951, 785, 805, 134, 257, 263, 1029, 367, 690, 766, 475, 946, 217, 745, 551, 1103, 668, 1751, 1131, 587, 210, 783, 645, 255, 463, 610, 182, 695, 939, 565, 138, 349, 421, 514, 666, 810, 1906, 274, 487, 1958, 431, 1545, 576, 206, 642, 1322, 680] payload_rows_per_dst=[8468, 8606, 10092, 9848, 10110, 9448, 8884, 9326] dup_rows_per_dst=[2666, 3654, 2984, 3582, 3646, 3542, 1970, 1478] combine_rows_per_src=[8468, 8606, 10092, 9848, 10110, 9448, 8884, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=259915776 dispatch_meta_bytes=329096 combine_payload_bytes=259915776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260244872 ingress_bytes=329121792 link_bytes=329121792 token_bytes=520160648 weight_movement_bytes=69206016 total_nvlink_bytes=589366664 weight_to_token_ratio=0.133047 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238458807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=341704 combine_payload_bytes=267632640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267974344 ingress_bytes=302235648 link_bytes=302235648 token_bytes=535606984 weight_movement_bytes=34603008 total_nvlink_bytes=570209992 weight_to_token_ratio=0.064605 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137288411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=62908 dup_rows=24148 dup_rows_remote=18452 dedup_ratio=0.245646 router_tokens_per_expert=[1423, 604, 588, 1177, 485, 1407, 1265, 174, 2062, 922, 1268, 290, 10, 2585, 730, 734, 673, 922, 746, 653, 129, 290, 277, 970, 334, 737, 758, 482, 948, 180, 689, 526, 1104, 676, 1603, 1166, 646, 260, 866, 597, 261, 482, 634, 159, 796, 845, 505, 116, 398, 413, 539, 582, 803, 2053, 314, 494, 1871, 468, 1558, 674, 195, 751, 1612, 673] payload_rows_per_dst=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] dup_rows_per_dst=[5696, 4940, 1854, 3332, 1968, 2224, 2102, 2032] combine_rows_per_src=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=342992 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268278736 ingress_bytes=302538752 link_bytes=302538752 token_bytes=536214480 weight_movement_bytes=34603008 total_nvlink_bytes=570817488 weight_to_token_ratio=0.064532 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138428967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272179200 dispatch_meta_bytes=342776 combine_payload_bytes=272179200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272521976 ingress_bytes=306782208 link_bytes=306782208 token_bytes=544701176 weight_movement_bytes=34603008 total_nvlink_bytes=579304184 weight_to_token_ratio=0.063527 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154396751e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268623872 dispatch_meta_bytes=342736 combine_payload_bytes=268623872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=268966608 ingress_bytes=372432896 link_bytes=372432896 token_bytes=537590480 weight_movement_bytes=103809024 total_nvlink_bytes=641399504 weight_to_token_ratio=0.193101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401435004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260055040 dispatch_meta_bytes=334048 combine_payload_bytes=260055040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260389088 ingress_bytes=294658048 link_bytes=294658048 token_bytes=520444128 weight_movement_bytes=34603008 total_nvlink_bytes=555047136 weight_to_token_ratio=0.066487 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108774513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76818 payload_rows_remote=66244 dup_rows=21486 dup_rows_remote=16540 dedup_ratio=0.218567 router_tokens_per_expert=[242, 545, 54, 252, 3660, 123, 452, 791, 4, 494, 2861, 384, 457, 652, 225, 3193, 303, 1033, 499, 255, 1443, 355, 1563, 1112, 1022, 473, 740, 2483, 4, 131, 72, 148, 6, 2, 277, 4, 532, 10, 467, 804, 1096, 565, 276, 837, 15, 1602, 1, 175, 5628, 411, 3040, 0, 11, 508, 84, 2147, 2002, 363, 849, 112, 487, 128, 489, 199] payload_rows_per_dst=[9694, 5148, 10666, 10678, 10600, 10574, 8552, 10906] dup_rows_per_dst=[2544, 1256, 2460, 3218, 1120, 4946, 1510, 4432] combine_rows_per_src=[9694, 5148, 10666, 10678, 10600, 10574, 8552, 10906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325440 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257996608 ingress_bytes=326877184 link_bytes=326877184 token_bytes=515667776 weight_movement_bytes=69206016 total_nvlink_bytes=584873792 weight_to_token_ratio=0.134207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230012528e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=338192 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265234704 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530131216 weight_movement_bytes=103809024 total_nvlink_bytes=633940240 weight_to_token_ratio=0.195818 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80160 payload_rows_remote=71170 dup_rows=18144 dup_rows_remote=16682 dedup_ratio=0.184570 router_tokens_per_expert=[182, 547, 48, 255, 3590, 87, 475, 739, 8, 429, 3027, 334, 424, 697, 258, 3217, 332, 964, 513, 274, 1409, 388, 1575, 1156, 1149, 369, 726, 2661, 0, 140, 40, 141, 1, 7, 247, 6, 546, 11, 437, 664, 1033, 554, 245, 742, 13, 1782, 0, 167, 5855, 326, 3317, 0, 9, 512, 61, 2009, 1989, 383, 752, 24, 517, 102, 504, 183] payload_rows_per_dst=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] dup_rows_per_dst=[2336, 4756, 2596, 1462, 1536, 2130, 1912, 1416] combine_rows_per_src=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75320 payload_rows_remote=64136 dup_rows=22984 dup_rows_remote=18654 dedup_ratio=0.233805 router_tokens_per_expert=[157, 590, 44, 252, 3827, 55, 635, 658, 5, 408, 3044, 256, 407, 620, 241, 3533, 321, 990, 419, 242, 1346, 458, 1613, 846, 1148, 342, 686, 2769, 0, 79, 25, 169, 9, 3, 261, 2, 537, 13, 485, 754, 957, 547, 267, 826, 21, 1746, 0, 176, 5972, 296, 3135, 0, 12, 495, 62, 1769, 2098, 344, 836, 45, 475, 93, 559, 172] payload_rows_per_dst=[9896, 3180, 10178, 12094, 4068, 11058, 13662, 11184] dup_rows_per_dst=[2540, 694, 2292, 4430, 60, 5088, 3550, 4330] combine_rows_per_src=[9896, 3180, 10178, 12094, 4068, 11058, 13662, 11184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80478 payload_rows_remote=70624 dup_rows=17826 dup_rows_remote=15512 dedup_ratio=0.181335 router_tokens_per_expert=[170, 515, 33, 241, 3966, 70, 318, 702, 2, 410, 3173, 283, 426, 606, 222, 3426, 325, 973, 548, 262, 1441, 392, 1474, 928, 1197, 312, 684, 2840, 0, 119, 20, 135, 14, 3, 236, 1, 527, 1, 464, 644, 1005, 499, 245, 770, 6, 1712, 0, 163, 6094, 292, 3393, 0, 0, 477, 117, 1817, 2056, 392, 750, 24, 445, 89, 536, 167] payload_rows_per_dst=[9854, 12388, 9854, 9166, 14392, 6818, 10348, 7658] dup_rows_per_dst=[2176, 5226, 2314, 1448, 1576, 1982, 1844, 1260] combine_rows_per_src=[9854, 12388, 9854, 9166, 14392, 6818, 10348, 7658] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76818 payload_rows_remote=66244 dup_rows=21486 dup_rows_remote=16540 dedup_ratio=0.218567 router_tokens_per_expert=[242, 545, 54, 252, 3660, 123, 452, 791, 4, 494, 2861, 384, 457, 652, 225, 3193, 303, 1033, 499, 255, 1443, 355, 1563, 1112, 1022, 473, 740, 2483, 4, 131, 72, 148, 6, 2, 277, 4, 532, 10, 467, 804, 1096, 565, 276, 837, 15, 1602, 1, 175, 5628, 411, 3040, 0, 11, 508, 84, 2147, 2002, 363, 849, 112, 487, 128, 489, 199] payload_rows_per_dst=[9694, 5148, 10666, 10678, 10600, 10574, 8552, 10906] dup_rows_per_dst=[2544, 1256, 2460, 3218, 1120, 4946, 1510, 4432] combine_rows_per_src=[9694, 5148, 10666, 10678, 10600, 10574, 8552, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=62908 dup_rows=24148 dup_rows_remote=18452 dedup_ratio=0.245646 router_tokens_per_expert=[1423, 604, 588, 1177, 485, 1407, 1265, 174, 2062, 922, 1268, 290, 10, 2585, 730, 734, 673, 922, 746, 653, 129, 290, 277, 970, 334, 737, 758, 482, 948, 180, 689, 526, 1104, 676, 1603, 1166, 646, 260, 866, 597, 261, 482, 634, 159, 796, 845, 505, 116, 398, 413, 539, 582, 803, 2053, 314, 494, 1871, 468, 1558, 674, 195, 751, 1612, 673] payload_rows_per_dst=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] dup_rows_per_dst=[5696, 4940, 1854, 3332, 1968, 2224, 2102, 2032] combine_rows_per_src=[11248, 9952, 7974, 9718, 8662, 7682, 9090, 9830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=351408 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291863728 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583376048 weight_movement_bytes=34603008 total_nvlink_bytes=617979056 weight_to_token_ratio=0.059315 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78578 payload_rows_remote=65766 dup_rows=19726 dup_rows_remote=12944 dedup_ratio=0.200663 router_tokens_per_expert=[240, 566, 47, 286, 3717, 77, 560, 798, 6, 468, 3026, 355, 445, 633, 286, 3047, 339, 1034, 533, 237, 1509, 437, 1531, 987, 1146, 374, 665, 2381, 0, 133, 39, 164, 9, 7, 260, 6, 476, 15, 569, 672, 1023, 508, 254, 731, 18, 1769, 0, 189, 5856, 384, 3164, 0, 12, 546, 96, 2005, 1991, 372, 841, 46, 494, 97, 492, 184] payload_rows_per_dst=[9970, 12812, 8456, 8400, 13998, 6926, 10340, 7676] dup_rows_per_dst=[2612, 6782, 1696, 1404, 1742, 2058, 2074, 1358] combine_rows_per_src=[9970, 12812, 8456, 8400, 13998, 6926, 10340, 7676] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75824 payload_rows_remote=62004 dup_rows=22480 dup_rows_remote=19020 dedup_ratio=0.228678 router_tokens_per_expert=[301, 587, 43, 217, 3886, 64, 368, 753, 10, 363, 3143, 380, 405, 540, 214, 3177, 314, 1137, 572, 249, 1330, 378, 1650, 1008, 1148, 345, 697, 2767, 1, 140, 31, 158, 7, 1, 262, 5, 522, 6, 533, 763, 990, 508, 263, 735, 7, 1608, 1, 211, 5893, 283, 3223, 0, 14, 467, 91, 1892, 1933, 402, 783, 47, 524, 88, 510, 204] payload_rows_per_dst=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] dup_rows_per_dst=[2466, 626, 2684, 4474, 82, 4622, 3460, 4066] combine_rows_per_src=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262701056 dispatch_meta_bytes=331160 combine_payload_bytes=262701056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263032216 ingress_bytes=297304064 link_bytes=297304064 token_bytes=525733272 weight_movement_bytes=34603008 total_nvlink_bytes=560336280 weight_to_token_ratio=0.065819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118731258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=289275904 dispatch_meta_bytes=344544 combine_payload_bytes=289275904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289620448 ingress_bytes=358481920 link_bytes=358481920 token_bytes=578896352 weight_movement_bytes=69206016 total_nvlink_bytes=648102368 weight_to_token_ratio=0.119548 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348938605e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271335424 dispatch_meta_bytes=331136 combine_payload_bytes=271335424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271666560 ingress_bytes=305938432 link_bytes=305938432 token_bytes=543001984 weight_movement_bytes=34603008 total_nvlink_bytes=577604992 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151221690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79228 payload_rows_remote=65014 dup_rows=19076 dup_rows_remote=17318 dedup_ratio=0.194051 router_tokens_per_expert=[157, 585, 36, 247, 3906, 36, 378, 675, 10, 407, 3125, 302, 390, 681, 285, 3326, 377, 1027, 489, 246, 1543, 414, 1401, 1188, 1059, 397, 692, 2498, 2, 127, 33, 142, 3, 1, 253, 4, 464, 31, 565, 707, 1005, 491, 252, 749, 15, 1755, 0, 151, 5958, 328, 3226, 0, 11, 623, 60, 1826, 2089, 365, 792, 42, 469, 115, 454, 167] payload_rows_per_dst=[9816, 8296, 10746, 8518, 14214, 10908, 5512, 11218] dup_rows_per_dst=[2224, 2104, 2624, 1382, 1758, 4580, 184, 4220] combine_rows_per_src=[9816, 8296, 10746, 8518, 14214, 10908, 5512, 11218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75320 payload_rows_remote=64136 dup_rows=22984 dup_rows_remote=18654 dedup_ratio=0.233805 router_tokens_per_expert=[157, 590, 44, 252, 3827, 55, 635, 658, 5, 408, 3044, 256, 407, 620, 241, 3533, 321, 990, 419, 242, 1346, 458, 1613, 846, 1148, 342, 686, 2769, 0, 79, 25, 169, 9, 3, 261, 2, 537, 13, 485, 754, 957, 547, 267, 826, 21, 1746, 0, 176, 5972, 296, 3135, 0, 12, 495, 62, 1769, 2098, 344, 836, 45, 475, 93, 559, 172] payload_rows_per_dst=[9896, 3180, 10178, 12094, 4068, 11058, 13662, 11184] dup_rows_per_dst=[2540, 694, 2292, 4430, 60, 5088, 3550, 4330] combine_rows_per_src=[9896, 3180, 10178, 12094, 4068, 11058, 13662, 11184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80160 payload_rows_remote=71170 dup_rows=18144 dup_rows_remote=16682 dedup_ratio=0.184570 router_tokens_per_expert=[182, 547, 48, 255, 3590, 87, 475, 739, 8, 429, 3027, 334, 424, 697, 258, 3217, 332, 964, 513, 274, 1409, 388, 1575, 1156, 1149, 369, 726, 2661, 0, 140, 40, 141, 1, 7, 247, 6, 546, 11, 437, 664, 1033, 554, 245, 742, 13, 1782, 0, 167, 5855, 326, 3317, 0, 9, 512, 61, 2009, 1989, 383, 752, 24, 517, 102, 504, 183] payload_rows_per_dst=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] dup_rows_per_dst=[2336, 4756, 2596, 1462, 1536, 2130, 1912, 1416] combine_rows_per_src=[9510, 11782, 10626, 8990, 14012, 7192, 10158, 7890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325440 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257996608 ingress_bytes=326877184 link_bytes=326877184 token_bytes=515667776 weight_movement_bytes=69206016 total_nvlink_bytes=584873792 weight_to_token_ratio=0.134207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230012528e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75824 payload_rows_remote=62004 dup_rows=22480 dup_rows_remote=19020 dedup_ratio=0.228678 router_tokens_per_expert=[301, 587, 43, 217, 3886, 64, 368, 753, 10, 363, 3143, 380, 405, 540, 214, 3177, 314, 1137, 572, 249, 1330, 378, 1650, 1008, 1148, 345, 697, 2767, 1, 140, 31, 158, 7, 1, 262, 5, 522, 6, 533, 763, 990, 508, 263, 735, 7, 1608, 1, 211, 5893, 283, 3223, 0, 14, 467, 91, 1892, 1933, 402, 783, 47, 524, 88, 510, 204] payload_rows_per_dst=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] dup_rows_per_dst=[2466, 626, 2684, 4474, 82, 4622, 3460, 4066] combine_rows_per_src=[9972, 3198, 10592, 12386, 4116, 10378, 13820, 11362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=329328 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266626672 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532924016 weight_movement_bytes=34603008 total_nvlink_bytes=567527024 weight_to_token_ratio=0.064930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271335424 dispatch_meta_bytes=331136 combine_payload_bytes=271335424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271666560 ingress_bytes=305938432 link_bytes=305938432 token_bytes=543001984 weight_movement_bytes=34603008 total_nvlink_bytes=577604992 weight_to_token_ratio=0.063725 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151221690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=289275904 dispatch_meta_bytes=344544 combine_payload_bytes=289275904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289620448 ingress_bytes=358481920 link_bytes=358481920 token_bytes=578896352 weight_movement_bytes=69206016 total_nvlink_bytes=648102368 weight_to_token_ratio=0.119548 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348938605e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269377536 dispatch_meta_bytes=314840 combine_payload_bytes=269377536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269692376 ingress_bytes=373186560 link_bytes=373186560 token_bytes=539069912 weight_movement_bytes=103809024 total_nvlink_bytes=642878936 weight_to_token_ratio=0.192571 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.404270981e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262701056 dispatch_meta_bytes=331160 combine_payload_bytes=262701056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263032216 ingress_bytes=297304064 link_bytes=297304064 token_bytes=525733272 weight_movement_bytes=34603008 total_nvlink_bytes=560336280 weight_to_token_ratio=0.065819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118731258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=351408 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291863728 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583376048 weight_movement_bytes=34603008 total_nvlink_bytes=617979056 weight_to_token_ratio=0.059315 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=253968384 dispatch_meta_bytes=324096 combine_payload_bytes=253968384 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=254292480 ingress_bytes=392380416 link_bytes=392380416 token_bytes=508260864 weight_movement_bytes=138412032 total_nvlink_bytes=646672896 weight_to_token_ratio=0.272325 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78048 payload_rows_remote=68206 dup_rows=20256 dup_rows_remote=17404 dedup_ratio=0.206055 router_tokens_per_expert=[257, 623, 61, 279, 3620, 107, 378, 772, 6, 459, 3006, 323, 447, 668, 273, 3221, 418, 1052, 539, 271, 1449, 434, 1625, 1049, 1081, 354, 740, 2543, 2, 161, 83, 163, 8, 6, 250, 10, 532, 16, 458, 744, 1108, 522, 258, 750, 21, 1688, 1, 174, 5491, 376, 2782, 1, 13, 610, 157, 2048, 2049, 384, 818, 64, 559, 95, 519, 176] payload_rows_per_dst=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] dup_rows_per_dst=[2852, 7096, 1840, 1544, 1624, 2072, 1766, 1462] combine_rows_per_src=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78314 payload_rows_remote=70912 dup_rows=19990 dup_rows_remote=18190 dedup_ratio=0.203349 router_tokens_per_expert=[0, 1052, 87, 155, 0, 133, 127, 391, 54, 987, 629, 2413, 576, 332, 0, 0, 1666, 1668, 1696, 203, 0, 0, 4035, 959, 1037, 1635, 2851, 3892, 0, 63, 205, 0, 123, 798, 102, 68, 6219, 993, 188, 110, 789, 0, 5, 17, 818, 1644, 97, 96, 96, 0, 544, 620, 0, 0, 111, 2, 2993, 362, 0, 10, 0, 2671, 0, 2830] payload_rows_per_dst=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] dup_rows_per_dst=[1798, 1170, 2050, 2214, 1800, 3698, 1946, 5314] combine_rows_per_src=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=329328 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266626672 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532924016 weight_movement_bytes=34603008 total_nvlink_bytes=567527024 weight_to_token_ratio=0.064930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79454 payload_rows_remote=70030 dup_rows=18850 dup_rows_remote=16602 dedup_ratio=0.191752 router_tokens_per_expert=[0, 1018, 102, 117, 0, 132, 129, 352, 55, 1054, 551, 2415, 497, 355, 0, 0, 1619, 1780, 1566, 156, 0, 0, 4029, 956, 876, 1618, 3040, 4075, 0, 60, 242, 0, 122, 867, 171, 84, 6338, 909, 219, 95, 719, 0, 2, 15, 607, 1791, 80, 81, 124, 0, 472, 578, 0, 0, 116, 1, 2974, 305, 0, 7, 0, 2618, 0, 3063] payload_rows_per_dst=[10094, 9400, 9562, 9424, 13448, 5688, 9962, 11876] dup_rows_per_dst=[1756, 1930, 2592, 2248, 4162, 902, 678, 4582] combine_rows_per_src=[10094, 9400, 9562, 9424, 13448, 5688, 9962, 11876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77686 payload_rows_remote=65508 dup_rows=20618 dup_rows_remote=15806 dedup_ratio=0.209737 router_tokens_per_expert=[0, 987, 76, 177, 0, 97, 156, 298, 54, 1022, 655, 2247, 528, 316, 0, 0, 1917, 1714, 1694, 156, 3, 0, 3880, 1138, 907, 1748, 2929, 4216, 0, 38, 237, 0, 227, 940, 124, 63, 6441, 1041, 193, 98, 607, 0, 1, 16, 506, 1714, 63, 61, 45, 1, 721, 535, 0, 0, 70, 0, 2785, 319, 0, 14, 0, 2725, 0, 2652] payload_rows_per_dst=[10112, 8480, 7858, 9434, 4446, 14324, 10854, 12178] dup_rows_per_dst=[1902, 1164, 1552, 2284, 926, 4494, 3484, 4812] combine_rows_per_src=[10112, 8480, 7858, 9434, 4446, 14324, 10854, 12178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77504 payload_rows_remote=63580 dup_rows=20800 dup_rows_remote=16492 dedup_ratio=0.211589 router_tokens_per_expert=[0, 832, 155, 145, 0, 135, 124, 423, 44, 1179, 605, 2291, 560, 326, 0, 1, 1869, 1518, 1662, 249, 5, 0, 3820, 999, 1026, 1491, 2958, 3886, 4, 71, 241, 0, 220, 842, 198, 88, 6100, 1220, 259, 117, 600, 0, 8, 9, 880, 1384, 70, 65, 39, 1, 873, 620, 0, 1, 100, 4, 2811, 417, 0, 3, 0, 2745, 1, 2858] payload_rows_per_dst=[9668, 8744, 7528, 9258, 4880, 13924, 11062, 12440] dup_rows_per_dst=[1732, 1268, 1338, 2324, 1008, 4308, 3592, 5230] combine_rows_per_src=[9668, 8744, 7528, 9258, 4880, 13924, 11062, 12440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79864 payload_rows_remote=69606 dup_rows=18440 dup_rows_remote=15336 dedup_ratio=0.187581 router_tokens_per_expert=[0, 900, 103, 146, 0, 136, 112, 296, 44, 1154, 592, 2429, 468, 392, 0, 0, 1759, 1623, 1730, 153, 3, 0, 3925, 916, 844, 1722, 2972, 4225, 0, 66, 215, 0, 136, 853, 68, 38, 6507, 932, 183, 108, 655, 0, 3, 10, 561, 1761, 77, 75, 14, 0, 680, 597, 0, 0, 121, 0, 2859, 305, 1, 5, 0, 2701, 0, 2977] payload_rows_per_dst=[10148, 11720, 10258, 9382, 13588, 5430, 9912, 9426] dup_rows_per_dst=[1688, 4156, 3104, 2256, 4064, 854, 762, 1556] combine_rows_per_src=[10148, 11720, 10258, 9382, 13588, 5430, 9912, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279371776 dispatch_meta_bytes=342440 combine_payload_bytes=279371776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279714216 ingress_bytes=313974784 link_bytes=313974784 token_bytes=559085992 weight_movement_bytes=34603008 total_nvlink_bytes=593689000 weight_to_token_ratio=0.061892 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181461835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79454 payload_rows_remote=70030 dup_rows=18850 dup_rows_remote=16602 dedup_ratio=0.191752 router_tokens_per_expert=[0, 1018, 102, 117, 0, 132, 129, 352, 55, 1054, 551, 2415, 497, 355, 0, 0, 1619, 1780, 1566, 156, 0, 0, 4029, 956, 876, 1618, 3040, 4075, 0, 60, 242, 0, 122, 867, 171, 84, 6338, 909, 219, 95, 719, 0, 2, 15, 607, 1791, 80, 81, 124, 0, 472, 578, 0, 0, 116, 1, 2974, 305, 0, 7, 0, 2618, 0, 3063] payload_rows_per_dst=[10094, 9400, 9562, 9424, 13448, 5688, 9962, 11876] dup_rows_per_dst=[1756, 1930, 2592, 2248, 4162, 902, 678, 4582] combine_rows_per_src=[10094, 9400, 9562, 9424, 13448, 5688, 9962, 11876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286842880 dispatch_meta_bytes=346528 combine_payload_bytes=286842880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287189408 ingress_bytes=390651904 link_bytes=390651904 token_bytes=574032288 weight_movement_bytes=103809024 total_nvlink_bytes=677841312 weight_to_token_ratio=0.180842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469991665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74810 payload_rows_remote=63114 dup_rows=23494 dup_rows_remote=19350 dedup_ratio=0.238993 router_tokens_per_expert=[1, 1015, 70, 166, 0, 149, 124, 350, 40, 959, 552, 2351, 585, 340, 1, 0, 1660, 1738, 1661, 172, 1, 0, 3894, 910, 1040, 1623, 2964, 3815, 1, 74, 221, 0, 202, 920, 130, 68, 6138, 1122, 243, 114, 703, 0, 43, 21, 832, 1612, 83, 82, 81, 0, 581, 721, 0, 0, 108, 6, 3092, 375, 5, 6, 0, 2674, 0, 2713] payload_rows_per_dst=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] dup_rows_per_dst=[1780, 4144, 5186, 4770, 5742, 1036, 836, 0] combine_rows_per_src=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77686 payload_rows_remote=65508 dup_rows=20618 dup_rows_remote=15806 dedup_ratio=0.209737 router_tokens_per_expert=[0, 987, 76, 177, 0, 97, 156, 298, 54, 1022, 655, 2247, 528, 316, 0, 0, 1917, 1714, 1694, 156, 3, 0, 3880, 1138, 907, 1748, 2929, 4216, 0, 38, 237, 0, 227, 940, 124, 63, 6441, 1041, 193, 98, 607, 0, 1, 16, 506, 1714, 63, 61, 45, 1, 721, 535, 0, 0, 70, 0, 2785, 319, 0, 14, 0, 2725, 0, 2652] payload_rows_per_dst=[10112, 8480, 7858, 9434, 4446, 14324, 10854, 12178] dup_rows_per_dst=[1902, 1164, 1552, 2284, 926, 4494, 3484, 4812] combine_rows_per_src=[10112, 8480, 7858, 9434, 4446, 14324, 10854, 12178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290455552 dispatch_meta_bytes=356408 combine_payload_bytes=290455552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290811960 ingress_bytes=359661568 link_bytes=359661568 token_bytes=581267512 weight_movement_bytes=69206016 total_nvlink_bytes=650473528 weight_to_token_ratio=0.119061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353377525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268320768 dispatch_meta_bytes=325256 combine_payload_bytes=268320768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268646024 ingress_bytes=406732800 link_bytes=406732800 token_bytes=536966792 weight_movement_bytes=138412032 total_nvlink_bytes=675378824 weight_to_token_ratio=0.257766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530502782e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74810 payload_rows_remote=63114 dup_rows=23494 dup_rows_remote=19350 dedup_ratio=0.238993 router_tokens_per_expert=[1, 1015, 70, 166, 0, 149, 124, 350, 40, 959, 552, 2351, 585, 340, 1, 0, 1660, 1738, 1661, 172, 1, 0, 3894, 910, 1040, 1623, 2964, 3815, 1, 74, 221, 0, 202, 920, 130, 68, 6138, 1122, 243, 114, 703, 0, 43, 21, 832, 1612, 83, 82, 81, 0, 581, 721, 0, 0, 108, 6, 3092, 375, 5, 6, 0, 2674, 0, 2713] payload_rows_per_dst=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] dup_rows_per_dst=[1780, 4144, 5186, 4770, 5742, 1036, 836, 0] combine_rows_per_src=[9600, 11696, 12524, 10944, 13612, 5716, 9946, 772] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78084 payload_rows_remote=67166 dup_rows=20220 dup_rows_remote=17112 dedup_ratio=0.205688 router_tokens_per_expert=[0, 915, 90, 155, 0, 132, 127, 311, 62, 990, 655, 2656, 502, 357, 0, 0, 1723, 1649, 1815, 156, 0, 0, 3874, 847, 884, 1647, 2980, 4191, 0, 91, 211, 1, 195, 900, 139, 46, 6414, 1081, 204, 85, 640, 0, 3, 17, 850, 1481, 85, 88, 20, 0, 574, 710, 0, 0, 112, 0, 2883, 271, 2, 5, 0, 2709, 0, 2617] payload_rows_per_dst=[10086, 9114, 7584, 9274, 4550, 14486, 10918, 12072] dup_rows_per_dst=[1756, 1330, 1350, 2354, 750, 4670, 3108, 4902] combine_rows_per_src=[10086, 9114, 7584, 9274, 4550, 14486, 10918, 12072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=260423680 dispatch_meta_bytes=320288 combine_payload_bytes=260423680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260743968 ingress_bytes=295026688 link_bytes=295026688 token_bytes=521167648 weight_movement_bytes=34603008 total_nvlink_bytes=555770656 weight_to_token_ratio=0.066395 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110161675e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=285106176 dispatch_meta_bytes=339768 combine_payload_bytes=285106176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=285445944 ingress_bytes=388915200 link_bytes=388915200 token_bytes=570552120 weight_movement_bytes=103809024 total_nvlink_bytes=674361144 weight_to_token_ratio=0.181945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.463456587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78314 payload_rows_remote=70912 dup_rows=19990 dup_rows_remote=18190 dedup_ratio=0.203349 router_tokens_per_expert=[0, 1052, 87, 155, 0, 133, 127, 391, 54, 987, 629, 2413, 576, 332, 0, 0, 1666, 1668, 1696, 203, 0, 0, 4035, 959, 1037, 1635, 2851, 3892, 0, 63, 205, 0, 123, 798, 102, 68, 6219, 993, 188, 110, 789, 0, 5, 17, 818, 1644, 97, 96, 96, 0, 544, 620, 0, 0, 111, 2, 2993, 362, 0, 10, 0, 2671, 0, 2830] payload_rows_per_dst=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] dup_rows_per_dst=[1798, 1170, 2050, 2214, 1800, 3698, 1946, 5314] combine_rows_per_src=[9876, 8812, 8832, 9368, 7402, 11234, 10372, 12418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78048 payload_rows_remote=68206 dup_rows=20256 dup_rows_remote=17404 dedup_ratio=0.206055 router_tokens_per_expert=[257, 623, 61, 279, 3620, 107, 378, 772, 6, 459, 3006, 323, 447, 668, 273, 3221, 418, 1052, 539, 271, 1449, 434, 1625, 1049, 1081, 354, 740, 2543, 2, 161, 83, 163, 8, 6, 250, 10, 532, 16, 458, 744, 1108, 522, 258, 750, 21, 1688, 1, 174, 5491, 376, 2782, 1, 13, 610, 157, 2048, 2049, 384, 818, 64, 559, 95, 519, 176] payload_rows_per_dst=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] dup_rows_per_dst=[2852, 7096, 1840, 1544, 1624, 2072, 1766, 1462] combine_rows_per_src=[9842, 12960, 8680, 8710, 13406, 6972, 9612, 7866] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268320768 dispatch_meta_bytes=325256 combine_payload_bytes=268320768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=268646024 ingress_bytes=406732800 link_bytes=406732800 token_bytes=536966792 weight_movement_bytes=138412032 total_nvlink_bytes=675378824 weight_to_token_ratio=0.257766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530502782e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=286842880 dispatch_meta_bytes=346528 combine_payload_bytes=286842880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287189408 ingress_bytes=390651904 link_bytes=390651904 token_bytes=574032288 weight_movement_bytes=103809024 total_nvlink_bytes=677841312 weight_to_token_ratio=0.180842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469991665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258514944 dispatch_meta_bytes=329856 combine_payload_bytes=258514944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258844800 ingress_bytes=293117952 link_bytes=293117952 token_bytes=517359744 weight_movement_bytes=34603008 total_nvlink_bytes=551962752 weight_to_token_ratio=0.066884 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102979255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76370 payload_rows_remote=64278 dup_rows=21934 dup_rows_remote=17718 dedup_ratio=0.223124 router_tokens_per_expert=[0, 804, 91, 159, 0, 176, 159, 366, 39, 1060, 554, 2388, 544, 346, 2, 1, 1628, 1570, 1581, 188, 3, 2, 3887, 887, 816, 1622, 3118, 4020, 3, 71, 242, 0, 243, 916, 177, 70, 6207, 1002, 240, 108, 676, 3, 34, 16, 728, 2128, 94, 91, 161, 2, 580, 645, 1, 0, 155, 5, 2654, 345, 3, 10, 5, 2892, 0, 2634] payload_rows_per_dst=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] dup_rows_per_dst=[4216, 3602, 4790, 2570, 4546, 1304, 904, 2] combine_rows_per_src=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275111936 dispatch_meta_bytes=337112 combine_payload_bytes=275111936 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275449048 ingress_bytes=344317952 link_bytes=344317952 token_bytes=550560984 weight_movement_bytes=69206016 total_nvlink_bytes=619767000 weight_to_token_ratio=0.125701 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295640734e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83874 payload_rows_remote=68912 dup_rows=14430 dup_rows_remote=13904 dedup_ratio=0.146790 router_tokens_per_expert=[0, 0, 158, 0, 1367, 1047, 1173, 298, 4852, 193, 356, 1, 258, 0, 7068, 43, 2248, 0, 756, 318, 444, 219, 0, 81, 328, 1705, 428, 349, 188, 2633, 474, 62, 0, 2, 94, 0, 433, 21, 126, 0, 22, 4529, 92, 334, 99, 0, 0, 5193, 0, 108, 94, 621, 0, 3448, 0, 0, 0, 0, 913, 0, 2177, 3565, 0, 234] payload_rows_per_dst=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] dup_rows_per_dst=[1478, 564, 1388, 1730, 526, 2770, 3482, 2492] combine_rows_per_src=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263282688 dispatch_meta_bytes=327984 combine_payload_bytes=263282688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=263610672 ingress_bytes=332488704 link_bytes=332488704 token_bytes=526893360 weight_movement_bytes=69206016 total_nvlink_bytes=596099376 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251128226e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83888 payload_rows_remote=72988 dup_rows=14416 dup_rows_remote=12202 dedup_ratio=0.146647 router_tokens_per_expert=[0, 0, 178, 0, 1411, 956, 1243, 253, 4783, 200, 349, 0, 252, 0, 7107, 31, 2544, 2, 770, 314, 367, 209, 0, 84, 418, 1604, 514, 367, 210, 2865, 486, 93, 0, 3, 80, 0, 491, 39, 157, 0, 16, 4180, 82, 378, 99, 0, 0, 5163, 0, 99, 80, 592, 1, 3217, 0, 0, 0, 0, 874, 0, 2220, 3545, 0, 226] payload_rows_per_dst=[11318, 10656, 7212, 10900, 15010, 9938, 7708, 11146] dup_rows_per_dst=[4234, 574, 1368, 2214, 744, 2428, 270, 2584] combine_rows_per_src=[11318, 10656, 7212, 10900, 15010, 9938, 7708, 11146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=290455552 dispatch_meta_bytes=356408 combine_payload_bytes=290455552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=290811960 ingress_bytes=359661568 link_bytes=359661568 token_bytes=581267512 weight_movement_bytes=69206016 total_nvlink_bytes=650473528 weight_to_token_ratio=0.119061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353377525e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=279371776 dispatch_meta_bytes=342440 combine_payload_bytes=279371776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279714216 ingress_bytes=313974784 link_bytes=313974784 token_bytes=559085992 weight_movement_bytes=34603008 total_nvlink_bytes=593689000 weight_to_token_ratio=0.061892 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181461835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78708 payload_rows_remote=68116 dup_rows=19596 dup_rows_remote=17644 dedup_ratio=0.199341 router_tokens_per_expert=[0, 0, 153, 0, 1469, 813, 1147, 193, 4859, 147, 320, 0, 273, 0, 7256, 19, 2892, 6, 800, 302, 538, 203, 0, 50, 233, 1534, 485, 323, 191, 2869, 543, 38, 1, 3, 62, 0, 412, 40, 115, 0, 16, 4486, 79, 412, 74, 0, 1, 5261, 0, 141, 63, 531, 4, 3523, 0, 0, 0, 0, 781, 0, 1888, 3390, 1, 212] payload_rows_per_dst=[6250, 1506, 12892, 6522, 15296, 13256, 12394, 10592] dup_rows_per_dst=[1300, 12, 6408, 172, 482, 4168, 5102, 1952] combine_rows_per_src=[6250, 1506, 12892, 6522, 15296, 13256, 12394, 10592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=298958848 dispatch_meta_bytes=340760 combine_payload_bytes=298958848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299299608 ingress_bytes=368164864 link_bytes=368164864 token_bytes=598258456 weight_movement_bytes=69206016 total_nvlink_bytes=667464472 weight_to_token_ratio=0.115679 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385374744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80998 payload_rows_remote=70222 dup_rows=17306 dup_rows_remote=16456 dedup_ratio=0.176046 router_tokens_per_expert=[1, 0, 165, 0, 1368, 858, 1287, 194, 4591, 230, 399, 1, 285, 1, 7088, 39, 2167, 2, 773, 365, 486, 198, 0, 64, 604, 1476, 575, 401, 211, 2859, 438, 35, 0, 8, 70, 0, 440, 38, 172, 0, 24, 4582, 80, 400, 86, 1, 0, 5222, 0, 103, 70, 715, 0, 3367, 0, 0, 0, 0, 861, 3, 2140, 3364, 0, 245] payload_rows_per_dst=[6426, 1900, 12236, 10878, 15086, 10776, 12622, 11074] dup_rows_per_dst=[1320, 10, 5056, 2320, 546, 850, 5052, 2152] combine_rows_per_src=[6426, 1900, 12236, 10878, 15086, 10776, 12622, 11074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282263552 dispatch_meta_bytes=331264 combine_payload_bytes=282263552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282594816 ingress_bytes=316866560 link_bytes=316866560 token_bytes=564858368 weight_movement_bytes=34603008 total_nvlink_bytes=599461376 weight_to_token_ratio=0.061260 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192343355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86156 payload_rows_remote=78752 dup_rows=12148 dup_rows_remote=10716 dedup_ratio=0.123576 router_tokens_per_expert=[0, 0, 123, 0, 1309, 806, 1117, 260, 5028, 179, 267, 0, 251, 0, 7110, 16, 2553, 1, 756, 313, 568, 187, 0, 40, 295, 1465, 480, 410, 185, 3086, 452, 68, 0, 2, 72, 0, 417, 28, 136, 2, 27, 4602, 77, 378, 87, 0, 0, 4793, 0, 78, 76, 645, 0, 3400, 0, 0, 0, 0, 899, 0, 2030, 3840, 0, 238] payload_rows_per_dst=[12634, 11050, 7404, 11006, 15024, 9268, 8180, 11590] dup_rows_per_dst=[4182, 432, 1432, 1876, 510, 1074, 218, 2424] combine_rows_per_src=[12634, 11050, 7404, 11006, 15024, 9268, 8180, 11590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279003136 dispatch_meta_bytes=343040 combine_payload_bytes=279003136 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279346176 ingress_bytes=348209152 link_bytes=348209152 token_bytes=558349312 weight_movement_bytes=69206016 total_nvlink_bytes=627555328 weight_to_token_ratio=0.123948 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310283006e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83888 payload_rows_remote=72988 dup_rows=14416 dup_rows_remote=12202 dedup_ratio=0.146647 router_tokens_per_expert=[0, 0, 178, 0, 1411, 956, 1243, 253, 4783, 200, 349, 0, 252, 0, 7107, 31, 2544, 2, 770, 314, 367, 209, 0, 84, 418, 1604, 514, 367, 210, 2865, 486, 93, 0, 3, 80, 0, 491, 39, 157, 0, 16, 4180, 82, 378, 99, 0, 0, 5163, 0, 99, 80, 592, 1, 3217, 0, 0, 0, 0, 874, 0, 2220, 3545, 0, 226] payload_rows_per_dst=[11318, 10656, 7212, 10900, 15010, 9938, 7708, 11146] dup_rows_per_dst=[4234, 574, 1368, 2214, 744, 2428, 270, 2584] combine_rows_per_src=[11318, 10656, 7212, 10900, 15010, 9938, 7708, 11146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78708 payload_rows_remote=68116 dup_rows=19596 dup_rows_remote=17644 dedup_ratio=0.199341 router_tokens_per_expert=[0, 0, 153, 0, 1469, 813, 1147, 193, 4859, 147, 320, 0, 273, 0, 7256, 19, 2892, 6, 800, 302, 538, 203, 0, 50, 233, 1534, 485, 323, 191, 2869, 543, 38, 1, 3, 62, 0, 412, 40, 115, 0, 16, 4486, 79, 412, 74, 0, 1, 5261, 0, 141, 63, 531, 4, 3523, 0, 0, 0, 0, 781, 0, 1888, 3390, 1, 212] payload_rows_per_dst=[6250, 1506, 12892, 6522, 15296, 13256, 12394, 10592] dup_rows_per_dst=[1300, 12, 6408, 172, 482, 4168, 5102, 1952] combine_rows_per_src=[6250, 1506, 12892, 6522, 15296, 13256, 12394, 10592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73318 payload_rows_remote=58144 dup_rows=24986 dup_rows_remote=12610 dedup_ratio=0.254171 router_tokens_per_expert=[0, 0, 186, 0, 1205, 1094, 1224, 259, 4753, 221, 376, 1, 268, 0, 6985, 31, 2378, 6, 817, 350, 467, 247, 0, 95, 630, 1587, 495, 377, 207, 2961, 545, 91, 0, 11, 82, 0, 540, 46, 159, 1, 23, 4026, 80, 338, 100, 1, 1, 4901, 0, 121, 84, 635, 4, 2888, 0, 0, 2, 0, 990, 1, 2278, 3730, 0, 254] payload_rows_per_dst=[12850, 15174, 7224, 11886, 2468, 8132, 7196, 8388] dup_rows_per_dst=[4888, 12376, 1496, 3830, 48, 1006, 268, 1074] combine_rows_per_src=[12850, 15174, 7224, 11886, 2468, 8132, 7196, 8388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73318 payload_rows_remote=58144 dup_rows=24986 dup_rows_remote=12610 dedup_ratio=0.254171 router_tokens_per_expert=[0, 0, 186, 0, 1205, 1094, 1224, 259, 4753, 221, 376, 1, 268, 0, 6985, 31, 2378, 6, 817, 350, 467, 247, 0, 95, 630, 1587, 495, 377, 207, 2961, 545, 91, 0, 11, 82, 0, 540, 46, 159, 1, 23, 4026, 80, 338, 100, 1, 1, 4901, 0, 121, 84, 635, 4, 2888, 0, 0, 2, 0, 990, 1, 2278, 3730, 0, 254] payload_rows_per_dst=[12850, 15174, 7224, 11886, 2468, 8132, 7196, 8388] dup_rows_per_dst=[4888, 12376, 1496, 3830, 48, 1006, 268, 1074] combine_rows_per_src=[12850, 15174, 7224, 11886, 2468, 8132, 7196, 8388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=287629312 dispatch_meta_bytes=346712 combine_payload_bytes=287629312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287976024 ingress_bytes=391438336 link_bytes=391438336 token_bytes=575605336 weight_movement_bytes=103809024 total_nvlink_bytes=679414360 weight_to_token_ratio=0.180348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.472950945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79346 payload_rows_remote=67618 dup_rows=18958 dup_rows_remote=14264 dedup_ratio=0.192851 router_tokens_per_expert=[0, 0, 142, 0, 1187, 1026, 1171, 259, 4854, 232, 273, 0, 269, 0, 7042, 30, 2455, 11, 777, 373, 423, 267, 0, 67, 330, 1462, 500, 388, 197, 3007, 505, 82, 0, 5, 81, 0, 595, 42, 170, 0, 29, 3967, 56, 335, 99, 0, 0, 4759, 0, 110, 85, 646, 1, 3402, 0, 0, 0, 0, 948, 0, 2388, 3808, 0, 297] payload_rows_per_dst=[6236, 1590, 12574, 9266, 15022, 11058, 11728, 11872] dup_rows_per_dst=[1334, 18, 5880, 1358, 848, 1816, 4694, 3010] combine_rows_per_src=[6236, 1590, 12574, 9266, 15022, 11058, 11728, 11872] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=322568192 dispatch_meta_bytes=357872 combine_payload_bytes=322568192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322926064 ingress_bytes=357171200 link_bytes=357171200 token_bytes=645494256 weight_movement_bytes=34603008 total_nvlink_bytes=680097264 weight_to_token_ratio=0.053607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344006471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=298958848 dispatch_meta_bytes=340760 combine_payload_bytes=298958848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=299299608 ingress_bytes=368164864 link_bytes=368164864 token_bytes=598258456 weight_movement_bytes=69206016 total_nvlink_bytes=667464472 weight_to_token_ratio=0.115679 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385374744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71802 payload_rows_remote=59050 dup_rows=26502 dup_rows_remote=22008 dedup_ratio=0.269592 router_tokens_per_expert=[0, 0, 155, 2, 1224, 877, 1229, 275, 4271, 230, 420, 5, 259, 2, 7001, 39, 2455, 8, 757, 354, 517, 255, 0, 97, 635, 1715, 559, 481, 226, 2688, 453, 114, 0, 4, 90, 4, 536, 56, 169, 7, 31, 4328, 71, 365, 132, 3, 0, 4861, 5, 114, 88, 653, 4, 3364, 0, 5, 5, 6, 1065, 3, 1997, 3625, 0, 258] payload_rows_per_dst=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] dup_rows_per_dst=[4494, 15982, 1570, 2738, 16, 1088, 312, 302] combine_rows_per_src=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=279003136 dispatch_meta_bytes=343040 combine_payload_bytes=279003136 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279346176 ingress_bytes=348209152 link_bytes=348209152 token_bytes=558349312 weight_movement_bytes=69206016 total_nvlink_bytes=627555328 weight_to_token_ratio=0.123948 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310283006e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=238157824 dispatch_meta_bytes=283016 combine_payload_bytes=238157824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=238440840 ingress_bytes=341966848 link_bytes=341966848 token_bytes=476598664 weight_movement_bytes=103809024 total_nvlink_bytes=580407688 weight_to_token_ratio=0.217812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286793718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83874 payload_rows_remote=68912 dup_rows=14430 dup_rows_remote=13904 dedup_ratio=0.146790 router_tokens_per_expert=[0, 0, 158, 0, 1367, 1047, 1173, 298, 4852, 193, 356, 1, 258, 0, 7068, 43, 2248, 0, 756, 318, 444, 219, 0, 81, 328, 1705, 428, 349, 188, 2633, 474, 62, 0, 2, 94, 0, 433, 21, 126, 0, 22, 4529, 92, 334, 99, 0, 0, 5193, 0, 108, 94, 621, 0, 3448, 0, 0, 0, 0, 913, 0, 2177, 3565, 0, 234] payload_rows_per_dst=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] dup_rows_per_dst=[1478, 564, 1388, 1730, 526, 2770, 3482, 2492] combine_rows_per_src=[6608, 10738, 6848, 10604, 14962, 11904, 10924, 11286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76370 payload_rows_remote=64278 dup_rows=21934 dup_rows_remote=17718 dedup_ratio=0.223124 router_tokens_per_expert=[0, 804, 91, 159, 0, 176, 159, 366, 39, 1060, 554, 2388, 544, 346, 2, 1, 1628, 1570, 1581, 188, 3, 2, 3887, 887, 816, 1622, 3118, 4020, 3, 71, 242, 0, 243, 916, 177, 70, 6207, 1002, 240, 108, 676, 3, 34, 16, 728, 2128, 94, 91, 161, 2, 580, 645, 1, 0, 155, 5, 2654, 345, 3, 10, 5, 2892, 0, 2634] payload_rows_per_dst=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] dup_rows_per_dst=[4216, 3602, 4790, 2570, 4546, 1304, 904, 2] combine_rows_per_src=[12092, 11574, 12196, 10200, 13380, 6236, 9968, 724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85952 payload_rows_remote=74062 dup_rows=12352 dup_rows_remote=11518 dedup_ratio=0.125651 router_tokens_per_expert=[0, 1943, 1238, 47, 154, 2537, 0, 2594, 0, 359, 397, 60, 74, 0, 0, 0, 0, 364, 0, 174, 0, 813, 5313, 14, 188, 0, 5737, 80, 148, 84, 125, 0, 55, 0, 0, 161, 1047, 0, 13, 1380, 0, 1542, 0, 6042, 60, 0, 420, 0, 0, 1277, 2603, 0, 0, 5697, 5233, 437, 0, 76, 0, 164, 502, 0, 0, 0] payload_rows_per_dst=[8540, 11310, 11806, 11890, 9092, 14052, 7602, 11660] dup_rows_per_dst=[3298, 936, 1550, 834, 1408, 2076, 1032, 1218] combine_rows_per_src=[8540, 11310, 11806, 11890, 9092, 14052, 7602, 11660] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85894 payload_rows_remote=76382 dup_rows=12410 dup_rows_remote=10548 dedup_ratio=0.126241 router_tokens_per_expert=[0, 1915, 1185, 45, 138, 2504, 0, 2733, 0, 359, 392, 59, 69, 0, 0, 0, 0, 335, 0, 154, 0, 816, 5289, 8, 234, 0, 5718, 94, 171, 97, 150, 0, 57, 0, 0, 173, 1406, 0, 21, 1297, 0, 1370, 0, 6045, 59, 0, 439, 0, 0, 1200, 2492, 0, 0, 5756, 5253, 422, 1, 69, 0, 134, 493, 0, 0, 0] payload_rows_per_dst=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] dup_rows_per_dst=[3152, 942, 1422, 994, 1862, 1930, 962, 1146] combine_rows_per_src=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=238157824 dispatch_meta_bytes=283016 combine_payload_bytes=238157824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=238440840 ingress_bytes=341966848 link_bytes=341966848 token_bytes=476598664 weight_movement_bytes=103809024 total_nvlink_bytes=580407688 weight_to_token_ratio=0.217812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286793718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276963328 dispatch_meta_bytes=327528 combine_payload_bytes=276963328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277290856 ingress_bytes=311566336 link_bytes=311566336 token_bytes=554254184 weight_movement_bytes=34603008 total_nvlink_bytes=588857192 weight_to_token_ratio=0.062432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172399039e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=241868800 dispatch_meta_bytes=324232 combine_payload_bytes=241868800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242193032 ingress_bytes=276471808 link_bytes=276471808 token_bytes=484061832 weight_movement_bytes=34603008 total_nvlink_bytes=518664840 weight_to_token_ratio=0.071485 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.040341156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82110 payload_rows_remote=70318 dup_rows=16194 dup_rows_remote=15066 dedup_ratio=0.164734 router_tokens_per_expert=[0, 1902, 1214, 57, 135, 2746, 1, 2750, 0, 327, 473, 35, 66, 0, 0, 0, 0, 324, 0, 179, 1, 670, 5554, 14, 152, 0, 6031, 63, 174, 73, 116, 0, 19, 0, 0, 152, 1253, 0, 26, 1177, 0, 1269, 0, 6057, 35, 0, 455, 0, 0, 1139, 2355, 0, 1, 5796, 5228, 469, 0, 44, 0, 157, 462, 0, 1, 0] payload_rows_per_dst=[5054, 11218, 12112, 12430, 12160, 3446, 13898, 11792] dup_rows_per_dst=[1564, 1040, 1372, 788, 4086, 72, 6144, 1128] combine_rows_per_src=[5054, 11218, 12112, 12430, 12160, 3446, 13898, 11792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=282263552 dispatch_meta_bytes=331264 combine_payload_bytes=282263552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282594816 ingress_bytes=316866560 link_bytes=316866560 token_bytes=564858368 weight_movement_bytes=34603008 total_nvlink_bytes=599461376 weight_to_token_ratio=0.061260 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192343355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263282688 dispatch_meta_bytes=327984 combine_payload_bytes=263282688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=263610672 ingress_bytes=332488704 link_bytes=332488704 token_bytes=526893360 weight_movement_bytes=69206016 total_nvlink_bytes=596099376 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251128226e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=303357952 dispatch_meta_bytes=342320 combine_payload_bytes=303357952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303700272 ingress_bytes=372563968 link_bytes=372563968 token_bytes=607058224 weight_movement_bytes=69206016 total_nvlink_bytes=676264240 weight_to_token_ratio=0.114002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401928218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84826 payload_rows_remote=72924 dup_rows=13478 dup_rows_remote=11712 dedup_ratio=0.137105 router_tokens_per_expert=[0, 1831, 1323, 95, 122, 2176, 0, 2558, 0, 436, 521, 70, 110, 0, 1, 0, 0, 349, 0, 183, 0, 818, 5494, 3, 206, 0, 5678, 70, 165, 72, 167, 0, 58, 0, 0, 101, 1177, 0, 16, 1264, 0, 1479, 0, 6040, 70, 0, 421, 0, 0, 1530, 2616, 0, 0, 5696, 5040, 484, 1, 96, 0, 172, 435, 0, 8, 0] payload_rows_per_dst=[7124, 11162, 12108, 11922, 9634, 11902, 9334, 11640] dup_rows_per_dst=[2890, 1194, 1586, 794, 1794, 1766, 2278, 1176] combine_rows_per_src=[7124, 11162, 12108, 11922, 9634, 11902, 9334, 11640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312860672 dispatch_meta_bytes=347720 combine_payload_bytes=312860672 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=313208392 ingress_bytes=382066688 link_bytes=382066688 token_bytes=626069064 weight_movement_bytes=69206016 total_nvlink_bytes=695275080 weight_to_token_ratio=0.110541 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437686188e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85952 payload_rows_remote=74062 dup_rows=12352 dup_rows_remote=11518 dedup_ratio=0.125651 router_tokens_per_expert=[0, 1943, 1238, 47, 154, 2537, 0, 2594, 0, 359, 397, 60, 74, 0, 0, 0, 0, 364, 0, 174, 0, 813, 5313, 14, 188, 0, 5737, 80, 148, 84, 125, 0, 55, 0, 0, 161, 1047, 0, 13, 1380, 0, 1542, 0, 6042, 60, 0, 420, 0, 0, 1277, 2603, 0, 0, 5697, 5233, 437, 0, 76, 0, 164, 502, 0, 0, 0] payload_rows_per_dst=[8540, 11310, 11806, 11890, 9092, 14052, 7602, 11660] dup_rows_per_dst=[3298, 936, 1550, 834, 1408, 2076, 1032, 1218] combine_rows_per_src=[8540, 11310, 11806, 11890, 9092, 14052, 7602, 11660] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=74546 dup_rows=11870 dup_rows_remote=10304 dedup_ratio=0.120748 router_tokens_per_expert=[0, 1909, 1274, 43, 118, 2676, 0, 2640, 0, 306, 417, 64, 86, 0, 0, 0, 0, 395, 0, 174, 0, 732, 5421, 5, 169, 0, 5666, 75, 181, 78, 128, 0, 34, 0, 0, 129, 1049, 0, 18, 1367, 0, 1220, 0, 6382, 64, 0, 380, 0, 0, 1475, 2186, 0, 0, 5916, 5311, 436, 0, 77, 0, 120, 431, 0, 0, 0] payload_rows_per_dst=[8666, 11404, 11888, 11758, 9114, 14298, 7268, 12038] dup_rows_per_dst=[3374, 964, 1566, 836, 1360, 1794, 926, 1050] combine_rows_per_src=[8666, 11404, 11888, 11758, 9114, 14298, 7268, 12038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82110 payload_rows_remote=70318 dup_rows=16194 dup_rows_remote=15066 dedup_ratio=0.164734 router_tokens_per_expert=[0, 1902, 1214, 57, 135, 2746, 1, 2750, 0, 327, 473, 35, 66, 0, 0, 0, 0, 324, 0, 179, 1, 670, 5554, 14, 152, 0, 6031, 63, 174, 73, 116, 0, 19, 0, 0, 152, 1253, 0, 26, 1177, 0, 1269, 0, 6057, 35, 0, 455, 0, 0, 1139, 2355, 0, 1, 5796, 5228, 469, 0, 44, 0, 157, 462, 0, 1, 0] payload_rows_per_dst=[5054, 11218, 12112, 12430, 12160, 3446, 13898, 11792] dup_rows_per_dst=[1564, 1040, 1372, 788, 4086, 72, 6144, 1128] combine_rows_per_src=[5054, 11218, 12112, 12430, 12160, 3446, 13898, 11792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83922 payload_rows_remote=72976 dup_rows=14382 dup_rows_remote=13308 dedup_ratio=0.146301 router_tokens_per_expert=[1, 1927, 1257, 87, 159, 2539, 1, 2904, 0, 365, 561, 69, 88, 0, 0, 0, 0, 353, 0, 153, 0, 918, 5209, 15, 207, 0, 5637, 83, 197, 94, 153, 0, 43, 0, 0, 170, 1231, 0, 18, 1206, 0, 1400, 0, 6104, 71, 0, 399, 1, 0, 1292, 2250, 0, 0, 5861, 4927, 427, 6, 88, 0, 185, 490, 0, 6, 0] payload_rows_per_dst=[11070, 10946, 7530, 11824, 9494, 14072, 6968, 12018] dup_rows_per_dst=[5732, 1074, 906, 918, 1650, 1878, 970, 1254] combine_rows_per_src=[11070, 10946, 7530, 11824, 9494, 14072, 6968, 12018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=288022528 dispatch_meta_bytes=341536 combine_payload_bytes=288022528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288364064 ingress_bytes=322625536 link_bytes=322625536 token_bytes=576386592 weight_movement_bytes=34603008 total_nvlink_bytes=610989600 weight_to_token_ratio=0.060034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214013918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298696704 dispatch_meta_bytes=338544 combine_payload_bytes=298696704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=299035248 ingress_bytes=367902720 link_bytes=367902720 token_bytes=597731952 weight_movement_bytes=69206016 total_nvlink_bytes=666937968 weight_to_token_ratio=0.115781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384388317e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83922 payload_rows_remote=72976 dup_rows=14382 dup_rows_remote=13308 dedup_ratio=0.146301 router_tokens_per_expert=[1, 1927, 1257, 87, 159, 2539, 1, 2904, 0, 365, 561, 69, 88, 0, 0, 0, 0, 353, 0, 153, 0, 918, 5209, 15, 207, 0, 5637, 83, 197, 94, 153, 0, 43, 0, 0, 170, 1231, 0, 18, 1206, 0, 1400, 0, 6104, 71, 0, 399, 1, 0, 1292, 2250, 0, 0, 5861, 4927, 427, 6, 88, 0, 185, 490, 0, 6, 0] payload_rows_per_dst=[11070, 10946, 7530, 11824, 9494, 14072, 6968, 12018] dup_rows_per_dst=[5732, 1074, 906, 918, 1650, 1878, 970, 1254] combine_rows_per_src=[11070, 10946, 7530, 11824, 9494, 14072, 6968, 12018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=305340416 dispatch_meta_bytes=339400 combine_payload_bytes=305340416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305679816 ingress_bytes=374546432 link_bytes=374546432 token_bytes=611020232 weight_movement_bytes=69206016 total_nvlink_bytes=680226248 weight_to_token_ratio=0.113263 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409388070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81104 payload_rows_remote=67098 dup_rows=17200 dup_rows_remote=10614 dedup_ratio=0.174967 router_tokens_per_expert=[0, 1859, 1268, 68, 124, 2816, 0, 2844, 0, 303, 453, 90, 126, 0, 0, 0, 0, 368, 0, 189, 0, 817, 5166, 12, 186, 0, 5685, 85, 188, 92, 164, 0, 5, 0, 0, 140, 1032, 0, 12, 1309, 0, 1408, 0, 6142, 90, 0, 401, 0, 0, 1313, 2414, 0, 0, 5727, 5054, 427, 0, 119, 0, 203, 453, 0, 0, 0] payload_rows_per_dst=[5234, 11062, 11536, 12012, 11882, 3612, 14006, 11760] dup_rows_per_dst=[1404, 990, 1568, 788, 4434, 186, 6586, 1244] combine_rows_per_src=[5234, 11062, 11536, 12012, 11882, 3612, 14006, 11760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=303357952 dispatch_meta_bytes=342320 combine_payload_bytes=303357952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303700272 ingress_bytes=372563968 link_bytes=372563968 token_bytes=607058224 weight_movement_bytes=69206016 total_nvlink_bytes=676264240 weight_to_token_ratio=0.114002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401928218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=305340416 dispatch_meta_bytes=339400 combine_payload_bytes=305340416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305679816 ingress_bytes=374546432 link_bytes=374546432 token_bytes=611020232 weight_movement_bytes=69206016 total_nvlink_bytes=680226248 weight_to_token_ratio=0.113263 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409388070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=288022528 dispatch_meta_bytes=341536 combine_payload_bytes=288022528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288364064 ingress_bytes=322625536 link_bytes=322625536 token_bytes=576386592 weight_movement_bytes=34603008 total_nvlink_bytes=610989600 weight_to_token_ratio=0.060034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.214013918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68780 dup_rows=18592 dup_rows_remote=15524 dedup_ratio=0.189128 router_tokens_per_expert=[172, 3532, 80, 46, 74, 2209, 1442, 92, 0, 0, 0, 537, 1932, 0, 0, 792, 346, 3804, 7089, 78, 0, 1589, 5341, 0, 0, 2065, 839, 0, 0, 0, 1231, 0, 3662, 4, 0, 0, 0, 0, 312, 379, 585, 0, 0, 87, 0, 0, 146, 1178, 4, 378, 930, 860, 0, 0, 0, 1087, 202, 0, 80, 0, 1, 334, 0, 5633] payload_rows_per_dst=[10660, 13084, 3870, 10932, 8672, 10370, 10422, 11702] dup_rows_per_dst=[3168, 4120, 156, 3068, 1508, 2070, 3704, 798] combine_rows_per_src=[10660, 13084, 3870, 10932, 8672, 10370, 10422, 11702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85894 payload_rows_remote=76382 dup_rows=12410 dup_rows_remote=10548 dedup_ratio=0.126241 router_tokens_per_expert=[0, 1915, 1185, 45, 138, 2504, 0, 2733, 0, 359, 392, 59, 69, 0, 0, 0, 0, 335, 0, 154, 0, 816, 5289, 8, 234, 0, 5718, 94, 171, 97, 150, 0, 57, 0, 0, 173, 1406, 0, 21, 1297, 0, 1370, 0, 6045, 59, 0, 439, 0, 0, 1200, 2492, 0, 0, 5756, 5253, 422, 1, 69, 0, 134, 493, 0, 0, 0] payload_rows_per_dst=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] dup_rows_per_dst=[3152, 942, 1422, 994, 1862, 1930, 962, 1146] combine_rows_per_src=[8422, 11322, 11782, 11934, 9512, 13896, 7266, 11760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71802 payload_rows_remote=59050 dup_rows=26502 dup_rows_remote=22008 dedup_ratio=0.269592 router_tokens_per_expert=[0, 0, 155, 2, 1224, 877, 1229, 275, 4271, 230, 420, 5, 259, 2, 7001, 39, 2455, 8, 757, 354, 517, 255, 0, 97, 635, 1715, 559, 481, 226, 2688, 453, 114, 0, 4, 90, 4, 536, 56, 169, 7, 31, 4328, 71, 365, 132, 3, 0, 4861, 5, 114, 88, 653, 4, 3364, 0, 5, 5, 6, 1065, 3, 1997, 3625, 0, 258] payload_rows_per_dst=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] dup_rows_per_dst=[4494, 15982, 1570, 2738, 16, 1088, 312, 302] combine_rows_per_src=[12752, 15722, 7316, 11004, 1716, 8772, 8154, 6366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298909696 dispatch_meta_bytes=345136 combine_payload_bytes=298909696 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=299254832 ingress_bytes=368115712 link_bytes=368115712 token_bytes=598164528 weight_movement_bytes=69206016 total_nvlink_bytes=667370544 weight_to_token_ratio=0.115697 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385189789e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79958 payload_rows_remote=66084 dup_rows=18346 dup_rows_remote=9440 dedup_ratio=0.186625 router_tokens_per_expert=[5, 1956, 1288, 86, 124, 2625, 1, 2548, 0, 279, 611, 66, 91, 0, 3, 2, 0, 373, 7, 182, 1, 792, 5305, 4, 167, 7, 5805, 88, 202, 98, 143, 5, 106, 4, 0, 148, 1238, 0, 19, 1348, 2, 1425, 0, 5970, 66, 4, 396, 2, 1, 1261, 2085, 2, 5, 5751, 5168, 436, 0, 86, 0, 181, 583, 0, 1, 0] payload_rows_per_dst=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] dup_rows_per_dst=[8906, 1322, 928, 632, 1572, 1518, 2800, 668] combine_rows_per_src=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81712 payload_rows_remote=69802 dup_rows=16592 dup_rows_remote=12722 dedup_ratio=0.168783 router_tokens_per_expert=[186, 3488, 94, 69, 83, 2327, 1373, 92, 0, 0, 0, 501, 1837, 0, 0, 773, 326, 4077, 7070, 13, 0, 1419, 5381, 0, 0, 1840, 785, 0, 0, 0, 1182, 0, 3748, 9, 0, 0, 0, 0, 330, 315, 534, 0, 0, 96, 0, 0, 227, 1252, 81, 335, 936, 767, 0, 0, 0, 1083, 307, 0, 94, 0, 2, 414, 0, 5706] payload_rows_per_dst=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] dup_rows_per_dst=[922, 2176, 1152, 630, 3870, 3142, 3570, 1130] combine_rows_per_src=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=274833408 dispatch_meta_bytes=310848 combine_payload_bytes=274833408 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=275144256 ingress_bytes=413245440 link_bytes=413245440 token_bytes=549977664 weight_movement_bytes=138412032 total_nvlink_bytes=688389696 weight_to_token_ratio=0.251668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555009322e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270680064 dispatch_meta_bytes=302096 combine_payload_bytes=270680064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270982160 ingress_bytes=374489088 link_bytes=374489088 token_bytes=541662224 weight_movement_bytes=103809024 total_nvlink_bytes=645471248 weight_to_token_ratio=0.191649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409172289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281722880 dispatch_meta_bytes=337216 combine_payload_bytes=281722880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282060096 ingress_bytes=316325888 link_bytes=316325888 token_bytes=563782976 weight_movement_bytes=34603008 total_nvlink_bytes=598385984 weight_to_token_ratio=0.061376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190308850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=66808 dup_rows=16266 dup_rows_remote=13078 dedup_ratio=0.165466 router_tokens_per_expert=[131, 3560, 70, 52, 61, 2482, 1163, 60, 0, 0, 0, 574, 1853, 0, 0, 803, 323, 4164, 7127, 32, 0, 1422, 5198, 0, 0, 1858, 781, 0, 0, 0, 1375, 0, 3789, 3, 0, 0, 0, 0, 253, 183, 579, 0, 0, 81, 2, 0, 200, 1220, 38, 400, 1070, 798, 0, 0, 0, 1105, 325, 1, 70, 0, 1, 336, 0, 5609] payload_rows_per_dst=[7204, 5520, 12070, 7338, 11838, 15230, 11186, 11652] dup_rows_per_dst=[834, 940, 1880, 690, 3738, 3188, 3964, 1032] combine_rows_per_src=[7204, 5520, 12070, 7338, 11838, 15230, 11186, 11652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285908992 dispatch_meta_bytes=330096 combine_payload_bytes=285908992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286239088 ingress_bytes=320512000 link_bytes=320512000 token_bytes=572148080 weight_movement_bytes=34603008 total_nvlink_bytes=606751088 weight_to_token_ratio=0.060479 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206060853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81140 payload_rows_remote=69136 dup_rows=17164 dup_rows_remote=14908 dedup_ratio=0.174601 router_tokens_per_expert=[109, 3753, 63, 57, 73, 1972, 1413, 46, 0, 0, 0, 587, 2008, 0, 0, 792, 312, 4138, 7190, 56, 0, 1395, 5307, 0, 0, 2074, 684, 0, 0, 0, 1441, 0, 3560, 0, 0, 0, 0, 0, 267, 185, 581, 0, 0, 66, 1, 0, 151, 1292, 10, 410, 1077, 816, 0, 0, 0, 1003, 300, 0, 63, 0, 0, 354, 1, 5545] payload_rows_per_dst=[6756, 5872, 10796, 7588, 11900, 15320, 10904, 12004] dup_rows_per_dst=[710, 902, 1610, 810, 3630, 3242, 4004, 2256] combine_rows_per_src=[6756, 5872, 10796, 7588, 11900, 15320, 10904, 12004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=312860672 dispatch_meta_bytes=347720 combine_payload_bytes=312860672 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=313208392 ingress_bytes=382066688 link_bytes=382066688 token_bytes=626069064 weight_movement_bytes=69206016 total_nvlink_bytes=695275080 weight_to_token_ratio=0.110541 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437686188e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=241868800 dispatch_meta_bytes=324232 combine_payload_bytes=241868800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=242193032 ingress_bytes=276471808 link_bytes=276471808 token_bytes=484061832 weight_movement_bytes=34603008 total_nvlink_bytes=518664840 weight_to_token_ratio=0.071485 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.040341156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68780 dup_rows=18592 dup_rows_remote=15524 dedup_ratio=0.189128 router_tokens_per_expert=[172, 3532, 80, 46, 74, 2209, 1442, 92, 0, 0, 0, 537, 1932, 0, 0, 792, 346, 3804, 7089, 78, 0, 1589, 5341, 0, 0, 2065, 839, 0, 0, 0, 1231, 0, 3662, 4, 0, 0, 0, 0, 312, 379, 585, 0, 0, 87, 0, 0, 146, 1178, 4, 378, 930, 860, 0, 0, 0, 1087, 202, 0, 80, 0, 1, 334, 0, 5633] payload_rows_per_dst=[10660, 13084, 3870, 10932, 8672, 10370, 10422, 11702] dup_rows_per_dst=[3168, 4120, 156, 3068, 1508, 2070, 3704, 798] combine_rows_per_src=[10660, 13084, 3870, 10932, 8672, 10370, 10422, 11702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78698 payload_rows_remote=73078 dup_rows=19606 dup_rows_remote=18812 dedup_ratio=0.199443 router_tokens_per_expert=[165, 3486, 76, 58, 62, 2255, 1195, 90, 0, 0, 0, 543, 1833, 0, 0, 789, 392, 4168, 7042, 49, 0, 1501, 5344, 0, 0, 1761, 860, 0, 0, 0, 1359, 0, 3558, 5, 0, 0, 0, 0, 275, 191, 533, 0, 0, 77, 0, 0, 164, 1401, 28, 376, 1012, 782, 0, 0, 0, 1103, 369, 0, 76, 0, 0, 333, 0, 5841] payload_rows_per_dst=[11232, 13028, 5620, 15790, 7630, 3842, 9484, 12072] dup_rows_per_dst=[3542, 3990, 794, 6254, 428, 508, 2924, 1166] combine_rows_per_src=[11232, 13028, 5620, 15790, 7630, 3842, 9484, 12072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81140 payload_rows_remote=69136 dup_rows=17164 dup_rows_remote=14908 dedup_ratio=0.174601 router_tokens_per_expert=[109, 3753, 63, 57, 73, 1972, 1413, 46, 0, 0, 0, 587, 2008, 0, 0, 792, 312, 4138, 7190, 56, 0, 1395, 5307, 0, 0, 2074, 684, 0, 0, 0, 1441, 0, 3560, 0, 0, 0, 0, 0, 267, 185, 581, 0, 0, 66, 1, 0, 151, 1292, 10, 410, 1077, 816, 0, 0, 0, 1003, 300, 0, 63, 0, 0, 354, 1, 5545] payload_rows_per_dst=[6756, 5872, 10796, 7588, 11900, 15320, 10904, 12004] dup_rows_per_dst=[710, 902, 1610, 810, 3630, 3242, 4004, 2256] combine_rows_per_src=[6756, 5872, 10796, 7588, 11900, 15320, 10904, 12004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78922 payload_rows_remote=65926 dup_rows=19382 dup_rows_remote=15386 dedup_ratio=0.197164 router_tokens_per_expert=[170, 3515, 83, 72, 69, 2401, 1413, 99, 0, 0, 0, 527, 2013, 0, 1, 637, 335, 4000, 6979, 9, 0, 1452, 5318, 2, 1, 1748, 832, 1, 0, 0, 1287, 0, 3687, 153, 0, 0, 0, 1, 326, 260, 545, 0, 0, 110, 2, 0, 234, 1241, 75, 338, 874, 780, 0, 0, 0, 1166, 332, 0, 83, 0, 1, 424, 0, 5556] payload_rows_per_dst=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] dup_rows_per_dst=[3980, 3996, 2100, 5996, 580, 568, 1000, 1162] combine_rows_per_src=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=273645568 dispatch_meta_bytes=319544 combine_payload_bytes=273645568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273965112 ingress_bytes=308248576 link_bytes=308248576 token_bytes=547610680 weight_movement_bytes=34603008 total_nvlink_bytes=582213688 weight_to_token_ratio=0.063189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159914575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283181056 dispatch_meta_bytes=336176 combine_payload_bytes=283181056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283517232 ingress_bytes=352387072 link_bytes=352387072 token_bytes=566698288 weight_movement_bytes=69206016 total_nvlink_bytes=635904304 weight_to_token_ratio=0.122121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326004182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78922 payload_rows_remote=65926 dup_rows=19382 dup_rows_remote=15386 dedup_ratio=0.197164 router_tokens_per_expert=[170, 3515, 83, 72, 69, 2401, 1413, 99, 0, 0, 0, 527, 2013, 0, 1, 637, 335, 4000, 6979, 9, 0, 1452, 5318, 2, 1, 1748, 832, 1, 0, 0, 1287, 0, 3687, 153, 0, 0, 0, 1, 326, 260, 545, 0, 0, 110, 2, 0, 234, 1241, 75, 338, 874, 780, 0, 0, 0, 1166, 332, 0, 83, 0, 1, 424, 0, 5556] payload_rows_per_dst=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] dup_rows_per_dst=[3980, 3996, 2100, 5996, 580, 568, 1000, 1162] combine_rows_per_src=[11664, 12996, 9636, 15560, 8274, 3696, 5466, 11630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82412 payload_rows_remote=71834 dup_rows=15892 dup_rows_remote=12288 dedup_ratio=0.161662 router_tokens_per_expert=[171, 3480, 85, 54, 42, 2580, 1406, 92, 0, 0, 0, 534, 1989, 0, 0, 763, 353, 3918, 6966, 43, 0, 1580, 5289, 0, 0, 1922, 924, 0, 0, 0, 1237, 0, 3654, 7, 0, 0, 0, 0, 265, 275, 567, 0, 0, 89, 0, 0, 179, 1208, 42, 334, 891, 763, 0, 0, 0, 1143, 302, 0, 85, 0, 2, 360, 0, 5558] payload_rows_per_dst=[7884, 5730, 12414, 7488, 11790, 14952, 10578, 11576] dup_rows_per_dst=[976, 842, 2116, 678, 3572, 3066, 3604, 1038] combine_rows_per_src=[7884, 5730, 12414, 7488, 11790, 14952, 10578, 11576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281722880 dispatch_meta_bytes=337216 combine_payload_bytes=281722880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282060096 ingress_bytes=316325888 link_bytes=316325888 token_bytes=563782976 weight_movement_bytes=34603008 total_nvlink_bytes=598385984 weight_to_token_ratio=0.061376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190308850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283181056 dispatch_meta_bytes=336176 combine_payload_bytes=283181056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283517232 ingress_bytes=352387072 link_bytes=352387072 token_bytes=566698288 weight_movement_bytes=69206016 total_nvlink_bytes=635904304 weight_to_token_ratio=0.122121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.326004182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299327488 dispatch_meta_bytes=367560 combine_payload_bytes=299327488 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=299695048 ingress_bytes=472342528 link_bytes=472342528 token_bytes=599022536 weight_movement_bytes=173015040 total_nvlink_bytes=772037576 weight_to_token_ratio=0.288829 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.777386906e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72940 payload_rows_remote=59874 dup_rows=25364 dup_rows_remote=17184 dedup_ratio=0.258016 router_tokens_per_expert=[194, 3312, 92, 69, 57, 2503, 1426, 105, 0, 5, 1, 542, 1614, 0, 1, 771, 341, 4111, 6778, 79, 2, 1537, 5224, 7, 5, 2072, 850, 6, 0, 1, 1309, 6, 3679, 284, 3, 0, 1, 7, 306, 202, 529, 2, 0, 127, 13, 2, 220, 1054, 13, 361, 935, 823, 0, 1, 8, 1112, 348, 1, 89, 0, 6, 404, 1, 5601] payload_rows_per_dst=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] dup_rows_per_dst=[8180, 3478, 10212, 848, 452, 514, 996, 684] combine_rows_per_src=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89276 payload_rows_remote=76142 dup_rows=9028 dup_rows_remote=9028 dedup_ratio=0.091838 router_tokens_per_expert=[2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 2022, 6671, 0, 82, 0, 0, 0, 0, 0, 277, 426, 345, 0, 0, 0, 0, 0, 80, 0, 0, 0, 5785, 3051, 4, 0, 0, 0, 0, 757, 929, 0, 37, 0, 80, 2324, 2693, 0, 10, 0, 225, 92, 6066, 6487, 0, 394, 0, 0, 0, 0, 0, 8079, 0] payload_rows_per_dst=[14530, 4406, 12356, 13134, 6102, 9966, 12624, 16158] dup_rows_per_dst=[3086, 0, 1310, 0, 8, 3674, 162, 788] combine_rows_per_src=[14530, 4406, 12356, 13134, 6102, 9966, 12624, 16158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81712 payload_rows_remote=69802 dup_rows=16592 dup_rows_remote=12722 dedup_ratio=0.168783 router_tokens_per_expert=[186, 3488, 94, 69, 83, 2327, 1373, 92, 0, 0, 0, 501, 1837, 0, 0, 773, 326, 4077, 7070, 13, 0, 1419, 5381, 0, 0, 1840, 785, 0, 0, 0, 1182, 0, 3748, 9, 0, 0, 0, 0, 330, 315, 534, 0, 0, 96, 0, 0, 227, 1252, 81, 335, 936, 767, 0, 0, 0, 1083, 307, 0, 94, 0, 2, 414, 0, 5706] payload_rows_per_dst=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] dup_rows_per_dst=[922, 2176, 1152, 630, 3870, 3142, 3570, 1130] combine_rows_per_src=[7526, 8792, 8380, 6984, 11910, 15216, 10988, 11916] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270032896 dispatch_meta_bytes=325248 combine_payload_bytes=270032896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270358144 ingress_bytes=304635904 link_bytes=304635904 token_bytes=540391040 weight_movement_bytes=34603008 total_nvlink_bytes=574994048 weight_to_token_ratio=0.064033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.146320382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89380 payload_rows_remote=83578 dup_rows=8924 dup_rows_remote=8924 dedup_ratio=0.090780 router_tokens_per_expert=[2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1707, 6740, 0, 126, 0, 0, 0, 0, 0, 332, 419, 381, 0, 0, 7, 0, 0, 94, 0, 0, 0, 5674, 2901, 0, 0, 0, 0, 0, 748, 953, 0, 5, 0, 94, 2382, 2528, 0, 5, 0, 234, 92, 6346, 6615, 0, 376, 0, 0, 0, 0, 0, 8038, 0] payload_rows_per_dst=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] dup_rows_per_dst=[2882, 468, 1272, 14, 0, 3388, 148, 752] combine_rows_per_src=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79958 payload_rows_remote=66084 dup_rows=18346 dup_rows_remote=9440 dedup_ratio=0.186625 router_tokens_per_expert=[5, 1956, 1288, 86, 124, 2625, 1, 2548, 0, 279, 611, 66, 91, 0, 3, 2, 0, 373, 7, 182, 1, 792, 5305, 4, 167, 7, 5805, 88, 202, 98, 143, 5, 106, 4, 0, 148, 1238, 0, 19, 1348, 2, 1425, 0, 5970, 66, 4, 396, 2, 1, 1261, 2085, 2, 5, 5751, 5168, 436, 0, 86, 0, 181, 583, 0, 1, 0] payload_rows_per_dst=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] dup_rows_per_dst=[8906, 1322, 928, 632, 1572, 1518, 2800, 668] combine_rows_per_src=[13874, 11418, 8644, 7950, 9250, 11806, 10264, 6752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=294232064 dispatch_meta_bytes=336488 combine_payload_bytes=294232064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294568552 ingress_bytes=328835072 link_bytes=328835072 token_bytes=588800616 weight_movement_bytes=34603008 total_nvlink_bytes=623403624 weight_to_token_ratio=0.058769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237379903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=245243904 dispatch_meta_bytes=308232 combine_payload_bytes=245243904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245552136 ingress_bytes=314449920 link_bytes=314449920 token_bytes=490796040 weight_movement_bytes=69206016 total_nvlink_bytes=560002056 weight_to_token_ratio=0.141008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183249734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=311877632 dispatch_meta_bytes=340680 combine_payload_bytes=311877632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312218312 ingress_bytes=346480640 link_bytes=346480640 token_bytes=624095944 weight_movement_bytes=34603008 total_nvlink_bytes=658698952 weight_to_token_ratio=0.055445 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303778754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90042 payload_rows_remote=79994 dup_rows=8262 dup_rows_remote=4494 dedup_ratio=0.084045 router_tokens_per_expert=[2277, 0, 0, 0, 0, 0, 0, 0, 0, 3, 67, 0, 1585, 6709, 0, 42, 0, 0, 0, 0, 0, 273, 392, 319, 0, 0, 0, 0, 0, 70, 0, 0, 0, 5563, 3436, 4, 0, 0, 0, 0, 805, 984, 0, 0, 0, 70, 2388, 2661, 0, 1, 0, 292, 60, 6235, 6601, 0, 358, 0, 0, 0, 0, 0, 7957, 0] payload_rows_per_dst=[4554, 14468, 11888, 13342, 6872, 10048, 12956, 15914] dup_rows_per_dst=[0, 2344, 1206, 0, 8, 3768, 220, 716] combine_rows_per_src=[4554, 14468, 11888, 13342, 6872, 10048, 12956, 15914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342335488 dispatch_meta_bytes=370008 combine_payload_bytes=342335488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342705496 ingress_bytes=411541504 link_bytes=411541504 token_bytes=685040984 weight_movement_bytes=69206016 total_nvlink_bytes=754247000 weight_to_token_ratio=0.101025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548597548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89276 payload_rows_remote=76142 dup_rows=9028 dup_rows_remote=9028 dedup_ratio=0.091838 router_tokens_per_expert=[2137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 2022, 6671, 0, 82, 0, 0, 0, 0, 0, 277, 426, 345, 0, 0, 0, 0, 0, 80, 0, 0, 0, 5785, 3051, 4, 0, 0, 0, 0, 757, 929, 0, 37, 0, 80, 2324, 2693, 0, 10, 0, 225, 92, 6066, 6487, 0, 394, 0, 0, 0, 0, 0, 8079, 0] payload_rows_per_dst=[14530, 4406, 12356, 13134, 6102, 9966, 12624, 16158] dup_rows_per_dst=[3086, 0, 1310, 0, 8, 3674, 162, 788] combine_rows_per_src=[14530, 4406, 12356, 13134, 6102, 9966, 12624, 16158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74866 payload_rows_remote=58618 dup_rows=23438 dup_rows_remote=18444 dedup_ratio=0.238424 router_tokens_per_expert=[2125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 1805, 6751, 0, 88, 0, 0, 0, 0, 0, 393, 374, 279, 0, 0, 0, 0, 0, 63, 0, 0, 0, 5781, 3079, 4, 0, 0, 0, 0, 779, 1096, 0, 0, 0, 63, 2393, 2748, 0, 18, 0, 177, 46, 6147, 6503, 0, 266, 0, 0, 0, 0, 0, 8122, 0] payload_rows_per_dst=[4250, 14558, 4530, 4542, 11896, 3962, 14880, 16248] dup_rows_per_dst=[0, 2834, 382, 0, 3012, 234, 11982, 4994] combine_rows_per_src=[4250, 14558, 4530, 4542, 11896, 3962, 14880, 16248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285908992 dispatch_meta_bytes=330096 combine_payload_bytes=285908992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286239088 ingress_bytes=320512000 link_bytes=320512000 token_bytes=572148080 weight_movement_bytes=34603008 total_nvlink_bytes=606751088 weight_to_token_ratio=0.060479 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=29440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206060853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270680064 dispatch_meta_bytes=302096 combine_payload_bytes=270680064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270982160 ingress_bytes=374489088 link_bytes=374489088 token_bytes=541662224 weight_movement_bytes=103809024 total_nvlink_bytes=645471248 weight_to_token_ratio=0.191649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409172289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86900 payload_rows_remote=74342 dup_rows=11404 dup_rows_remote=10034 dedup_ratio=0.116007 router_tokens_per_expert=[2084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1787, 6682, 0, 68, 0, 0, 0, 0, 0, 387, 469, 297, 0, 0, 2, 0, 0, 76, 0, 0, 0, 5811, 3170, 2, 0, 0, 0, 0, 759, 1108, 0, 6, 0, 76, 2337, 2468, 0, 14, 0, 232, 89, 6170, 6627, 0, 249, 0, 0, 0, 0, 0, 8089, 0] payload_rows_per_dst=[14518, 3894, 12558, 13406, 11504, 10034, 12824, 8162] dup_rows_per_dst=[3014, 2, 1370, 4, 3072, 3474, 186, 282] combine_rows_per_src=[14518, 3894, 12558, 13406, 11504, 10034, 12824, 8162] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=327655424 dispatch_meta_bytes=337952 combine_payload_bytes=327655424 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=327993376 ingress_bytes=396861440 link_bytes=396861440 token_bytes=655648800 weight_movement_bytes=69206016 total_nvlink_bytes=724854816 weight_to_token_ratio=0.105553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.493357649e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=240099328 dispatch_meta_bytes=308248 combine_payload_bytes=240099328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=240407576 ingress_bytes=309305344 link_bytes=309305344 token_bytes=480506904 weight_movement_bytes=69206016 total_nvlink_bytes=549712920 weight_to_token_ratio=0.144027 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163891108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82470 payload_rows_remote=68722 dup_rows=15834 dup_rows_remote=13714 dedup_ratio=0.161072 router_tokens_per_expert=[1949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 1903, 6703, 0, 110, 0, 0, 1, 0, 0, 291, 417, 357, 0, 0, 7, 1, 0, 84, 0, 0, 0, 5311, 3047, 0, 0, 0, 0, 1, 691, 918, 0, 41, 0, 83, 2559, 2643, 0, 14, 0, 265, 89, 6541, 6699, 1, 376, 0, 0, 0, 0, 1, 7946, 0] payload_rows_per_dst=[14538, 13748, 11464, 13566, 15996, 10246, 2158, 754] dup_rows_per_dst=[2766, 2120, 1290, 16, 5992, 3624, 26, 0] combine_rows_per_src=[14538, 13748, 11464, 13566, 15996, 10246, 2158, 754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88492 payload_rows_remote=74798 dup_rows=9812 dup_rows_remote=7494 dedup_ratio=0.099813 router_tokens_per_expert=[2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 1928, 6787, 0, 64, 0, 0, 0, 0, 0, 337, 437, 321, 0, 0, 1, 0, 0, 85, 0, 0, 0, 5277, 3081, 0, 0, 0, 0, 0, 716, 975, 0, 8, 1, 85, 2370, 2746, 0, 21, 0, 299, 86, 6329, 6714, 0, 343, 0, 0, 0, 0, 0, 7988, 0] payload_rows_per_dst=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] dup_rows_per_dst=[0, 2924, 1288, 2, 0, 2594, 2318, 686] combine_rows_per_src=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=311877632 dispatch_meta_bytes=340680 combine_payload_bytes=311877632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312218312 ingress_bytes=346480640 link_bytes=346480640 token_bytes=624095944 weight_movement_bytes=34603008 total_nvlink_bytes=658698952 weight_to_token_ratio=0.055445 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303778754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=240099328 dispatch_meta_bytes=308248 combine_payload_bytes=240099328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=240407576 ingress_bytes=309305344 link_bytes=309305344 token_bytes=480506904 weight_movement_bytes=69206016 total_nvlink_bytes=549712920 weight_to_token_ratio=0.144027 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163891108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88492 payload_rows_remote=74798 dup_rows=9812 dup_rows_remote=7494 dedup_ratio=0.099813 router_tokens_per_expert=[2064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 1928, 6787, 0, 64, 0, 0, 0, 0, 0, 337, 437, 321, 0, 0, 1, 0, 0, 85, 0, 0, 0, 5277, 3081, 0, 0, 0, 0, 0, 716, 975, 0, 8, 1, 85, 2370, 2746, 0, 21, 0, 299, 86, 6329, 6714, 0, 343, 0, 0, 0, 0, 0, 7988, 0] payload_rows_per_dst=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] dup_rows_per_dst=[0, 2924, 1288, 2, 0, 2594, 2318, 686] combine_rows_per_src=[4128, 14812, 11456, 13598, 6162, 8666, 13694, 15976] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=304504832 dispatch_meta_bytes=337504 combine_payload_bytes=304504832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304842336 ingress_bytes=339107840 link_bytes=339107840 token_bytes=609347168 weight_movement_bytes=34603008 total_nvlink_bytes=643950176 weight_to_token_ratio=0.056787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276035501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82584 payload_rows_remote=68222 dup_rows=15720 dup_rows_remote=12984 dedup_ratio=0.159912 router_tokens_per_expert=[1965, 0, 2, 1, 5, 0, 3, 2, 3, 2, 139, 1, 1763, 6571, 0, 80, 0, 0, 5, 1, 4, 277, 428, 365, 1, 2, 2, 0, 0, 90, 0, 0, 1, 5717, 2981, 57, 0, 7, 3, 0, 760, 1051, 7, 41, 0, 88, 2472, 2878, 3, 61, 7, 268, 113, 6092, 6536, 8, 384, 0, 0, 2, 9, 4, 7890, 0] payload_rows_per_dst=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] dup_rows_per_dst=[2736, 1644, 1314, 4, 5902, 4010, 104, 6] combine_rows_per_src=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89772 payload_rows_remote=78576 dup_rows=8532 dup_rows_remote=7556 dedup_ratio=0.086792 router_tokens_per_expert=[251, 407, 0, 24, 251, 0, 0, 0, 0, 3902, 98, 5087, 0, 0, 2012, 4479, 69, 357, 0, 0, 1020, 0, 0, 0, 257, 248, 0, 6554, 45, 0, 0, 0, 0, 0, 0, 0, 4509, 1333, 244, 0, 3, 0, 0, 0, 0, 92, 0, 0, 4210, 0, 0, 6938, 94, 2611, 925, 0, 0, 0, 0, 0, 0, 312, 2734, 86] payload_rows_per_dst=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] dup_rows_per_dst=[1814, 1432, 1166, 290, 976, 0, 262, 2592] combine_rows_per_src=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90096 payload_rows_remote=76058 dup_rows=8208 dup_rows_remote=7952 dedup_ratio=0.083496 router_tokens_per_expert=[245, 448, 0, 60, 245, 0, 0, 0, 0, 3668, 122, 4995, 0, 0, 1824, 4284, 74, 397, 0, 0, 1063, 0, 0, 0, 253, 252, 0, 6628, 14, 0, 0, 0, 0, 0, 0, 0, 4797, 1400, 171, 0, 1, 0, 0, 0, 0, 92, 0, 0, 4420, 0, 0, 6815, 82, 2650, 1005, 0, 0, 0, 0, 0, 0, 314, 2741, 92] payload_rows_per_dst=[13764, 10060, 10646, 14038, 11804, 10176, 7212, 12396] dup_rows_per_dst=[1862, 1168, 1262, 256, 932, 0, 262, 2466] combine_rows_per_src=[13764, 10060, 10646, 14038, 11804, 10176, 7212, 12396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89380 payload_rows_remote=83578 dup_rows=8924 dup_rows_remote=8924 dedup_ratio=0.090780 router_tokens_per_expert=[2262, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 1707, 6740, 0, 126, 0, 0, 0, 0, 0, 332, 419, 381, 0, 0, 7, 0, 0, 94, 0, 0, 0, 5674, 2901, 0, 0, 0, 0, 0, 748, 953, 0, 5, 0, 94, 2382, 2528, 0, 5, 0, 234, 92, 6346, 6615, 0, 376, 0, 0, 0, 0, 0, 8038, 0] payload_rows_per_dst=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] dup_rows_per_dst=[2882, 468, 1272, 14, 0, 3388, 148, 752] combine_rows_per_src=[12940, 5566, 12340, 13418, 5802, 10032, 13206, 16076] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=281485312 dispatch_meta_bytes=329744 combine_payload_bytes=281485312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281815056 ingress_bytes=350691328 link_bytes=350691328 token_bytes=563300368 weight_movement_bytes=69206016 total_nvlink_bytes=632506384 weight_to_token_ratio=0.122858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319623234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72940 payload_rows_remote=59874 dup_rows=25364 dup_rows_remote=17184 dedup_ratio=0.258016 router_tokens_per_expert=[194, 3312, 92, 69, 57, 2503, 1426, 105, 0, 5, 1, 542, 1614, 0, 1, 771, 341, 4111, 6778, 79, 2, 1537, 5224, 7, 5, 2072, 850, 6, 0, 1, 1309, 6, 3679, 284, 3, 0, 1, 7, 306, 202, 529, 2, 0, 127, 13, 2, 220, 1054, 13, 361, 935, 823, 0, 1, 8, 1112, 348, 1, 89, 0, 6, 404, 1, 5601] payload_rows_per_dst=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] dup_rows_per_dst=[8180, 3478, 10212, 848, 452, 514, 996, 684] combine_rows_per_src=[13066, 12838, 15498, 7650, 8512, 3380, 5510, 6486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306372608 dispatch_meta_bytes=329168 combine_payload_bytes=306372608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=306701776 ingress_bytes=410181632 link_bytes=410181632 token_bytes=613074384 weight_movement_bytes=103809024 total_nvlink_bytes=716883408 weight_to_token_ratio=0.169325 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.543480459e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=324824 combine_payload_bytes=279437312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279762136 ingress_bytes=314040320 link_bytes=314040320 token_bytes=559199448 weight_movement_bytes=34603008 total_nvlink_bytes=593802456 weight_to_token_ratio=0.061880 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181708442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=321847296 dispatch_meta_bytes=344528 combine_payload_bytes=321847296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322191824 ingress_bytes=391053312 link_bytes=391053312 token_bytes=644039120 weight_movement_bytes=69206016 total_nvlink_bytes=713245136 weight_to_token_ratio=0.107456 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471502131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311533568 dispatch_meta_bytes=336040 combine_payload_bytes=311533568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311869608 ingress_bytes=380739584 link_bytes=380739584 token_bytes=623403176 weight_movement_bytes=69206016 total_nvlink_bytes=692609192 weight_to_token_ratio=0.111013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432692403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=342335488 dispatch_meta_bytes=370008 combine_payload_bytes=342335488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342705496 ingress_bytes=411541504 link_bytes=411541504 token_bytes=685040984 weight_movement_bytes=69206016 total_nvlink_bytes=754247000 weight_to_token_ratio=0.101025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548597548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=79046 dup_rows=8814 dup_rows_remote=8558 dedup_ratio=0.089661 router_tokens_per_expert=[274, 433, 0, 89, 274, 0, 0, 0, 0, 3584, 115, 5096, 0, 0, 1811, 4657, 110, 434, 0, 0, 1018, 0, 0, 4, 276, 282, 0, 6551, 11, 0, 0, 0, 0, 0, 1, 0, 4684, 1266, 266, 0, 0, 0, 0, 0, 0, 64, 0, 0, 4428, 0, 0, 6641, 87, 2773, 961, 0, 0, 0, 0, 0, 0, 299, 2603, 60] payload_rows_per_dst=[13490, 9252, 10610, 14008, 11644, 10444, 7350, 12692] dup_rows_per_dst=[1932, 956, 1378, 232, 1222, 256, 292, 2546] combine_rows_per_src=[13490, 9252, 10610, 14008, 11644, 10444, 7350, 12692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90096 payload_rows_remote=76058 dup_rows=8208 dup_rows_remote=7952 dedup_ratio=0.083496 router_tokens_per_expert=[245, 448, 0, 60, 245, 0, 0, 0, 0, 3668, 122, 4995, 0, 0, 1824, 4284, 74, 397, 0, 0, 1063, 0, 0, 0, 253, 252, 0, 6628, 14, 0, 0, 0, 0, 0, 0, 0, 4797, 1400, 171, 0, 1, 0, 0, 0, 0, 92, 0, 0, 4420, 0, 0, 6815, 82, 2650, 1005, 0, 0, 0, 0, 0, 0, 314, 2741, 92] payload_rows_per_dst=[13764, 10060, 10646, 14038, 11804, 10176, 7212, 12396] dup_rows_per_dst=[1862, 1168, 1262, 256, 932, 0, 262, 2466] combine_rows_per_src=[13764, 10060, 10646, 14038, 11804, 10176, 7212, 12396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69676 payload_rows_remote=57480 dup_rows=28628 dup_rows_remote=26186 dedup_ratio=0.291219 router_tokens_per_expert=[264, 432, 0, 47, 264, 0, 0, 0, 0, 3843, 67, 5183, 0, 0, 1819, 4157, 65, 451, 0, 0, 1020, 0, 0, 0, 269, 199, 0, 6647, 17, 0, 0, 0, 0, 0, 0, 0, 4690, 1403, 289, 1, 4, 0, 0, 0, 0, 47, 0, 0, 4426, 0, 0, 6809, 63, 2604, 910, 0, 0, 0, 0, 0, 0, 274, 2843, 45] payload_rows_per_dst=[1486, 3702, 10548, 936, 11650, 13364, 15794, 12196] dup_rows_per_dst=[528, 70, 1376, 34, 1116, 4790, 18272, 2442] combine_rows_per_src=[1486, 3702, 10548, 936, 11650, 13364, 15794, 12196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=245243904 dispatch_meta_bytes=308232 combine_payload_bytes=245243904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=245552136 ingress_bytes=314449920 link_bytes=314449920 token_bytes=490796040 weight_movement_bytes=69206016 total_nvlink_bytes=560002056 weight_to_token_ratio=0.141008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183249734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89052 payload_rows_remote=78840 dup_rows=9252 dup_rows_remote=7988 dedup_ratio=0.094116 router_tokens_per_expert=[244, 447, 0, 77, 245, 0, 0, 0, 0, 3692, 117, 5078, 0, 0, 1951, 4171, 86, 409, 0, 0, 1081, 0, 0, 0, 249, 238, 0, 6735, 9, 0, 0, 0, 0, 0, 0, 0, 4785, 1361, 289, 0, 1, 0, 0, 0, 0, 90, 0, 0, 4162, 0, 0, 6771, 83, 2624, 1004, 0, 0, 0, 0, 0, 0, 269, 2795, 89] payload_rows_per_dst=[13702, 12340, 10212, 14258, 11756, 10338, 7196, 9250] dup_rows_per_dst=[1866, 3422, 1264, 204, 1114, 0, 226, 1156] combine_rows_per_src=[13702, 12340, 10212, 14258, 11756, 10338, 7196, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=350416 combine_payload_bytes=323772416 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=324122832 ingress_bytes=392978432 link_bytes=392978432 token_bytes=647895248 weight_movement_bytes=69206016 total_nvlink_bytes=717101264 weight_to_token_ratio=0.106817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478746202e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=235438080 dispatch_meta_bytes=334664 combine_payload_bytes=235438080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235772744 ingress_bytes=270041088 link_bytes=270041088 token_bytes=471210824 weight_movement_bytes=34603008 total_nvlink_bytes=505813832 weight_to_token_ratio=0.073434 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016142874e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86986 payload_rows_remote=72672 dup_rows=11318 dup_rows_remote=4776 dedup_ratio=0.115133 router_tokens_per_expert=[252, 433, 1, 38, 252, 0, 0, 0, 0, 3861, 121, 5031, 0, 0, 2015, 4431, 87, 375, 1, 0, 1013, 0, 0, 0, 258, 260, 0, 6371, 48, 0, 0, 0, 0, 0, 0, 1, 4627, 1368, 260, 0, 7, 0, 1, 0, 0, 89, 0, 1, 4323, 1, 0, 6758, 83, 2745, 934, 0, 0, 0, 0, 0, 0, 273, 2744, 89] payload_rows_per_dst=[13620, 14314, 10406, 13576, 11452, 10254, 7240, 6124] dup_rows_per_dst=[1848, 6542, 1192, 298, 1060, 4, 286, 88] combine_rows_per_src=[13620, 14314, 10406, 13576, 11452, 10254, 7240, 6124] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78446 payload_rows_remote=63522 dup_rows=19858 dup_rows_remote=10176 dedup_ratio=0.202006 router_tokens_per_expert=[260, 434, 0, 90, 260, 0, 0, 0, 0, 3771, 138, 5034, 0, 0, 2006, 4248, 126, 357, 0, 0, 1150, 0, 0, 0, 265, 270, 0, 6559, 36, 0, 0, 0, 0, 0, 0, 0, 4720, 1279, 136, 0, 0, 0, 0, 0, 0, 89, 0, 0, 3952, 0, 0, 6808, 91, 2718, 963, 0, 0, 0, 0, 0, 0, 389, 2917, 86] payload_rows_per_dst=[2236, 4108, 10032, 9840, 11580, 13088, 14924, 12638] dup_rows_per_dst=[604, 180, 1138, 222, 690, 4700, 9682, 2642] combine_rows_per_src=[2236, 4108, 10032, 9840, 11580, 13088, 14924, 12638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311533568 dispatch_meta_bytes=336040 combine_payload_bytes=311533568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311869608 ingress_bytes=380739584 link_bytes=380739584 token_bytes=623403176 weight_movement_bytes=69206016 total_nvlink_bytes=692609192 weight_to_token_ratio=0.111013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432692403e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=235438080 dispatch_meta_bytes=334664 combine_payload_bytes=235438080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235772744 ingress_bytes=270041088 link_bytes=270041088 token_bytes=471210824 weight_movement_bytes=34603008 total_nvlink_bytes=505813832 weight_to_token_ratio=0.073434 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.016142874e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83676 payload_rows_remote=69956 dup_rows=14628 dup_rows_remote=12724 dedup_ratio=0.148804 router_tokens_per_expert=[250, 460, 12, 85, 248, 0, 0, 0, 1, 3641, 113, 5133, 4, 1, 1895, 4428, 91, 404, 9, 3, 1064, 0, 0, 11, 269, 300, 0, 6454, 11, 3, 0, 0, 2, 2, 6, 7, 4504, 1364, 231, 0, 12, 0, 1, 4, 0, 104, 0, 9, 4454, 12, 0, 6757, 112, 2449, 1009, 0, 0, 3, 6, 0, 1, 347, 2762, 104] payload_rows_per_dst=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] dup_rows_per_dst=[1904, 5998, 3952, 1876, 672, 12, 46, 168] combine_rows_per_src=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=322928640 dispatch_meta_bytes=347312 combine_payload_bytes=322928640 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=323275952 ingress_bytes=392134656 link_bytes=392134656 token_bytes=646204592 weight_movement_bytes=69206016 total_nvlink_bytes=715410608 weight_to_token_ratio=0.107096 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.475571141e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88306 payload_rows_remote=74556 dup_rows=9998 dup_rows_remote=8524 dedup_ratio=0.101705 router_tokens_per_expert=[6809, 80, 0, 215, 4320, 0, 391, 185, 0, 80, 0, 0, 0, 72, 0, 7763, 0, 661, 2985, 0, 0, 0, 378, 0, 0, 663, 0, 0, 0, 140, 0, 0, 3060, 0, 0, 0, 0, 377, 5064, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 1571, 0, 0, 0, 0, 4814, 1, 0, 3475, 0, 0, 720, 277, 2662, 0, 482] payload_rows_per_dst=[9342, 15526, 7740, 13750, 10824, 11838, 11506, 7780] dup_rows_per_dst=[1040, 304, 308, 1474, 2384, 2104, 1266, 1118] combine_rows_per_src=[9342, 15526, 7740, 13750, 10824, 11838, 11506, 7780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89772 payload_rows_remote=78576 dup_rows=8532 dup_rows_remote=7556 dedup_ratio=0.086792 router_tokens_per_expert=[251, 407, 0, 24, 251, 0, 0, 0, 0, 3902, 98, 5087, 0, 0, 2012, 4479, 69, 357, 0, 0, 1020, 0, 0, 0, 257, 248, 0, 6554, 45, 0, 0, 0, 0, 0, 0, 0, 4509, 1333, 244, 0, 3, 0, 0, 0, 0, 92, 0, 0, 4210, 0, 0, 6938, 94, 2611, 925, 0, 0, 0, 0, 0, 0, 312, 2734, 86] payload_rows_per_dst=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] dup_rows_per_dst=[1814, 1432, 1166, 290, 976, 0, 262, 2592] combine_rows_per_src=[13928, 10592, 10146, 13918, 11196, 10364, 6998, 12630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88330 payload_rows_remote=81290 dup_rows=9974 dup_rows_remote=9968 dedup_ratio=0.101461 router_tokens_per_expert=[6901, 94, 0, 185, 4678, 0, 389, 196, 0, 94, 0, 0, 0, 6, 0, 7770, 0, 563, 2667, 0, 0, 0, 286, 0, 0, 686, 0, 0, 0, 153, 0, 0, 3163, 0, 0, 0, 0, 360, 4941, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 1251, 0, 0, 0, 0, 4940, 9, 0, 3887, 0, 0, 689, 238, 2352, 0, 562] payload_rows_per_dst=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] dup_rows_per_dst=[982, 200, 216, 1506, 6, 2082, 1234, 3748] combine_rows_per_src=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=297664512 dispatch_meta_bytes=309792 combine_payload_bytes=297664512 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=297974304 ingress_bytes=436076544 link_bytes=436076544 token_bytes=595638816 weight_movement_bytes=138412032 total_nvlink_bytes=734050848 weight_to_token_ratio=0.232376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.640920928e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82584 payload_rows_remote=68222 dup_rows=15720 dup_rows_remote=12984 dedup_ratio=0.159912 router_tokens_per_expert=[1965, 0, 2, 1, 5, 0, 3, 2, 3, 2, 139, 1, 1763, 6571, 0, 80, 0, 0, 5, 1, 4, 277, 428, 365, 1, 2, 2, 0, 0, 90, 0, 0, 1, 5717, 2981, 57, 0, 7, 3, 0, 760, 1051, 7, 41, 0, 88, 2472, 2878, 3, 61, 7, 268, 113, 6092, 6536, 8, 384, 0, 0, 2, 9, 4, 7890, 0] payload_rows_per_dst=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] dup_rows_per_dst=[2736, 1644, 1314, 4, 5902, 4010, 104, 6] combine_rows_per_src=[14362, 14516, 12280, 13258, 15976, 10584, 816, 792] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=260186112 dispatch_meta_bytes=294792 combine_payload_bytes=260186112 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=260480904 ingress_bytes=398598144 link_bytes=398598144 token_bytes=520667016 weight_movement_bytes=138412032 total_nvlink_bytes=659079048 weight_to_token_ratio=0.265836 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499892726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286539776 dispatch_meta_bytes=330720 combine_payload_bytes=286539776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286870496 ingress_bytes=321142784 link_bytes=321142784 token_bytes=573410272 weight_movement_bytes=34603008 total_nvlink_bytes=608013280 weight_to_token_ratio=0.060346 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208434442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305381376 dispatch_meta_bytes=332320 combine_payload_bytes=305381376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305713696 ingress_bytes=339984384 link_bytes=339984384 token_bytes=611095072 weight_movement_bytes=34603008 total_nvlink_bytes=645698080 weight_to_token_ratio=0.056625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279333866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=321847296 dispatch_meta_bytes=344528 combine_payload_bytes=321847296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322191824 ingress_bytes=391053312 link_bytes=391053312 token_bytes=644039120 weight_movement_bytes=69206016 total_nvlink_bytes=713245136 weight_to_token_ratio=0.107456 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471502131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85360 payload_rows_remote=74016 dup_rows=12944 dup_rows_remote=11010 dedup_ratio=0.131673 router_tokens_per_expert=[6728, 70, 0, 202, 4677, 0, 388, 150, 0, 70, 0, 3, 0, 7, 0, 7795, 0, 634, 2789, 0, 0, 0, 25, 1, 0, 725, 0, 0, 0, 140, 0, 0, 3154, 0, 0, 0, 0, 353, 4669, 0, 0, 0, 0, 0, 0, 0, 0, 1970, 1630, 0, 0, 0, 0, 4891, 26, 0, 4624, 0, 0, 597, 307, 1983, 0, 544] payload_rows_per_dst=[9890, 8082, 11286, 13620, 7002, 11344, 11750, 12386] dup_rows_per_dst=[1084, 76, 3204, 1566, 12, 1934, 1344, 3724] combine_rows_per_src=[9890, 8082, 11286, 13620, 7002, 11344, 11750, 12386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=332963840 dispatch_meta_bytes=365032 combine_payload_bytes=332963840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333328872 ingress_bytes=402169856 link_bytes=402169856 token_bytes=666292712 weight_movement_bytes=69206016 total_nvlink_bytes=735498728 weight_to_token_ratio=0.103867 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513332791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88306 payload_rows_remote=74556 dup_rows=9998 dup_rows_remote=8524 dedup_ratio=0.101705 router_tokens_per_expert=[6809, 80, 0, 215, 4320, 0, 391, 185, 0, 80, 0, 0, 0, 72, 0, 7763, 0, 661, 2985, 0, 0, 0, 378, 0, 0, 663, 0, 0, 0, 140, 0, 0, 3060, 0, 0, 0, 0, 377, 5064, 0, 0, 0, 0, 0, 0, 0, 0, 1907, 1571, 0, 0, 0, 0, 4814, 1, 0, 3475, 0, 0, 720, 277, 2662, 0, 482] payload_rows_per_dst=[9342, 15526, 7740, 13750, 10824, 11838, 11506, 7780] dup_rows_per_dst=[1040, 304, 308, 1474, 2384, 2104, 1266, 1118] combine_rows_per_src=[9342, 15526, 7740, 13750, 10824, 11838, 11506, 7780] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80388 payload_rows_remote=67810 dup_rows=17916 dup_rows_remote=11706 dedup_ratio=0.182251 router_tokens_per_expert=[6978, 63, 0, 211, 4281, 0, 432, 114, 0, 64, 0, 1, 0, 8, 0, 7814, 0, 571, 2734, 0, 0, 0, 428, 0, 0, 697, 0, 0, 0, 98, 0, 0, 3210, 0, 0, 1, 0, 343, 5091, 0, 0, 0, 0, 0, 0, 0, 0, 2016, 1534, 0, 0, 0, 0, 4555, 8, 0, 3841, 0, 0, 574, 304, 2531, 0, 650] payload_rows_per_dst=[1316, 8708, 16256, 14040, 7072, 11964, 8454, 12578] dup_rows_per_dst=[324, 0, 6838, 1506, 36, 2250, 752, 6210] combine_rows_per_src=[1316, 8708, 16256, 14040, 7072, 11964, 8454, 12578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=303169536 dispatch_meta_bytes=340104 combine_payload_bytes=303169536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303509640 ingress_bytes=337772544 link_bytes=337772544 token_bytes=606679176 weight_movement_bytes=34603008 total_nvlink_bytes=641282184 weight_to_token_ratio=0.057037 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271010890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88084 payload_rows_remote=81022 dup_rows=10220 dup_rows_remote=9826 dedup_ratio=0.103963 router_tokens_per_expert=[6928, 76, 0, 228, 4304, 0, 433, 174, 0, 73, 0, 1, 0, 21, 0, 7722, 0, 705, 2595, 0, 0, 0, 428, 0, 0, 710, 0, 0, 0, 140, 0, 0, 2972, 0, 0, 0, 1, 422, 5087, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 1439, 0, 0, 0, 0, 4740, 22, 0, 3779, 0, 0, 672, 334, 2558, 0, 738] payload_rows_per_dst=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] dup_rows_per_dst=[1126, 190, 394, 1556, 2832, 1988, 1184, 950] combine_rows_per_src=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=279437312 dispatch_meta_bytes=324824 combine_payload_bytes=279437312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279762136 ingress_bytes=314040320 link_bytes=314040320 token_bytes=559199448 weight_movement_bytes=34603008 total_nvlink_bytes=593802456 weight_to_token_ratio=0.061880 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181708442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88084 payload_rows_remote=81022 dup_rows=10220 dup_rows_remote=9826 dedup_ratio=0.103963 router_tokens_per_expert=[6928, 76, 0, 228, 4304, 0, 433, 174, 0, 73, 0, 1, 0, 21, 0, 7722, 0, 705, 2595, 0, 0, 0, 428, 0, 0, 710, 0, 0, 0, 140, 0, 0, 2972, 0, 0, 0, 1, 422, 5087, 0, 0, 0, 0, 0, 0, 0, 0, 1850, 1439, 0, 0, 0, 0, 4740, 22, 0, 3779, 0, 0, 672, 334, 2558, 0, 738] payload_rows_per_dst=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] dup_rows_per_dst=[1126, 190, 394, 1556, 2832, 1988, 1184, 950] combine_rows_per_src=[9304, 15444, 7062, 14000, 11516, 11886, 11218, 7654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=303169536 dispatch_meta_bytes=340104 combine_payload_bytes=303169536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303509640 ingress_bytes=337772544 link_bytes=337772544 token_bytes=606679176 weight_movement_bytes=34603008 total_nvlink_bytes=641282184 weight_to_token_ratio=0.057037 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.022500 gemm_active_groups=9 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271010890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=318064 combine_payload_bytes=277749760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278067824 ingress_bytes=346955776 link_bytes=346955776 token_bytes=555817584 weight_movement_bytes=69206016 total_nvlink_bytes=625023600 weight_to_token_ratio=0.124512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.305566653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82062 payload_rows_remote=66520 dup_rows=16242 dup_rows_remote=15982 dedup_ratio=0.165222 router_tokens_per_expert=[6672, 83, 0, 210, 4500, 0, 391, 186, 1, 85, 1, 0, 1, 45, 0, 7768, 0, 585, 2818, 0, 0, 0, 304, 0, 1, 604, 2, 0, 0, 152, 0, 1, 3287, 0, 0, 2, 0, 378, 5016, 1, 0, 0, 0, 0, 0, 0, 0, 1779, 1451, 0, 0, 0, 0, 5036, 18, 1, 4098, 0, 0, 689, 259, 2207, 1, 519] payload_rows_per_dst=[14290, 15542, 7124, 3964, 12674, 10344, 11570, 6554] dup_rows_per_dst=[7030, 260, 290, 320, 4546, 1558, 1442, 796] combine_rows_per_src=[14290, 15542, 7124, 3964, 12674, 10344, 11570, 6554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=331866112 dispatch_meta_bytes=363392 combine_payload_bytes=331866112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332229504 ingress_bytes=366469120 link_bytes=366469120 token_bytes=664095616 weight_movement_bytes=34603008 total_nvlink_bytes=698698624 weight_to_token_ratio=0.052105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378993795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81922 payload_rows_remote=70444 dup_rows=16382 dup_rows_remote=15050 dedup_ratio=0.166646 router_tokens_per_expert=[6789, 85, 0, 251, 4394, 0, 393, 194, 0, 85, 0, 0, 0, 66, 0, 7747, 0, 671, 2888, 0, 0, 0, 323, 0, 0, 668, 0, 0, 0, 175, 0, 0, 3403, 0, 0, 0, 0, 393, 4969, 0, 0, 0, 0, 0, 0, 0, 0, 1774, 1475, 0, 0, 0, 0, 4930, 0, 0, 3568, 0, 0, 711, 307, 2281, 0, 612] payload_rows_per_dst=[7352, 2578, 16212, 13778, 7566, 11604, 11478, 11354] dup_rows_per_dst=[1006, 0, 7046, 1486, 26, 1882, 1332, 3604] combine_rows_per_src=[7352, 2578, 16212, 13778, 7566, 11604, 11478, 11354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305381376 dispatch_meta_bytes=332320 combine_payload_bytes=305381376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305713696 ingress_bytes=339984384 link_bytes=339984384 token_bytes=611095072 weight_movement_bytes=34603008 total_nvlink_bytes=645698080 weight_to_token_ratio=0.056625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279333866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277749760 dispatch_meta_bytes=318064 combine_payload_bytes=277749760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278067824 ingress_bytes=346955776 link_bytes=346955776 token_bytes=555817584 weight_movement_bytes=69206016 total_nvlink_bytes=625023600 weight_to_token_ratio=0.124512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.305566653e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94418 payload_rows_remote=78194 dup_rows=3886 dup_rows_remote=3506 dedup_ratio=0.039530 router_tokens_per_expert=[0, 7766, 92, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1278, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 8043, 0, 190, 0, 69, 0, 0, 0, 0, 0, 8, 80, 0, 65, 0, 7561, 0, 137, 0, 0, 0, 87, 0, 0, 0, 6232, 0, 80, 0, 0, 0, 0, 0, 0, 0, 7352, 1858, 0, 0, 0, 0] payload_rows_per_dst=[15532, 15120, 0, 16224, 15138, 16224, 12464, 3716] dup_rows_per_dst=[188, 2420, 0, 380, 290, 448, 160, 0] combine_rows_per_src=[15532, 15120, 0, 16224, 15138, 16224, 12464, 3716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88330 payload_rows_remote=81290 dup_rows=9974 dup_rows_remote=9968 dedup_ratio=0.101461 router_tokens_per_expert=[6901, 94, 0, 185, 4678, 0, 389, 196, 0, 94, 0, 0, 0, 6, 0, 7770, 0, 563, 2667, 0, 0, 0, 286, 0, 0, 686, 0, 0, 0, 153, 0, 0, 3163, 0, 0, 0, 0, 360, 4941, 0, 0, 0, 0, 0, 0, 0, 0, 2092, 1251, 0, 0, 0, 0, 4940, 9, 0, 3887, 0, 0, 689, 238, 2352, 0, 562] payload_rows_per_dst=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] dup_rows_per_dst=[982, 200, 216, 1506, 6, 2082, 1234, 3748] combine_rows_per_src=[10102, 15540, 6816, 13974, 7040, 11984, 11166, 11708] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75874 payload_rows_remote=60464 dup_rows=22430 dup_rows_remote=13724 dedup_ratio=0.228170 router_tokens_per_expert=[6787, 88, 0, 211, 4088, 1, 395, 209, 6, 93, 9, 2, 14, 64, 4, 7735, 0, 658, 2699, 0, 10, 0, 371, 18, 1, 713, 0, 0, 0, 156, 8, 4, 3145, 3, 2, 2, 0, 374, 5198, 8, 1, 1, 1, 1, 0, 10, 11, 1746, 1608, 2, 0, 0, 0, 4943, 22, 8, 3652, 0, 0, 700, 314, 2512, 7, 537] payload_rows_per_dst=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] dup_rows_per_dst=[8706, 342, 364, 8, 10780, 14, 1300, 916] combine_rows_per_src=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=331866112 dispatch_meta_bytes=363392 combine_payload_bytes=331866112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332229504 ingress_bytes=366469120 link_bytes=366469120 token_bytes=664095616 weight_movement_bytes=34603008 total_nvlink_bytes=698698624 weight_to_token_ratio=0.052105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378993795e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94466 payload_rows_remote=79252 dup_rows=3838 dup_rows_remote=3588 dedup_ratio=0.039042 router_tokens_per_expert=[0, 7775, 109, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1246, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 8017, 0, 237, 0, 81, 0, 0, 0, 0, 0, 1, 94, 0, 31, 0, 7606, 0, 112, 0, 0, 0, 97, 0, 0, 0, 6134, 0, 94, 0, 0, 0, 0, 0, 0, 0, 7350, 1915, 0, 0, 0, 0] payload_rows_per_dst=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] dup_rows_per_dst=[222, 2286, 0, 474, 250, 418, 188, 0] combine_rows_per_src=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=272465920 dispatch_meta_bytes=330008 combine_payload_bytes=272465920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272795928 ingress_bytes=341671936 link_bytes=341671936 token_bytes=545261848 weight_movement_bytes=69206016 total_nvlink_bytes=614467864 weight_to_token_ratio=0.126923 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285683988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288538624 dispatch_meta_bytes=341976 combine_payload_bytes=288538624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288880600 ingress_bytes=357744640 link_bytes=357744640 token_bytes=577419224 weight_movement_bytes=69206016 total_nvlink_bytes=646625240 weight_to_token_ratio=0.119854 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346164280e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=247660544 dispatch_meta_bytes=296752 combine_payload_bytes=247660544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247957296 ingress_bytes=351469568 link_bytes=351469568 token_bytes=495617840 weight_movement_bytes=103809024 total_nvlink_bytes=599426864 weight_to_token_ratio=0.209454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83676 payload_rows_remote=69956 dup_rows=14628 dup_rows_remote=12724 dedup_ratio=0.148804 router_tokens_per_expert=[250, 460, 12, 85, 248, 0, 0, 0, 1, 3641, 113, 5133, 4, 1, 1895, 4428, 91, 404, 9, 3, 1064, 0, 0, 11, 269, 300, 0, 6454, 11, 3, 0, 0, 2, 2, 6, 7, 4504, 1364, 231, 0, 12, 0, 1, 4, 0, 104, 0, 9, 4454, 12, 0, 6757, 112, 2449, 1009, 0, 0, 3, 6, 0, 1, 347, 2762, 104] payload_rows_per_dst=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] dup_rows_per_dst=[1904, 5998, 3952, 1876, 672, 12, 46, 168] combine_rows_per_src=[13720, 14168, 12976, 14318, 9440, 10514, 2262, 6278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94280 payload_rows_remote=78036 dup_rows=4024 dup_rows_remote=3708 dedup_ratio=0.040934 router_tokens_per_expert=[0, 7799, 106, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1290, 2, 140, 0, 0, 0, 8122, 0, 0, 0, 8060, 0, 271, 0, 61, 0, 0, 0, 0, 0, 9, 70, 0, 45, 0, 7519, 0, 70, 0, 0, 0, 88, 0, 0, 0, 6110, 0, 70, 0, 0, 0, 0, 0, 0, 0, 7580, 1725, 0, 0, 0, 0] payload_rows_per_dst=[15598, 4792, 0, 16242, 15044, 16244, 12220, 14140] dup_rows_per_dst=[242, 374, 0, 542, 242, 316, 140, 2168] combine_rows_per_src=[15598, 4792, 0, 16242, 15044, 16244, 12220, 14140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=324616192 dispatch_meta_bytes=331360 combine_payload_bytes=324616192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324947552 ingress_bytes=393822208 link_bytes=393822208 token_bytes=649563744 weight_movement_bytes=69206016 total_nvlink_bytes=718769760 weight_to_token_ratio=0.106542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481921263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94418 payload_rows_remote=78194 dup_rows=3886 dup_rows_remote=3506 dedup_ratio=0.039530 router_tokens_per_expert=[0, 7766, 92, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1278, 0, 140, 0, 0, 0, 8112, 0, 0, 0, 8043, 0, 190, 0, 69, 0, 0, 0, 0, 0, 8, 80, 0, 65, 0, 7561, 0, 137, 0, 0, 0, 87, 0, 0, 0, 6232, 0, 80, 0, 0, 0, 0, 0, 0, 0, 7352, 1858, 0, 0, 0, 0] payload_rows_per_dst=[15532, 15120, 0, 16224, 15138, 16224, 12464, 3716] dup_rows_per_dst=[188, 2420, 0, 380, 290, 448, 160, 0] combine_rows_per_src=[15532, 15120, 0, 16224, 15138, 16224, 12464, 3716] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=319635456 dispatch_meta_bytes=326976 combine_payload_bytes=319635456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319962432 ingress_bytes=354238464 link_bytes=354238464 token_bytes=639597888 weight_movement_bytes=34603008 total_nvlink_bytes=674200896 weight_to_token_ratio=0.054101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332970821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=288538624 dispatch_meta_bytes=341976 combine_payload_bytes=288538624 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=288880600 ingress_bytes=357744640 link_bytes=357744640 token_bytes=577419224 weight_movement_bytes=69206016 total_nvlink_bytes=646625240 weight_to_token_ratio=0.119854 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346164280e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=63692 dup_rows=18228 dup_rows_remote=918 dedup_ratio=0.185425 router_tokens_per_expert=[0, 7823, 108, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1279, 0, 98, 0, 0, 0, 8122, 0, 0, 0, 8064, 0, 203, 0, 64, 0, 0, 0, 0, 0, 0, 63, 0, 53, 0, 7519, 0, 124, 0, 0, 0, 69, 0, 0, 0, 6170, 0, 63, 0, 0, 0, 0, 0, 0, 0, 7461, 1867, 0, 0, 0, 0] payload_rows_per_dst=[220, 2754, 15646, 16256, 232, 16244, 12340, 16384] dup_rows_per_dst=[0, 0, 0, 406, 0, 386, 126, 17310] combine_rows_per_src=[220, 2754, 15646, 16256, 232, 16244, 12340, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92278 payload_rows_remote=89310 dup_rows=6026 dup_rows_remote=3686 dedup_ratio=0.061300 router_tokens_per_expert=[0, 7725, 93, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1331, 0, 139, 0, 0, 0, 8116, 0, 0, 0, 8055, 0, 204, 0, 61, 0, 0, 0, 0, 0, 0, 76, 0, 26, 0, 7570, 0, 116, 0, 0, 0, 82, 0, 0, 0, 6228, 0, 78, 0, 0, 0, 0, 0, 0, 0, 7358, 1893, 0, 0, 0, 0] payload_rows_per_dst=[15450, 15172, 2968, 13324, 15140, 13978, 12460, 3786] dup_rows_per_dst=[188, 2484, 2340, 372, 204, 286, 152, 0] combine_rows_per_src=[15450, 15172, 2968, 13324, 15140, 13978, 12460, 3786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=320282624 dispatch_meta_bytes=326800 combine_payload_bytes=320282624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320609424 ingress_bytes=354885632 link_bytes=354885632 token_bytes=640892048 weight_movement_bytes=34603008 total_nvlink_bytes=675495056 weight_to_token_ratio=0.053992 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335406063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=332963840 dispatch_meta_bytes=365032 combine_payload_bytes=332963840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333328872 ingress_bytes=402169856 link_bytes=402169856 token_bytes=666292712 weight_movement_bytes=69206016 total_nvlink_bytes=735498728 weight_to_token_ratio=0.103867 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513332791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79288 payload_rows_remote=64152 dup_rows=19016 dup_rows_remote=16676 dedup_ratio=0.193441 router_tokens_per_expert=[0, 7776, 92, 0, 0, 0, 0, 3, 0, 0, 1, 1, 0, 1250, 0, 152, 0, 1, 0, 8103, 0, 1, 0, 8038, 0, 247, 1, 68, 0, 0, 0, 0, 0, 12, 83, 1, 42, 0, 7589, 0, 89, 0, 1, 0, 92, 0, 0, 1, 6264, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7334, 1824, 3, 0, 0, 0] payload_rows_per_dst=[15552, 15136, 16218, 632, 15208, 360, 12528, 3654] dup_rows_per_dst=[190, 2340, 16068, 0, 246, 6, 166, 0] combine_rows_per_src=[15552, 15136, 16218, 632, 15208, 360, 12528, 3654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83984 payload_rows_remote=71902 dup_rows=14320 dup_rows_remote=14150 dedup_ratio=0.145671 router_tokens_per_expert=[0, 7755, 92, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1225, 0, 175, 0, 0, 0, 8107, 0, 0, 0, 8067, 0, 226, 0, 40, 0, 0, 0, 0, 0, 0, 85, 0, 55, 0, 7598, 0, 123, 0, 0, 0, 93, 0, 0, 0, 6041, 0, 85, 0, 0, 0, 0, 0, 0, 0, 7334, 2050, 0, 0, 0, 0] payload_rows_per_dst=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] dup_rows_per_dst=[30, 0, 0, 452, 80, 432, 170, 13156] combine_rows_per_src=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=320282624 dispatch_meta_bytes=326800 combine_payload_bytes=320282624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320609424 ingress_bytes=354885632 link_bytes=354885632 token_bytes=640892048 weight_movement_bytes=34603008 total_nvlink_bytes=675495056 weight_to_token_ratio=0.053992 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335406063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286539776 dispatch_meta_bytes=330720 combine_payload_bytes=286539776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286870496 ingress_bytes=321142784 link_bytes=321142784 token_bytes=573410272 weight_movement_bytes=34603008 total_nvlink_bytes=608013280 weight_to_token_ratio=0.060346 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208434442e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=319635456 dispatch_meta_bytes=326976 combine_payload_bytes=319635456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319962432 ingress_bytes=354238464 link_bytes=354238464 token_bytes=639597888 weight_movement_bytes=34603008 total_nvlink_bytes=674200896 weight_to_token_ratio=0.054101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332970821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260882432 dispatch_meta_bytes=258440 combine_payload_bytes=260882432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=261140872 ingress_bytes=330088448 link_bytes=330088448 token_bytes=522023304 weight_movement_bytes=69206016 total_nvlink_bytes=591229320 weight_to_token_ratio=0.132573 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242096256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77638 payload_rows_remote=62094 dup_rows=20666 dup_rows_remote=19622 dedup_ratio=0.210225 router_tokens_per_expert=[3, 7748, 146, 0, 0, 0, 0, 11, 0, 3, 7, 1, 0, 1263, 2, 154, 3, 1, 1, 8069, 2, 9, 0, 7991, 0, 239, 2, 53, 0, 0, 1, 0, 7, 39, 88, 6, 45, 0, 7520, 0, 135, 0, 16, 0, 98, 6, 5, 16, 6125, 5, 88, 2, 0, 2, 0, 12, 0, 6, 7277, 1942, 1, 0, 2, 0] payload_rows_per_dst=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] dup_rows_per_dst=[1044, 2348, 15974, 0, 1078, 48, 172, 2] combine_rows_per_src=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=79166 dup_rows=9674 dup_rows_remote=7620 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 0, 430, 0, 0, 0, 0, 338, 0, 0, 0, 1617, 0, 0, 62, 0, 0, 2158, 0, 303, 0, 0, 0, 3395, 7945, 0, 4517, 0, 7958, 0, 0, 456, 1233, 0, 6928, 0, 7849, 0, 0, 0, 0, 319, 0, 0, 609, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 88, 59, 0, 0, 0] payload_rows_per_dst=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] dup_rows_per_dst=[742, 2006, 540, 0, 2054, 2870, 0, 1462] combine_rows_per_src=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=365813760 dispatch_meta_bytes=371984 combine_payload_bytes=365813760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=366185744 ingress_bytes=469622784 link_bytes=469622784 token_bytes=731999504 weight_movement_bytes=103809024 total_nvlink_bytes=835808528 weight_to_token_ratio=0.141816 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.767152728e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294510592 dispatch_meta_bytes=344208 combine_payload_bytes=294510592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=294854800 ingress_bytes=363716608 link_bytes=363716608 token_bytes=589365392 weight_movement_bytes=69206016 total_nvlink_bytes=658571408 weight_to_token_ratio=0.117425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.368636314e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=262766592 dispatch_meta_bytes=323312 combine_payload_bytes=262766592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263089904 ingress_bytes=297369600 link_bytes=297369600 token_bytes=525856496 weight_movement_bytes=34603008 total_nvlink_bytes=560459504 weight_to_token_ratio=0.065803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118977865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84300 payload_rows_remote=68048 dup_rows=14004 dup_rows_remote=11522 dedup_ratio=0.142456 router_tokens_per_expert=[0, 0, 0, 475, 0, 0, 0, 0, 452, 3, 0, 0, 1641, 0, 0, 57, 0, 0, 2822, 0, 283, 0, 0, 0, 3310, 7982, 0, 3558, 0, 7991, 0, 0, 522, 1164, 0, 6683, 0, 7853, 0, 0, 0, 0, 348, 0, 0, 574, 592, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 104, 72, 0, 0, 0] payload_rows_per_dst=[11340, 3642, 6210, 5036, 14438, 16252, 15982, 11400] dup_rows_per_dst=[538, 664, 0, 0, 8920, 2482, 42, 1358] combine_rows_per_src=[11340, 3642, 6210, 5036, 14438, 16252, 15982, 11400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=254337024 dispatch_meta_bytes=326864 combine_payload_bytes=254337024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254663888 ingress_bytes=323543040 link_bytes=323543040 token_bytes=509000912 weight_movement_bytes=69206016 total_nvlink_bytes=578206928 weight_to_token_ratio=0.135964 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217466412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=324263936 dispatch_meta_bytes=347144 combine_payload_bytes=324263936 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=324611080 ingress_bytes=497278976 link_bytes=497278976 token_bytes=648875016 weight_movement_bytes=173015040 total_nvlink_bytes=821890056 weight_to_token_ratio=0.266638 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.871220752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=278724608 dispatch_meta_bytes=318280 combine_payload_bytes=278724608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279042888 ingress_bytes=313327616 link_bytes=313327616 token_bytes=557767496 weight_movement_bytes=34603008 total_nvlink_bytes=592370504 weight_to_token_ratio=0.062038 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179026594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79288 payload_rows_remote=64152 dup_rows=19016 dup_rows_remote=16676 dedup_ratio=0.193441 router_tokens_per_expert=[0, 7776, 92, 0, 0, 0, 0, 3, 0, 0, 1, 1, 0, 1250, 0, 152, 0, 1, 0, 8103, 0, 1, 0, 8038, 0, 247, 1, 68, 0, 0, 0, 0, 0, 12, 83, 1, 42, 0, 7589, 0, 89, 0, 1, 0, 92, 0, 0, 1, 6264, 0, 83, 0, 0, 0, 0, 0, 0, 0, 7334, 1824, 3, 0, 0, 0] payload_rows_per_dst=[15552, 15136, 16218, 632, 15208, 360, 12528, 3654] dup_rows_per_dst=[190, 2340, 16068, 0, 246, 6, 166, 0] combine_rows_per_src=[15552, 15136, 16218, 632, 15208, 360, 12528, 3654] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83984 payload_rows_remote=71902 dup_rows=14320 dup_rows_remote=14150 dedup_ratio=0.145671 router_tokens_per_expert=[0, 7755, 92, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1225, 0, 175, 0, 0, 0, 8107, 0, 0, 0, 8067, 0, 226, 0, 40, 0, 0, 0, 0, 0, 0, 85, 0, 55, 0, 7598, 0, 123, 0, 0, 0, 93, 0, 0, 0, 6041, 0, 85, 0, 0, 0, 0, 0, 0, 0, 7334, 2050, 0, 0, 0, 0] payload_rows_per_dst=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] dup_rows_per_dst=[30, 0, 0, 452, 80, 432, 170, 13156] combine_rows_per_src=[2754, 2800, 12912, 16214, 4750, 16214, 12082, 16258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94466 payload_rows_remote=79252 dup_rows=3838 dup_rows_remote=3588 dedup_ratio=0.039042 router_tokens_per_expert=[0, 7775, 109, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1246, 0, 153, 0, 0, 0, 8098, 0, 0, 0, 8017, 0, 237, 0, 81, 0, 0, 0, 0, 0, 1, 94, 0, 31, 0, 7606, 0, 112, 0, 0, 0, 97, 0, 0, 0, 6134, 0, 94, 0, 0, 0, 0, 0, 0, 0, 7350, 1915, 0, 0, 0, 0] payload_rows_per_dst=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] dup_rows_per_dst=[222, 2286, 0, 474, 250, 418, 188, 0] combine_rows_per_src=[15550, 15212, 0, 16196, 15214, 16196, 12268, 3830] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89168 payload_rows_remote=84070 dup_rows=9136 dup_rows_remote=9136 dedup_ratio=0.092936 router_tokens_per_expert=[0, 0, 0, 527, 0, 0, 0, 0, 291, 0, 0, 0, 1646, 0, 0, 21, 0, 0, 2369, 0, 340, 0, 0, 0, 3101, 7972, 0, 4294, 0, 7983, 0, 0, 574, 1258, 0, 6934, 0, 7873, 0, 0, 0, 0, 278, 0, 0, 592, 817, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2175, 47, 60, 0, 0, 0] payload_rows_per_dst=[16064, 15446, 9932, 5098, 4334, 10480, 15966, 11848] dup_rows_per_dst=[934, 2338, 2338, 0, 434, 1788, 0, 1304] combine_rows_per_src=[16064, 15446, 9932, 5098, 4334, 10480, 15966, 11848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84772 payload_rows_remote=72770 dup_rows=13532 dup_rows_remote=12188 dedup_ratio=0.137655 router_tokens_per_expert=[0, 0, 0, 517, 0, 0, 0, 0, 290, 1, 0, 0, 1547, 0, 0, 14, 0, 0, 2277, 0, 296, 4, 0, 1, 3398, 8031, 0, 4457, 0, 8029, 0, 0, 400, 1205, 0, 6980, 0, 7875, 0, 0, 0, 0, 251, 0, 0, 642, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2117, 63, 35, 0, 0, 0] payload_rows_per_dst=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] dup_rows_per_dst=[0, 408, 0, 0, 9082, 2698, 0, 1344] combine_rows_per_src=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=262766592 dispatch_meta_bytes=323312 combine_payload_bytes=262766592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263089904 ingress_bytes=297369600 link_bytes=297369600 token_bytes=525856496 weight_movement_bytes=34603008 total_nvlink_bytes=560459504 weight_to_token_ratio=0.065803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118977865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75874 payload_rows_remote=60464 dup_rows=22430 dup_rows_remote=13724 dedup_ratio=0.228170 router_tokens_per_expert=[6787, 88, 0, 211, 4088, 1, 395, 209, 6, 93, 9, 2, 14, 64, 4, 7735, 0, 658, 2699, 0, 10, 0, 371, 18, 1, 713, 0, 0, 0, 156, 8, 4, 3145, 3, 2, 2, 0, 374, 5198, 8, 1, 1, 1, 1, 0, 10, 11, 1746, 1608, 2, 0, 0, 0, 4943, 22, 8, 3652, 0, 0, 700, 314, 2512, 7, 537] payload_rows_per_dst=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] dup_rows_per_dst=[8706, 342, 364, 8, 10780, 14, 1300, 916] combine_rows_per_src=[15410, 15512, 7148, 1756, 13988, 3528, 11308, 7224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84772 payload_rows_remote=72770 dup_rows=13532 dup_rows_remote=12188 dedup_ratio=0.137655 router_tokens_per_expert=[0, 0, 0, 517, 0, 0, 0, 0, 290, 1, 0, 0, 1547, 0, 0, 14, 0, 0, 2277, 0, 296, 4, 0, 1, 3398, 8031, 0, 4457, 0, 8029, 0, 0, 400, 1205, 0, 6980, 0, 7875, 0, 0, 0, 0, 251, 0, 0, 642, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2117, 63, 35, 0, 0, 0] payload_rows_per_dst=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] dup_rows_per_dst=[0, 408, 0, 0, 9082, 2698, 0, 1344] combine_rows_per_src=[1034, 3296, 5156, 16062, 14884, 16280, 16058, 12002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84300 payload_rows_remote=68048 dup_rows=14004 dup_rows_remote=11522 dedup_ratio=0.142456 router_tokens_per_expert=[0, 0, 0, 475, 0, 0, 0, 0, 452, 3, 0, 0, 1641, 0, 0, 57, 0, 0, 2822, 0, 283, 0, 0, 0, 3310, 7982, 0, 3558, 0, 7991, 0, 0, 522, 1164, 0, 6683, 0, 7853, 0, 0, 0, 0, 348, 0, 0, 574, 592, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 2645, 104, 72, 0, 0, 0] payload_rows_per_dst=[11340, 3642, 6210, 5036, 14438, 16252, 15982, 11400] dup_rows_per_dst=[538, 664, 0, 0, 8920, 2482, 42, 1358] combine_rows_per_src=[11340, 3642, 6210, 5036, 14438, 16252, 15982, 11400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88126 payload_rows_remote=72354 dup_rows=10178 dup_rows_remote=4790 dedup_ratio=0.103536 router_tokens_per_expert=[0, 0, 0, 479, 0, 0, 0, 0, 370, 0, 0, 0, 1648, 0, 0, 34, 0, 0, 2336, 0, 372, 0, 0, 0, 3365, 7976, 0, 4540, 0, 7991, 0, 0, 500, 1099, 0, 6818, 0, 7872, 0, 0, 0, 0, 304, 0, 0, 504, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2113, 30, 65, 0, 0, 0] payload_rows_per_dst=[16080, 15478, 15772, 6730, 3008, 2876, 15982, 12200] dup_rows_per_dst=[830, 2262, 5388, 0, 190, 212, 0, 1296] combine_rows_per_src=[16080, 15478, 15772, 6730, 3008, 2876, 15982, 12200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84282 payload_rows_remote=68330 dup_rows=14022 dup_rows_remote=14022 dedup_ratio=0.142639 router_tokens_per_expert=[0, 0, 0, 551, 0, 0, 0, 0, 453, 0, 0, 0, 1591, 0, 0, 67, 0, 0, 2179, 0, 334, 0, 0, 0, 3235, 7932, 0, 4344, 0, 7976, 0, 0, 638, 1303, 0, 6846, 0, 7855, 0, 0, 0, 0, 321, 0, 0, 540, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 61, 90, 0, 0, 0] payload_rows_per_dst=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] dup_rows_per_dst=[0, 608, 0, 0, 9404, 2632, 0, 1378] combine_rows_per_src=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82728 payload_rows_remote=67248 dup_rows=15576 dup_rows_remote=12936 dedup_ratio=0.158447 router_tokens_per_expert=[0, 0, 0, 469, 0, 0, 0, 1, 407, 1, 0, 0, 1761, 2, 1, 79, 0, 0, 2293, 0, 325, 0, 0, 0, 3038, 7956, 1, 4353, 0, 7967, 1, 0, 551, 1281, 0, 6809, 0, 7852, 0, 0, 0, 0, 371, 0, 0, 612, 794, 1, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1, 2082, 69, 70, 0, 0, 1] payload_rows_per_dst=[16052, 15480, 15712, 14634, 3448, 3298, 2260, 11844] dup_rows_per_dst=[800, 2640, 5228, 5126, 216, 258, 0, 1308] combine_rows_per_src=[16052, 15480, 15712, 14634, 3448, 3298, 2260, 11844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=324616192 dispatch_meta_bytes=331360 combine_payload_bytes=324616192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324947552 ingress_bytes=393822208 link_bytes=393822208 token_bytes=649563744 weight_movement_bytes=69206016 total_nvlink_bytes=718769760 weight_to_token_ratio=0.106542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481921263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294510592 dispatch_meta_bytes=344208 combine_payload_bytes=294510592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=294854800 ingress_bytes=363716608 link_bytes=363716608 token_bytes=589365392 weight_movement_bytes=69206016 total_nvlink_bytes=658571408 weight_to_token_ratio=0.117425 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.368636314e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=344350720 dispatch_meta_bytes=372824 combine_payload_bytes=344350720 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=344723544 ingress_bytes=482762752 link_bytes=482762752 token_bytes=689074264 weight_movement_bytes=138412032 total_nvlink_bytes=827486296 weight_to_token_ratio=0.200867 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.816597370e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75600 payload_rows_remote=59630 dup_rows=22704 dup_rows_remote=21894 dedup_ratio=0.230957 router_tokens_per_expert=[5, 1, 0, 443, 3, 0, 1, 18, 405, 21, 0, 12, 1677, 1, 7, 48, 0, 2, 2160, 1, 358, 10, 0, 3, 3369, 7919, 0, 3824, 14, 7941, 3, 5, 527, 1355, 0, 6726, 1, 7823, 9, 2, 0, 0, 350, 5, 10, 645, 858, 6, 0, 52, 14, 1, 2, 0, 0, 4, 1, 12, 2330, 82, 66, 0, 0, 20] payload_rows_per_dst=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] dup_rows_per_dst=[810, 2470, 5034, 13628, 232, 288, 6, 236] combine_rows_per_src=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87090 payload_rows_remote=70706 dup_rows=11214 dup_rows_remote=9370 dedup_ratio=0.114075 router_tokens_per_expert=[0, 0, 4904, 69, 3509, 5171, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 8192, 0, 403, 0, 0, 22, 8192, 0, 0, 0, 0, 251, 0, 0, 184, 0, 0, 71, 416, 317, 0, 0, 5914, 0, 0, 0, 0, 7912, 0, 0, 92, 467, 0, 17, 0, 8, 0, 971, 487, 970, 95, 0, 94] payload_rows_per_dst=[9456, 12766, 1004, 16384, 16384, 12272, 6550, 12274] dup_rows_per_dst=[2046, 3688, 0, 44, 1844, 190, 300, 3102] combine_rows_per_src=[9456, 12766, 1004, 16384, 16384, 12272, 6550, 12274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=247660544 dispatch_meta_bytes=296752 combine_payload_bytes=247660544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=247957296 ingress_bytes=351469568 link_bytes=351469568 token_bytes=495617840 weight_movement_bytes=103809024 total_nvlink_bytes=599426864 weight_to_token_ratio=0.209454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322551689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298065920 dispatch_meta_bytes=339832 combine_payload_bytes=298065920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298405752 ingress_bytes=332668928 link_bytes=332668928 token_bytes=596471672 weight_movement_bytes=34603008 total_nvlink_bytes=631074680 weight_to_token_ratio=0.058013 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251806394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86078 payload_rows_remote=73558 dup_rows=12226 dup_rows_remote=12078 dedup_ratio=0.124369 router_tokens_per_expert=[0, 0, 5115, 110, 3237, 5252, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 8192, 0, 429, 0, 0, 11, 8192, 0, 0, 0, 0, 274, 0, 0, 130, 0, 0, 114, 462, 325, 0, 0, 6009, 0, 0, 0, 0, 7992, 0, 0, 66, 465, 0, 10, 0, 14, 0, 689, 428, 983, 64, 0, 70] payload_rows_per_dst=[6800, 14090, 1250, 16384, 16384, 12520, 2266, 16384] dup_rows_per_dst=[540, 5454, 0, 22, 1960, 148, 6, 4096] combine_rows_per_src=[6800, 14090, 1250, 16384, 16384, 12520, 2266, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=278724608 dispatch_meta_bytes=318280 combine_payload_bytes=278724608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279042888 ingress_bytes=313327616 link_bytes=313327616 token_bytes=557767496 weight_movement_bytes=34603008 total_nvlink_bytes=592370504 weight_to_token_ratio=0.062038 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179026594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296361984 dispatch_meta_bytes=308576 combine_payload_bytes=296361984 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296670560 ingress_bytes=365568000 link_bytes=365568000 token_bytes=593032544 weight_movement_bytes=69206016 total_nvlink_bytes=662238560 weight_to_token_ratio=0.116699 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375602953e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=279879680 dispatch_meta_bytes=329408 combine_payload_bytes=279879680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280209088 ingress_bytes=314482688 link_bytes=314482688 token_bytes=560088768 weight_movement_bytes=34603008 total_nvlink_bytes=594691776 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183373037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275447808 dispatch_meta_bytes=320736 combine_payload_bytes=275447808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275768544 ingress_bytes=310050816 link_bytes=310050816 token_bytes=551216352 weight_movement_bytes=34603008 total_nvlink_bytes=585819360 weight_to_token_ratio=0.062776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166696259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=244244480 dispatch_meta_bytes=326096 combine_payload_bytes=244244480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244570576 ingress_bytes=278847488 link_bytes=278847488 token_bytes=488815056 weight_movement_bytes=34603008 total_nvlink_bytes=523418064 weight_to_token_ratio=0.070790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.049280648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=289611776 dispatch_meta_bytes=320304 combine_payload_bytes=289611776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289932080 ingress_bytes=324214784 link_bytes=324214784 token_bytes=579543856 weight_movement_bytes=34603008 total_nvlink_bytes=614146864 weight_to_token_ratio=0.059707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219994131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301293568 dispatch_meta_bytes=342544 combine_payload_bytes=301293568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301636112 ingress_bytes=335896576 link_bytes=335896576 token_bytes=602929680 weight_movement_bytes=34603008 total_nvlink_bytes=637532688 weight_to_token_ratio=0.057391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82728 payload_rows_remote=67248 dup_rows=15576 dup_rows_remote=12936 dedup_ratio=0.158447 router_tokens_per_expert=[0, 0, 0, 469, 0, 0, 0, 1, 407, 1, 0, 0, 1761, 2, 1, 79, 0, 0, 2293, 0, 325, 0, 0, 0, 3038, 7956, 1, 4353, 0, 7967, 1, 0, 551, 1281, 0, 6809, 0, 7852, 0, 0, 0, 0, 371, 0, 0, 612, 794, 1, 0, 0, 1, 2, 0, 0, 0, 0, 0, 1, 2082, 69, 70, 0, 0, 1] payload_rows_per_dst=[16052, 15480, 15712, 14634, 3448, 3298, 2260, 11844] dup_rows_per_dst=[800, 2640, 5228, 5126, 216, 258, 0, 1308] combine_rows_per_src=[16052, 15480, 15712, 14634, 3448, 3298, 2260, 11844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=79166 dup_rows=9674 dup_rows_remote=7620 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 0, 430, 0, 0, 0, 0, 338, 0, 0, 0, 1617, 0, 0, 62, 0, 0, 2158, 0, 303, 0, 0, 0, 3395, 7945, 0, 4517, 0, 7958, 0, 0, 456, 1233, 0, 6928, 0, 7849, 0, 0, 0, 0, 319, 0, 0, 609, 758, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2130, 88, 59, 0, 0, 0] payload_rows_per_dst=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] dup_rows_per_dst=[742, 2006, 540, 0, 2054, 2870, 0, 1462] combine_rows_per_src=[16008, 12628, 6288, 0, 9464, 16200, 15916, 12126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84282 payload_rows_remote=68330 dup_rows=14022 dup_rows_remote=14022 dedup_ratio=0.142639 router_tokens_per_expert=[0, 0, 0, 551, 0, 0, 0, 0, 453, 0, 0, 0, 1591, 0, 0, 67, 0, 0, 2179, 0, 334, 0, 0, 0, 3235, 7932, 0, 4344, 0, 7976, 0, 0, 638, 1303, 0, 6846, 0, 7855, 0, 0, 0, 0, 321, 0, 0, 540, 715, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2121, 61, 90, 0, 0, 0] payload_rows_per_dst=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] dup_rows_per_dst=[0, 608, 0, 0, 9404, 2632, 0, 1378] combine_rows_per_src=[1102, 3614, 5026, 15864, 14640, 16230, 15952, 11854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275447808 dispatch_meta_bytes=320736 combine_payload_bytes=275447808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275768544 ingress_bytes=310050816 link_bytes=310050816 token_bytes=551216352 weight_movement_bytes=34603008 total_nvlink_bytes=585819360 weight_to_token_ratio=0.062776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166696259e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90576 payload_rows_remote=74192 dup_rows=7728 dup_rows_remote=7686 dedup_ratio=0.078613 router_tokens_per_expert=[0, 0, 4800, 74, 3522, 5108, 0, 346, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 8192, 0, 447, 0, 0, 21, 8192, 0, 0, 0, 0, 264, 0, 0, 172, 0, 0, 75, 447, 295, 0, 0, 5957, 0, 0, 0, 0, 7940, 0, 0, 79, 468, 0, 1, 0, 8, 0, 872, 540, 977, 96, 0, 80] payload_rows_per_dst=[13256, 10216, 1252, 16384, 16384, 12342, 16038, 4704] dup_rows_per_dst=[4228, 0, 0, 42, 1916, 162, 938, 442] combine_rows_per_src=[13256, 10216, 1252, 16384, 16384, 12342, 16038, 4704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84056 payload_rows_remote=67672 dup_rows=14248 dup_rows_remote=9808 dedup_ratio=0.144938 router_tokens_per_expert=[0, 0, 4732, 65, 3871, 5244, 0, 311, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 8192, 0, 400, 0, 0, 21, 8192, 0, 0, 0, 0, 264, 0, 0, 108, 0, 0, 69, 475, 226, 0, 0, 5949, 0, 0, 0, 0, 8020, 0, 0, 58, 471, 0, 6, 0, 5, 0, 887, 435, 948, 54, 0, 63] payload_rows_per_dst=[752, 10488, 972, 16384, 16384, 12226, 10466, 16384] dup_rows_per_dst=[0, 0, 0, 7784, 1832, 124, 68, 4440] combine_rows_per_src=[752, 10488, 972, 16384, 16384, 12226, 10466, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77638 payload_rows_remote=62094 dup_rows=20666 dup_rows_remote=19622 dedup_ratio=0.210225 router_tokens_per_expert=[3, 7748, 146, 0, 0, 0, 0, 11, 0, 3, 7, 1, 0, 1263, 2, 154, 3, 1, 1, 8069, 2, 9, 0, 7991, 0, 239, 2, 53, 0, 0, 1, 0, 7, 39, 88, 6, 45, 0, 7520, 0, 135, 0, 16, 0, 98, 6, 5, 16, 6125, 5, 88, 2, 0, 2, 0, 12, 0, 6, 7277, 1942, 1, 0, 2, 0] payload_rows_per_dst=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] dup_rows_per_dst=[1044, 2348, 15974, 0, 1078, 48, 172, 2] combine_rows_per_src=[15544, 15066, 16178, 590, 15188, 504, 10668, 3900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86078 payload_rows_remote=73558 dup_rows=12226 dup_rows_remote=12078 dedup_ratio=0.124369 router_tokens_per_expert=[0, 0, 5115, 110, 3237, 5252, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 0, 0, 0, 8192, 0, 429, 0, 0, 11, 8192, 0, 0, 0, 0, 274, 0, 0, 130, 0, 0, 114, 462, 325, 0, 0, 6009, 0, 0, 0, 0, 7992, 0, 0, 66, 465, 0, 10, 0, 14, 0, 689, 428, 983, 64, 0, 70] payload_rows_per_dst=[6800, 14090, 1250, 16384, 16384, 12520, 2266, 16384] dup_rows_per_dst=[540, 5454, 0, 22, 1960, 148, 6, 4096] combine_rows_per_src=[6800, 14090, 1250, 16384, 16384, 12520, 2266, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82880 payload_rows_remote=67552 dup_rows=15424 dup_rows_remote=5846 dedup_ratio=0.156901 router_tokens_per_expert=[0, 0, 4533, 86, 3801, 5423, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 8192, 0, 444, 0, 0, 23, 8192, 0, 0, 0, 0, 244, 0, 0, 165, 0, 0, 88, 439, 301, 0, 0, 5844, 0, 0, 0, 0, 7951, 0, 0, 72, 441, 0, 12, 0, 4, 0, 794, 382, 1052, 96, 0, 76] payload_rows_per_dst=[13294, 10846, 15328, 7836, 2924, 12140, 16070, 4442] dup_rows_per_dst=[4326, 0, 9578, 26, 96, 150, 882, 366] combine_rows_per_src=[13294, 10846, 15328, 7836, 2924, 12140, 16070, 4442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=324263936 dispatch_meta_bytes=347144 combine_payload_bytes=324263936 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=324611080 ingress_bytes=497278976 link_bytes=497278976 token_bytes=648875016 weight_movement_bytes=173015040 total_nvlink_bytes=821890056 weight_to_token_ratio=0.266638 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.871220752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=279879680 dispatch_meta_bytes=329408 combine_payload_bytes=279879680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280209088 ingress_bytes=314482688 link_bytes=314482688 token_bytes=560088768 weight_movement_bytes=34603008 total_nvlink_bytes=594691776 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183373037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90248 payload_rows_remote=79694 dup_rows=8056 dup_rows_remote=7980 dedup_ratio=0.081950 router_tokens_per_expert=[0, 0, 4788, 126, 3519, 5088, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8192, 0, 412, 0, 0, 27, 8192, 1, 0, 0, 0, 288, 0, 0, 171, 0, 0, 124, 375, 389, 0, 0, 5813, 0, 0, 0, 0, 7936, 0, 0, 85, 440, 0, 2, 0, 8, 0, 908, 550, 1056, 88, 0, 85] payload_rows_per_dst=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] dup_rows_per_dst=[536, 0, 0, 474, 1916, 176, 76, 4878] combine_rows_per_src=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=64500 dup_rows=23432 dup_rows_remote=23432 dedup_ratio=0.238363 router_tokens_per_expert=[0, 0, 5082, 88, 3319, 5186, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 1, 8187, 0, 435, 0, 0, 23, 8191, 0, 0, 0, 1, 262, 0, 0, 171, 0, 1, 89, 405, 323, 1, 1, 5802, 0, 0, 1, 0, 7936, 0, 5, 81, 528, 0, 5, 0, 15, 0, 932, 497, 944, 91, 1, 83] payload_rows_per_dst=[13380, 10372, 16384, 48, 1834, 12088, 16038, 4728] dup_rows_per_dst=[4266, 0, 17506, 0, 22, 168, 1072, 398] combine_rows_per_src=[13380, 10372, 16384, 48, 1834, 12088, 16038, 4728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=303890432 dispatch_meta_bytes=327512 combine_payload_bytes=303890432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304217944 ingress_bytes=373096448 link_bytes=373096448 token_bytes=608108376 weight_movement_bytes=69206016 total_nvlink_bytes=677314392 weight_to_token_ratio=0.113805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403931897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=60568 dup_rows=24214 dup_rows_remote=19354 dedup_ratio=0.246318 router_tokens_per_expert=[1, 2, 5135, 97, 3115, 5083, 6, 340, 0, 0, 0, 3, 3, 10, 0, 5, 164, 0, 1, 0, 19, 8148, 7, 456, 0, 0, 44, 8172, 1, 5, 0, 4, 297, 5, 1, 192, 0, 10, 103, 412, 340, 13, 26, 5785, 0, 0, 15, 1, 7888, 0, 0, 77, 520, 0, 1, 4, 20, 0, 1027, 461, 927, 111, 7, 88] payload_rows_per_dst=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] dup_rows_per_dst=[4860, 8, 17550, 0, 98, 224, 1040, 434] combine_rows_per_src=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94182 payload_rows_remote=78228 dup_rows=4122 dup_rows_remote=2510 dedup_ratio=0.041931 router_tokens_per_expert=[875, 0, 0, 0, 64, 0, 0, 0, 0, 899, 0, 0, 95, 0, 7198, 8098, 7384, 5334, 0, 94, 0, 0, 0, 0, 180, 0, 0, 55, 0, 287, 7726, 1679, 251, 153, 0, 653, 0, 0, 0, 0, 0, 0, 94, 0, 0, 111, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0] payload_rows_per_dst=[14968, 1800, 10668, 4154, 15954, 15100, 16196, 15342] dup_rows_per_dst=[1306, 188, 188, 248, 1612, 580, 0, 0] combine_rows_per_src=[14968, 1800, 10668, 4154, 15954, 15100, 16196, 15342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277184512 dispatch_meta_bytes=309920 combine_payload_bytes=277184512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277494432 ingress_bytes=311787520 link_bytes=311787520 token_bytes=554678944 weight_movement_bytes=34603008 total_nvlink_bytes=589281952 weight_to_token_ratio=0.062384 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173231337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94492 payload_rows_remote=79306 dup_rows=3812 dup_rows_remote=3388 dedup_ratio=0.038778 router_tokens_per_expert=[657, 0, 0, 0, 77, 0, 0, 0, 0, 971, 0, 0, 73, 0, 7270, 8122, 7239, 5472, 0, 70, 0, 0, 0, 0, 214, 0, 0, 133, 0, 292, 7787, 1601, 275, 140, 0, 477, 0, 0, 0, 0, 1, 0, 70, 0, 0, 221, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0] payload_rows_per_dst=[12984, 3996, 10944, 3990, 15716, 15186, 16244, 15432] dup_rows_per_dst=[836, 280, 140, 490, 1642, 424, 0, 0] combine_rows_per_src=[12984, 3996, 10944, 3990, 15716, 15186, 16244, 15432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=254337024 dispatch_meta_bytes=326864 combine_payload_bytes=254337024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254663888 ingress_bytes=323543040 link_bytes=323543040 token_bytes=509000912 weight_movement_bytes=69206016 total_nvlink_bytes=578206928 weight_to_token_ratio=0.135964 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217466412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=301293568 dispatch_meta_bytes=342544 combine_payload_bytes=301293568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=301636112 ingress_bytes=335896576 link_bytes=335896576 token_bytes=602929680 weight_movement_bytes=34603008 total_nvlink_bytes=637532688 weight_to_token_ratio=0.057391 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=293592 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276986584 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553679576 weight_movement_bytes=69206016 total_nvlink_bytes=622885592 weight_to_token_ratio=0.124993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=326426624 dispatch_meta_bytes=350696 combine_payload_bytes=326426624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326777320 ingress_bytes=395632640 link_bytes=395632640 token_bytes=653203944 weight_movement_bytes=69206016 total_nvlink_bytes=722409960 weight_to_token_ratio=0.105949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488733773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74872 payload_rows_remote=64500 dup_rows=23432 dup_rows_remote=23432 dedup_ratio=0.238363 router_tokens_per_expert=[0, 0, 5082, 88, 3319, 5186, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 1, 8187, 0, 435, 0, 0, 23, 8191, 0, 0, 0, 1, 262, 0, 0, 171, 0, 1, 89, 405, 323, 1, 1, 5802, 0, 0, 1, 0, 7936, 0, 5, 81, 528, 0, 5, 0, 15, 0, 932, 497, 944, 91, 1, 83] payload_rows_per_dst=[13380, 10372, 16384, 48, 1834, 12088, 16038, 4728] dup_rows_per_dst=[4266, 0, 17506, 0, 22, 168, 1072, 398] combine_rows_per_src=[13380, 10372, 16384, 48, 1834, 12088, 16038, 4728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264192000 dispatch_meta_bytes=351728 combine_payload_bytes=264192000 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=264543728 ingress_bytes=333398016 link_bytes=333398016 token_bytes=528735728 weight_movement_bytes=69206016 total_nvlink_bytes=597941744 weight_to_token_ratio=0.130890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254549893e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=248086528 dispatch_meta_bytes=319688 combine_payload_bytes=248086528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=248406216 ingress_bytes=421101568 link_bytes=421101568 token_bytes=496492744 weight_movement_bytes=173015040 total_nvlink_bytes=669507784 weight_to_token_ratio=0.348474 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584571299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=320421888 dispatch_meta_bytes=322952 combine_payload_bytes=320421888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320744840 ingress_bytes=355024896 link_bytes=355024896 token_bytes=641166728 weight_movement_bytes=34603008 total_nvlink_bytes=675769736 weight_to_token_ratio=0.053969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=324837376 dispatch_meta_bytes=330776 combine_payload_bytes=324837376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325168152 ingress_bytes=359440384 link_bytes=359440384 token_bytes=650005528 weight_movement_bytes=34603008 total_nvlink_bytes=684608536 weight_to_token_ratio=0.053235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352545228e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87090 payload_rows_remote=70706 dup_rows=11214 dup_rows_remote=9370 dedup_ratio=0.114075 router_tokens_per_expert=[0, 0, 4904, 69, 3509, 5171, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 8192, 0, 403, 0, 0, 22, 8192, 0, 0, 0, 0, 251, 0, 0, 184, 0, 0, 71, 416, 317, 0, 0, 5914, 0, 0, 0, 0, 7912, 0, 0, 92, 467, 0, 17, 0, 8, 0, 971, 487, 970, 95, 0, 94] payload_rows_per_dst=[9456, 12766, 1004, 16384, 16384, 12272, 6550, 12274] dup_rows_per_dst=[2046, 3688, 0, 44, 1844, 190, 300, 3102] combine_rows_per_src=[9456, 12766, 1004, 16384, 16384, 12272, 6550, 12274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90248 payload_rows_remote=79694 dup_rows=8056 dup_rows_remote=7980 dedup_ratio=0.081950 router_tokens_per_expert=[0, 0, 4788, 126, 3519, 5088, 0, 353, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 8192, 0, 412, 0, 0, 27, 8192, 1, 0, 0, 0, 288, 0, 0, 171, 0, 0, 124, 375, 389, 0, 0, 5813, 0, 0, 0, 0, 7936, 0, 0, 85, 440, 0, 2, 0, 8, 0, 908, 550, 1056, 88, 0, 85] payload_rows_per_dst=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] dup_rows_per_dst=[536, 0, 0, 474, 1916, 176, 76, 4878] combine_rows_per_src=[7042, 10176, 1096, 16384, 16384, 12228, 10554, 16384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94162 payload_rows_remote=90192 dup_rows=4142 dup_rows_remote=3982 dedup_ratio=0.042135 router_tokens_per_expert=[861, 0, 0, 0, 67, 0, 0, 0, 0, 887, 0, 0, 82, 0, 7229, 8112, 7395, 5383, 0, 80, 0, 0, 0, 0, 201, 0, 0, 10, 0, 289, 7732, 1565, 244, 140, 0, 757, 0, 0, 0, 0, 0, 0, 80, 0, 0, 74, 245, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 0, 0, 0, 0] payload_rows_per_dst=[14982, 1778, 10766, 3970, 16046, 14958, 16224, 15438] dup_rows_per_dst=[1332, 160, 160, 160, 1700, 630, 0, 0] combine_rows_per_src=[14982, 1778, 10766, 3970, 16046, 14958, 16224, 15438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94690 payload_rows_remote=79214 dup_rows=3614 dup_rows_remote=3614 dedup_ratio=0.036764 router_tokens_per_expert=[639, 0, 0, 0, 51, 0, 0, 0, 0, 832, 0, 0, 69, 0, 7322, 8129, 7380, 5401, 0, 63, 0, 0, 0, 0, 207, 0, 0, 37, 0, 228, 7963, 1786, 264, 99, 0, 568, 0, 0, 0, 0, 1, 0, 61, 0, 0, 50, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0] payload_rows_per_dst=[1380, 15418, 10802, 4318, 16104, 14934, 16258, 15476] dup_rows_per_dst=[0, 1028, 126, 198, 1684, 578, 0, 0] combine_rows_per_src=[1380, 15418, 10802, 4318, 16104, 14934, 16258, 15476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75600 payload_rows_remote=59630 dup_rows=22704 dup_rows_remote=21894 dedup_ratio=0.230957 router_tokens_per_expert=[5, 1, 0, 443, 3, 0, 1, 18, 405, 21, 0, 12, 1677, 1, 7, 48, 0, 2, 2160, 1, 358, 10, 0, 3, 3369, 7919, 0, 3824, 14, 7941, 3, 5, 527, 1355, 0, 6726, 1, 7823, 9, 2, 0, 0, 350, 5, 10, 645, 858, 6, 0, 52, 14, 1, 2, 0, 0, 4, 1, 12, 2330, 82, 66, 0, 0, 20] payload_rows_per_dst=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] dup_rows_per_dst=[810, 2470, 5034, 13628, 232, 288, 6, 236] combine_rows_per_src=[15970, 15324, 15680, 16126, 3556, 3460, 140, 5344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94492 payload_rows_remote=79306 dup_rows=3812 dup_rows_remote=3388 dedup_ratio=0.038778 router_tokens_per_expert=[657, 0, 0, 0, 77, 0, 0, 0, 0, 971, 0, 0, 73, 0, 7270, 8122, 7239, 5472, 0, 70, 0, 0, 0, 0, 214, 0, 0, 133, 0, 292, 7787, 1601, 275, 140, 0, 477, 0, 0, 0, 0, 1, 0, 70, 0, 0, 221, 274, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7716, 0, 0, 0, 0, 0] payload_rows_per_dst=[12984, 3996, 10944, 3990, 15716, 15186, 16244, 15432] dup_rows_per_dst=[836, 280, 140, 490, 1642, 424, 0, 0] combine_rows_per_src=[12984, 3996, 10944, 3990, 15716, 15186, 16244, 15432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92916 payload_rows_remote=82110 dup_rows=5388 dup_rows_remote=5236 dedup_ratio=0.054810 router_tokens_per_expert=[833, 0, 0, 0, 66, 0, 0, 0, 0, 871, 0, 0, 76, 0, 7186, 8116, 7323, 5403, 0, 76, 0, 0, 0, 0, 204, 0, 0, 1, 0, 262, 7704, 1754, 244, 140, 0, 765, 0, 0, 0, 0, 0, 0, 74, 0, 0, 73, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 0, 0] payload_rows_per_dst=[14908, 1744, 10806, 13100, 5858, 14794, 16232, 15474] dup_rows_per_dst=[1262, 150, 152, 2836, 354, 634, 0, 0] combine_rows_per_src=[14908, 1744, 10806, 13100, 5858, 14794, 16232, 15474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=78400 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[904, 0, 0, 0, 97, 0, 0, 0, 0, 881, 0, 0, 87, 0, 7169, 8107, 7303, 5419, 0, 85, 0, 0, 0, 0, 169, 0, 0, 16, 0, 313, 7567, 1679, 260, 175, 0, 765, 0, 0, 0, 0, 0, 0, 85, 0, 0, 90, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0] payload_rows_per_dst=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] dup_rows_per_dst=[0, 1010, 170, 274, 1578, 658, 0, 0] combine_rows_per_src=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=369426432 dispatch_meta_bytes=376696 combine_payload_bytes=369426432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=369803128 ingress_bytes=438632448 link_bytes=438632448 token_bytes=739229560 weight_movement_bytes=69206016 total_nvlink_bytes=808435576 weight_to_token_ratio=0.093619 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650538589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=289611776 dispatch_meta_bytes=320304 combine_payload_bytes=289611776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289932080 ingress_bytes=324214784 link_bytes=324214784 token_bytes=579543856 weight_movement_bytes=34603008 total_nvlink_bytes=614146864 weight_to_token_ratio=0.059707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219994131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=326426624 dispatch_meta_bytes=350696 combine_payload_bytes=326426624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326777320 ingress_bytes=395632640 link_bytes=395632640 token_bytes=653203944 weight_movement_bytes=69206016 total_nvlink_bytes=722409960 weight_to_token_ratio=0.105949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.488733773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72384 payload_rows_remote=56824 dup_rows=25920 dup_rows_remote=14532 dedup_ratio=0.263672 router_tokens_per_expert=[768, 0, 0, 0, 75, 0, 0, 0, 0, 975, 0, 0, 85, 0, 7198, 8108, 7330, 5523, 0, 83, 0, 0, 0, 1, 194, 1, 0, 48, 1, 257, 7695, 1559, 253, 153, 1, 686, 0, 0, 0, 0, 2, 0, 81, 0, 0, 116, 251, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7706, 0, 0, 0, 0, 0] payload_rows_per_dst=[14936, 15560, 13126, 16184, 2182, 3832, 922, 5642] dup_rows_per_dst=[1146, 11388, 150, 13096, 4, 136, 0, 0] combine_rows_per_src=[14936, 15560, 13126, 16184, 2182, 3832, 922, 5642] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93688 payload_rows_remote=78156 dup_rows=4616 dup_rows_remote=4616 dedup_ratio=0.046956 router_tokens_per_expert=[0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 848, 0, 8039, 492, 0, 0, 7765, 0, 0, 0, 1354, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 0, 0, 0, 7846, 0, 89, 92, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 0, 0, 0, 153, 0, 0, 0, 0, 494, 0, 0] payload_rows_per_dst=[16266, 13448, 984, 13630, 15532, 3072, 15226, 15530] dup_rows_per_dst=[382, 692, 0, 2228, 0, 20, 0, 1294] combine_rows_per_src=[16266, 13448, 984, 13630, 15532, 3072, 15226, 15530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65310 payload_rows_remote=50410 dup_rows=32994 dup_rows_remote=31632 dedup_ratio=0.335632 router_tokens_per_expert=[935, 0, 0, 0, 74, 2, 0, 0, 3, 1052, 0, 0, 103, 0, 7120, 8072, 7222, 5040, 8, 89, 0, 0, 5, 5, 189, 16, 0, 146, 16, 362, 7788, 1700, 248, 155, 4, 534, 10, 5, 0, 1, 4, 0, 88, 9, 2, 145, 251, 22, 0, 0, 0, 0, 1, 0, 6, 0, 10, 0, 7691, 0, 4, 9, 0, 6] payload_rows_per_dst=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] dup_rows_per_dst=[1362, 12212, 4300, 15066, 10, 26, 2, 16] combine_rows_per_src=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81048 payload_rows_remote=64720 dup_rows=17256 dup_rows_remote=2586 dedup_ratio=0.175537 router_tokens_per_expert=[0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 5802, 0, 0, 0, 992, 0, 8052, 506, 0, 0, 7795, 0, 0, 0, 1631, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 7789, 0, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 140, 0, 0, 0, 0, 521, 0, 0] payload_rows_per_dst=[13608, 12886, 3556, 3418, 438, 16328, 15224, 15590] dup_rows_per_dst=[400, 702, 162, 0, 0, 14670, 0, 1322] combine_rows_per_src=[13608, 12886, 3556, 3418, 438, 16328, 15224, 15590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=369426432 dispatch_meta_bytes=376696 combine_payload_bytes=369426432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=369803128 ingress_bytes=438632448 link_bytes=438632448 token_bytes=739229560 weight_movement_bytes=69206016 total_nvlink_bytes=808435576 weight_to_token_ratio=0.093619 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650538589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=244244480 dispatch_meta_bytes=326096 combine_payload_bytes=244244480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=244570576 ingress_bytes=278847488 link_bytes=278847488 token_bytes=488815056 weight_movement_bytes=34603008 total_nvlink_bytes=523418064 weight_to_token_ratio=0.070790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.049280648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=324837376 dispatch_meta_bytes=330776 combine_payload_bytes=324837376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325168152 ingress_bytes=359440384 link_bytes=359440384 token_bytes=650005528 weight_movement_bytes=34603008 total_nvlink_bytes=684608536 weight_to_token_ratio=0.053235 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352545228e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=324460544 dispatch_meta_bytes=331312 combine_payload_bytes=324460544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324791856 ingress_bytes=393666560 link_bytes=393666560 token_bytes=649252400 weight_movement_bytes=69206016 total_nvlink_bytes=718458416 weight_to_token_ratio=0.106593 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481335573e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72384 payload_rows_remote=56824 dup_rows=25920 dup_rows_remote=14532 dedup_ratio=0.263672 router_tokens_per_expert=[768, 0, 0, 0, 75, 0, 0, 0, 0, 975, 0, 0, 85, 0, 7198, 8108, 7330, 5523, 0, 83, 0, 0, 0, 1, 194, 1, 0, 48, 1, 257, 7695, 1559, 253, 153, 1, 686, 0, 0, 0, 0, 2, 0, 81, 0, 0, 116, 251, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7706, 0, 0, 0, 0, 0] payload_rows_per_dst=[14936, 15560, 13126, 16184, 2182, 3832, 922, 5642] dup_rows_per_dst=[1146, 11388, 150, 13096, 4, 136, 0, 0] combine_rows_per_src=[14936, 15560, 13126, 16184, 2182, 3832, 922, 5642] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=336322560 dispatch_meta_bytes=349384 combine_payload_bytes=336322560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=336671944 ingress_bytes=440131584 link_bytes=440131584 token_bytes=672994504 weight_movement_bytes=103809024 total_nvlink_bytes=776803528 weight_to_token_ratio=0.154249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.656179717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=321126400 dispatch_meta_bytes=328360 combine_payload_bytes=321126400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321454760 ingress_bytes=355729408 link_bytes=355729408 token_bytes=642581160 weight_movement_bytes=34603008 total_nvlink_bytes=677184168 weight_to_token_ratio=0.053850 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338581124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=232751104 dispatch_meta_bytes=285424 combine_payload_bytes=232751104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=233036528 ingress_bytes=336560128 link_bytes=336560128 token_bytes=465787632 weight_movement_bytes=103809024 total_nvlink_bytes=569596656 weight_to_token_ratio=0.222868 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266448666e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320126976 dispatch_meta_bytes=331088 combine_payload_bytes=320126976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320458064 ingress_bytes=389332992 link_bytes=389332992 token_bytes=640585040 weight_movement_bytes=69206016 total_nvlink_bytes=709791056 weight_to_token_ratio=0.108036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465028705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206479360 dispatch_meta_bytes=328168 combine_payload_bytes=206479360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=206807528 ingress_bytes=241082368 link_bytes=241082368 token_bytes=413286888 weight_movement_bytes=34603008 total_nvlink_bytes=447889896 weight_to_token_ratio=0.083726 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.071735421e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=269224 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265362344 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530455464 weight_movement_bytes=69206016 total_nvlink_bytes=599661480 weight_to_token_ratio=0.130465 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94182 payload_rows_remote=78228 dup_rows=4122 dup_rows_remote=2510 dedup_ratio=0.041931 router_tokens_per_expert=[875, 0, 0, 0, 64, 0, 0, 0, 0, 899, 0, 0, 95, 0, 7198, 8098, 7384, 5334, 0, 94, 0, 0, 0, 0, 180, 0, 0, 55, 0, 287, 7726, 1679, 251, 153, 0, 653, 0, 0, 0, 0, 0, 0, 94, 0, 0, 111, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7671, 0, 0, 0, 0, 0] payload_rows_per_dst=[14968, 1800, 10668, 4154, 15954, 15100, 16196, 15342] dup_rows_per_dst=[1306, 188, 188, 248, 1612, 580, 0, 0] combine_rows_per_src=[14968, 1800, 10668, 4154, 15954, 15100, 16196, 15342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94614 payload_rows_remote=78400 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.037537 router_tokens_per_expert=[904, 0, 0, 0, 97, 0, 0, 0, 0, 881, 0, 0, 87, 0, 7169, 8107, 7303, 5419, 0, 85, 0, 0, 0, 0, 169, 0, 0, 16, 0, 313, 7567, 1679, 260, 175, 0, 765, 0, 0, 0, 0, 0, 0, 85, 0, 0, 90, 260, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7721, 0, 0, 0, 0, 0] payload_rows_per_dst=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] dup_rows_per_dst=[0, 1010, 170, 274, 1578, 658, 0, 0] combine_rows_per_src=[2002, 15264, 10838, 4080, 15956, 14818, 16214, 15442] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93204 payload_rows_remote=77364 dup_rows=5100 dup_rows_remote=2424 dedup_ratio=0.051880 router_tokens_per_expert=[0, 0, 0, 296, 0, 0, 0, 0, 0, 0, 6080, 0, 0, 0, 905, 0, 8052, 545, 0, 0, 7786, 0, 0, 0, 1300, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 7799, 0, 0, 0, 7875, 0, 72, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7627, 0, 0, 0, 140, 0, 0, 0, 0, 527, 0, 0] payload_rows_per_dst=[16264, 13312, 1090, 15840, 15598, 274, 15254, 15572] dup_rows_per_dst=[432, 658, 0, 2676, 0, 0, 0, 1334] combine_rows_per_src=[16264, 13312, 1090, 15840, 15598, 274, 15254, 15572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=60568 dup_rows=24214 dup_rows_remote=19354 dedup_ratio=0.246318 router_tokens_per_expert=[1, 2, 5135, 97, 3115, 5083, 6, 340, 0, 0, 0, 3, 3, 10, 0, 5, 164, 0, 1, 0, 19, 8148, 7, 456, 0, 0, 44, 8172, 1, 5, 0, 4, 297, 5, 1, 192, 0, 10, 103, 412, 340, 13, 26, 5785, 0, 0, 15, 1, 7888, 0, 0, 77, 520, 0, 1, 4, 20, 0, 1027, 461, 927, 111, 7, 88] payload_rows_per_dst=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] dup_rows_per_dst=[4860, 8, 17550, 0, 98, 224, 1040, 434] combine_rows_per_src=[13522, 10200, 16384, 108, 1942, 11146, 15940, 4848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81048 payload_rows_remote=64720 dup_rows=17256 dup_rows_remote=2586 dedup_ratio=0.175537 router_tokens_per_expert=[0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 5802, 0, 0, 0, 992, 0, 8052, 506, 0, 0, 7795, 0, 0, 0, 1631, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 7787, 0, 0, 0, 7789, 0, 67, 75, 0, 0, 0, 0, 0, 0, 0, 0, 7612, 0, 0, 0, 140, 0, 0, 0, 0, 521, 0, 0] payload_rows_per_dst=[13608, 12886, 3556, 3418, 438, 16328, 15224, 15590] dup_rows_per_dst=[400, 702, 162, 0, 0, 14670, 0, 1322] combine_rows_per_src=[13608, 12886, 3556, 3418, 438, 16328, 15224, 15590] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68408 payload_rows_remote=52772 dup_rows=29896 dup_rows_remote=28684 dedup_ratio=0.304118 router_tokens_per_expert=[0, 0, 0, 253, 0, 0, 0, 0, 0, 0, 6329, 0, 0, 0, 845, 0, 8094, 486, 0, 0, 7818, 0, 0, 0, 1188, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7801, 0, 0, 0, 7847, 0, 65, 38, 0, 0, 0, 0, 0, 0, 0, 0, 7717, 0, 0, 0, 98, 0, 0, 0, 0, 508, 0, 0] payload_rows_per_dst=[506, 1690, 16314, 2506, 0, 16322, 15434, 15636] dup_rows_per_dst=[0, 0, 13504, 0, 0, 15180, 0, 1212] combine_rows_per_src=[506, 1690, 16314, 2506, 0, 16322, 15434, 15636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316882944 dispatch_meta_bytes=319152 combine_payload_bytes=316882944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317202096 ingress_bytes=351485952 link_bytes=351485952 token_bytes=634085040 weight_movement_bytes=34603008 total_nvlink_bytes=668688048 weight_to_token_ratio=0.054572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322613340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=320421888 dispatch_meta_bytes=322952 combine_payload_bytes=320421888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320744840 ingress_bytes=355024896 link_bytes=355024896 token_bytes=641166728 weight_movement_bytes=34603008 total_nvlink_bytes=675769736 weight_to_token_ratio=0.053969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94038 payload_rows_remote=87290 dup_rows=4266 dup_rows_remote=4150 dedup_ratio=0.043396 router_tokens_per_expert=[0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 6415, 0, 0, 0, 863, 0, 8052, 501, 0, 0, 7795, 0, 0, 0, 1097, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 7871, 0, 61, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 139, 0, 0, 0, 0, 584, 0, 0] payload_rows_per_dst=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] dup_rows_per_dst=[338, 640, 116, 2244, 0, 0, 0, 928] combine_rows_per_src=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=321126400 dispatch_meta_bytes=328360 combine_payload_bytes=321126400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321454760 ingress_bytes=355729408 link_bytes=355729408 token_bytes=642581160 weight_movement_bytes=34603008 total_nvlink_bytes=677184168 weight_to_token_ratio=0.053850 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338581124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74774 payload_rows_remote=59682 dup_rows=23530 dup_rows_remote=23530 dedup_ratio=0.239360 router_tokens_per_expert=[0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 6341, 0, 0, 0, 898, 0, 8017, 520, 0, 0, 7805, 0, 0, 0, 1183, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 7829, 0, 41, 83, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 0, 0, 0, 175, 0, 0, 0, 0, 516, 0, 0] payload_rows_per_dst=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] dup_rows_per_dst=[0, 412, 6592, 0, 0, 15144, 0, 1382] combine_rows_per_src=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=316882944 dispatch_meta_bytes=319152 combine_payload_bytes=316882944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317202096 ingress_bytes=351485952 link_bytes=351485952 token_bytes=634085040 weight_movement_bytes=34603008 total_nvlink_bytes=668688048 weight_to_token_ratio=0.054572 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.322613340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86078 payload_rows_remote=72724 dup_rows=12226 dup_rows_remote=11552 dedup_ratio=0.124369 router_tokens_per_expert=[0, 0, 0, 299, 0, 0, 0, 0, 0, 0, 6177, 0, 0, 1, 836, 0, 8038, 485, 1, 0, 7779, 0, 0, 0, 1410, 0, 0, 90, 0, 0, 0, 1, 0, 0, 0, 0, 7789, 0, 0, 0, 7829, 1, 72, 76, 0, 0, 0, 1, 1, 1, 0, 0, 7611, 1, 0, 0, 152, 0, 0, 0, 1, 500, 0, 0] payload_rows_per_dst=[16242, 13354, 16200, 15874, 15946, 298, 6862, 1302] dup_rows_per_dst=[432, 674, 330, 2786, 7996, 2, 2, 4] combine_rows_per_src=[16242, 13354, 16200, 15874, 15946, 298, 6862, 1302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94038 payload_rows_remote=87290 dup_rows=4266 dup_rows_remote=4150 dedup_ratio=0.043396 router_tokens_per_expert=[0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 6415, 0, 0, 0, 863, 0, 8052, 501, 0, 0, 7795, 0, 0, 0, 1097, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 7812, 0, 0, 0, 7871, 0, 61, 65, 0, 0, 0, 0, 0, 0, 0, 0, 7577, 0, 0, 0, 139, 0, 0, 0, 0, 584, 0, 0] payload_rows_per_dst=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] dup_rows_per_dst=[338, 640, 116, 2244, 0, 0, 0, 928] combine_rows_per_src=[16254, 13916, 6748, 15844, 15624, 252, 15154, 10246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94120 payload_rows_remote=84918 dup_rows=4184 dup_rows_remote=3202 dedup_ratio=0.042562 router_tokens_per_expert=[0, 0, 0, 69, 1, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 8191, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 70, 0, 107, 0, 0, 0, 0, 0, 0, 2036, 0, 382, 599, 3435, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 5, 0, 0, 0, 0, 631, 8122, 0, 0] payload_rows_per_dst=[15044, 15772, 4642, 16384, 4072, 9202, 12760, 16244] dup_rows_per_dst=[128, 0, 0, 1806, 0, 982, 6, 1262] combine_rows_per_src=[15044, 15772, 4642, 16384, 4072, 9202, 12760, 16244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94776 payload_rows_remote=90028 dup_rows=3528 dup_rows_remote=3288 dedup_ratio=0.035889 router_tokens_per_expert=[0, 0, 0, 94, 0, 0, 0, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 94, 0, 63, 0, 0, 0, 0, 0, 0, 1995, 0, 342, 690, 3259, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6546, 7, 0, 0, 0, 0, 563, 8098, 0, 0] payload_rows_per_dst=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] dup_rows_per_dst=[188, 0, 0, 1866, 240, 176, 2, 1056] combine_rows_per_src=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79820 payload_rows_remote=63666 dup_rows=18484 dup_rows_remote=18064 dedup_ratio=0.188029 router_tokens_per_expert=[4, 3, 0, 298, 0, 0, 0, 0, 0, 0, 6015, 9, 5, 23, 926, 5, 7982, 522, 17, 0, 7758, 0, 6, 0, 1418, 0, 0, 103, 2, 1, 1, 5, 0, 0, 4, 0, 7771, 2, 0, 1, 7827, 16, 55, 91, 1, 0, 0, 21, 0, 5, 0, 1, 7561, 16, 6, 0, 154, 0, 1, 0, 0, 516, 0, 0] payload_rows_per_dst=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] dup_rows_per_dst=[420, 3530, 446, 2888, 11150, 34, 14, 2] combine_rows_per_src=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=248086528 dispatch_meta_bytes=319688 combine_payload_bytes=248086528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=248406216 ingress_bytes=421101568 link_bytes=421101568 token_bytes=496492744 weight_movement_bytes=173015040 total_nvlink_bytes=669507784 weight_to_token_ratio=0.348474 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.584571299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=269224 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265362344 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530455464 weight_movement_bytes=69206016 total_nvlink_bytes=599661480 weight_to_token_ratio=0.130465 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=216154112 dispatch_meta_bytes=325824 combine_payload_bytes=216154112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216479936 ingress_bytes=250757120 link_bytes=250757120 token_bytes=432634048 weight_movement_bytes=34603008 total_nvlink_bytes=467237056 weight_to_token_ratio=0.079982 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.435788550e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=244457472 dispatch_meta_bytes=332848 combine_payload_bytes=244457472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=244790320 ingress_bytes=313663488 link_bytes=313663488 token_bytes=489247792 weight_movement_bytes=69206016 total_nvlink_bytes=558453808 weight_to_token_ratio=0.141454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180290453e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=297877504 dispatch_meta_bytes=337104 combine_payload_bytes=297877504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=298214608 ingress_bytes=367083520 link_bytes=367083520 token_bytes=596092112 weight_movement_bytes=69206016 total_nvlink_bytes=665298128 weight_to_token_ratio=0.116100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381305733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=357539840 dispatch_meta_bytes=365760 combine_payload_bytes=357539840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357905600 ingress_bytes=461348864 link_bytes=461348864 token_bytes=715445440 weight_movement_bytes=103809024 total_nvlink_bytes=819254464 weight_to_token_ratio=0.145097 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.736018634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347824128 dispatch_meta_bytes=352480 combine_payload_bytes=347824128 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348176608 ingress_bytes=451633152 link_bytes=451633152 token_bytes=696000736 weight_movement_bytes=103809024 total_nvlink_bytes=799809760 weight_to_token_ratio=0.149151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699459192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=368754688 dispatch_meta_bytes=373264 combine_payload_bytes=368754688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=369127952 ingress_bytes=403357696 link_bytes=403357696 token_bytes=737882640 weight_movement_bytes=34603008 total_nvlink_bytes=772485648 weight_to_token_ratio=0.046895 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517802537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260775936 dispatch_meta_bytes=326920 combine_payload_bytes=260775936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261102856 ingress_bytes=295378944 link_bytes=295378944 token_bytes=521878792 weight_movement_bytes=34603008 total_nvlink_bytes=556481800 weight_to_token_ratio=0.066305 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111487186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=297877504 dispatch_meta_bytes=337104 combine_payload_bytes=297877504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=298214608 ingress_bytes=367083520 link_bytes=367083520 token_bytes=596092112 weight_movement_bytes=69206016 total_nvlink_bytes=665298128 weight_to_token_ratio=0.116100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381305733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93688 payload_rows_remote=78156 dup_rows=4616 dup_rows_remote=4616 dedup_ratio=0.046956 router_tokens_per_expert=[0, 0, 0, 285, 0, 0, 0, 0, 0, 0, 6222, 0, 0, 0, 848, 0, 8039, 492, 0, 0, 7765, 0, 0, 0, 1354, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 0, 7766, 0, 0, 0, 7846, 0, 89, 92, 0, 0, 0, 0, 0, 0, 0, 0, 7613, 0, 0, 0, 153, 0, 0, 0, 0, 494, 0, 0] payload_rows_per_dst=[16266, 13448, 984, 13630, 15532, 3072, 15226, 15530] dup_rows_per_dst=[382, 692, 0, 2228, 0, 20, 0, 1294] combine_rows_per_src=[16266, 13448, 984, 13630, 15532, 3072, 15226, 15530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74774 payload_rows_remote=59682 dup_rows=23530 dup_rows_remote=23530 dedup_ratio=0.239360 router_tokens_per_expert=[0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 6341, 0, 0, 0, 898, 0, 8017, 520, 0, 0, 7805, 0, 0, 0, 1183, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 7829, 0, 41, 83, 0, 0, 0, 0, 0, 0, 0, 0, 7546, 0, 0, 0, 175, 0, 0, 0, 0, 516, 0, 0] payload_rows_per_dst=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] dup_rows_per_dst=[0, 412, 6592, 0, 0, 15144, 0, 1382] combine_rows_per_src=[606, 8446, 16102, 2544, 0, 16374, 15092, 15610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92152 payload_rows_remote=75768 dup_rows=6152 dup_rows_remote=4428 dedup_ratio=0.062581 router_tokens_per_expert=[0, 0, 0, 80, 0, 0, 0, 0, 8006, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 80, 0, 69, 0, 0, 0, 0, 0, 0, 2058, 0, 292, 788, 3181, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6465, 1, 0, 0, 0, 0, 602, 8112, 0, 0] payload_rows_per_dst=[16384, 16012, 4642, 16384, 16224, 8372, 12930, 1204] dup_rows_per_dst=[160, 0, 0, 1724, 4116, 150, 2, 0] combine_rows_per_src=[16384, 16012, 4642, 16384, 16224, 8372, 12930, 1204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=65310 payload_rows_remote=50410 dup_rows=32994 dup_rows_remote=31632 dedup_ratio=0.335632 router_tokens_per_expert=[935, 0, 0, 0, 74, 2, 0, 0, 3, 1052, 0, 0, 103, 0, 7120, 8072, 7222, 5040, 8, 89, 0, 0, 5, 5, 189, 16, 0, 146, 16, 362, 7788, 1700, 248, 155, 4, 534, 10, 5, 0, 1, 4, 0, 88, 9, 2, 145, 251, 22, 0, 0, 0, 0, 1, 0, 6, 0, 10, 0, 7691, 0, 4, 9, 0, 6] payload_rows_per_dst=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] dup_rows_per_dst=[1362, 12212, 4300, 15066, 10, 26, 2, 16] combine_rows_per_src=[14900, 16328, 14968, 16140, 1904, 1016, 12, 42] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94120 payload_rows_remote=84918 dup_rows=4184 dup_rows_remote=3202 dedup_ratio=0.042562 router_tokens_per_expert=[0, 0, 0, 69, 1, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 0, 0, 2321, 0, 8191, 0, 0, 0, 0, 0, 727, 0, 0, 0, 0, 70, 0, 107, 0, 0, 0, 0, 0, 0, 2036, 0, 382, 599, 3435, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 5, 0, 0, 0, 0, 631, 8122, 0, 0] payload_rows_per_dst=[15044, 15772, 4642, 16384, 4072, 9202, 12760, 16244] dup_rows_per_dst=[128, 0, 0, 1806, 0, 982, 6, 1262] combine_rows_per_src=[15044, 15772, 4642, 16384, 4072, 9202, 12760, 16244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73654 payload_rows_remote=57396 dup_rows=24650 dup_rows_remote=7688 dedup_ratio=0.250753 router_tokens_per_expert=[0, 0, 0, 63, 0, 3, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 2430, 0, 8191, 0, 0, 0, 0, 0, 717, 0, 0, 0, 0, 63, 0, 52, 0, 0, 0, 0, 0, 0, 2083, 0, 326, 597, 3171, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6649, 5, 0, 0, 0, 0, 571, 8129, 0, 0] payload_rows_per_dst=[132, 0, 4860, 16382, 4166, 16384, 15472, 16258] dup_rows_per_dst=[0, 0, 0, 1664, 0, 1846, 4178, 16962] combine_rows_per_src=[132, 0, 4860, 16382, 4166, 16384, 15472, 16258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=310345728 dispatch_meta_bytes=320784 combine_payload_bytes=310345728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310666512 ingress_bytes=344948736 link_bytes=344948736 token_bytes=621012240 weight_movement_bytes=34603008 total_nvlink_bytes=655615248 weight_to_token_ratio=0.055720 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298014323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75476 payload_rows_remote=61564 dup_rows=22828 dup_rows_remote=21830 dedup_ratio=0.232218 router_tokens_per_expert=[0, 0, 0, 85, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 8192, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 85, 0, 81, 0, 0, 0, 0, 0, 0, 1977, 0, 390, 744, 2892, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 1, 0, 0, 0, 0, 640, 8107, 0, 0] payload_rows_per_dst=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] dup_rows_per_dst=[0, 0, 0, 1938, 0, 9864, 998, 10028] combine_rows_per_src=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=320126976 dispatch_meta_bytes=331088 combine_payload_bytes=320126976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320458064 ingress_bytes=389332992 link_bytes=389332992 token_bytes=640585040 weight_movement_bytes=69206016 total_nvlink_bytes=709791056 weight_to_token_ratio=0.108036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465028705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89636 payload_rows_remote=74658 dup_rows=8668 dup_rows_remote=4586 dedup_ratio=0.088175 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 76, 0, 67, 0, 0, 0, 0, 0, 0, 1913, 0, 452, 636, 3304, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6574, 6, 0, 0, 0, 0, 669, 8117, 0, 0] payload_rows_per_dst=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] dup_rows_per_dst=[152, 0, 4082, 236, 3826, 364, 8, 0] combine_rows_per_src=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=244457472 dispatch_meta_bytes=332848 combine_payload_bytes=244457472 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=244790320 ingress_bytes=313663488 link_bytes=313663488 token_bytes=489247792 weight_movement_bytes=69206016 total_nvlink_bytes=558453808 weight_to_token_ratio=0.141454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180290453e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87762 payload_rows_remote=72022 dup_rows=10542 dup_rows_remote=10540 dedup_ratio=0.107239 router_tokens_per_expert=[1, 0, 1, 83, 1, 0, 0, 0, 7870, 0, 0, 0, 0, 0, 0, 1, 2377, 0, 8191, 0, 0, 0, 1, 0, 666, 0, 0, 0, 0, 83, 0, 84, 1, 0, 0, 0, 0, 1, 1910, 0, 558, 678, 3248, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6505, 7, 0, 1, 0, 0, 585, 8107, 0, 0] payload_rows_per_dst=[16384, 15740, 16384, 1666, 16216, 8480, 11720, 1172] dup_rows_per_dst=[170, 2, 6044, 0, 3822, 490, 14, 0] combine_rows_per_src=[16384, 15740, 16384, 1666, 16216, 8480, 11720, 1172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89636 payload_rows_remote=74658 dup_rows=8668 dup_rows_remote=4586 dedup_ratio=0.088175 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 7855, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 713, 0, 0, 0, 0, 76, 0, 67, 0, 0, 0, 0, 0, 0, 1913, 0, 452, 636, 3304, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6574, 6, 0, 0, 0, 0, 669, 8117, 0, 0] payload_rows_per_dst=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] dup_rows_per_dst=[152, 0, 4082, 236, 3826, 364, 8, 0] combine_rows_per_src=[16384, 15710, 14978, 3420, 16234, 8420, 13152, 1338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91944 payload_rows_remote=85966 dup_rows=6360 dup_rows_remote=5410 dedup_ratio=0.064697 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 87, 0, 0, 242, 0, 1407, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 5819, 0, 0, 0, 0, 8052, 0, 0, 0, 7, 5427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5332, 8192, 1482, 1982, 0, 0, 0, 0, 568, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[15960, 12770, 0, 10916, 16384, 5978, 14124, 15812] dup_rows_per_dst=[174, 1192, 0, 96, 0, 950, 3826, 122] combine_rows_per_src=[15960, 12770, 0, 10916, 16384, 5978, 14124, 15812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91712 payload_rows_remote=75328 dup_rows=6592 dup_rows_remote=6592 dedup_ratio=0.067057 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 106, 0, 0, 235, 0, 1381, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 5964, 0, 0, 0, 0, 8004, 0, 0, 0, 3, 4910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5714, 8192, 1562, 1884, 0, 0, 0, 0, 586, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] payload_rows_per_dst=[16008, 13254, 0, 9826, 16384, 6058, 14304, 15878] dup_rows_per_dst=[212, 1406, 0, 0, 0, 834, 3978, 162] combine_rows_per_src=[16008, 13254, 0, 9826, 16384, 6058, 14304, 15878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76168 payload_rows_remote=59790 dup_rows=22136 dup_rows_remote=21890 dedup_ratio=0.225179 router_tokens_per_expert=[0, 0, 24, 87, 3, 14, 19, 3, 7765, 0, 0, 0, 3, 3, 0, 1, 2129, 0, 8169, 6, 3, 1, 12, 0, 844, 0, 0, 10, 1, 88, 0, 90, 15, 0, 0, 0, 0, 7, 2265, 0, 410, 652, 3317, 8162, 0, 13, 0, 0, 5, 3, 0, 0, 0, 6, 6308, 9, 6, 20, 0, 9, 607, 8056, 0, 7] payload_rows_per_dst=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] dup_rows_per_dst=[246, 0, 16886, 10, 4540, 408, 4, 42] combine_rows_per_src=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=310345728 dispatch_meta_bytes=320784 combine_payload_bytes=310345728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310666512 ingress_bytes=344948736 link_bytes=344948736 token_bytes=621012240 weight_movement_bytes=34603008 total_nvlink_bytes=655615248 weight_to_token_ratio=0.055720 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298014323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=206479360 dispatch_meta_bytes=328168 combine_payload_bytes=206479360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=206807528 ingress_bytes=241082368 link_bytes=241082368 token_bytes=413286888 weight_movement_bytes=34603008 total_nvlink_bytes=447889896 weight_to_token_ratio=0.083726 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.071735421e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=347824128 dispatch_meta_bytes=352480 combine_payload_bytes=347824128 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348176608 ingress_bytes=451633152 link_bytes=451633152 token_bytes=696000736 weight_movement_bytes=103809024 total_nvlink_bytes=799809760 weight_to_token_ratio=0.149151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699459192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=235094016 dispatch_meta_bytes=260336 combine_payload_bytes=235094016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=235354352 ingress_bytes=304300032 link_bytes=304300032 token_bytes=470448368 weight_movement_bytes=69206016 total_nvlink_bytes=539654384 weight_to_token_ratio=0.147107 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145056522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=252166144 dispatch_meta_bytes=333576 combine_payload_bytes=252166144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252499720 ingress_bytes=321372160 link_bytes=321372160 token_bytes=504665864 weight_movement_bytes=69206016 total_nvlink_bytes=573871880 weight_to_token_ratio=0.137132 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209297565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295002112 dispatch_meta_bytes=330248 combine_payload_bytes=295002112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295332360 ingress_bytes=364208128 link_bytes=364208128 token_bytes=590334472 weight_movement_bytes=69206016 total_nvlink_bytes=659540488 weight_to_token_ratio=0.117232 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370485865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=305799168 dispatch_meta_bytes=316976 combine_payload_bytes=305799168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306116144 ingress_bytes=375005184 link_bytes=375005184 token_bytes=611915312 weight_movement_bytes=69206016 total_nvlink_bytes=681121328 weight_to_token_ratio=0.113097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.411114317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=352116736 dispatch_meta_bytes=365504 combine_payload_bytes=352116736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=352482240 ingress_bytes=455925760 link_bytes=455925760 token_bytes=704598976 weight_movement_bytes=103809024 total_nvlink_bytes=808408000 weight_to_token_ratio=0.147331 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.715611930e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308543488 dispatch_meta_bytes=327680 combine_payload_bytes=308543488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308871168 ingress_bytes=343146496 link_bytes=343146496 token_bytes=617414656 weight_movement_bytes=34603008 total_nvlink_bytes=652017664 weight_to_token_ratio=0.056045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291232639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=244899840 dispatch_meta_bytes=326720 combine_payload_bytes=244899840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245226560 ingress_bytes=279502848 link_bytes=279502848 token_bytes=490126400 weight_movement_bytes=34603008 total_nvlink_bytes=524729408 weight_to_token_ratio=0.070600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051746715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295002112 dispatch_meta_bytes=330248 combine_payload_bytes=295002112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=295332360 ingress_bytes=364208128 link_bytes=364208128 token_bytes=590334472 weight_movement_bytes=69206016 total_nvlink_bytes=659540488 weight_to_token_ratio=0.117232 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370485865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94776 payload_rows_remote=90028 dup_rows=3528 dup_rows_remote=3288 dedup_ratio=0.035889 router_tokens_per_expert=[0, 0, 0, 94, 0, 0, 0, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 2310, 0, 8192, 0, 0, 0, 0, 0, 776, 0, 0, 0, 0, 94, 0, 63, 0, 0, 0, 0, 0, 0, 1995, 0, 342, 690, 3259, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6546, 7, 0, 0, 0, 0, 563, 8098, 0, 0] payload_rows_per_dst=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] dup_rows_per_dst=[188, 0, 0, 1866, 240, 176, 2, 1056] combine_rows_per_src=[16384, 15862, 4620, 16384, 4748, 8406, 13104, 15268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75476 payload_rows_remote=61564 dup_rows=22828 dup_rows_remote=21830 dedup_ratio=0.232218 router_tokens_per_expert=[0, 0, 0, 85, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 2398, 0, 8192, 0, 0, 0, 0, 0, 803, 0, 0, 0, 0, 85, 0, 81, 0, 0, 0, 0, 0, 0, 1977, 0, 390, 744, 2892, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6713, 1, 0, 0, 0, 0, 640, 8107, 0, 0] payload_rows_per_dst=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] dup_rows_per_dst=[0, 0, 0, 1938, 0, 9864, 998, 10028] combine_rows_per_src=[170, 3662, 4796, 16384, 3954, 16384, 13912, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=368754688 dispatch_meta_bytes=373264 combine_payload_bytes=368754688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=369127952 ingress_bytes=403357696 link_bytes=403357696 token_bytes=737882640 weight_movement_bytes=34603008 total_nvlink_bytes=772485648 weight_to_token_ratio=0.046895 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517802537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95250 payload_rows_remote=85042 dup_rows=3054 dup_rows_remote=3054 dedup_ratio=0.031067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 128, 0, 0, 256, 0, 1175, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 6096, 0, 0, 0, 0, 8032, 0, 0, 0, 10, 5094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5602, 8192, 1491, 1854, 0, 0, 0, 0, 475, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[16064, 13048, 11068, 10208, 16384, 5716, 6868, 15894] dup_rows_per_dst=[256, 1018, 0, 0, 0, 974, 668, 138] combine_rows_per_src=[16064, 13048, 11068, 10208, 16384, 5716, 6868, 15894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=348332032 dispatch_meta_bytes=352384 combine_payload_bytes=348332032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348684416 ingress_bytes=452141056 link_bytes=452141056 token_bytes=697016448 weight_movement_bytes=103809024 total_nvlink_bytes=800825472 weight_to_token_ratio=0.148933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.701370393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79820 payload_rows_remote=63666 dup_rows=18484 dup_rows_remote=18064 dedup_ratio=0.188029 router_tokens_per_expert=[4, 3, 0, 298, 0, 0, 0, 0, 0, 0, 6015, 9, 5, 23, 926, 5, 7982, 522, 17, 0, 7758, 0, 6, 0, 1418, 0, 0, 103, 2, 1, 1, 5, 0, 0, 4, 0, 7771, 2, 0, 1, 7827, 16, 55, 91, 1, 0, 0, 21, 0, 5, 0, 1, 7561, 16, 6, 0, 154, 0, 1, 0, 0, 516, 0, 0] payload_rows_per_dst=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] dup_rows_per_dst=[420, 3530, 446, 2888, 11150, 34, 14, 2] combine_rows_per_src=[16154, 13944, 16160, 15826, 16020, 334, 42, 1340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66538 payload_rows_remote=50690 dup_rows=31766 dup_rows_remote=31638 dedup_ratio=0.323140 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 202, 0, 0, 235, 0, 1500, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 6185, 0, 0, 0, 0, 8066, 0, 0, 0, 7, 5048, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5408, 8192, 1543, 1904, 0, 0, 0, 0, 458, 0, 2415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 64] payload_rows_per_dst=[404, 3470, 0, 16132, 2, 16384, 14298, 15848] dup_rows_per_dst=[0, 0, 0, 10110, 0, 17710, 3818, 128] combine_rows_per_src=[404, 3470, 0, 16132, 2, 16384, 14298, 15848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73700 payload_rows_remote=59302 dup_rows=24604 dup_rows_remote=20518 dedup_ratio=0.250285 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 65, 0, 0, 205, 0, 1415, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 5935, 0, 0, 0, 0, 8022, 0, 0, 0, 3, 4862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 8192, 1650, 1918, 0, 0, 0, 0, 606, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] dup_rows_per_dst=[0, 736, 0, 9730, 0, 9970, 4086, 82] combine_rows_per_src=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91944 payload_rows_remote=85966 dup_rows=6360 dup_rows_remote=5410 dedup_ratio=0.064697 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 87, 0, 0, 242, 0, 1407, 0, 0, 0, 0, 0, 7906, 0, 0, 0, 5819, 0, 0, 0, 0, 8052, 0, 0, 0, 7, 5427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5332, 8192, 1482, 1982, 0, 0, 0, 0, 568, 0, 2588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[15960, 12770, 0, 10916, 16384, 5978, 14124, 15812] dup_rows_per_dst=[174, 1192, 0, 96, 0, 950, 3826, 122] combine_rows_per_src=[15960, 12770, 0, 10916, 16384, 5978, 14124, 15812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95302 payload_rows_remote=82488 dup_rows=3002 dup_rows_remote=2766 dedup_ratio=0.030538 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 209, 0, 0, 246, 0, 1451, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 6376, 0, 0, 0, 0, 8040, 0, 0, 0, 9, 4820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 8192, 1587, 1802, 0, 0, 0, 0, 510, 0, 2385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] dup_rows_per_dst=[418, 1068, 236, 0, 0, 898, 260, 122] combine_rows_per_src=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=252166144 dispatch_meta_bytes=333576 combine_payload_bytes=252166144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252499720 ingress_bytes=321372160 link_bytes=321372160 token_bytes=504665864 weight_movement_bytes=69206016 total_nvlink_bytes=573871880 weight_to_token_ratio=0.137132 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209297565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75790 payload_rows_remote=62926 dup_rows=22514 dup_rows_remote=21438 dedup_ratio=0.229024 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 161, 0, 0, 236, 0, 1417, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 5908, 0, 1, 1, 0, 8024, 0, 0, 0, 3, 5243, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 5317, 8191, 1439, 1928, 0, 0, 0, 0, 563, 0, 2705, 0, 0, 0, 1, 0, 2, 0, 0, 0, 68] payload_rows_per_dst=[16052, 12864, 16176, 8, 16384, 5922, 6196, 2188] dup_rows_per_dst=[322, 1076, 19936, 0, 2, 814, 340, 24] combine_rows_per_src=[16052, 12864, 16176, 8, 16384, 5922, 6196, 2188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95302 payload_rows_remote=82488 dup_rows=3002 dup_rows_remote=2766 dedup_ratio=0.030538 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 209, 0, 0, 246, 0, 1451, 0, 0, 0, 0, 0, 7946, 0, 0, 0, 6376, 0, 0, 0, 0, 8040, 0, 0, 0, 9, 4820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5518, 8192, 1587, 1802, 0, 0, 0, 0, 510, 0, 2385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61] payload_rows_per_dst=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] dup_rows_per_dst=[418, 1068, 236, 0, 0, 898, 260, 122] combine_rows_per_src=[16080, 13362, 12814, 9360, 16384, 5880, 5530, 15892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94588 payload_rows_remote=78204 dup_rows=3716 dup_rows_remote=36 dedup_ratio=0.037801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5332, 0, 0, 6806, 0, 0, 0, 0, 0, 6792, 0, 0, 0, 0, 0, 198, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 3279, 8192, 0, 0] payload_rows_per_dst=[13612, 10664, 13584, 396, 16384, 16384, 16302, 7262] dup_rows_per_dst=[0, 0, 0, 0, 0, 3680, 0, 36] combine_rows_per_src=[13612, 10664, 13584, 396, 16384, 16384, 16302, 7262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94854 payload_rows_remote=78470 dup_rows=3450 dup_rows_remote=3450 dedup_ratio=0.035095 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 0, 0, 7074, 0, 0, 0, 0, 0, 6831, 0, 0, 0, 0, 0, 175, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 3011, 8192, 0, 0] payload_rows_per_dst=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] dup_rows_per_dst=[0, 0, 0, 0, 0, 3450, 0, 0] combine_rows_per_src=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73750 payload_rows_remote=57788 dup_rows=24554 dup_rows_remote=24292 dedup_ratio=0.249776 router_tokens_per_expert=[2, 1, 1, 0, 1, 0, 133, 0, 14, 219, 6, 1429, 1, 0, 0, 0, 8, 7914, 0, 0, 1, 5822, 0, 5, 2, 5, 7974, 1, 0, 4, 45, 5287, 25, 30, 3, 0, 17, 0, 0, 0, 8, 18, 5, 5557, 8163, 1520, 2035, 6, 1, 0, 5, 546, 0, 2253, 0, 0, 0, 0, 0, 27, 0, 2, 0, 56] payload_rows_per_dst=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] dup_rows_per_dst=[262, 1288, 21718, 56, 48, 914, 264, 4] combine_rows_per_src=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=348332032 dispatch_meta_bytes=352384 combine_payload_bytes=348332032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=348684416 ingress_bytes=452141056 link_bytes=452141056 token_bytes=697016448 weight_movement_bytes=103809024 total_nvlink_bytes=800825472 weight_to_token_ratio=0.148933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.701370393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=260775936 dispatch_meta_bytes=326920 combine_payload_bytes=260775936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261102856 ingress_bytes=295378944 link_bytes=295378944 token_bytes=521878792 weight_movement_bytes=34603008 total_nvlink_bytes=556481800 weight_to_token_ratio=0.066305 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.111487186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=207626240 dispatch_meta_bytes=329312 combine_payload_bytes=207626240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=207955552 ingress_bytes=242229248 link_bytes=242229248 token_bytes=415581792 weight_movement_bytes=34603008 total_nvlink_bytes=450184800 weight_to_token_ratio=0.083264 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.114891592e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=352116736 dispatch_meta_bytes=365504 combine_payload_bytes=352116736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=352482240 ingress_bytes=455925760 link_bytes=455925760 token_bytes=704598976 weight_movement_bytes=103809024 total_nvlink_bytes=808408000 weight_to_token_ratio=0.147331 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.715611930e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242900992 dispatch_meta_bytes=319280 combine_payload_bytes=242900992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243220272 ingress_bytes=277504000 link_bytes=277504000 token_bytes=486121264 weight_movement_bytes=34603008 total_nvlink_bytes=520724272 weight_to_token_ratio=0.071182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044225211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257744896 dispatch_meta_bytes=337456 combine_payload_bytes=257744896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258082352 ingress_bytes=292347904 link_bytes=292347904 token_bytes=515827248 weight_movement_bytes=34603008 total_nvlink_bytes=550430256 weight_to_token_ratio=0.067083 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100081627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337870848 dispatch_meta_bytes=341016 combine_payload_bytes=337870848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=338211864 ingress_bytes=441679872 link_bytes=441679872 token_bytes=676082712 weight_movement_bytes=103809024 total_nvlink_bytes=779891736 weight_to_token_ratio=0.153545 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.662005800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320323584 dispatch_meta_bytes=312960 combine_payload_bytes=320323584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320636544 ingress_bytes=354926592 link_bytes=354926592 token_bytes=640960128 weight_movement_bytes=34603008 total_nvlink_bytes=675563136 weight_to_token_ratio=0.053986 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335560192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=257744896 dispatch_meta_bytes=337456 combine_payload_bytes=257744896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258082352 ingress_bytes=292347904 link_bytes=292347904 token_bytes=515827248 weight_movement_bytes=34603008 total_nvlink_bytes=550430256 weight_to_token_ratio=0.067083 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100081627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91712 payload_rows_remote=75328 dup_rows=6592 dup_rows_remote=6592 dedup_ratio=0.067057 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 106, 0, 0, 235, 0, 1381, 0, 0, 0, 0, 0, 7939, 0, 0, 0, 5964, 0, 0, 0, 0, 8004, 0, 0, 0, 3, 4910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5714, 8192, 1562, 1884, 0, 0, 0, 0, 586, 0, 2591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81] payload_rows_per_dst=[16008, 13254, 0, 9826, 16384, 6058, 14304, 15878] dup_rows_per_dst=[212, 1406, 0, 0, 0, 834, 3978, 162] combine_rows_per_src=[16008, 13254, 0, 9826, 16384, 6058, 14304, 15878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308543488 dispatch_meta_bytes=327680 combine_payload_bytes=308543488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308871168 ingress_bytes=343146496 link_bytes=343146496 token_bytes=617414656 weight_movement_bytes=34603008 total_nvlink_bytes=652017664 weight_to_token_ratio=0.056045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291232639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73700 payload_rows_remote=59302 dup_rows=24604 dup_rows_remote=20518 dedup_ratio=0.250285 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 65, 0, 0, 205, 0, 1415, 0, 0, 0, 0, 0, 7922, 0, 0, 0, 5935, 0, 0, 0, 0, 8022, 0, 0, 0, 3, 4862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5615, 8192, 1650, 1918, 0, 0, 0, 0, 606, 0, 2701, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41] payload_rows_per_dst=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] dup_rows_per_dst=[0, 736, 0, 9730, 0, 9970, 4086, 82] combine_rows_per_src=[130, 8588, 0, 16044, 5664, 13032, 14398, 15844] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97146 payload_rows_remote=85232 dup_rows=1158 dup_rows_remote=944 dedup_ratio=0.011780 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5315, 0, 0, 6931, 0, 0, 0, 0, 0, 6794, 0, 0, 0, 0, 0, 164, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 326, 0, 0, 3194, 8192, 0, 0] payload_rows_per_dst=[13862, 10630, 13588, 11914, 16384, 7344, 16384, 7040] dup_rows_per_dst=[0, 0, 0, 214, 0, 944, 0, 0] combine_rows_per_src=[13862, 10630, 13588, 11914, 16384, 7344, 16384, 7040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=321413120 dispatch_meta_bytes=327680 combine_payload_bytes=321413120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321740800 ingress_bytes=390619136 link_bytes=390619136 token_bytes=643153920 weight_movement_bytes=69206016 total_nvlink_bytes=712359936 weight_to_token_ratio=0.107604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469868361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=236699648 dispatch_meta_bytes=328320 combine_payload_bytes=236699648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=237027968 ingress_bytes=271302656 link_bytes=271302656 token_bytes=473727616 weight_movement_bytes=34603008 total_nvlink_bytes=508330624 weight_to_token_ratio=0.073044 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.020890053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=349110272 dispatch_meta_bytes=344704 combine_payload_bytes=349110272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=349454976 ingress_bytes=418316288 link_bytes=418316288 token_bytes=698565248 weight_movement_bytes=69206016 total_nvlink_bytes=767771264 weight_to_token_ratio=0.099069 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.574090515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70310 payload_rows_remote=53926 dup_rows=27994 dup_rows_remote=12714 dedup_ratio=0.284770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5325, 0, 0, 7144, 0, 0, 0, 0, 0, 6857, 0, 0, 0, 0, 0, 148, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 0, 0, 3137, 8192, 0, 0] payload_rows_per_dst=[1852, 15426, 3584, 296, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 7660, 0, 0, 1708, 3346, 0, 15280] combine_rows_per_src=[1852, 15426, 3584, 296, 16384, 16384, 0, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87146 payload_rows_remote=87146 dup_rows=11158 dup_rows_remote=11158 dedup_ratio=0.113505 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5313, 0, 0, 6707, 0, 0, 0, 0, 0, 6756, 0, 0, 0, 0, 0, 221, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 3489, 8192, 0, 0] payload_rows_per_dst=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 3544, 0, 7614] combine_rows_per_src=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76168 payload_rows_remote=59790 dup_rows=22136 dup_rows_remote=21890 dedup_ratio=0.225179 router_tokens_per_expert=[0, 0, 24, 87, 3, 14, 19, 3, 7765, 0, 0, 0, 3, 3, 0, 1, 2129, 0, 8169, 6, 3, 1, 12, 0, 844, 0, 0, 10, 1, 88, 0, 90, 15, 0, 0, 0, 0, 7, 2265, 0, 410, 652, 3317, 8162, 0, 13, 0, 0, 5, 3, 0, 0, 0, 6, 6308, 9, 6, 20, 0, 9, 607, 8056, 0, 7] payload_rows_per_dst=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] dup_rows_per_dst=[246, 0, 16886, 10, 4540, 408, 4, 42] combine_rows_per_src=[16378, 15544, 16370, 2056, 16146, 8376, 42, 1256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94588 payload_rows_remote=78204 dup_rows=3716 dup_rows_remote=36 dedup_ratio=0.037801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5332, 0, 0, 6806, 0, 0, 0, 0, 0, 6792, 0, 0, 0, 0, 0, 198, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 329, 0, 0, 3279, 8192, 0, 0] payload_rows_per_dst=[13612, 10664, 13584, 396, 16384, 16384, 16302, 7262] dup_rows_per_dst=[0, 0, 0, 0, 0, 3680, 0, 36] combine_rows_per_src=[13612, 10664, 13584, 396, 16384, 16384, 16302, 7262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81526 payload_rows_remote=70592 dup_rows=16778 dup_rows_remote=16776 dedup_ratio=0.170675 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 5464, 1, 1, 6893, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 1, 0, 196, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1716, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 327, 0, 0, 3184, 8191, 0, 0] payload_rows_per_dst=[13788, 10934, 13576, 16384, 0, 3436, 16384, 7024] dup_rows_per_dst=[0, 2, 0, 16774, 0, 0, 2, 0] combine_rows_per_src=[13788, 10934, 13576, 16384, 0, 3436, 16384, 7024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96032 payload_rows_remote=82956 dup_rows=2272 dup_rows_remote=2272 dedup_ratio=0.023112 router_tokens_per_expert=[7591, 0, 0, 0, 0, 0, 0, 528, 0, 0, 21, 0, 0, 0, 0, 0, 72, 0, 772, 1848, 8192, 0, 0, 0, 0, 0, 223, 0, 509, 0, 0, 0, 451, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7952, 7071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6402, 7142, 0, 70, 0, 0, 238] payload_rows_per_dst=[3152, 14324, 5234, 15014, 16384, 13076, 15904, 12944] dup_rows_per_dst=[10, 2, 150, 592, 1042, 0, 0, 476] combine_rows_per_src=[3152, 14324, 5234, 15014, 16384, 13076, 15904, 12944] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96254 payload_rows_remote=82734 dup_rows=2050 dup_rows_remote=2050 dedup_ratio=0.020854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5271, 0, 0, 7057, 0, 0, 0, 0, 0, 6760, 0, 0, 0, 0, 0, 165, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1655, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 3343, 8192, 0, 0] payload_rows_per_dst=[14114, 10542, 13520, 16384, 14664, 3310, 16384, 7336] dup_rows_per_dst=[0, 0, 0, 2050, 0, 0, 0, 0] combine_rows_per_src=[14114, 10542, 13520, 16384, 14664, 3310, 16384, 7336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95884 payload_rows_remote=79500 dup_rows=2420 dup_rows_remote=1386 dedup_ratio=0.024618 router_tokens_per_expert=[7652, 0, 0, 0, 0, 0, 0, 497, 0, 0, 29, 0, 0, 0, 0, 0, 101, 0, 742, 1730, 8192, 0, 0, 0, 0, 0, 158, 0, 506, 0, 0, 0, 422, 0, 1, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6304, 7229, 0, 114, 0, 0, 265] payload_rows_per_dst=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] dup_rows_per_dst=[122, 0, 126, 568, 1034, 0, 0, 570] combine_rows_per_src=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81344 payload_rows_remote=67834 dup_rows=16960 dup_rows_remote=16956 dedup_ratio=0.172526 router_tokens_per_expert=[1, 4, 0, 0, 1, 3, 3, 0, 0, 0, 3, 0, 5358, 10, 27, 6745, 3, 0, 2, 2, 10, 6702, 1, 6, 0, 0, 0, 199, 0, 27, 8162, 0, 0, 0, 1, 0, 0, 8162, 2, 3, 0, 0, 1802, 0, 0, 28, 27, 0, 11, 35, 0, 0, 0, 3, 14, 4, 8, 340, 0, 3, 3271, 8165, 4, 0] payload_rows_per_dst=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] dup_rows_per_dst=[4, 102, 4, 16720, 0, 30, 98, 2] combine_rows_per_src=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81526 payload_rows_remote=70592 dup_rows=16778 dup_rows_remote=16776 dedup_ratio=0.170675 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 5464, 1, 1, 6893, 0, 0, 0, 0, 0, 6788, 0, 0, 0, 1, 0, 196, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1716, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 327, 0, 0, 3184, 8191, 0, 0] payload_rows_per_dst=[13788, 10934, 13576, 16384, 0, 3436, 16384, 7024] dup_rows_per_dst=[0, 2, 0, 16774, 0, 0, 2, 0] combine_rows_per_src=[13788, 10934, 13576, 16384, 0, 3436, 16384, 7024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=242900992 dispatch_meta_bytes=319280 combine_payload_bytes=242900992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243220272 ingress_bytes=277504000 link_bytes=277504000 token_bytes=486121264 weight_movement_bytes=34603008 total_nvlink_bytes=520724272 weight_to_token_ratio=0.071182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044225211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=349110272 dispatch_meta_bytes=344704 combine_payload_bytes=349110272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=349454976 ingress_bytes=418316288 link_bytes=418316288 token_bytes=698565248 weight_movement_bytes=69206016 total_nvlink_bytes=767771264 weight_to_token_ratio=0.099069 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.574090515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=220880896 dispatch_meta_bytes=266560 combine_payload_bytes=220880896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221147456 ingress_bytes=290086912 link_bytes=290086912 token_bytes=442028352 weight_movement_bytes=69206016 total_nvlink_bytes=511234368 weight_to_token_ratio=0.156565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.091573696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=356950016 dispatch_meta_bytes=393216 combine_payload_bytes=356950016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=357343232 ingress_bytes=391553024 link_bytes=391553024 token_bytes=714293248 weight_movement_bytes=34603008 total_nvlink_bytes=748896256 weight_to_token_ratio=0.048444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473382507e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96042 payload_rows_remote=81036 dup_rows=2262 dup_rows_remote=1728 dedup_ratio=0.023010 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 507, 0, 0, 24, 0, 0, 0, 0, 0, 103, 0, 708, 1739, 8192, 0, 0, 0, 0, 0, 180, 0, 509, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 7081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6436, 7251, 0, 81, 0, 0, 242] payload_rows_per_dst=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] dup_rows_per_dst=[122, 0, 124, 534, 996, 0, 0, 486] combine_rows_per_src=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=289144832 dispatch_meta_bytes=349472 combine_payload_bytes=289144832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289494304 ingress_bytes=358350848 link_bytes=358350848 token_bytes=578639136 weight_movement_bytes=69206016 total_nvlink_bytes=647845152 weight_to_token_ratio=0.119601 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348445391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=339787776 dispatch_meta_bytes=340912 combine_payload_bytes=339787776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=340128688 ingress_bytes=443596800 link_bytes=443596800 token_bytes=679916464 weight_movement_bytes=103809024 total_nvlink_bytes=783725488 weight_to_token_ratio=0.152679 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.669219046e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=244899840 dispatch_meta_bytes=326720 combine_payload_bytes=244899840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=245226560 ingress_bytes=279502848 link_bytes=279502848 token_bytes=490126400 weight_movement_bytes=34603008 total_nvlink_bytes=524729408 weight_to_token_ratio=0.070600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.051746715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320323584 dispatch_meta_bytes=312960 combine_payload_bytes=320323584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320636544 ingress_bytes=354926592 link_bytes=354926592 token_bytes=640960128 weight_movement_bytes=34603008 total_nvlink_bytes=675563136 weight_to_token_ratio=0.053986 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335560192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94854 payload_rows_remote=78470 dup_rows=3450 dup_rows_remote=3450 dedup_ratio=0.035095 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5425, 0, 0, 7074, 0, 0, 0, 0, 0, 6831, 0, 0, 0, 0, 0, 175, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 335, 0, 0, 3011, 8192, 0, 0] payload_rows_per_dst=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] dup_rows_per_dst=[0, 0, 0, 0, 0, 3450, 0, 0] combine_rows_per_src=[14148, 10850, 13662, 350, 16384, 16384, 16384, 6692] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=338878464 dispatch_meta_bytes=339136 combine_payload_bytes=338878464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=339217600 ingress_bytes=442687488 link_bytes=442687488 token_bytes=678096064 weight_movement_bytes=103809024 total_nvlink_bytes=781905088 weight_to_token_ratio=0.153089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665797378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=325632000 dispatch_meta_bytes=323544 combine_payload_bytes=325632000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325955544 ingress_bytes=360235008 link_bytes=360235008 token_bytes=651587544 weight_movement_bytes=34603008 total_nvlink_bytes=686190552 weight_to_token_ratio=0.053106 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355535334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=277848064 dispatch_meta_bytes=339160 combine_payload_bytes=277848064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278187224 ingress_bytes=312451072 link_bytes=312451072 token_bytes=556035288 weight_movement_bytes=34603008 total_nvlink_bytes=590638296 weight_to_token_ratio=0.062232 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175728230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331923456 dispatch_meta_bytes=331056 combine_payload_bytes=331923456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332254512 ingress_bytes=366526464 link_bytes=366526464 token_bytes=664177968 weight_movement_bytes=34603008 total_nvlink_bytes=698780976 weight_to_token_ratio=0.052099 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379209576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=289144832 dispatch_meta_bytes=349472 combine_payload_bytes=289144832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289494304 ingress_bytes=358350848 link_bytes=358350848 token_bytes=578639136 weight_movement_bytes=69206016 total_nvlink_bytes=647845152 weight_to_token_ratio=0.119601 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348445391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=321413120 dispatch_meta_bytes=327680 combine_payload_bytes=321413120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321740800 ingress_bytes=390619136 link_bytes=390619136 token_bytes=643153920 weight_movement_bytes=69206016 total_nvlink_bytes=712359936 weight_to_token_ratio=0.107604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469868361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87146 payload_rows_remote=87146 dup_rows=11158 dup_rows_remote=11158 dedup_ratio=0.113505 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5313, 0, 0, 6707, 0, 0, 0, 0, 0, 6756, 0, 0, 0, 0, 0, 221, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1772, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 318, 0, 0, 3489, 8192, 0, 0] payload_rows_per_dst=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 3544, 0, 7614] combine_rows_per_src=[13414, 10626, 13512, 442, 16384, 16384, 0, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96042 payload_rows_remote=81036 dup_rows=2262 dup_rows_remote=1728 dedup_ratio=0.023010 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 507, 0, 0, 24, 0, 0, 0, 0, 0, 103, 0, 708, 1739, 8192, 0, 0, 0, 0, 0, 180, 0, 509, 0, 0, 0, 418, 0, 0, 0, 0, 0, 0, 80, 0, 0, 0, 0, 7966, 7081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6436, 7251, 0, 81, 0, 0, 242] payload_rows_per_dst=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] dup_rows_per_dst=[122, 0, 124, 534, 996, 0, 0, 486] combine_rows_per_src=[16162, 14550, 4976, 15006, 16384, 0, 15932, 13032] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71922 payload_rows_remote=55820 dup_rows=26382 dup_rows_remote=14572 dedup_ratio=0.268372 router_tokens_per_expert=[7635, 0, 0, 0, 0, 0, 0, 603, 0, 0, 47, 0, 0, 0, 0, 0, 60, 0, 788, 1684, 8192, 0, 0, 0, 0, 0, 183, 0, 425, 0, 0, 0, 379, 0, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0, 8044, 7091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6365, 7370, 0, 63, 0, 0, 158] payload_rows_per_dst=[1206, 94, 4976, 1216, 16384, 15856, 16088, 16102] dup_rows_per_dst=[0, 0, 88, 0, 888, 13596, 0, 11810] combine_rows_per_src=[1206, 94, 4976, 1216, 16384, 15856, 16088, 16102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82546 payload_rows_remote=66764 dup_rows=15758 dup_rows_remote=15758 dedup_ratio=0.160299 router_tokens_per_expert=[7570, 0, 0, 0, 0, 0, 0, 554, 0, 0, 5, 0, 0, 0, 0, 0, 98, 0, 753, 1697, 8192, 0, 0, 0, 0, 0, 207, 0, 567, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7891, 7051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6411, 7179, 0, 86, 0, 0, 330] payload_rows_per_dst=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] dup_rows_per_dst=[0, 0, 190, 106, 1122, 10958, 0, 3382] combine_rows_per_src=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73750 payload_rows_remote=57788 dup_rows=24554 dup_rows_remote=24292 dedup_ratio=0.249776 router_tokens_per_expert=[2, 1, 1, 0, 1, 0, 133, 0, 14, 219, 6, 1429, 1, 0, 0, 0, 8, 7914, 0, 0, 1, 5822, 0, 5, 2, 5, 7974, 1, 0, 4, 45, 5287, 25, 30, 3, 0, 17, 0, 0, 0, 8, 18, 5, 5557, 8163, 1520, 2035, 6, 1, 0, 5, 546, 0, 2253, 0, 0, 0, 0, 0, 27, 0, 2, 0, 56] payload_rows_per_dst=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] dup_rows_per_dst=[262, 1288, 21718, 56, 48, 914, 264, 4] combine_rows_per_src=[15962, 13164, 16356, 5818, 10668, 6270, 5346, 166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96032 payload_rows_remote=82956 dup_rows=2272 dup_rows_remote=2272 dedup_ratio=0.023112 router_tokens_per_expert=[7591, 0, 0, 0, 0, 0, 0, 528, 0, 0, 21, 0, 0, 0, 0, 0, 72, 0, 772, 1848, 8192, 0, 0, 0, 0, 0, 223, 0, 509, 0, 0, 0, 451, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 7952, 7071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6402, 7142, 0, 70, 0, 0, 238] payload_rows_per_dst=[3152, 14324, 5234, 15014, 16384, 13076, 15904, 12944] dup_rows_per_dst=[10, 2, 150, 592, 1042, 0, 0, 476] combine_rows_per_src=[3152, 14324, 5234, 15014, 16384, 13076, 15904, 12944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91920 payload_rows_remote=81324 dup_rows=6384 dup_rows_remote=1638 dedup_ratio=0.064941 router_tokens_per_expert=[7616, 0, 0, 0, 0, 0, 0, 594, 0, 0, 65, 0, 0, 0, 0, 0, 104, 0, 764, 1624, 8192, 0, 0, 0, 0, 0, 146, 0, 329, 0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 75, 0, 0, 0, 0, 7952, 7087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 7406, 0, 76, 0, 0, 297] payload_rows_per_dst=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] dup_rows_per_dst=[242, 0, 4746, 416, 482, 0, 0, 498] combine_rows_per_src=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88064 payload_rows_remote=79826 dup_rows=10240 dup_rows_remote=10240 dedup_ratio=0.104167 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 0, 0, 0, 79, 0, 2335, 6584, 0, 0, 7359, 0, 0, 0, 0, 663, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 1032, 0, 7777, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 70, 0, 0, 0, 309, 1335, 0, 7868, 0, 0, 70, 0, 0] payload_rows_per_dst=[15554, 2860, 4702, 13556, 16256, 8238, 12522, 14376] dup_rows_per_dst=[0, 0, 126, 1102, 3438, 0, 512, 5062] combine_rows_per_src=[15554, 2860, 4702, 13556, 16256, 8238, 12522, 14376] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79742 payload_rows_remote=65220 dup_rows=18562 dup_rows_remote=18562 dedup_ratio=0.188822 router_tokens_per_expert=[7602, 0, 0, 0, 1, 0, 0, 546, 0, 0, 5, 0, 0, 0, 0, 0, 106, 0, 762, 1738, 8190, 0, 1, 0, 0, 0, 139, 0, 592, 0, 0, 0, 436, 1, 0, 0, 1, 0, 0, 83, 0, 1, 0, 1, 7946, 7104, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6289, 7256, 0, 83, 1, 0, 267] payload_rows_per_dst=[16142, 14522, 16384, 15054, 1042, 13198, 2698, 702] dup_rows_per_dst=[156, 0, 17788, 616, 0, 2, 0, 0] combine_rows_per_src=[16142, 14522, 16384, 15054, 1042, 13198, 2698, 702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91920 payload_rows_remote=81324 dup_rows=6384 dup_rows_remote=1638 dedup_ratio=0.064941 router_tokens_per_expert=[7616, 0, 0, 0, 0, 0, 0, 594, 0, 0, 65, 0, 0, 0, 0, 0, 104, 0, 764, 1624, 8192, 0, 0, 0, 0, 0, 146, 0, 329, 0, 0, 0, 443, 0, 4, 0, 0, 0, 0, 75, 0, 0, 0, 0, 7952, 7087, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6378, 7406, 0, 76, 0, 0, 297] payload_rows_per_dst=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] dup_rows_per_dst=[242, 0, 4746, 416, 482, 0, 0, 498] combine_rows_per_src=[16178, 14942, 10596, 14708, 8632, 0, 15904, 10960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79790 payload_rows_remote=63438 dup_rows=18514 dup_rows_remote=7338 dedup_ratio=0.188334 router_tokens_per_expert=[7554, 0, 20, 0, 28, 0, 0, 505, 0, 0, 32, 0, 0, 0, 0, 0, 112, 2, 792, 1621, 8143, 0, 13, 0, 0, 0, 164, 0, 540, 12, 1, 0, 437, 11, 35, 0, 2, 0, 0, 88, 0, 11, 0, 21, 7898, 7012, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 7, 6474, 7129, 0, 91, 26, 1, 344] payload_rows_per_dst=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] dup_rows_per_dst=[11176, 0, 6654, 656, 4, 8, 2, 14] combine_rows_per_src=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89012 payload_rows_remote=84728 dup_rows=9292 dup_rows_remote=9022 dedup_ratio=0.094523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 102, 0, 2439, 6593, 0, 0, 7422, 0, 0, 0, 0, 661, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 964, 0, 7753, 0, 0, 0, 0, 4242, 0, 0, 0, 0, 94, 0, 0, 0, 338, 1274, 4, 7927, 0, 0, 94, 0, 0] payload_rows_per_dst=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] dup_rows_per_dst=[0, 1856, 1440, 1204, 270, 3708, 676, 138] combine_rows_per_src=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=356950016 dispatch_meta_bytes=393216 combine_payload_bytes=356950016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=357343232 ingress_bytes=391553024 link_bytes=391553024 token_bytes=714293248 weight_movement_bytes=34603008 total_nvlink_bytes=748896256 weight_to_token_ratio=0.048444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473382507e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331923456 dispatch_meta_bytes=331056 combine_payload_bytes=331923456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332254512 ingress_bytes=366526464 link_bytes=366526464 token_bytes=664177968 weight_movement_bytes=34603008 total_nvlink_bytes=698780976 weight_to_token_ratio=0.052099 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379209576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95884 payload_rows_remote=79500 dup_rows=2420 dup_rows_remote=1386 dedup_ratio=0.024618 router_tokens_per_expert=[7652, 0, 0, 0, 0, 0, 0, 497, 0, 0, 29, 0, 0, 0, 0, 0, 101, 0, 742, 1730, 8192, 0, 0, 0, 0, 0, 158, 0, 506, 0, 0, 0, 422, 0, 1, 0, 0, 0, 0, 94, 0, 0, 0, 0, 7972, 7144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6304, 7229, 0, 114, 0, 0, 265] payload_rows_per_dst=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] dup_rows_per_dst=[122, 0, 126, 568, 1034, 0, 0, 570] combine_rows_per_src=[16176, 14516, 5020, 15048, 16384, 0, 15944, 12796] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=228638720 dispatch_meta_bytes=281568 combine_payload_bytes=228638720 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=228920288 ingress_bytes=332447744 link_bytes=332447744 token_bytes=457559008 weight_movement_bytes=103809024 total_nvlink_bytes=561368032 weight_to_token_ratio=0.226876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250974096e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90408 payload_rows_remote=75610 dup_rows=7896 dup_rows_remote=6522 dedup_ratio=0.080322 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 112, 0, 2685, 6399, 0, 0, 7338, 0, 0, 0, 0, 748, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 959, 0, 7758, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 80, 0, 0, 0, 344, 1331, 1, 7971, 0, 0, 80, 0, 0] payload_rows_per_dst=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] dup_rows_per_dst=[0, 2334, 3468, 1374, 0, 0, 688, 32] combine_rows_per_src=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=273465344 dispatch_meta_bytes=330088 combine_payload_bytes=273465344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273795432 ingress_bytes=308068352 link_bytes=308068352 token_bytes=547260776 weight_movement_bytes=34603008 total_nvlink_bytes=581863784 weight_to_token_ratio=0.063229 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159236407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326967296 dispatch_meta_bytes=360264 combine_payload_bytes=326967296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=327327560 ingress_bytes=396173312 link_bytes=396173312 token_bytes=654294856 weight_movement_bytes=69206016 total_nvlink_bytes=723500872 weight_to_token_ratio=0.105772 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490768278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=335128 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476248 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534617368 weight_movement_bytes=34603008 total_nvlink_bytes=569220376 weight_to_token_ratio=0.064725 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=236699648 dispatch_meta_bytes=328320 combine_payload_bytes=236699648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=237027968 ingress_bytes=271302656 link_bytes=271302656 token_bytes=473727616 weight_movement_bytes=34603008 total_nvlink_bytes=508330624 weight_to_token_ratio=0.073044 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.020890053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=339787776 dispatch_meta_bytes=340912 combine_payload_bytes=339787776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=340128688 ingress_bytes=443596800 link_bytes=443596800 token_bytes=679916464 weight_movement_bytes=103809024 total_nvlink_bytes=783725488 weight_to_token_ratio=0.152679 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.669219046e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=333103104 dispatch_meta_bytes=331848 combine_payload_bytes=333103104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=333434952 ingress_bytes=402309120 link_bytes=402309120 token_bytes=666538056 weight_movement_bytes=69206016 total_nvlink_bytes=735744072 weight_to_token_ratio=0.103829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.513856830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=259842048 dispatch_meta_bytes=283104 combine_payload_bytes=259842048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260125152 ingress_bytes=329048064 link_bytes=329048064 token_bytes=519967200 weight_movement_bytes=69206016 total_nvlink_bytes=589173216 weight_to_token_ratio=0.133097 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238181374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347045888 dispatch_meta_bytes=375000 combine_payload_bytes=347045888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347420888 ingress_bytes=416251904 link_bytes=416251904 token_bytes=694466776 weight_movement_bytes=69206016 total_nvlink_bytes=763672792 weight_to_token_ratio=0.099653 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.566322404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309698560 dispatch_meta_bytes=328528 combine_payload_bytes=309698560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310027088 ingress_bytes=378904576 link_bytes=378904576 token_bytes=619725648 weight_movement_bytes=69206016 total_nvlink_bytes=688931664 weight_to_token_ratio=0.111672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425787415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=335128 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476248 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534617368 weight_movement_bytes=34603008 total_nvlink_bytes=569220376 weight_to_token_ratio=0.064725 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=325632000 dispatch_meta_bytes=323544 combine_payload_bytes=325632000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325955544 ingress_bytes=360235008 link_bytes=360235008 token_bytes=651587544 weight_movement_bytes=34603008 total_nvlink_bytes=686190552 weight_to_token_ratio=0.053106 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355535334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82546 payload_rows_remote=66764 dup_rows=15758 dup_rows_remote=15758 dedup_ratio=0.160299 router_tokens_per_expert=[7570, 0, 0, 0, 0, 0, 0, 554, 0, 0, 5, 0, 0, 0, 0, 0, 98, 0, 753, 1697, 8192, 0, 0, 0, 0, 0, 207, 0, 567, 0, 0, 0, 476, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 7891, 7051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6411, 7179, 0, 86, 0, 0, 330] payload_rows_per_dst=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] dup_rows_per_dst=[0, 0, 190, 106, 1122, 10958, 0, 3382] combine_rows_per_src=[1108, 11032, 4906, 3968, 16384, 15758, 15782, 13608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90408 payload_rows_remote=75610 dup_rows=7896 dup_rows_remote=6522 dedup_ratio=0.080322 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1167, 0, 0, 0, 112, 0, 2685, 6399, 0, 0, 7338, 0, 0, 0, 0, 748, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 959, 0, 7758, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 80, 0, 0, 0, 344, 1331, 1, 7971, 0, 0, 80, 0, 0] payload_rows_per_dst=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] dup_rows_per_dst=[0, 2334, 3468, 1374, 0, 0, 688, 32] combine_rows_per_src=[15516, 16224, 14924, 14798, 1918, 8134, 16102, 2792] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57754 payload_rows_remote=41396 dup_rows=40550 dup_rows_remote=24474 dedup_ratio=0.412496 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1148, 0, 0, 0, 66, 0, 2428, 6803, 0, 0, 7391, 0, 0, 0, 0, 733, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 879, 0, 7831, 0, 0, 0, 0, 4005, 0, 0, 0, 0, 63, 0, 0, 0, 262, 1339, 6, 8006, 0, 0, 63, 0, 0] payload_rows_per_dst=[0, 2296, 4892, 1466, 16368, 15724, 650, 16358] dup_rows_per_dst=[0, 0, 96, 0, 16430, 7948, 0, 16076] combine_rows_per_src=[0, 2296, 4892, 1466, 16368, 15724, 650, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95078 payload_rows_remote=78694 dup_rows=3226 dup_rows_remote=1824 dedup_ratio=0.032817 router_tokens_per_expert=[0, 140, 0, 8071, 0, 70, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 7452, 0, 0, 6878, 0, 0, 713, 0, 0, 0, 353, 0, 8192, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 548, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] dup_rows_per_dst=[142, 0, 96, 604, 0, 1402, 0, 982] combine_rows_per_src=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60822 payload_rows_remote=59986 dup_rows=37482 dup_rows_remote=37482 dedup_ratio=0.381287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 99, 0, 2449, 6422, 0, 0, 7378, 0, 0, 0, 0, 738, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 1016, 0, 7721, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 85, 0, 0, 0, 333, 1250, 1, 7994, 0, 0, 85, 0, 0] payload_rows_per_dst=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] dup_rows_per_dst=[0, 0, 174, 0, 16626, 5530, 0, 15152] combine_rows_per_src=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81344 payload_rows_remote=67834 dup_rows=16960 dup_rows_remote=16956 dedup_ratio=0.172526 router_tokens_per_expert=[1, 4, 0, 0, 1, 3, 3, 0, 0, 0, 3, 0, 5358, 10, 27, 6745, 3, 0, 2, 2, 10, 6702, 1, 6, 0, 0, 0, 199, 0, 27, 8162, 0, 0, 0, 1, 0, 0, 8162, 2, 3, 0, 0, 1802, 0, 0, 28, 27, 0, 11, 35, 0, 0, 0, 3, 14, 4, 8, 340, 0, 3, 3271, 8165, 4, 0] payload_rows_per_dst=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] dup_rows_per_dst=[4, 102, 4, 16720, 0, 30, 98, 2] combine_rows_per_src=[13510, 10868, 13274, 16380, 12, 3684, 16366, 7250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90038 payload_rows_remote=74978 dup_rows=8266 dup_rows_remote=5454 dedup_ratio=0.084086 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 101, 0, 2006, 6829, 0, 0, 7270, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 990, 0, 7783, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 76, 0, 0, 0, 337, 1378, 13, 7990, 0, 0, 76, 0, 0] payload_rows_per_dst=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] dup_rows_per_dst=[0, 2666, 2812, 2082, 0, 0, 674, 32] combine_rows_per_src=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88064 payload_rows_remote=79826 dup_rows=10240 dup_rows_remote=10240 dedup_ratio=0.104167 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1430, 0, 0, 0, 79, 0, 2335, 6584, 0, 0, 7359, 0, 0, 0, 0, 663, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 1032, 0, 7777, 0, 0, 0, 0, 4119, 0, 0, 0, 0, 70, 0, 0, 0, 309, 1335, 0, 7868, 0, 0, 70, 0, 0] payload_rows_per_dst=[15554, 2860, 4702, 13556, 16256, 8238, 12522, 14376] dup_rows_per_dst=[0, 0, 126, 1102, 3438, 0, 512, 5062] combine_rows_per_src=[15554, 2860, 4702, 13556, 16256, 8238, 12522, 14376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90672 payload_rows_remote=74454 dup_rows=7632 dup_rows_remote=5172 dedup_ratio=0.077637 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1229, 0, 0, 0, 106, 0, 2544, 6419, 1, 0, 7354, 0, 0, 0, 0, 654, 0, 0, 0, 1, 8108, 1, 0, 0, 0, 963, 0, 7728, 0, 0, 0, 1, 4312, 0, 0, 0, 0, 83, 1, 0, 0, 353, 1269, 9, 7929, 0, 1, 83, 0, 0] payload_rows_per_dst=[15458, 16218, 14886, 14846, 1928, 8626, 16024, 2686] dup_rows_per_dst=[0, 2460, 3254, 1170, 2, 0, 708, 38] combine_rows_per_src=[15458, 16218, 14886, 14846, 1928, 8626, 16024, 2686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90038 payload_rows_remote=74978 dup_rows=8266 dup_rows_remote=5454 dedup_ratio=0.084086 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1333, 0, 0, 0, 101, 0, 2006, 6829, 0, 0, 7270, 0, 0, 0, 0, 1107, 0, 0, 0, 0, 8117, 0, 0, 0, 0, 990, 0, 7783, 0, 0, 0, 0, 3746, 0, 0, 0, 0, 76, 0, 0, 0, 337, 1378, 13, 7990, 0, 0, 76, 0, 0] payload_rows_per_dst=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] dup_rows_per_dst=[0, 2666, 2812, 2082, 0, 0, 674, 32] combine_rows_per_src=[15566, 16234, 15060, 14672, 1980, 7492, 16132, 2902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89808 payload_rows_remote=74376 dup_rows=8496 dup_rows_remote=8492 dedup_ratio=0.086426 router_tokens_per_expert=[0, 0, 22, 0, 2, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1216, 16, 0, 0, 109, 0, 2604, 6356, 14, 0, 7229, 17, 0, 0, 0, 810, 2, 0, 0, 10, 8071, 26, 0, 0, 0, 1178, 11, 7691, 3, 27, 1, 0, 3837, 0, 8, 6, 2, 88, 6, 7, 0, 410, 1337, 72, 7842, 0, 29, 88, 0, 0] payload_rows_per_dst=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] dup_rows_per_dst=[4, 2454, 3752, 1262, 14, 26, 826, 158] combine_rows_per_src=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95500 payload_rows_remote=79274 dup_rows=2804 dup_rows_remote=2804 dedup_ratio=0.028524 router_tokens_per_expert=[0, 153, 0, 8099, 0, 94, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 7553, 0, 0, 6828, 0, 0, 663, 0, 0, 0, 356, 0, 8192, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] dup_rows_per_dst=[366, 0, 46, 0, 0, 1432, 0, 960] combine_rows_per_src=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95238 payload_rows_remote=94550 dup_rows=3066 dup_rows_remote=3066 dedup_ratio=0.031189 router_tokens_per_expert=[0, 140, 0, 8103, 0, 80, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 7497, 0, 0, 6700, 0, 0, 669, 0, 0, 0, 344, 0, 8192, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] dup_rows_per_dst=[302, 0, 66, 0, 0, 1478, 0, 1220] combine_rows_per_src=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=273465344 dispatch_meta_bytes=330088 combine_payload_bytes=273465344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273795432 ingress_bytes=308068352 link_bytes=308068352 token_bytes=547260776 weight_movement_bytes=34603008 total_nvlink_bytes=581863784 weight_to_token_ratio=0.063229 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159236407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89012 payload_rows_remote=84728 dup_rows=9292 dup_rows_remote=9022 dedup_ratio=0.094523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 0, 0, 0, 102, 0, 2439, 6593, 0, 0, 7422, 0, 0, 0, 0, 661, 0, 0, 0, 0, 8098, 0, 0, 0, 0, 964, 0, 7753, 0, 0, 0, 0, 4242, 0, 0, 0, 0, 94, 0, 0, 0, 338, 1274, 4, 7927, 0, 0, 94, 0, 0] payload_rows_per_dst=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] dup_rows_per_dst=[0, 1856, 1440, 1204, 270, 3708, 676, 138] combine_rows_per_src=[15506, 14008, 8862, 14962, 4284, 12092, 16042, 3256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309698560 dispatch_meta_bytes=328528 combine_payload_bytes=309698560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310027088 ingress_bytes=378904576 link_bytes=378904576 token_bytes=619725648 weight_movement_bytes=69206016 total_nvlink_bytes=688931664 weight_to_token_ratio=0.111672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425787415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=169558016 dispatch_meta_bytes=263480 combine_payload_bytes=169558016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=169821496 ingress_bytes=238764032 link_bytes=238764032 token_bytes=339379512 weight_movement_bytes=69206016 total_nvlink_bytes=408585528 weight_to_token_ratio=0.203919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.984498303e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=322330624 dispatch_meta_bytes=322072 combine_payload_bytes=322330624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322652696 ingress_bytes=356933632 link_bytes=356933632 token_bytes=644983320 weight_movement_bytes=34603008 total_nvlink_bytes=679586328 weight_to_token_ratio=0.053649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343112522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=245702656 dispatch_meta_bytes=389872 combine_payload_bytes=245702656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246092528 ingress_bytes=280305664 link_bytes=280305664 token_bytes=491795184 weight_movement_bytes=34603008 total_nvlink_bytes=526398192 weight_to_token_ratio=0.070361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054767647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=318504 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305282088 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610245672 weight_movement_bytes=34603008 total_nvlink_bytes=644848680 weight_to_token_ratio=0.056703 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307109888 dispatch_meta_bytes=321728 combine_payload_bytes=307109888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307431616 ingress_bytes=410918912 link_bytes=410918912 token_bytes=614541504 weight_movement_bytes=103809024 total_nvlink_bytes=718350528 weight_to_token_ratio=0.168921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546254784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304644096 dispatch_meta_bytes=331472 combine_payload_bytes=304644096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304975568 ingress_bytes=339247104 link_bytes=339247104 token_bytes=609619664 weight_movement_bytes=34603008 total_nvlink_bytes=644222672 weight_to_token_ratio=0.056762 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276559541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=324706304 dispatch_meta_bytes=328312 combine_payload_bytes=324706304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325034616 ingress_bytes=393912320 link_bytes=393912320 token_bytes=649740920 weight_movement_bytes=69206016 total_nvlink_bytes=718946936 weight_to_token_ratio=0.106513 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482260348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=387276800 dispatch_meta_bytes=390464 combine_payload_bytes=387276800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=387667264 ingress_bytes=456482816 link_bytes=456482816 token_bytes=774944064 weight_movement_bytes=69206016 total_nvlink_bytes=844150080 weight_to_token_ratio=0.089305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.717708087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=277848064 dispatch_meta_bytes=339160 combine_payload_bytes=277848064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278187224 ingress_bytes=312451072 link_bytes=312451072 token_bytes=556035288 weight_movement_bytes=34603008 total_nvlink_bytes=590638296 weight_to_token_ratio=0.062232 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.175728230e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=326967296 dispatch_meta_bytes=360264 combine_payload_bytes=326967296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=327327560 ingress_bytes=396173312 link_bytes=396173312 token_bytes=654294856 weight_movement_bytes=69206016 total_nvlink_bytes=723500872 weight_to_token_ratio=0.105772 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.490768278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304963584 dispatch_meta_bytes=318504 combine_payload_bytes=304963584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305282088 ingress_bytes=339566592 link_bytes=339566592 token_bytes=610245672 weight_movement_bytes=34603008 total_nvlink_bytes=644848680 weight_to_token_ratio=0.056703 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277761748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=347045888 dispatch_meta_bytes=375000 combine_payload_bytes=347045888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347420888 ingress_bytes=416251904 link_bytes=416251904 token_bytes=694466776 weight_movement_bytes=69206016 total_nvlink_bytes=763672792 weight_to_token_ratio=0.099653 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.566322404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60822 payload_rows_remote=59986 dup_rows=37482 dup_rows_remote=37482 dedup_ratio=0.381287 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1269, 0, 0, 0, 99, 0, 2449, 6422, 0, 0, 7378, 0, 0, 0, 0, 738, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 1016, 0, 7721, 0, 0, 0, 0, 4205, 0, 0, 0, 0, 85, 0, 0, 0, 333, 1250, 1, 7994, 0, 0, 85, 0, 0] payload_rows_per_dst=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] dup_rows_per_dst=[0, 0, 174, 0, 16626, 5530, 0, 15152] combine_rows_per_src=[5268, 2538, 4922, 1476, 16376, 13054, 836, 16352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95238 payload_rows_remote=94550 dup_rows=3066 dup_rows_remote=3066 dedup_ratio=0.031189 router_tokens_per_expert=[0, 140, 0, 8103, 0, 80, 0, 0, 0, 0, 7865, 0, 0, 0, 0, 7497, 0, 0, 6700, 0, 0, 669, 0, 0, 0, 344, 0, 8192, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 152, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] dup_rows_per_dst=[302, 0, 66, 0, 0, 1478, 0, 1220] combine_rows_per_src=[16344, 0, 14672, 688, 16278, 16384, 15730, 15142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88658 payload_rows_remote=73410 dup_rows=9646 dup_rows_remote=8632 dedup_ratio=0.098124 router_tokens_per_expert=[0, 98, 0, 8101, 1, 63, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 7588, 0, 0, 6817, 0, 0, 676, 0, 0, 0, 338, 0, 8192, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[198, 0, 8416, 16306, 16378, 16384, 15728, 15248] dup_rows_per_dst=[126, 0, 8, 572, 6546, 1380, 0, 1014] combine_rows_per_src=[198, 0, 8416, 16306, 16378, 16384, 15728, 15248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92236 payload_rows_remote=76774 dup_rows=6068 dup_rows_remote=5936 dedup_ratio=0.061727 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 650, 0, 1067, 0, 0, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 4589, 0, 0, 0, 0, 517, 0, 8192, 0, 0, 0, 8192, 0, 0, 71, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 611, 0, 7129] payload_rows_per_dst=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] dup_rows_per_dst=[0, 5216, 84, 0, 72, 132, 0, 564] combine_rows_per_src=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94772 payload_rows_remote=79078 dup_rows=3532 dup_rows_remote=3532 dedup_ratio=0.035929 router_tokens_per_expert=[0, 175, 0, 8134, 0, 85, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7474, 0, 0, 6686, 0, 0, 672, 0, 0, 0, 364, 0, 8192, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] dup_rows_per_dst=[170, 0, 26, 720, 0, 1492, 0, 1124] combine_rows_per_src=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69270 payload_rows_remote=53034 dup_rows=29034 dup_rows_remote=2158 dedup_ratio=0.295349 router_tokens_per_expert=[0, 152, 0, 8100, 0, 83, 0, 0, 0, 0, 7854, 0, 0, 0, 1, 7538, 1, 0, 6768, 0, 0, 651, 0, 0, 0, 340, 1, 8190, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 174, 0, 1, 0, 527, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 650, 0, 0, 0, 0, 1, 1] payload_rows_per_dst=[16358, 16236, 13850, 16382, 0, 1230, 3910, 1304] dup_rows_per_dst=[1296, 26876, 6, 680, 0, 174, 2, 0] combine_rows_per_src=[16358, 16236, 13850, 16382, 0, 1230, 3910, 1304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94018 payload_rows_remote=79024 dup_rows=4286 dup_rows_remote=4252 dedup_ratio=0.043599 router_tokens_per_expert=[0, 139, 0, 8112, 0, 75, 0, 0, 0, 0, 7871, 0, 0, 0, 0, 7506, 0, 0, 6845, 0, 0, 669, 0, 0, 0, 322, 0, 8192, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 171, 0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16354, 5966, 14994, 12120, 10340, 5506, 15742, 12996] dup_rows_per_dst=[298, 2386, 34, 458, 0, 486, 0, 624] combine_rows_per_src=[16354, 5966, 14994, 12120, 10340, 5506, 15742, 12996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54038 payload_rows_remote=37766 dup_rows=44266 dup_rows_remote=31620 dedup_ratio=0.450297 router_tokens_per_expert=[0, 154, 0, 8014, 0, 88, 0, 0, 1, 1, 7822, 0, 17, 1, 20, 7492, 28, 4, 6203, 0, 19, 638, 0, 0, 0, 339, 0, 8160, 0, 0, 11, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 4, 0, 194, 0, 5, 1, 681, 17, 3, 18, 1, 0, 5, 31, 0, 0, 0, 1084, 0, 7, 0, 2, 0, 21] payload_rows_per_dst=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] dup_rows_per_dst=[12646, 30624, 42, 678, 0, 252, 16, 8] combine_rows_per_src=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93292 payload_rows_remote=83308 dup_rows=5012 dup_rows_remote=4924 dedup_ratio=0.050985 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 656, 0, 1095, 0, 0, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 522, 0, 8192, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 618, 0, 7179] payload_rows_per_dst=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] dup_rows_per_dst=[0, 2418, 1932, 0, 88, 0, 0, 574] combine_rows_per_src=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=245702656 dispatch_meta_bytes=389872 combine_payload_bytes=245702656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246092528 ingress_bytes=280305664 link_bytes=280305664 token_bytes=491795184 weight_movement_bytes=34603008 total_nvlink_bytes=526398192 weight_to_token_ratio=0.070361 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054767647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79790 payload_rows_remote=63438 dup_rows=18514 dup_rows_remote=7338 dedup_ratio=0.188334 router_tokens_per_expert=[7554, 0, 20, 0, 28, 0, 0, 505, 0, 0, 32, 0, 0, 0, 0, 0, 112, 2, 792, 1621, 8143, 0, 13, 0, 0, 0, 164, 0, 540, 12, 1, 0, 437, 11, 35, 0, 2, 0, 0, 88, 0, 11, 0, 21, 7898, 7012, 0, 0, 0, 0, 2, 0, 0, 24, 0, 0, 7, 6474, 7129, 0, 91, 26, 1, 344] payload_rows_per_dst=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] dup_rows_per_dst=[11176, 0, 6654, 656, 4, 8, 2, 14] combine_rows_per_src=[16352, 14322, 16346, 14802, 1142, 15852, 50, 924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95078 payload_rows_remote=78694 dup_rows=3226 dup_rows_remote=1824 dedup_ratio=0.032817 router_tokens_per_expert=[0, 140, 0, 8071, 0, 70, 0, 0, 0, 0, 7844, 0, 0, 0, 0, 7452, 0, 0, 6878, 0, 0, 713, 0, 0, 0, 353, 0, 8192, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 0, 548, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 579, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] dup_rows_per_dst=[142, 0, 96, 604, 0, 1402, 0, 982] combine_rows_per_src=[756, 0, 15086, 15766, 16316, 16384, 15690, 15080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69270 payload_rows_remote=53034 dup_rows=29034 dup_rows_remote=2158 dedup_ratio=0.295349 router_tokens_per_expert=[0, 152, 0, 8100, 0, 83, 0, 0, 0, 0, 7854, 0, 0, 0, 1, 7538, 1, 0, 6768, 0, 0, 651, 0, 0, 0, 340, 1, 8190, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 174, 0, 1, 0, 527, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 650, 0, 0, 0, 0, 1, 1] payload_rows_per_dst=[16358, 16236, 13850, 16382, 0, 1230, 3910, 1304] dup_rows_per_dst=[1296, 26876, 6, 680, 0, 174, 2, 0] combine_rows_per_src=[16358, 16236, 13850, 16382, 0, 1230, 3910, 1304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95500 payload_rows_remote=79274 dup_rows=2804 dup_rows_remote=2804 dedup_ratio=0.028524 router_tokens_per_expert=[0, 153, 0, 8099, 0, 94, 0, 0, 0, 0, 7846, 0, 0, 0, 0, 7553, 0, 0, 6828, 0, 0, 663, 0, 0, 0, 356, 0, 8192, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 0, 0, 0, 0, 161, 0, 0, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] dup_rows_per_dst=[366, 0, 46, 0, 0, 1432, 0, 960] combine_rows_per_src=[16326, 0, 14936, 712, 16226, 16384, 15692, 15224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=387276800 dispatch_meta_bytes=390464 combine_payload_bytes=387276800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=387667264 ingress_bytes=456482816 link_bytes=456482816 token_bytes=774944064 weight_movement_bytes=69206016 total_nvlink_bytes=844150080 weight_to_token_ratio=0.089305 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.717708087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=300687360 dispatch_meta_bytes=328168 combine_payload_bytes=300687360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301015528 ingress_bytes=335290368 link_bytes=335290368 token_bytes=601702888 weight_movement_bytes=34603008 total_nvlink_bytes=636305896 weight_to_token_ratio=0.057508 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261670662e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323682304 dispatch_meta_bytes=333104 combine_payload_bytes=323682304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324015408 ingress_bytes=392888320 link_bytes=392888320 token_bytes=647697712 weight_movement_bytes=69206016 total_nvlink_bytes=716903728 weight_to_token_ratio=0.106849 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478407118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=259842048 dispatch_meta_bytes=283104 combine_payload_bytes=259842048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260125152 ingress_bytes=329048064 link_bytes=329048064 token_bytes=519967200 weight_movement_bytes=69206016 total_nvlink_bytes=589173216 weight_to_token_ratio=0.133097 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238181374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=322330624 dispatch_meta_bytes=322072 combine_payload_bytes=322330624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322652696 ingress_bytes=356933632 link_bytes=356933632 token_bytes=644983320 weight_movement_bytes=34603008 total_nvlink_bytes=679586328 weight_to_token_ratio=0.053649 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343112522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=217227264 dispatch_meta_bytes=220768 combine_payload_bytes=217227264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217448032 ingress_bytes=321036288 link_bytes=321036288 token_bytes=434675296 weight_movement_bytes=103809024 total_nvlink_bytes=538484320 weight_to_token_ratio=0.238820 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208033706e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=324706304 dispatch_meta_bytes=328312 combine_payload_bytes=324706304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325034616 ingress_bytes=393912320 link_bytes=393912320 token_bytes=649740920 weight_movement_bytes=69206016 total_nvlink_bytes=718946936 weight_to_token_ratio=0.106513 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.482260348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94772 payload_rows_remote=79078 dup_rows=3532 dup_rows_remote=3532 dedup_ratio=0.035929 router_tokens_per_expert=[0, 175, 0, 8134, 0, 85, 0, 0, 0, 0, 7847, 0, 0, 0, 0, 7474, 0, 0, 6686, 0, 0, 672, 0, 0, 0, 364, 0, 8192, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 197, 0, 0, 0, 549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 667, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] dup_rows_per_dst=[170, 0, 26, 720, 0, 1492, 0, 1124] combine_rows_per_src=[350, 0, 14690, 16276, 16220, 16384, 15694, 15158] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94924 payload_rows_remote=78540 dup_rows=3380 dup_rows_remote=3380 dedup_ratio=0.034383 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 714, 0, 1075, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 979, 0, 0, 0, 0, 0, 0, 0, 0, 4586, 0, 0, 0, 0, 488, 0, 8192, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 0, 0, 8147, 0, 0, 0, 0, 0, 0, 0, 643, 0, 7134] payload_rows_per_dst=[16384, 4290, 16384, 16384, 10052, 332, 16112, 14986] dup_rows_per_dst=[0, 758, 1958, 0, 96, 0, 0, 568] combine_rows_per_src=[16384, 4290, 16384, 16384, 10052, 332, 16112, 14986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314466304 dispatch_meta_bytes=330840 combine_payload_bytes=314466304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314797144 ingress_bytes=383672320 link_bytes=383672320 token_bytes=629263448 weight_movement_bytes=69206016 total_nvlink_bytes=698469464 weight_to_token_ratio=0.109979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443728052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323903488 dispatch_meta_bytes=330440 combine_payload_bytes=323903488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324233928 ingress_bytes=358506496 link_bytes=358506496 token_bytes=648137416 weight_movement_bytes=34603008 total_nvlink_bytes=682740424 weight_to_token_ratio=0.053388 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349031082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=217227264 dispatch_meta_bytes=220768 combine_payload_bytes=217227264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=217448032 ingress_bytes=321036288 link_bytes=321036288 token_bytes=434675296 weight_movement_bytes=103809024 total_nvlink_bytes=538484320 weight_to_token_ratio=0.238820 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208033706e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323682304 dispatch_meta_bytes=333104 combine_payload_bytes=323682304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324015408 ingress_bytes=392888320 link_bytes=392888320 token_bytes=647697712 weight_movement_bytes=69206016 total_nvlink_bytes=716903728 weight_to_token_ratio=0.106849 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478407118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=154689536 dispatch_meta_bytes=277544 combine_payload_bytes=154689536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=154967080 ingress_bytes=223895552 link_bytes=223895552 token_bytes=309656616 weight_movement_bytes=69206016 total_nvlink_bytes=378862632 weight_to_token_ratio=0.223493 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.425009371e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=341229568 dispatch_meta_bytes=352928 combine_payload_bytes=341229568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=341582496 ingress_bytes=445038592 link_bytes=445038592 token_bytes=682812064 weight_movement_bytes=103809024 total_nvlink_bytes=786621088 weight_to_token_ratio=0.152032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674644393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321699840 dispatch_meta_bytes=327680 combine_payload_bytes=321699840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322027520 ingress_bytes=356302848 link_bytes=356302848 token_bytes=643727360 weight_movement_bytes=34603008 total_nvlink_bytes=678330368 weight_to_token_ratio=0.053754 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340738932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82482 payload_rows_remote=67540 dup_rows=15822 dup_rows_remote=15294 dedup_ratio=0.160950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 679, 0, 1210, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 0, 0, 1002, 0, 0, 0, 0, 0, 0, 0, 0, 4385, 0, 0, 0, 0, 544, 0, 8192, 0, 0, 0, 8192, 0, 0, 75, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 580, 0, 7155] payload_rows_per_dst=[0, 16384, 2004, 0, 16384, 16384, 16384, 14942] dup_rows_per_dst=[0, 5302, 0, 0, 1088, 150, 8754, 528] combine_rows_per_src=[0, 16384, 2004, 0, 16384, 16384, 16384, 14942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95052 payload_rows_remote=89210 dup_rows=3252 dup_rows_remote=1928 dedup_ratio=0.033081 router_tokens_per_expert=[0, 0, 3451, 6674, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 58, 0, 0, 70, 0, 0, 0, 8192, 8192, 0, 0, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 4332, 0, 0, 0] payload_rows_per_dst=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] dup_rows_per_dst=[0, 0, 454, 108, 116, 1324, 1250, 0] combine_rows_per_src=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94452 payload_rows_remote=78068 dup_rows=3852 dup_rows_remote=1602 dedup_ratio=0.039185 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 670, 0, 1167, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 4121, 0, 0, 0, 0, 654, 0, 8192, 0, 0, 0, 8192, 0, 0, 66, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 682, 0, 7103] payload_rows_per_dst=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] dup_rows_per_dst=[0, 768, 2250, 0, 118, 132, 0, 584] combine_rows_per_src=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89808 payload_rows_remote=74376 dup_rows=8496 dup_rows_remote=8492 dedup_ratio=0.086426 router_tokens_per_expert=[0, 0, 22, 0, 2, 3, 0, 0, 0, 0, 0, 0, 2, 0, 1216, 16, 0, 0, 109, 0, 2604, 6356, 14, 0, 7229, 17, 0, 0, 0, 810, 2, 0, 0, 10, 8071, 26, 0, 0, 0, 1178, 11, 7691, 3, 27, 1, 0, 3837, 0, 8, 6, 2, 88, 6, 7, 0, 410, 1337, 72, 7842, 0, 29, 88, 0, 0] payload_rows_per_dst=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] dup_rows_per_dst=[4, 2454, 3752, 1262, 14, 26, 826, 158] combine_rows_per_src=[15432, 16156, 14934, 14334, 2414, 7732, 15912, 2894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92236 payload_rows_remote=76774 dup_rows=6068 dup_rows_remote=5936 dedup_ratio=0.061727 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 650, 0, 1067, 0, 0, 0, 0, 891, 0, 0, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 0, 0, 0, 0, 4589, 0, 0, 0, 0, 517, 0, 8192, 0, 0, 0, 8192, 0, 0, 71, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 611, 0, 7129] payload_rows_per_dst=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] dup_rows_per_dst=[0, 5216, 84, 0, 72, 132, 0, 564] combine_rows_per_src=[0, 16384, 2588, 16384, 10140, 15462, 16362, 14916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82452 payload_rows_remote=66068 dup_rows=15852 dup_rows_remote=7904 dedup_ratio=0.161255 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 685, 0, 1244, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 4468, 0, 0, 0, 0, 424, 0, 8192, 0, 0, 0, 8192, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 691, 0, 7196] payload_rows_per_dst=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] dup_rows_per_dst=[0, 5376, 0, 0, 1752, 108, 7948, 668] combine_rows_per_src=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84210 payload_rows_remote=79666 dup_rows=14094 dup_rows_remote=13404 dedup_ratio=0.143372 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 8190, 656, 1, 1227, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 985, 0, 0, 0, 0, 0, 0, 0, 1, 4387, 0, 1, 0, 0, 497, 0, 8190, 0, 1, 0, 8190, 1, 1, 70, 0, 0, 0, 1, 1, 1, 8163, 0, 1, 0, 0, 0, 0, 0, 625, 0, 7228] payload_rows_per_dst=[16382, 4544, 16380, 16382, 10730, 16328, 6, 3458] dup_rows_per_dst=[9344, 690, 1970, 0, 1852, 144, 2, 92] combine_rows_per_src=[16382, 4544, 16380, 16382, 10730, 16328, 6, 3458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94452 payload_rows_remote=78068 dup_rows=3852 dup_rows_remote=1602 dedup_ratio=0.039185 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 670, 0, 1167, 0, 0, 0, 0, 838, 0, 0, 0, 0, 0, 0, 0, 1125, 0, 0, 0, 0, 0, 0, 0, 0, 4121, 0, 0, 0, 0, 654, 0, 8192, 0, 0, 0, 8192, 0, 0, 66, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 682, 0, 7103] payload_rows_per_dst=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] dup_rows_per_dst=[0, 768, 2250, 0, 118, 132, 0, 584] combine_rows_per_src=[16384, 4582, 16384, 16384, 9432, 16300, 0, 14986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81578 payload_rows_remote=65234 dup_rows=16726 dup_rows_remote=2978 dedup_ratio=0.170146 router_tokens_per_expert=[11, 20, 0, 0, 0, 0, 3, 8134, 584, 29, 1580, 0, 0, 0, 0, 727, 0, 8, 2, 0, 0, 0, 1, 941, 0, 0, 0, 0, 0, 0, 3, 15, 4637, 15, 0, 0, 0, 429, 0, 8155, 0, 29, 0, 8141, 20, 28, 52, 0, 0, 0, 0, 9, 17, 8010, 0, 18, 0, 9, 4, 2, 1, 640, 0, 6878] payload_rows_per_dst=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] dup_rows_per_dst=[13748, 732, 1874, 14, 122, 194, 32, 10] combine_rows_per_src=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=79718 dup_rows=2202 dup_rows_remote=2018 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 3516, 6499, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 75, 0, 0, 0, 8192, 8192, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4391, 0, 0, 0] payload_rows_per_dst=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] dup_rows_per_dst=[0, 0, 734, 184, 138, 0, 1146, 0] combine_rows_per_src=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=365404160 dispatch_meta_bytes=364552 combine_payload_bytes=365404160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=365768712 ingress_bytes=469213184 link_bytes=469213184 token_bytes=731172872 weight_movement_bytes=103809024 total_nvlink_bytes=834981896 weight_to_token_ratio=0.141976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.765611437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93292 payload_rows_remote=83308 dup_rows=5012 dup_rows_remote=4924 dedup_ratio=0.050985 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 656, 0, 1095, 0, 0, 0, 0, 809, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 4514, 0, 0, 0, 0, 522, 0, 8192, 0, 0, 0, 8192, 0, 0, 85, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 618, 0, 7179] payload_rows_per_dst=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] dup_rows_per_dst=[0, 2418, 1932, 0, 88, 0, 0, 574] combine_rows_per_src=[9718, 9368, 16384, 16384, 9984, 170, 16264, 15020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=276643840 dispatch_meta_bytes=331336 combine_payload_bytes=276643840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276975176 ingress_bytes=345849856 link_bytes=345849856 token_bytes=553619016 weight_movement_bytes=69206016 total_nvlink_bytes=622825032 weight_to_token_ratio=0.125007 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301405165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95832 payload_rows_remote=79448 dup_rows=2472 dup_rows_remote=2310 dedup_ratio=0.025146 router_tokens_per_expert=[0, 0, 3317, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 0, 0, 106, 0, 0, 0, 8192, 8192, 0, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 4357, 0, 0, 0] payload_rows_per_dst=[6634, 16384, 16384, 16384, 16384, 212, 14400, 9050] dup_rows_per_dst=[0, 0, 676, 184, 162, 0, 1450, 0] combine_rows_per_src=[6634, 16384, 16384, 16384, 16384, 212, 14400, 9050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=321699840 dispatch_meta_bytes=327680 combine_payload_bytes=321699840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322027520 ingress_bytes=356302848 link_bytes=356302848 token_bytes=643727360 weight_movement_bytes=34603008 total_nvlink_bytes=678330368 weight_to_token_ratio=0.053754 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340738932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=304644096 dispatch_meta_bytes=331472 combine_payload_bytes=304644096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304975568 ingress_bytes=339247104 link_bytes=339247104 token_bytes=609619664 weight_movement_bytes=34603008 total_nvlink_bytes=644222672 weight_to_token_ratio=0.056762 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276559541e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270614528 dispatch_meta_bytes=295888 combine_payload_bytes=270614528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270910416 ingress_bytes=339820544 link_bytes=339820544 token_bytes=541524944 weight_movement_bytes=69206016 total_nvlink_bytes=610730960 weight_to_token_ratio=0.127798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278717349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=326311936 dispatch_meta_bytes=372280 combine_payload_bytes=326311936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326684216 ingress_bytes=360914944 link_bytes=360914944 token_bytes=652996152 weight_movement_bytes=34603008 total_nvlink_bytes=687599160 weight_to_token_ratio=0.052991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358093878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=319766528 dispatch_meta_bytes=318680 combine_payload_bytes=319766528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320085208 ingress_bytes=354369536 link_bytes=354369536 token_bytes=639851736 weight_movement_bytes=34603008 total_nvlink_bytes=674454744 weight_to_token_ratio=0.054080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333464035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=272848 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267471312 ingress_bytes=336404480 link_bytes=336404480 token_bytes=534669776 weight_movement_bytes=69206016 total_nvlink_bytes=603875792 weight_to_token_ratio=0.129437 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=326524928 dispatch_meta_bytes=326944 combine_payload_bytes=326524928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326851872 ingress_bytes=361127936 link_bytes=361127936 token_bytes=653376800 weight_movement_bytes=34603008 total_nvlink_bytes=687979808 weight_to_token_ratio=0.052960 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325419008 dispatch_meta_bytes=327032 combine_payload_bytes=325419008 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=325746040 ingress_bytes=394625024 link_bytes=394625024 token_bytes=651165048 weight_movement_bytes=69206016 total_nvlink_bytes=720371064 weight_to_token_ratio=0.106280 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484942195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97250 payload_rows_remote=80866 dup_rows=1054 dup_rows_remote=1054 dedup_ratio=0.010722 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 240, 0, 0, 0, 0, 0, 0, 0, 200, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 61, 0, 0, 0, 0, 293, 72, 0, 7604, 0, 0] payload_rows_per_dst=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] dup_rows_per_dst=[0, 0, 0, 0, 400, 0, 654, 0] combine_rows_per_src=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314466304 dispatch_meta_bytes=330840 combine_payload_bytes=314466304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314797144 ingress_bytes=383672320 link_bytes=383672320 token_bytes=629263448 weight_movement_bytes=69206016 total_nvlink_bytes=698469464 weight_to_token_ratio=0.109979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443728052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=326311936 dispatch_meta_bytes=372280 combine_payload_bytes=326311936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326684216 ingress_bytes=360914944 link_bytes=360914944 token_bytes=652996152 weight_movement_bytes=34603008 total_nvlink_bytes=687599160 weight_to_token_ratio=0.052991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358093878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82452 payload_rows_remote=66068 dup_rows=15852 dup_rows_remote=7904 dedup_ratio=0.161255 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8192, 685, 0, 1244, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 4468, 0, 0, 0, 0, 424, 0, 8192, 0, 0, 0, 8192, 0, 0, 54, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 691, 0, 7196] payload_rows_per_dst=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] dup_rows_per_dst=[0, 5376, 0, 0, 1752, 108, 7948, 668] combine_rows_per_src=[0, 16384, 1800, 40, 16354, 16384, 16384, 15106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=341229568 dispatch_meta_bytes=352928 combine_payload_bytes=341229568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=341582496 ingress_bytes=445038592 link_bytes=445038592 token_bytes=682812064 weight_movement_bytes=103809024 total_nvlink_bytes=786621088 weight_to_token_ratio=0.152032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.674644393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=270614528 dispatch_meta_bytes=295888 combine_payload_bytes=270614528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270910416 ingress_bytes=339820544 link_bytes=339820544 token_bytes=541524944 weight_movement_bytes=69206016 total_nvlink_bytes=610730960 weight_to_token_ratio=0.127798 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278717349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54516 payload_rows_remote=38132 dup_rows=43788 dup_rows_remote=34676 dedup_ratio=0.445435 router_tokens_per_expert=[0, 0, 3254, 6613, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 64, 0, 0, 29, 0, 0, 0, 8192, 8192, 0, 0, 1500, 0, 0, 0, 0, 0, 0, 0, 0, 194, 0, 4362, 0, 0, 0] payload_rows_per_dst=[6508, 0, 644, 92, 128, 16384, 14376, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 32826, 1850, 9112] combine_rows_per_src=[6508, 0, 644, 92, 128, 16384, 14376, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66366 payload_rows_remote=52074 dup_rows=31938 dup_rows_remote=30464 dedup_ratio=0.324890 router_tokens_per_expert=[0, 0, 3449, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 129, 0, 0, 0, 8192, 8192, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 4264, 0, 0, 0] payload_rows_per_dst=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] dup_rows_per_dst=[0, 0, 0, 0, 36, 24680, 1474, 5748] combine_rows_per_src=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91838 payload_rows_remote=75454 dup_rows=6466 dup_rows_remote=6466 dedup_ratio=0.065776 router_tokens_per_expert=[0, 0, 3700, 6476, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 1, 366, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 68, 1, 0, 52, 1, 0, 1, 8189, 8190, 0, 1, 1367, 0, 1, 1, 0, 0, 0, 0, 1, 164, 0, 4100, 0, 0, 0] payload_rows_per_dst=[14934, 16384, 16382, 16380, 16384, 106, 2738, 8530] dup_rows_per_dst=[5418, 0, 730, 178, 136, 2, 2, 0] combine_rows_per_src=[14934, 16384, 16382, 16380, 16384, 106, 2738, 8530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331227136 dispatch_meta_bytes=327680 combine_payload_bytes=331227136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331554816 ingress_bytes=365830144 link_bytes=365830144 token_bytes=662781952 weight_movement_bytes=34603008 total_nvlink_bytes=697384960 weight_to_token_ratio=0.052209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376589380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=79718 dup_rows=2202 dup_rows_remote=2018 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 3516, 6499, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 367, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 69, 0, 0, 75, 0, 0, 0, 8192, 8192, 0, 0, 1194, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4391, 0, 0, 0] payload_rows_per_dst=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] dup_rows_per_dst=[0, 0, 734, 184, 138, 0, 1146, 0] combine_rows_per_src=[7032, 16384, 16384, 16384, 16384, 150, 14240, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54038 payload_rows_remote=37766 dup_rows=44266 dup_rows_remote=31620 dedup_ratio=0.450297 router_tokens_per_expert=[0, 154, 0, 8014, 0, 88, 0, 0, 1, 1, 7822, 0, 17, 1, 20, 7492, 28, 4, 6203, 0, 19, 638, 0, 0, 0, 339, 0, 8160, 0, 0, 11, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 4, 0, 194, 0, 5, 1, 681, 17, 3, 18, 1, 0, 5, 31, 0, 0, 0, 1084, 0, 7, 0, 2, 0, 21] payload_rows_per_dst=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] dup_rows_per_dst=[12646, 30624, 42, 678, 0, 252, 16, 8] combine_rows_per_src=[16272, 16216, 1336, 16342, 0, 1552, 100, 2220] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95000 payload_rows_remote=78616 dup_rows=3304 dup_rows_remote=2486 dedup_ratio=0.033610 router_tokens_per_expert=[0, 0, 3226, 6587, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 409, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 61, 0, 0, 81, 0, 0, 0, 8192, 8192, 0, 0, 1451, 0, 0, 0, 0, 0, 0, 0, 0, 191, 0, 4289, 0, 0, 0] payload_rows_per_dst=[7812, 16384, 16384, 16384, 16384, 162, 12530, 8960] dup_rows_per_dst=[690, 0, 818, 178, 122, 0, 1496, 0] combine_rows_per_src=[7812, 16384, 16384, 16384, 16384, 162, 12530, 8960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97310 payload_rows_remote=89698 dup_rows=994 dup_rows_remote=588 dedup_ratio=0.010111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 388, 0, 0, 0, 0, 0, 0, 0, 180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 92, 0, 0, 0, 0, 250, 63, 0, 7629, 0, 0] payload_rows_per_dst=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] dup_rows_per_dst=[0, 0, 0, 406, 210, 0, 378, 0] combine_rows_per_src=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77880 payload_rows_remote=61672 dup_rows=20424 dup_rows_remote=1156 dedup_ratio=0.207764 router_tokens_per_expert=[7, 0, 3869, 6429, 0, 0, 0, 1, 32, 0, 0, 0, 8153, 0, 0, 0, 15, 399, 0, 0, 0, 0, 0, 0, 115, 1, 11, 0, 0, 0, 0, 0, 8142, 1, 0, 0, 32, 0, 54, 0, 0, 84, 1, 0, 31, 8146, 8149, 0, 31, 1245, 0, 0, 2, 0, 0, 0, 0, 31, 167, 23, 3974, 0, 0, 7] payload_rows_per_dst=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] dup_rows_per_dst=[19268, 0, 806, 222, 72, 0, 4, 52] combine_rows_per_src=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95052 payload_rows_remote=89210 dup_rows=3252 dup_rows_remote=1928 dedup_ratio=0.033081 router_tokens_per_expert=[0, 0, 3451, 6674, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 58, 0, 0, 70, 0, 0, 0, 8192, 8192, 0, 0, 1203, 0, 0, 0, 0, 0, 0, 0, 0, 196, 0, 4332, 0, 0, 0] payload_rows_per_dst=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] dup_rows_per_dst=[0, 0, 454, 108, 116, 1324, 1250, 0] combine_rows_per_src=[6902, 16384, 10856, 15124, 16384, 5842, 14504, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91838 payload_rows_remote=75454 dup_rows=6466 dup_rows_remote=6466 dedup_ratio=0.065776 router_tokens_per_expert=[0, 0, 3700, 6476, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 1, 366, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 68, 1, 0, 52, 1, 0, 1, 8189, 8190, 0, 1, 1367, 0, 1, 1, 0, 0, 0, 0, 1, 164, 0, 4100, 0, 0, 0] payload_rows_per_dst=[14934, 16384, 16382, 16380, 16384, 106, 2738, 8530] dup_rows_per_dst=[5418, 0, 730, 178, 136, 2, 2, 0] combine_rows_per_src=[14934, 16384, 16382, 16380, 16384, 106, 2738, 8530] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=156188672 dispatch_meta_bytes=291232 combine_payload_bytes=156188672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=156479904 ingress_bytes=190791680 link_bytes=190791680 token_bytes=312668576 weight_movement_bytes=34603008 total_nvlink_bytes=347271584 weight_to_token_ratio=0.110670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.179337318e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95832 payload_rows_remote=79448 dup_rows=2472 dup_rows_remote=2310 dedup_ratio=0.025146 router_tokens_per_expert=[0, 0, 3317, 6511, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 81, 0, 0, 106, 0, 0, 0, 8192, 8192, 0, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 0, 168, 0, 4357, 0, 0, 0] payload_rows_per_dst=[6634, 16384, 16384, 16384, 16384, 212, 14400, 9050] dup_rows_per_dst=[0, 0, 676, 184, 162, 0, 1450, 0] combine_rows_per_src=[6634, 16384, 16384, 16384, 16384, 212, 14400, 9050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=326524928 dispatch_meta_bytes=326944 combine_payload_bytes=326524928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326851872 ingress_bytes=361127936 link_bytes=361127936 token_bytes=653376800 weight_movement_bytes=34603008 total_nvlink_bytes=687979808 weight_to_token_ratio=0.052960 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=154689536 dispatch_meta_bytes=277544 combine_payload_bytes=154689536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=154967080 ingress_bytes=223895552 link_bytes=223895552 token_bytes=309656616 weight_movement_bytes=69206016 total_nvlink_bytes=378862632 weight_to_token_ratio=0.223493 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.425009371e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=322011136 dispatch_meta_bytes=324408 combine_payload_bytes=322011136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322335544 ingress_bytes=356614144 link_bytes=356614144 token_bytes=644346680 weight_movement_bytes=34603008 total_nvlink_bytes=678949688 weight_to_token_ratio=0.053702 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341910314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97358 payload_rows_remote=80974 dup_rows=946 dup_rows_remote=612 dedup_ratio=0.009623 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 312, 0, 0, 0, 0, 0, 0, 0, 167, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 92, 0, 0, 0, 0, 255, 63, 0, 7605, 0, 0] payload_rows_per_dst=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] dup_rows_per_dst=[0, 0, 0, 0, 334, 0, 612, 0] combine_rows_per_src=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=213295104 dispatch_meta_bytes=330152 combine_payload_bytes=213295104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=213625256 ingress_bytes=247898112 link_bytes=247898112 token_bytes=426920360 weight_movement_bytes=34603008 total_nvlink_bytes=461523368 weight_to_token_ratio=0.081053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.328206380e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=365404160 dispatch_meta_bytes=364552 combine_payload_bytes=365404160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=365768712 ingress_bytes=469213184 link_bytes=469213184 token_bytes=731172872 weight_movement_bytes=103809024 total_nvlink_bytes=834981896 weight_to_token_ratio=0.141976 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19328 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.765611437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=309059584 dispatch_meta_bytes=327680 combine_payload_bytes=309059584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309387264 ingress_bytes=412868608 link_bytes=412868608 token_bytes=618446848 weight_movement_bytes=103809024 total_nvlink_bytes=722255872 weight_to_token_ratio=0.167854 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553591333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14264770808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=7136231672 ingress_bytes=8478056448 link_bytes=8478056448 total_nvlink_bytes=15614288120 weight_to_token_ratio=0.094605 prefetch_share_of_ingress=0.081630 dedup_ratio=0.103379 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=23876 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.387054081e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=322011136 dispatch_meta_bytes=324408 combine_payload_bytes=322011136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322335544 ingress_bytes=356614144 link_bytes=356614144 token_bytes=644346680 weight_movement_bytes=34603008 total_nvlink_bytes=678949688 weight_to_token_ratio=0.053702 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341910314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=367403008 dispatch_meta_bytes=361144 combine_payload_bytes=367403008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=367764152 ingress_bytes=436609024 link_bytes=436609024 token_bytes=735167160 weight_movement_bytes=69206016 total_nvlink_bytes=804373176 weight_to_token_ratio=0.094136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642924607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252608512 dispatch_meta_bytes=251312 combine_payload_bytes=252608512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252859824 ingress_bytes=321814528 link_bytes=321814528 token_bytes=505468336 weight_movement_bytes=69206016 total_nvlink_bytes=574674352 weight_to_token_ratio=0.136915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210962161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=326344 combine_payload_bytes=331669504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331995848 ingress_bytes=400875520 link_bytes=400875520 token_bytes=663665352 weight_movement_bytes=69206016 total_nvlink_bytes=732871368 weight_to_token_ratio=0.104278 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508462309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14739811392 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=7373843520 ingress_bytes=8715485184 link_bytes=8715485184 total_nvlink_bytes=16089328704 weight_to_token_ratio=0.091556 prefetch_share_of_ingress=0.083376 dedup_ratio=0.085407 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.425898656e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66366 payload_rows_remote=52074 dup_rows=31938 dup_rows_remote=30464 dedup_ratio=0.324890 router_tokens_per_expert=[0, 0, 3449, 6465, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 41, 0, 0, 129, 0, 0, 0, 8192, 8192, 0, 0, 1418, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 4264, 0, 0, 0] payload_rows_per_dst=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] dup_rows_per_dst=[0, 0, 0, 0, 36, 24680, 1474, 5748] combine_rows_per_src=[6898, 8346, 738, 172, 5620, 16384, 14292, 13916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325419008 dispatch_meta_bytes=327032 combine_payload_bytes=325419008 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=325746040 ingress_bytes=394625024 link_bytes=394625024 token_bytes=651165048 weight_movement_bytes=69206016 total_nvlink_bytes=720371064 weight_to_token_ratio=0.106280 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484942195e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=213295104 dispatch_meta_bytes=330152 combine_payload_bytes=213295104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=213625256 ingress_bytes=247898112 link_bytes=247898112 token_bytes=426920360 weight_movement_bytes=34603008 total_nvlink_bytes=461523368 weight_to_token_ratio=0.081053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.328206380e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97394 payload_rows_remote=81382 dup_rows=910 dup_rows_remote=910 dedup_ratio=0.009257 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 251, 0, 0, 0, 0, 0, 0, 0, 193, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 46, 0, 0, 0, 0, 264, 35, 0, 7707, 0, 0] payload_rows_per_dst=[0, 15344, 16384, 502, 16384, 16384, 16384, 16012] dup_rows_per_dst=[0, 0, 0, 0, 386, 0, 524, 0] combine_rows_per_src=[0, 15344, 16384, 502, 16384, 16384, 16384, 16012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14554394376 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=7281110792 ingress_bytes=8761212928 link_bytes=8761212928 total_nvlink_bytes=16042323720 weight_to_token_ratio=0.102232 prefetch_share_of_ingress=0.090840 dedup_ratio=0.089974 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=22860 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.433379953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97306 payload_rows_remote=80922 dup_rows=998 dup_rows_remote=324 dedup_ratio=0.010152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 230, 0, 0, 0, 0, 0, 0, 0, 162, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 86, 0, 0, 0, 0, 268, 91, 0, 7612, 0, 0] payload_rows_per_dst=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] dup_rows_per_dst=[0, 0, 0, 0, 324, 0, 674, 0] combine_rows_per_src=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64712 payload_rows_remote=48330 dup_rows=33592 dup_rows_remote=17200 dedup_ratio=0.341715 router_tokens_per_expert=[0, 0, 0, 1, 1, 1, 0, 0, 2, 8191, 7690, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 1, 0, 0, 8188, 262, 1, 0, 1, 1, 0, 0, 0, 164, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 234, 0, 0, 1, 0, 89, 0, 0, 0, 0, 263, 71, 0, 7605, 0, 0] payload_rows_per_dst=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] dup_rows_per_dst=[4, 16392, 0, 16900, 296, 0, 0, 0] combine_rows_per_src=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97310 payload_rows_remote=89698 dup_rows=994 dup_rows_remote=588 dedup_ratio=0.010111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 388, 0, 0, 0, 0, 0, 0, 0, 180, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 0, 0, 92, 0, 0, 0, 0, 250, 63, 0, 7629, 0, 0] payload_rows_per_dst=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] dup_rows_per_dst=[0, 0, 0, 406, 210, 0, 378, 0] combine_rows_per_src=[15370, 0, 16384, 7612, 9292, 16384, 16384, 15884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81578 payload_rows_remote=65234 dup_rows=16726 dup_rows_remote=2978 dedup_ratio=0.170146 router_tokens_per_expert=[11, 20, 0, 0, 0, 0, 3, 8134, 584, 29, 1580, 0, 0, 0, 0, 727, 0, 8, 2, 0, 0, 0, 1, 941, 0, 0, 0, 0, 0, 0, 3, 15, 4637, 15, 0, 0, 0, 429, 0, 8155, 0, 29, 0, 8141, 20, 28, 52, 0, 0, 0, 0, 9, 17, 8010, 0, 18, 0, 9, 4, 2, 1, 640, 0, 6878] payload_rows_per_dst=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] dup_rows_per_dst=[13748, 732, 1874, 14, 122, 194, 32, 10] combine_rows_per_src=[16344, 5108, 16312, 16332, 10040, 16084, 56, 1302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92446 payload_rows_remote=84246 dup_rows=5858 dup_rows_remote=5858 dedup_ratio=0.059591 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 260, 0, 0, 0, 0, 0, 0, 0, 189, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 206, 0, 0, 0, 0, 90, 0, 0, 0, 0, 293, 80, 0, 7599, 0, 0] payload_rows_per_dst=[15334, 8184, 8200, 16384, 378, 16208, 11814, 15944] dup_rows_per_dst=[0, 176, 0, 5240, 0, 0, 442, 0] combine_rows_per_src=[15334, 8184, 8200, 16384, 378, 16208, 11814, 15944] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64688 payload_rows_remote=49374 dup_rows=33616 dup_rows_remote=33506 dedup_ratio=0.341960 router_tokens_per_expert=[0, 0, 17, 23, 35, 15, 0, 4, 1, 8152, 7618, 0, 0, 0, 2, 0, 0, 5, 0, 5, 31, 8145, 3, 0, 6, 22, 0, 0, 8150, 309, 14, 0, 18, 29, 0, 0, 0, 153, 0, 8156, 1, 0, 5, 0, 16, 1, 64, 0, 214, 1, 0, 0, 0, 104, 0, 0, 0, 0, 287, 73, 0, 7473, 0, 0] payload_rows_per_dst=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] dup_rows_per_dst=[110, 16292, 6, 16976, 200, 32, 0, 0] combine_rows_per_src=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97250 payload_rows_remote=80866 dup_rows=1054 dup_rows_remote=1054 dedup_ratio=0.010722 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 240, 0, 0, 0, 0, 0, 0, 0, 200, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 0, 0, 61, 0, 0, 0, 0, 293, 72, 0, 7604, 0, 0] payload_rows_per_dst=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] dup_rows_per_dst=[0, 0, 0, 0, 400, 0, 654, 0] combine_rows_per_src=[15296, 0, 16384, 480, 16384, 16384, 16384, 15938] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=333340672 dispatch_meta_bytes=329168 combine_payload_bytes=333340672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=333669840 ingress_bytes=402546688 link_bytes=402546688 token_bytes=667010512 weight_movement_bytes=69206016 total_nvlink_bytes=736216528 weight_to_token_ratio=0.103756 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514750779e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331456512 dispatch_meta_bytes=324984 combine_payload_bytes=331456512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331781496 ingress_bytes=366059520 link_bytes=366059520 token_bytes=663238008 weight_movement_bytes=34603008 total_nvlink_bytes=697841016 weight_to_token_ratio=0.052173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=197959680 dispatch_meta_bytes=262120 combine_payload_bytes=197959680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=198221800 ingress_bytes=301768704 link_bytes=301768704 token_bytes=396181480 weight_movement_bytes=103809024 total_nvlink_bytes=499990504 weight_to_token_ratio=0.262024 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64712 payload_rows_remote=48330 dup_rows=33592 dup_rows_remote=17200 dedup_ratio=0.341715 router_tokens_per_expert=[0, 0, 0, 1, 1, 1, 0, 0, 2, 8191, 7690, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 1, 1, 0, 0, 8188, 262, 1, 0, 1, 1, 0, 0, 0, 164, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 234, 0, 0, 1, 0, 89, 0, 0, 0, 0, 263, 71, 0, 7605, 0, 0] payload_rows_per_dst=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] dup_rows_per_dst=[4, 16392, 0, 16900, 296, 0, 0, 0] combine_rows_per_src=[15382, 16382, 2, 16382, 13896, 2, 648, 2018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=367403008 dispatch_meta_bytes=361144 combine_payload_bytes=367403008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=367764152 ingress_bytes=436609024 link_bytes=436609024 token_bytes=735167160 weight_movement_bytes=69206016 total_nvlink_bytes=804373176 weight_to_token_ratio=0.094136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642924607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97358 payload_rows_remote=80974 dup_rows=946 dup_rows_remote=612 dedup_ratio=0.009623 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7676, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 312, 0, 0, 0, 0, 0, 0, 0, 167, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 214, 0, 0, 0, 0, 92, 0, 0, 0, 0, 255, 63, 0, 7605, 0, 0] payload_rows_per_dst=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] dup_rows_per_dst=[0, 0, 0, 0, 334, 0, 612, 0] combine_rows_per_src=[15352, 0, 16384, 624, 16384, 16384, 16384, 15846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=272848 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267471312 ingress_bytes=336404480 link_bytes=336404480 token_bytes=534669776 weight_movement_bytes=69206016 total_nvlink_bytes=603875792 weight_to_token_ratio=0.129437 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265862975e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=345071616 dispatch_meta_bytes=360416 combine_payload_bytes=345071616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=345432032 ingress_bytes=414277632 link_bytes=414277632 token_bytes=690503648 weight_movement_bytes=69206016 total_nvlink_bytes=759709664 weight_to_token_ratio=0.100225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.558893377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=202235904 dispatch_meta_bytes=331520 combine_payload_bytes=202235904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202567424 ingress_bytes=236838912 link_bytes=236838912 token_bytes=404803328 weight_movement_bytes=34603008 total_nvlink_bytes=439406336 weight_to_token_ratio=0.085481 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.912057588e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=97306 payload_rows_remote=80922 dup_rows=998 dup_rows_remote=324 dedup_ratio=0.010152 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 7684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 230, 0, 0, 0, 0, 0, 0, 0, 162, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 86, 0, 0, 0, 0, 268, 91, 0, 7612, 0, 0] payload_rows_per_dst=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] dup_rows_per_dst=[0, 0, 0, 0, 324, 0, 674, 0] combine_rows_per_src=[6142, 9226, 16384, 460, 16384, 16384, 16384, 15942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331227136 dispatch_meta_bytes=327680 combine_payload_bytes=331227136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331554816 ingress_bytes=365830144 link_bytes=365830144 token_bytes=662781952 weight_movement_bytes=34603008 total_nvlink_bytes=697384960 weight_to_token_ratio=0.052209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376589380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11806888608 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=5907051168 ingress_bytes=7283957760 link_bytes=7283957760 total_nvlink_bytes=13191008928 weight_to_token_ratio=0.117230 prefetch_share_of_ingress=0.080760 dedup_ratio=0.217075 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191693332e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12941903184 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6474786128 ingress_bytes=7885840384 link_bytes=7885840384 total_nvlink_bytes=14360626512 weight_to_token_ratio=0.109622 prefetch_share_of_ingress=0.087760 dedup_ratio=0.173323 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290164457e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=197959680 dispatch_meta_bytes=262120 combine_payload_bytes=197959680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=198221800 ingress_bytes=301768704 link_bytes=301768704 token_bytes=396181480 weight_movement_bytes=103809024 total_nvlink_bytes=499990504 weight_to_token_ratio=0.262024 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14739811392 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=7373843520 ingress_bytes=8715485184 link_bytes=8715485184 total_nvlink_bytes=16089328704 weight_to_token_ratio=0.091556 prefetch_share_of_ingress=0.083376 dedup_ratio=0.085407 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.425898656e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=326344 combine_payload_bytes=331669504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331995848 ingress_bytes=400875520 link_bytes=400875520 token_bytes=663665352 weight_movement_bytes=69206016 total_nvlink_bytes=732871368 weight_to_token_ratio=0.104278 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508462309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14597572344 prefetch_bytes=761266176 reduce_bytes=968884224 weight_movement_bytes=1730150400 egress_bytes=7302694648 ingress_bytes=9025028096 link_bytes=9025028096 total_nvlink_bytes=16327722744 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.084351 dedup_ratio=0.097700 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=28 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=22987 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.476541485e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14264770808 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=7136231672 ingress_bytes=8478056448 link_bytes=8478056448 total_nvlink_bytes=15614288120 weight_to_token_ratio=0.094605 prefetch_share_of_ingress=0.081630 dedup_ratio=0.103379 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=23876 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.387054081e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=331456512 dispatch_meta_bytes=324984 combine_payload_bytes=331456512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=331781496 ingress_bytes=366059520 link_bytes=366059520 token_bytes=663238008 weight_movement_bytes=34603008 total_nvlink_bytes=697841016 weight_to_token_ratio=0.052173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377452504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12513027392 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6260204864 ingress_bytes=7879163904 link_bytes=7879163904 total_nvlink_bytes=14139368768 weight_to_token_ratio=0.129972 prefetch_share_of_ingress=0.096618 dedup_ratio=0.187887 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.289072150e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14554394376 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=7281110792 ingress_bytes=8761212928 link_bytes=8761212928 total_nvlink_bytes=16042323720 weight_to_token_ratio=0.102232 prefetch_share_of_ingress=0.090840 dedup_ratio=0.089974 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=22860 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.433379953e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77880 payload_rows_remote=61672 dup_rows=20424 dup_rows_remote=1156 dedup_ratio=0.207764 router_tokens_per_expert=[7, 0, 3869, 6429, 0, 0, 0, 1, 32, 0, 0, 0, 8153, 0, 0, 0, 15, 399, 0, 0, 0, 0, 0, 0, 115, 1, 11, 0, 0, 0, 0, 0, 8142, 1, 0, 0, 32, 0, 54, 0, 0, 84, 1, 0, 31, 8146, 8149, 0, 31, 1245, 0, 0, 2, 0, 0, 0, 0, 31, 167, 23, 3974, 0, 0, 7] payload_rows_per_dst=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] dup_rows_per_dst=[19268, 0, 806, 222, 72, 0, 4, 52] combine_rows_per_src=[16208, 8038, 16314, 16330, 9854, 232, 2552, 8352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12941903184 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6474786128 ingress_bytes=7885840384 link_bytes=7885840384 total_nvlink_bytes=14360626512 weight_to_token_ratio=0.109622 prefetch_share_of_ingress=0.087760 dedup_ratio=0.173323 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290164457e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=252608512 dispatch_meta_bytes=251312 combine_payload_bytes=252608512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=252859824 ingress_bytes=321814528 link_bytes=321814528 token_bytes=505468336 weight_movement_bytes=69206016 total_nvlink_bytes=574674352 weight_to_token_ratio=0.136915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210962161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64688 payload_rows_remote=49374 dup_rows=33616 dup_rows_remote=33506 dedup_ratio=0.341960 router_tokens_per_expert=[0, 0, 17, 23, 35, 15, 0, 4, 1, 8152, 7618, 0, 0, 0, 2, 0, 0, 5, 0, 5, 31, 8145, 3, 0, 6, 22, 0, 0, 8150, 309, 14, 0, 18, 29, 0, 0, 0, 153, 0, 8156, 1, 0, 5, 0, 16, 1, 64, 0, 214, 1, 0, 0, 0, 104, 0, 0, 0, 0, 287, 73, 0, 7473, 0, 0] payload_rows_per_dst=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] dup_rows_per_dst=[110, 16292, 6, 16976, 200, 32, 0, 0] combine_rows_per_src=[15314, 16308, 8176, 16338, 7052, 142, 638, 720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=202235904 dispatch_meta_bytes=331520 combine_payload_bytes=202235904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=202567424 ingress_bytes=236838912 link_bytes=236838912 token_bytes=404803328 weight_movement_bytes=34603008 total_nvlink_bytes=439406336 weight_to_token_ratio=0.085481 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.912057588e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11579943400 prefetch_bytes=830472192 reduce_bytes=519045120 weight_movement_bytes=1349517312 egress_bytes=5793597928 ingress_bytes=7135862784 link_bytes=7135862784 total_nvlink_bytes=12929460712 weight_to_token_ratio=0.116539 prefetch_share_of_ingress=0.116380 dedup_ratio=0.227370 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=15 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.167464225e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none - [2026-08-26 16:55:26] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 793.6 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.982457E+00 | loss scale: 1.0 | grad norm: 3.717 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:07:33] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 794.2 | throughput per GPU (TFLOP/s/GPU): 160.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.982457E+00 | loss scale: 1.0 | grad norm: 3.717 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74764 payload_rows_remote=65574 dup_rows=23540 dup_rows_remote=21214 dedup_ratio=0.239461 router_tokens_per_expert=[1174, 445, 801, 1296, 526, 1123, 919, 591, 1562, 637, 1060, 278, 73, 1948, 847, 901, 820, 1177, 831, 699, 218, 461, 372, 1180, 364, 728, 843, 405, 846, 117, 810, 580, 1062, 834, 1504, 1061, 496, 244, 648, 727, 391, 534, 620, 415, 1196, 735, 536, 316, 675, 528, 642, 598, 1037, 1674, 480, 576, 1560, 374, 1497, 630, 174, 724, 1286, 746] payload_rows_per_dst=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] dup_rows_per_dst=[3718, 3298, 2326, 3294, 3470, 3254, 2728, 1452] combine_rows_per_src=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=61938 dup_rows=24142 dup_rows_remote=18898 dedup_ratio=0.245585 router_tokens_per_expert=[1212, 479, 648, 1089, 623, 1300, 1082, 550, 1499, 816, 1152, 388, 91, 1757, 877, 949, 733, 931, 877, 764, 181, 499, 330, 1057, 411, 712, 826, 440, 944, 168, 637, 502, 1158, 801, 1455, 1110, 365, 231, 665, 703, 461, 528, 645, 377, 1105, 673, 662, 330, 661, 565, 663, 566, 1005, 1481, 506, 633, 1381, 474, 1550, 714, 307, 873, 1257, 723] payload_rows_per_dst=[6600, 9528, 10010, 9924, 8524, 7846, 9506, 12224] dup_rows_per_dst=[2046, 4194, 3158, 2870, 2200, 1716, 2714, 5244] combine_rows_per_src=[6600, 9528, 10010, 9924, 8524, 7846, 9506, 12224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65256 dup_rows=23688 dup_rows_remote=20308 dedup_ratio=0.240967 router_tokens_per_expert=[1204, 385, 647, 1088, 593, 1126, 1030, 441, 1698, 855, 1176, 235, 74, 1893, 876, 981, 803, 967, 1021, 747, 188, 502, 340, 1203, 442, 742, 797, 525, 879, 162, 656, 495, 1025, 711, 1588, 1069, 382, 257, 671, 808, 391, 483, 607, 371, 1189, 803, 656, 354, 738, 539, 697, 467, 846, 1574, 435, 592, 1516, 448, 1493, 568, 323, 797, 1205, 778] payload_rows_per_dst=[8118, 8058, 10474, 10050, 9600, 9360, 9354, 9602] dup_rows_per_dst=[2602, 3732, 3376, 3132, 3360, 3380, 2422, 1684] combine_rows_per_src=[8118, 8058, 10474, 10050, 9600, 9360, 9354, 9602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74768 payload_rows_remote=65776 dup_rows=23536 dup_rows_remote=19214 dedup_ratio=0.239421 router_tokens_per_expert=[1222, 546, 702, 1162, 528, 1238, 862, 524, 1554, 842, 1099, 306, 106, 1885, 767, 916, 795, 1129, 864, 684, 200, 504, 346, 1268, 430, 667, 808, 340, 848, 144, 764, 541, 1050, 838, 1531, 1042, 387, 219, 673, 758, 369, 471, 607, 408, 1052, 723, 666, 283, 609, 525, 710, 582, 974, 1574, 559, 607, 1738, 433, 1536, 584, 207, 760, 1235, 851] payload_rows_per_dst=[9938, 8992, 9254, 8534, 9530, 9344, 9546, 9630] dup_rows_per_dst=[3630, 4322, 2326, 2186, 3466, 3290, 2734, 1582] combine_rows_per_src=[9938, 8992, 9254, 8534, 9530, 9344, 9546, 9630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=64852 dup_rows=23756 dup_rows_remote=20816 dedup_ratio=0.241659 router_tokens_per_expert=[1247, 525, 807, 1295, 500, 1161, 847, 506, 1562, 713, 1075, 286, 93, 1809, 736, 995, 757, 1088, 800, 624, 233, 593, 328, 1370, 428, 716, 762, 291, 850, 138, 778, 565, 1011, 871, 1380, 1004, 582, 209, 617, 735, 484, 571, 665, 458, 1109, 706, 648, 333, 578, 581, 701, 674, 952, 1520, 607, 705, 1511, 377, 1435, 671, 186, 683, 1249, 861] payload_rows_per_dst=[7996, 8150, 10844, 9666, 8230, 9360, 9696, 10606] dup_rows_per_dst=[2530, 3526, 3236, 3008, 1828, 3094, 2940, 3594] combine_rows_per_src=[7996, 8150, 10844, 9666, 8230, 9360, 9696, 10606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=61938 dup_rows=24142 dup_rows_remote=18898 dedup_ratio=0.245585 router_tokens_per_expert=[1212, 479, 648, 1089, 623, 1300, 1082, 550, 1499, 816, 1152, 388, 91, 1757, 877, 949, 733, 931, 877, 764, 181, 499, 330, 1057, 411, 712, 826, 440, 944, 168, 637, 502, 1158, 801, 1455, 1110, 365, 231, 665, 703, 461, 528, 645, 377, 1105, 673, 662, 330, 661, 565, 663, 566, 1005, 1481, 506, 633, 1381, 474, 1550, 714, 307, 873, 1257, 723] payload_rows_per_dst=[6600, 9528, 10010, 9924, 8524, 7846, 9506, 12224] dup_rows_per_dst=[2046, 4194, 3158, 2870, 2200, 1716, 2714, 5244] combine_rows_per_src=[6600, 9528, 10010, 9924, 8524, 7846, 9506, 12224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74738 payload_rows_remote=64918 dup_rows=23566 dup_rows_remote=20474 dedup_ratio=0.239726 router_tokens_per_expert=[1154, 449, 744, 1315, 581, 1125, 1094, 563, 1589, 564, 1245, 319, 87, 1889, 888, 952, 710, 967, 904, 730, 186, 525, 339, 1168, 375, 756, 750, 425, 862, 116, 729, 554, 1039, 738, 1504, 1002, 641, 254, 532, 827, 463, 603, 601, 495, 1050, 593, 601, 324, 611, 509, 771, 653, 927, 1630, 527, 638, 1436, 366, 1473, 724, 292, 698, 1204, 772] payload_rows_per_dst=[10260, 7884, 8778, 9820, 9858, 9172, 9560, 9406] dup_rows_per_dst=[3790, 3404, 2280, 3092, 3216, 3160, 2972, 1652] combine_rows_per_src=[10260, 7884, 8778, 9820, 9858, 9172, 9560, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65280 dup_rows=23596 dup_rows_remote=20280 dedup_ratio=0.240031 router_tokens_per_expert=[1247, 516, 622, 1184, 488, 1046, 955, 595, 1625, 877, 1139, 307, 99, 1887, 806, 883, 746, 1139, 841, 655, 201, 498, 339, 1263, 447, 767, 874, 421, 865, 123, 730, 483, 1074, 769, 1420, 1066, 398, 228, 636, 781, 457, 530, 672, 365, 1156, 667, 634, 326, 691, 565, 720, 592, 916, 1644, 507, 600, 1567, 413, 1392, 673, 192, 721, 1308, 804] payload_rows_per_dst=[9832, 7708, 9142, 10064, 9428, 9408, 9642, 9484] dup_rows_per_dst=[3474, 3764, 2222, 3130, 3316, 3340, 2828, 1522] combine_rows_per_src=[9832, 7708, 9142, 10064, 9428, 9408, 9642, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=347152 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268938256 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537529360 weight_movement_bytes=34603008 total_nvlink_bytes=572132368 weight_to_token_ratio=0.064374 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=342256 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267630832 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534919408 weight_movement_bytes=34603008 total_nvlink_bytes=569522416 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=342672 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265976464 ingress_bytes=300236800 link_bytes=300236800 token_bytes=531610256 weight_movement_bytes=34603008 total_nvlink_bytes=566213264 weight_to_token_ratio=0.065091 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129766907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=339960 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269758456 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539176952 weight_movement_bytes=69206016 total_nvlink_bytes=608382968 weight_to_token_ratio=0.128355 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=253698048 dispatch_meta_bytes=323344 combine_payload_bytes=253698048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=254021392 ingress_bytes=322904064 link_bytes=322904064 token_bytes=507719440 weight_movement_bytes=69206016 total_nvlink_bytes=576925456 weight_to_token_ratio=0.136308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215061997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=339960 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269758456 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539176952 weight_movement_bytes=69206016 total_nvlink_bytes=608382968 weight_to_token_ratio=0.128355 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=342672 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265976464 ingress_bytes=300236800 link_bytes=300236800 token_bytes=531610256 weight_movement_bytes=34603008 total_nvlink_bytes=566213264 weight_to_token_ratio=0.065091 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129766907e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=341568 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266245696 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532149824 weight_movement_bytes=34603008 total_nvlink_bytes=566752832 weight_to_token_ratio=0.065025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74764 payload_rows_remote=65574 dup_rows=23540 dup_rows_remote=21214 dedup_ratio=0.239461 router_tokens_per_expert=[1174, 445, 801, 1296, 526, 1123, 919, 591, 1562, 637, 1060, 278, 73, 1948, 847, 901, 820, 1177, 831, 699, 218, 461, 372, 1180, 364, 728, 843, 405, 846, 117, 810, 580, 1062, 834, 1504, 1061, 496, 244, 648, 727, 391, 534, 620, 415, 1196, 735, 536, 316, 675, 528, 642, 598, 1037, 1674, 480, 576, 1560, 374, 1497, 630, 174, 724, 1286, 746] payload_rows_per_dst=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] dup_rows_per_dst=[3718, 3298, 2326, 3294, 3470, 3254, 2728, 1452] combine_rows_per_src=[10032, 7418, 9190, 9988, 9682, 9352, 9692, 9410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=342240 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267729120 ingress_bytes=336592896 link_bytes=336592896 token_bytes=535116000 weight_movement_bytes=69206016 total_nvlink_bytes=604322016 weight_to_token_ratio=0.129329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266571970e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74982 payload_rows_remote=64994 dup_rows=23322 dup_rows_remote=19572 dedup_ratio=0.237244 router_tokens_per_expert=[1237, 449, 766, 1169, 564, 1054, 1013, 617, 1517, 509, 1084, 280, 112, 1741, 904, 1016, 721, 1087, 949, 748, 189, 473, 335, 1342, 476, 750, 722, 374, 988, 126, 752, 585, 997, 779, 1335, 968, 662, 200, 579, 778, 421, 560, 587, 421, 1127, 775, 726, 358, 593, 494, 696, 607, 992, 1479, 498, 626, 1639, 420, 1417, 689, 276, 777, 1220, 807] payload_rows_per_dst=[9988, 9922, 9172, 7958, 9564, 9620, 9290, 9468] dup_rows_per_dst=[3750, 4404, 2516, 1588, 3032, 3608, 2680, 1744] combine_rows_per_src=[9988, 9922, 9172, 7958, 9564, 9620, 9290, 9468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=347152 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268938256 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537529360 weight_movement_bytes=34603008 total_nvlink_bytes=572132368 weight_to_token_ratio=0.064374 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76146 payload_rows_remote=65950 dup_rows=22158 dup_rows_remote=18614 dedup_ratio=0.225403 router_tokens_per_expert=[458, 647, 303, 465, 1945, 136, 496, 1268, 24, 728, 2378, 380, 522, 788, 523, 1975, 637, 1203, 583, 327, 1495, 262, 1893, 1789, 824, 651, 853, 1405, 29, 289, 87, 98, 110, 26, 271, 34, 879, 514, 880, 903, 1839, 1133, 363, 1048, 137, 2127, 6, 217, 2719, 666, 1940, 0, 64, 1033, 141, 1695, 1404, 405, 1088, 282, 757, 324, 324, 362] payload_rows_per_dst=[9018, 7844, 9762, 10742, 8870, 10196, 8886, 10828] dup_rows_per_dst=[2418, 2036, 2830, 3168, 2150, 3544, 2192, 3820] combine_rows_per_src=[9018, 7844, 9762, 10742, 8870, 10196, 8886, 10828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=66304 dup_rows=22272 dup_rows_remote=19698 dedup_ratio=0.226562 router_tokens_per_expert=[592, 795, 369, 442, 1750, 179, 611, 1086, 18, 748, 2383, 346, 584, 928, 560, 1933, 536, 1308, 739, 234, 1606, 163, 2005, 1565, 779, 575, 801, 1228, 25, 159, 97, 148, 135, 28, 306, 57, 920, 650, 965, 985, 1868, 953, 455, 1013, 151, 2050, 6, 180, 2354, 681, 1659, 0, 20, 1292, 224, 1938, 1408, 559, 949, 167, 1027, 259, 292, 309] payload_rows_per_dst=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] dup_rows_per_dst=[2806, 4230, 2574, 2740, 2460, 3188, 2288, 1986] combine_rows_per_src=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75160 payload_rows_remote=64006 dup_rows=23144 dup_rows_remote=18838 dedup_ratio=0.235433 router_tokens_per_expert=[471, 697, 372, 428, 2113, 157, 561, 1022, 23, 752, 2405, 387, 619, 945, 552, 2047, 527, 1176, 669, 281, 1580, 280, 2016, 1440, 817, 637, 815, 1235, 15, 220, 122, 144, 169, 45, 256, 37, 838, 584, 854, 991, 1593, 1126, 414, 1040, 147, 2041, 7, 223, 2414, 677, 1876, 0, 33, 1248, 173, 1834, 1320, 522, 912, 292, 966, 281, 431, 283] payload_rows_per_dst=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] dup_rows_per_dst=[4820, 4306, 2442, 2934, 2464, 1972, 2338, 1868] combine_rows_per_src=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75256 payload_rows_remote=65986 dup_rows=23048 dup_rows_remote=20770 dedup_ratio=0.234456 router_tokens_per_expert=[434, 844, 455, 381, 1794, 215, 557, 1110, 19, 727, 2477, 345, 590, 892, 641, 2035, 565, 1306, 660, 243, 1783, 244, 2011, 1474, 808, 628, 815, 1119, 15, 188, 90, 139, 222, 51, 294, 97, 936, 717, 852, 1016, 1733, 1148, 477, 1021, 158, 1807, 10, 193, 2109, 766, 1724, 0, 53, 1300, 182, 1749, 1216, 613, 889, 144, 1169, 291, 321, 290] payload_rows_per_dst=[8912, 6606, 10666, 9406, 9964, 9670, 9270, 10762] dup_rows_per_dst=[2668, 1594, 3758, 2416, 2852, 3424, 2278, 4058] combine_rows_per_src=[8912, 6606, 10666, 9406, 9964, 9670, 9270, 10762] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75370 payload_rows_remote=64342 dup_rows=22934 dup_rows_remote=18576 dedup_ratio=0.233297 router_tokens_per_expert=[484, 641, 286, 417, 1894, 170, 556, 1064, 26, 825, 2526, 391, 558, 900, 535, 1995, 647, 1191, 647, 424, 1435, 251, 1869, 1600, 797, 599, 821, 1394, 14, 310, 134, 166, 237, 37, 279, 58, 881, 555, 870, 888, 1777, 949, 395, 919, 155, 1914, 4, 226, 2400, 751, 1892, 0, 57, 1068, 197, 1909, 1436, 587, 946, 153, 1022, 345, 366, 312] payload_rows_per_dst=[8818, 5214, 8196, 8874, 10922, 9568, 12750, 11028] dup_rows_per_dst=[2206, 1256, 1426, 1952, 3296, 3110, 5330, 4358] combine_rows_per_src=[8818, 5214, 8196, 8874, 10922, 9568, 12750, 11028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75160 payload_rows_remote=64006 dup_rows=23144 dup_rows_remote=18838 dedup_ratio=0.235433 router_tokens_per_expert=[471, 697, 372, 428, 2113, 157, 561, 1022, 23, 752, 2405, 387, 619, 945, 552, 2047, 527, 1176, 669, 281, 1580, 280, 2016, 1440, 817, 637, 815, 1235, 15, 220, 122, 144, 169, 45, 256, 37, 838, 584, 854, 991, 1593, 1126, 414, 1040, 147, 2041, 7, 223, 2414, 677, 1876, 0, 33, 1248, 173, 1834, 1320, 522, 912, 292, 966, 281, 431, 283] payload_rows_per_dst=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] dup_rows_per_dst=[4820, 4306, 2442, 2934, 2464, 1972, 2338, 1868] combine_rows_per_src=[10904, 11154, 9464, 9904, 9116, 7128, 9344, 8146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75886 payload_rows_remote=66332 dup_rows=22418 dup_rows_remote=19910 dedup_ratio=0.228048 router_tokens_per_expert=[430, 771, 370, 423, 1471, 228, 550, 1073, 34, 743, 2272, 312, 686, 917, 655, 1918, 599, 1300, 693, 293, 1722, 211, 2072, 1382, 804, 572, 754, 1315, 15, 196, 141, 116, 238, 36, 323, 96, 904, 704, 993, 958, 2023, 1146, 544, 1173, 146, 1779, 13, 216, 2118, 728, 1494, 1, 57, 1373, 213, 1802, 1325, 306, 901, 410, 1140, 317, 320, 317] payload_rows_per_dst=[8430, 10946, 9668, 9554, 9852, 10268, 9110, 8058] dup_rows_per_dst=[2202, 4128, 2732, 2508, 2796, 3812, 2226, 2014] combine_rows_per_src=[8430, 10946, 9668, 9554, 9852, 10268, 9110, 8058] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76048 payload_rows_remote=66518 dup_rows=22256 dup_rows_remote=19742 dedup_ratio=0.226400 router_tokens_per_expert=[423, 654, 353, 422, 1818, 213, 612, 902, 30, 679, 2404, 326, 622, 849, 550, 1951, 571, 1289, 665, 218, 1673, 212, 1998, 1642, 752, 682, 898, 1295, 15, 172, 112, 137, 180, 31, 276, 34, 1009, 561, 997, 936, 1824, 1116, 450, 1152, 163, 1924, 8, 165, 2230, 741, 1689, 0, 40, 1330, 180, 1824, 1373, 527, 967, 235, 1065, 320, 356, 310] payload_rows_per_dst=[8488, 9526, 9712, 9918, 9530, 10042, 9330, 9502] dup_rows_per_dst=[2306, 3038, 2828, 2668, 2514, 3562, 2278, 3062] combine_rows_per_src=[8488, 9526, 9712, 9918, 9530, 10042, 9330, 9502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=338264 combine_payload_bytes=266215424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266553688 ingress_bytes=404627456 link_bytes=404627456 token_bytes=532769112 weight_movement_bytes=138412032 total_nvlink_bytes=671181144 weight_to_token_ratio=0.259797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522580542e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270131200 dispatch_meta_bytes=338256 combine_payload_bytes=270131200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270469456 ingress_bytes=339337216 link_bytes=339337216 token_bytes=540600656 weight_movement_bytes=69206016 total_nvlink_bytes=609806672 weight_to_token_ratio=0.128017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276898625e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270278656 dispatch_meta_bytes=347024 combine_payload_bytes=270278656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270625680 ingress_bytes=339484672 link_bytes=339484672 token_bytes=540904336 weight_movement_bytes=69206016 total_nvlink_bytes=610110352 weight_to_token_ratio=0.127945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277453490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271581184 dispatch_meta_bytes=344008 combine_payload_bytes=271581184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271925192 ingress_bytes=375390208 link_bytes=375390208 token_bytes=543506376 weight_movement_bytes=103809024 total_nvlink_bytes=647315400 weight_to_token_ratio=0.190999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412563131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263544832 dispatch_meta_bytes=331672 combine_payload_bytes=263544832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263876504 ingress_bytes=298147840 link_bytes=298147840 token_bytes=527421336 weight_movement_bytes=34603008 total_nvlink_bytes=562024344 weight_to_token_ratio=0.065608 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121906319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=262168576 dispatch_meta_bytes=331376 combine_payload_bytes=262168576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=262499952 ingress_bytes=400580608 link_bytes=400580608 token_bytes=524668528 weight_movement_bytes=138412032 total_nvlink_bytes=663080560 weight_to_token_ratio=0.263809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.507352579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263544832 dispatch_meta_bytes=331672 combine_payload_bytes=263544832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263876504 ingress_bytes=298147840 link_bytes=298147840 token_bytes=527421336 weight_movement_bytes=34603008 total_nvlink_bytes=562024344 weight_to_token_ratio=0.065608 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.121906319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270278656 dispatch_meta_bytes=347024 combine_payload_bytes=270278656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270625680 ingress_bytes=339484672 link_bytes=339484672 token_bytes=540904336 weight_movement_bytes=69206016 total_nvlink_bytes=610110352 weight_to_token_ratio=0.127945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277453490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271695872 dispatch_meta_bytes=344968 combine_payload_bytes=271695872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272040840 ingress_bytes=306298880 link_bytes=306298880 token_bytes=543736712 weight_movement_bytes=34603008 total_nvlink_bytes=578339720 weight_to_token_ratio=0.063639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152578026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=338264 combine_payload_bytes=266215424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266553688 ingress_bytes=404627456 link_bytes=404627456 token_bytes=532769112 weight_movement_bytes=138412032 total_nvlink_bytes=671181144 weight_to_token_ratio=0.259797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522580542e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272457728 dispatch_meta_bytes=345040 combine_payload_bytes=272457728 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=272802768 ingress_bytes=341663744 link_bytes=341663744 token_bytes=545260496 weight_movement_bytes=69206016 total_nvlink_bytes=614466512 weight_to_token_ratio=0.126923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285653162e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76032 payload_rows_remote=66304 dup_rows=22272 dup_rows_remote=19698 dedup_ratio=0.226562 router_tokens_per_expert=[592, 795, 369, 442, 1750, 179, 611, 1086, 18, 748, 2383, 346, 584, 928, 560, 1933, 536, 1308, 739, 234, 1606, 163, 2005, 1565, 779, 575, 801, 1228, 25, 159, 97, 148, 135, 28, 306, 57, 920, 650, 965, 985, 1868, 953, 455, 1013, 151, 2050, 6, 180, 2354, 681, 1659, 0, 20, 1292, 224, 1938, 1408, 559, 949, 167, 1027, 259, 292, 309] payload_rows_per_dst=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] dup_rows_per_dst=[2806, 4230, 2574, 2740, 2460, 3188, 2288, 1986] combine_rows_per_src=[9246, 10770, 9728, 9592, 9642, 9760, 9340, 7954] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77492 payload_rows_remote=67096 dup_rows=20812 dup_rows_remote=16562 dedup_ratio=0.211711 router_tokens_per_expert=[0, 1419, 394, 250, 0, 1154, 541, 178, 240, 513, 1226, 2179, 1181, 437, 0, 21, 960, 1637, 594, 505, 26, 0, 1619, 840, 1178, 1044, 1602, 2194, 18, 72, 658, 11, 325, 1426, 459, 364, 4744, 2197, 1304, 182, 984, 0, 139, 55, 1590, 1907, 114, 337, 344, 1, 404, 969, 0, 0, 195, 148, 2096, 645, 11, 139, 0, 2197, 4, 3181] payload_rows_per_dst=[9932, 9486, 9778, 10526, 6764, 10396, 11270, 9340] dup_rows_per_dst=[2728, 2108, 2584, 3028, 1356, 4250, 2340, 2418] combine_rows_per_src=[9932, 9486, 9778, 10526, 6764, 10396, 11270, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77296 payload_rows_remote=67270 dup_rows=21008 dup_rows_remote=18514 dedup_ratio=0.213704 router_tokens_per_expert=[0, 1635, 411, 268, 11, 979, 540, 176, 266, 461, 1181, 2142, 1116, 541, 0, 144, 1026, 1626, 327, 544, 36, 17, 1956, 728, 877, 1077, 1668, 1854, 21, 97, 698, 15, 403, 1237, 434, 458, 4692, 2285, 1125, 196, 738, 0, 196, 86, 1722, 1785, 116, 482, 382, 0, 652, 1135, 0, 3, 133, 151, 1743, 613, 23, 212, 0, 2594, 15, 3103] payload_rows_per_dst=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] dup_rows_per_dst=[3304, 2246, 2494, 2686, 2818, 2788, 3122, 1550] combine_rows_per_src=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75396 payload_rows_remote=62800 dup_rows=22908 dup_rows_remote=16192 dedup_ratio=0.233032 router_tokens_per_expert=[0, 1255, 440, 273, 1, 1096, 455, 212, 278, 500, 1116, 2260, 1043, 483, 0, 63, 1001, 1767, 646, 569, 41, 0, 1722, 837, 1095, 1193, 1760, 1997, 29, 63, 624, 20, 323, 1256, 504, 455, 4358, 2224, 1318, 185, 968, 0, 172, 165, 1640, 1999, 152, 321, 363, 2, 370, 921, 1, 0, 251, 121, 2028, 724, 33, 210, 0, 2183, 12, 3054] payload_rows_per_dst=[6522, 9386, 9762, 8924, 6836, 10714, 10656, 12596] dup_rows_per_dst=[942, 2100, 2748, 1814, 1902, 4568, 2118, 6716] combine_rows_per_src=[6522, 9386, 9762, 8924, 6836, 10714, 10656, 12596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75688 payload_rows_remote=65794 dup_rows=22616 dup_rows_remote=19548 dedup_ratio=0.230062 router_tokens_per_expert=[0, 1513, 436, 232, 3, 942, 444, 210, 310, 529, 1290, 2004, 1189, 524, 0, 90, 1027, 1811, 454, 600, 36, 4, 1705, 917, 1075, 1043, 1480, 1735, 25, 73, 649, 23, 326, 1121, 516, 471, 4708, 2535, 1236, 223, 906, 0, 170, 196, 1710, 1875, 99, 488, 337, 0, 580, 1044, 2, 0, 172, 127, 1895, 633, 13, 181, 0, 1973, 17, 3225] payload_rows_per_dst=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] dup_rows_per_dst=[3032, 3068, 2524, 2644, 4842, 3102, 2044, 1360] combine_rows_per_src=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76100 payload_rows_remote=64580 dup_rows=22204 dup_rows_remote=19188 dedup_ratio=0.225871 router_tokens_per_expert=[0, 1487, 432, 245, 1, 966, 650, 188, 300, 432, 1302, 1714, 1092, 542, 4, 284, 995, 1647, 476, 637, 31, 63, 2106, 1043, 987, 961, 1330, 1601, 21, 93, 840, 33, 349, 1258, 398, 441, 4955, 2167, 1264, 292, 797, 0, 167, 185, 1638, 1762, 106, 477, 375, 1, 576, 1011, 4, 1, 157, 188, 1682, 498, 39, 301, 0, 2420, 16, 3124] payload_rows_per_dst=[6802, 9128, 10556, 9304, 6524, 10386, 11520, 11880] dup_rows_per_dst=[1136, 2212, 3440, 2428, 1480, 4212, 3016, 4280] combine_rows_per_src=[6802, 9128, 10556, 9304, 6524, 10386, 11520, 11880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74988 payload_rows_remote=64422 dup_rows=23316 dup_rows_remote=18820 dedup_ratio=0.237183 router_tokens_per_expert=[487, 675, 437, 511, 1721, 176, 538, 1070, 49, 744, 2282, 335, 668, 967, 622, 1869, 709, 1282, 616, 365, 1663, 195, 1935, 1429, 778, 647, 838, 1211, 12, 301, 94, 121, 187, 16, 296, 105, 897, 672, 857, 973, 1830, 1061, 464, 1090, 150, 1916, 7, 220, 1962, 647, 1747, 2, 62, 1194, 242, 1778, 1405, 636, 965, 157, 1212, 289, 373, 393] payload_rows_per_dst=[10566, 11092, 9680, 9506, 9490, 7128, 9064, 8462] dup_rows_per_dst=[4496, 4564, 2838, 2422, 2386, 1932, 2280, 2398] combine_rows_per_src=[10566, 11092, 9680, 9506, 9490, 7128, 9064, 8462] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76644 payload_rows_remote=67082 dup_rows=21660 dup_rows_remote=19166 dedup_ratio=0.220337 router_tokens_per_expert=[0, 1318, 393, 258, 8, 1010, 585, 200, 280, 572, 1179, 2005, 1080, 571, 0, 302, 970, 1557, 358, 595, 42, 45, 1980, 890, 853, 998, 1420, 1742, 24, 313, 642, 36, 374, 1500, 366, 506, 4082, 1907, 1410, 330, 770, 2, 200, 132, 1820, 1883, 121, 586, 457, 10, 640, 1128, 1, 3, 185, 207, 1876, 620, 42, 330, 0, 2333, 33, 3072] payload_rows_per_dst=[10784, 9558, 10164, 9562, 9590, 7848, 10288, 8850] dup_rows_per_dst=[2904, 2420, 2710, 2494, 3196, 3180, 3138, 1618] combine_rows_per_src=[10784, 9558, 10164, 9562, 9590, 7848, 10288, 8850] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75688 payload_rows_remote=65794 dup_rows=22616 dup_rows_remote=19548 dedup_ratio=0.230062 router_tokens_per_expert=[0, 1513, 436, 232, 3, 942, 444, 210, 310, 529, 1290, 2004, 1189, 524, 0, 90, 1027, 1811, 454, 600, 36, 4, 1705, 917, 1075, 1043, 1480, 1735, 25, 73, 649, 23, 326, 1121, 516, 471, 4708, 2535, 1236, 223, 906, 0, 170, 196, 1710, 1875, 99, 488, 337, 0, 580, 1044, 2, 0, 172, 127, 1895, 633, 13, 181, 0, 1973, 17, 3225] payload_rows_per_dst=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] dup_rows_per_dst=[3032, 3068, 2524, 2644, 4842, 3102, 2044, 1360] combine_rows_per_src=[10978, 9894, 9494, 9562, 11212, 7786, 8698, 8064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271581184 dispatch_meta_bytes=344008 combine_payload_bytes=271581184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271925192 ingress_bytes=375390208 link_bytes=375390208 token_bytes=543506376 weight_movement_bytes=103809024 total_nvlink_bytes=647315400 weight_to_token_ratio=0.190999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412563131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77406 payload_rows_remote=67906 dup_rows=20898 dup_rows_remote=17996 dedup_ratio=0.212585 router_tokens_per_expert=[1, 1569, 411, 232, 1, 1051, 505, 195, 260, 488, 1302, 1922, 1109, 552, 0, 133, 960, 1823, 346, 618, 27, 30, 1900, 857, 1100, 958, 1380, 1830, 33, 97, 638, 19, 353, 1172, 382, 499, 4446, 2573, 1383, 226, 734, 0, 169, 92, 1623, 1881, 113, 454, 375, 1, 586, 1097, 3, 0, 141, 169, 1836, 571, 27, 231, 0, 2645, 12, 3011] payload_rows_per_dst=[10874, 9270, 10340, 9654, 9500, 7914, 10818, 9036] dup_rows_per_dst=[3078, 2262, 2782, 2456, 2902, 2992, 2818, 1608] combine_rows_per_src=[10874, 9270, 10340, 9654, 9500, 7914, 10818, 9036] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274825216 dispatch_meta_bytes=334632 combine_payload_bytes=274825216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275159848 ingress_bytes=309428224 link_bytes=309428224 token_bytes=549985064 weight_movement_bytes=34603008 total_nvlink_bytes=584588072 weight_to_token_ratio=0.062916 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164353496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274825216 dispatch_meta_bytes=334632 combine_payload_bytes=274825216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275159848 ingress_bytes=309428224 link_bytes=309428224 token_bytes=549985064 weight_movement_bytes=34603008 total_nvlink_bytes=584588072 weight_to_token_ratio=0.062916 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164353496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275537920 dispatch_meta_bytes=343136 combine_payload_bytes=275537920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275881056 ingress_bytes=344743936 link_bytes=344743936 token_bytes=551418976 weight_movement_bytes=69206016 total_nvlink_bytes=620624992 weight_to_token_ratio=0.125505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297243677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74988 payload_rows_remote=64422 dup_rows=23316 dup_rows_remote=18820 dedup_ratio=0.237183 router_tokens_per_expert=[487, 675, 437, 511, 1721, 176, 538, 1070, 49, 744, 2282, 335, 668, 967, 622, 1869, 709, 1282, 616, 365, 1663, 195, 1935, 1429, 778, 647, 838, 1211, 12, 301, 94, 121, 187, 16, 296, 105, 897, 672, 857, 973, 1830, 1061, 464, 1090, 150, 1916, 7, 220, 1962, 647, 1747, 2, 62, 1194, 242, 1778, 1405, 636, 965, 157, 1212, 289, 373, 393] payload_rows_per_dst=[10566, 11092, 9680, 9506, 9490, 7128, 9064, 8462] dup_rows_per_dst=[4496, 4564, 2838, 2422, 2386, 1932, 2280, 2398] combine_rows_per_src=[10566, 11092, 9680, 9506, 9490, 7128, 9064, 8462] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=257228800 dispatch_meta_bytes=315968 combine_payload_bytes=257228800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=257544768 ingress_bytes=326434816 link_bytes=326434816 token_bytes=514773568 weight_movement_bytes=69206016 total_nvlink_bytes=583979584 weight_to_token_ratio=0.134440 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228347932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=335072 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264854752 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529374432 weight_movement_bytes=34603008 total_nvlink_bytes=563977440 weight_to_token_ratio=0.065366 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263872512 dispatch_meta_bytes=332968 combine_payload_bytes=263872512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264205480 ingress_bytes=298475520 link_bytes=298475520 token_bytes=528077992 weight_movement_bytes=34603008 total_nvlink_bytes=562681000 weight_to_token_ratio=0.065526 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123139353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274767872 dispatch_meta_bytes=344992 combine_payload_bytes=274767872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275112864 ingress_bytes=343973888 link_bytes=343973888 token_bytes=549880736 weight_movement_bytes=69206016 total_nvlink_bytes=619086752 weight_to_token_ratio=0.125856 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294346048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269492224 dispatch_meta_bytes=341368 combine_payload_bytes=269492224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269833592 ingress_bytes=304095232 link_bytes=304095232 token_bytes=539325816 weight_movement_bytes=34603008 total_nvlink_bytes=573928824 weight_to_token_ratio=0.064160 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144285876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274767872 dispatch_meta_bytes=344992 combine_payload_bytes=274767872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275112864 ingress_bytes=343973888 link_bytes=343973888 token_bytes=549880736 weight_movement_bytes=69206016 total_nvlink_bytes=619086752 weight_to_token_ratio=0.125856 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294346048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=335072 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264854752 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529374432 weight_movement_bytes=34603008 total_nvlink_bytes=563977440 weight_to_token_ratio=0.065366 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278142976 dispatch_meta_bytes=343608 combine_payload_bytes=278142976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=278486584 ingress_bytes=416555008 link_bytes=416555008 token_bytes=556629560 weight_movement_bytes=138412032 total_nvlink_bytes=695041592 weight_to_token_ratio=0.248661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567462960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77296 payload_rows_remote=67270 dup_rows=21008 dup_rows_remote=18514 dedup_ratio=0.213704 router_tokens_per_expert=[0, 1635, 411, 268, 11, 979, 540, 176, 266, 461, 1181, 2142, 1116, 541, 0, 144, 1026, 1626, 327, 544, 36, 17, 1956, 728, 877, 1077, 1668, 1854, 21, 97, 698, 15, 403, 1237, 434, 458, 4692, 2285, 1125, 196, 738, 0, 196, 86, 1722, 1785, 116, 482, 382, 0, 652, 1135, 0, 3, 133, 151, 1743, 613, 23, 212, 0, 2594, 15, 3103] payload_rows_per_dst=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] dup_rows_per_dst=[3304, 2246, 2494, 2686, 2818, 2788, 3122, 1550] combine_rows_per_src=[10942, 9456, 10026, 9928, 9458, 7462, 11174, 8850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78906 payload_rows_remote=72068 dup_rows=19398 dup_rows_remote=18240 dedup_ratio=0.197327 router_tokens_per_expert=[0, 73, 1095, 1, 430, 1117, 808, 657, 4101, 2106, 221, 145, 617, 14, 5659, 239, 2700, 6, 1938, 360, 676, 250, 0, 212, 760, 1877, 472, 307, 242, 1656, 922, 254, 27, 157, 91, 64, 555, 165, 302, 198, 23, 3320, 66, 452, 96, 41, 0, 4890, 0, 114, 90, 1089, 135, 1769, 0, 161, 36, 0, 2181, 10, 1701, 1138, 83, 283] payload_rows_per_dst=[6634, 11180, 9366, 7886, 12822, 6838, 12862, 11318] dup_rows_per_dst=[1728, 3706, 2918, 1340, 1614, 1158, 3634, 3300] combine_rows_per_src=[6634, 11180, 9366, 7886, 12822, 6838, 12862, 11318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78982 payload_rows_remote=69228 dup_rows=19322 dup_rows_remote=16134 dedup_ratio=0.196554 router_tokens_per_expert=[0, 93, 1157, 0, 439, 1103, 936, 690, 3495, 1899, 187, 142, 744, 12, 5123, 170, 2350, 11, 2432, 407, 698, 242, 0, 331, 749, 1907, 456, 372, 243, 1964, 865, 160, 18, 198, 82, 42, 697, 192, 541, 206, 21, 2956, 54, 384, 99, 22, 0, 5190, 0, 144, 83, 1266, 161, 1759, 0, 72, 8, 1, 2304, 6, 1933, 904, 112, 320] payload_rows_per_dst=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] dup_rows_per_dst=[4072, 508, 3188, 2582, 1982, 4500, 632, 1858] combine_rows_per_src=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78626 payload_rows_remote=66928 dup_rows=19678 dup_rows_remote=15890 dedup_ratio=0.200175 router_tokens_per_expert=[0, 43, 1146, 1, 527, 1194, 1022, 743, 3665, 2012, 250, 170, 604, 6, 5431, 224, 2891, 8, 2203, 328, 552, 197, 0, 270, 606, 2104, 425, 342, 249, 1628, 941, 272, 51, 167, 96, 75, 606, 197, 236, 250, 17, 2943, 83, 395, 114, 46, 0, 4654, 0, 105, 96, 1269, 129, 1757, 0, 173, 36, 0, 2327, 125, 1805, 1041, 57, 248] payload_rows_per_dst=[7346, 10534, 7490, 9978, 12374, 6536, 12670, 11698] dup_rows_per_dst=[2006, 3328, 1580, 2214, 1844, 1222, 3696, 3788] combine_rows_per_src=[7346, 10534, 7490, 9978, 12374, 6536, 12670, 11698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78650 payload_rows_remote=65476 dup_rows=19654 dup_rows_remote=15894 dedup_ratio=0.199931 router_tokens_per_expert=[0, 66, 1300, 0, 382, 969, 956, 610, 3404, 2268, 208, 158, 619, 25, 5179, 282, 2661, 8, 2306, 359, 705, 287, 0, 268, 819, 1844, 488, 344, 282, 1683, 1164, 116, 27, 194, 65, 48, 658, 196, 542, 253, 21, 3190, 89, 453, 101, 55, 0, 5137, 0, 95, 63, 1057, 180, 1887, 0, 48, 40, 1, 2136, 9, 1724, 847, 92, 184] payload_rows_per_dst=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] dup_rows_per_dst=[1690, 3338, 3300, 1460, 2104, 1134, 3760, 2868] combine_rows_per_src=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79240 payload_rows_remote=68006 dup_rows=19064 dup_rows_remote=16206 dedup_ratio=0.193929 router_tokens_per_expert=[28, 67, 1237, 0, 369, 614, 1022, 600, 3041, 2205, 206, 145, 751, 17, 4902, 200, 2436, 14, 2540, 378, 658, 290, 0, 353, 848, 1999, 552, 361, 300, 1825, 993, 168, 41, 338, 69, 65, 757, 178, 583, 209, 40, 2943, 79, 469, 123, 60, 0, 5243, 0, 142, 75, 1146, 218, 1822, 0, 78, 28, 5, 2294, 173, 1691, 872, 79, 213] payload_rows_per_dst=[11100, 6546, 9864, 11234, 12038, 11356, 8138, 8964] dup_rows_per_dst=[2856, 502, 3474, 2858, 2246, 3834, 1548, 1746] combine_rows_per_src=[11100, 6546, 9864, 11234, 12038, 11356, 8138, 8964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73104 payload_rows_remote=62050 dup_rows=25200 dup_rows_remote=21908 dedup_ratio=0.256348 router_tokens_per_expert=[0, 1588, 436, 307, 5, 1179, 514, 195, 282, 412, 1228, 1997, 1220, 604, 2, 297, 1009, 1609, 341, 574, 26, 17, 1813, 875, 1036, 928, 1479, 1536, 44, 103, 705, 28, 332, 1340, 505, 575, 4174, 2133, 1448, 243, 892, 0, 148, 92, 1753, 1868, 117, 483, 382, 4, 725, 1103, 2, 1, 229, 196, 1748, 527, 27, 321, 0, 2425, 21, 2949] payload_rows_per_dst=[11054, 11172, 7904, 10050, 12236, 7650, 4436, 8602] dup_rows_per_dst=[3292, 4538, 998, 3244, 7688, 3056, 848, 1536] combine_rows_per_src=[11054, 11172, 7904, 10050, 12236, 7650, 4436, 8602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78640 payload_rows_remote=66400 dup_rows=19664 dup_rows_remote=17794 dedup_ratio=0.200033 router_tokens_per_expert=[1, 71, 1184, 0, 358, 946, 991, 648, 3357, 2043, 195, 172, 707, 14, 5349, 285, 2592, 17, 2193, 342, 693, 276, 0, 268, 860, 1998, 456, 361, 269, 1680, 967, 144, 41, 183, 78, 67, 632, 193, 305, 207, 21, 3030, 85, 456, 85, 56, 0, 5430, 0, 128, 78, 1196, 143, 1908, 0, 64, 28, 0, 2275, 46, 1769, 932, 69, 210] payload_rows_per_dst=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] dup_rows_per_dst=[1910, 2812, 3268, 2332, 1870, 1120, 4286, 2066] combine_rows_per_src=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77346 payload_rows_remote=67594 dup_rows=20958 dup_rows_remote=18508 dedup_ratio=0.213196 router_tokens_per_expert=[0, 64, 1198, 1, 451, 1158, 901, 714, 3896, 2079, 194, 147, 629, 12, 5240, 264, 2517, 10, 2091, 337, 684, 285, 0, 236, 884, 1929, 580, 349, 265, 1868, 769, 154, 20, 88, 72, 82, 732, 135, 375, 235, 34, 3104, 106, 513, 86, 37, 0, 5116, 0, 96, 71, 1233, 103, 1797, 0, 81, 42, 0, 2054, 12, 1688, 1011, 76, 247] payload_rows_per_dst=[12124, 9752, 9366, 11002, 7446, 12930, 6128, 8598] dup_rows_per_dst=[4642, 2450, 2954, 2594, 960, 5062, 634, 1662] combine_rows_per_src=[12124, 9752, 9366, 11002, 7446, 12930, 6128, 8598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295190528 dispatch_meta_bytes=361232 combine_payload_bytes=295190528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=295551760 ingress_bytes=398999552 link_bytes=398999552 token_bytes=590742288 weight_movement_bytes=103809024 total_nvlink_bytes=694551312 weight_to_token_ratio=0.175726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.501403192e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275537920 dispatch_meta_bytes=343136 combine_payload_bytes=275537920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275881056 ingress_bytes=344743936 link_bytes=344743936 token_bytes=551418976 weight_movement_bytes=69206016 total_nvlink_bytes=620624992 weight_to_token_ratio=0.125505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297243677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=283557888 dispatch_meta_bytes=341448 combine_payload_bytes=283557888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283899336 ingress_bytes=352763904 link_bytes=352763904 token_bytes=567457224 weight_movement_bytes=69206016 total_nvlink_bytes=636663240 weight_to_token_ratio=0.121958 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327422171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78650 payload_rows_remote=65476 dup_rows=19654 dup_rows_remote=15894 dedup_ratio=0.199931 router_tokens_per_expert=[0, 66, 1300, 0, 382, 969, 956, 610, 3404, 2268, 208, 158, 619, 25, 5179, 282, 2661, 8, 2306, 359, 705, 287, 0, 268, 819, 1844, 488, 344, 282, 1683, 1164, 116, 27, 194, 65, 48, 658, 196, 542, 253, 21, 3190, 89, 453, 101, 55, 0, 5137, 0, 95, 63, 1057, 180, 1887, 0, 48, 40, 1, 2136, 9, 1724, 847, 92, 184] payload_rows_per_dst=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] dup_rows_per_dst=[1690, 3338, 3300, 1460, 2104, 1134, 3760, 2868] combine_rows_per_src=[6876, 10590, 9888, 8332, 12220, 6684, 13174, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78640 payload_rows_remote=66400 dup_rows=19664 dup_rows_remote=17794 dedup_ratio=0.200033 router_tokens_per_expert=[1, 71, 1184, 0, 358, 946, 991, 648, 3357, 2043, 195, 172, 707, 14, 5349, 285, 2592, 17, 2193, 342, 693, 276, 0, 268, 860, 1998, 456, 361, 269, 1680, 967, 144, 41, 183, 78, 67, 632, 193, 305, 207, 21, 3030, 85, 456, 85, 56, 0, 5430, 0, 128, 78, 1196, 143, 1908, 0, 64, 28, 0, 2275, 46, 1769, 932, 69, 210] payload_rows_per_dst=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] dup_rows_per_dst=[1910, 2812, 3268, 2332, 1870, 1120, 4286, 2066] combine_rows_per_src=[7498, 9724, 9494, 10210, 12240, 6346, 13608, 9520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=336848 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278889424 ingress_bytes=347758592 link_bytes=347758592 token_bytes=557442000 weight_movement_bytes=69206016 total_nvlink_bytes=626648016 weight_to_token_ratio=0.124149 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308587585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274137088 dispatch_meta_bytes=331272 combine_payload_bytes=274137088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274468360 ingress_bytes=308740096 link_bytes=308740096 token_bytes=548605448 weight_movement_bytes=34603008 total_nvlink_bytes=583208456 weight_to_token_ratio=0.063074 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161764126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=344408 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277209432 ingress_bytes=380674048 link_bytes=380674048 token_bytes=554074456 weight_movement_bytes=103809024 total_nvlink_bytes=657883480 weight_to_token_ratio=0.187356 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432445796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=268189696 dispatch_meta_bytes=325480 combine_payload_bytes=268189696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268515176 ingress_bytes=302792704 link_bytes=302792704 token_bytes=536704872 weight_movement_bytes=34603008 total_nvlink_bytes=571307880 weight_to_token_ratio=0.064473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139384568e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263872512 dispatch_meta_bytes=332968 combine_payload_bytes=263872512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264205480 ingress_bytes=298475520 link_bytes=298475520 token_bytes=528077992 weight_movement_bytes=34603008 total_nvlink_bytes=562681000 weight_to_token_ratio=0.065526 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123139353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271974400 dispatch_meta_bytes=336776 combine_payload_bytes=271974400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272311176 ingress_bytes=306577408 link_bytes=306577408 token_bytes=544285576 weight_movement_bytes=34603008 total_nvlink_bytes=578888584 weight_to_token_ratio=0.063575 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153626105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278552576 dispatch_meta_bytes=336848 combine_payload_bytes=278552576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278889424 ingress_bytes=347758592 link_bytes=347758592 token_bytes=557442000 weight_movement_bytes=69206016 total_nvlink_bytes=626648016 weight_to_token_ratio=0.124149 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.308587585e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=254156800 dispatch_meta_bytes=335832 combine_payload_bytes=254156800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254492632 ingress_bytes=288759808 link_bytes=288759808 token_bytes=508649432 weight_movement_bytes=34603008 total_nvlink_bytes=543252440 weight_to_token_ratio=0.068029 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.086579910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=344408 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277209432 ingress_bytes=380674048 link_bytes=380674048 token_bytes=554074456 weight_movement_bytes=103809024 total_nvlink_bytes=657883480 weight_to_token_ratio=0.187356 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.432445796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82264 payload_rows_remote=70004 dup_rows=16040 dup_rows_remote=14494 dedup_ratio=0.163167 router_tokens_per_expert=[0, 827, 1211, 1047, 801, 2833, 0, 1802, 0, 1431, 2029, 66, 153, 135, 246, 9, 1450, 304, 0, 2, 22, 1106, 750, 1, 928, 0, 3579, 99, 4862, 103, 193, 0, 150, 0, 0, 246, 280, 0, 4, 742, 2, 648, 0, 1324, 160, 0, 4769, 0, 0, 1317, 127, 236, 4, 6089, 4300, 569, 311, 228, 5, 855, 642, 40, 4, 111] payload_rows_per_dst=[8070, 7008, 9994, 8648, 13910, 12260, 10548, 11826] dup_rows_per_dst=[3306, 1130, 2942, 1156, 1112, 1546, 2558, 2290] combine_rows_per_src=[8070, 7008, 9994, 8648, 13910, 12260, 10548, 11826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78982 payload_rows_remote=69228 dup_rows=19322 dup_rows_remote=16134 dedup_ratio=0.196554 router_tokens_per_expert=[0, 93, 1157, 0, 439, 1103, 936, 690, 3495, 1899, 187, 142, 744, 12, 5123, 170, 2350, 11, 2432, 407, 698, 242, 0, 331, 749, 1907, 456, 372, 243, 1964, 865, 160, 18, 198, 82, 42, 697, 192, 541, 206, 21, 2956, 54, 384, 99, 22, 0, 5190, 0, 144, 83, 1266, 161, 1759, 0, 72, 8, 1, 2304, 6, 1933, 904, 112, 320] payload_rows_per_dst=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] dup_rows_per_dst=[4072, 508, 3188, 2582, 1982, 4500, 632, 1858] combine_rows_per_src=[11754, 5800, 9754, 10850, 12216, 12952, 6338, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81336 payload_rows_remote=68336 dup_rows=16968 dup_rows_remote=16188 dedup_ratio=0.172607 router_tokens_per_expert=[0, 895, 1219, 1014, 943, 2927, 4, 1834, 5, 1475, 1972, 63, 132, 164, 246, 10, 1697, 343, 0, 1, 44, 1026, 742, 3, 1090, 0, 3874, 96, 4718, 107, 205, 0, 144, 0, 0, 183, 188, 0, 0, 686, 28, 704, 0, 1205, 143, 0, 4477, 0, 0, 1369, 127, 169, 0, 5689, 4147, 662, 341, 189, 4, 924, 707, 64, 42, 111] payload_rows_per_dst=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] dup_rows_per_dst=[3524, 2084, 3322, 1464, 780, 1576, 1834, 2384] combine_rows_per_src=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68686 payload_rows_remote=56620 dup_rows=29618 dup_rows_remote=27210 dedup_ratio=0.301290 router_tokens_per_expert=[0, 827, 1162, 1210, 892, 2541, 0, 1851, 1, 1230, 1879, 43, 96, 45, 254, 2, 1496, 337, 0, 1, 7, 1190, 654, 2, 1181, 0, 3923, 108, 5042, 114, 168, 0, 167, 0, 0, 233, 319, 0, 3, 764, 22, 662, 0, 1338, 146, 0, 4540, 0, 0, 1233, 155, 272, 0, 5950, 4301, 596, 331, 152, 5, 872, 677, 36, 7, 115] payload_rows_per_dst=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] dup_rows_per_dst=[3028, 732, 2856, 152, 384, 1196, 18862, 2408] combine_rows_per_src=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80698 payload_rows_remote=70928 dup_rows=17606 dup_rows_remote=14684 dedup_ratio=0.179097 router_tokens_per_expert=[0, 929, 1282, 934, 905, 2609, 0, 1737, 3, 1415, 2016, 90, 203, 57, 264, 12, 1641, 361, 0, 3, 2, 1046, 681, 3, 1073, 0, 4129, 101, 5099, 101, 214, 0, 126, 0, 0, 282, 201, 0, 5, 637, 4, 714, 0, 1235, 182, 0, 4467, 0, 0, 1314, 133, 221, 17, 5506, 4087, 618, 315, 272, 3, 1031, 689, 56, 13, 114] payload_rows_per_dst=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] dup_rows_per_dst=[3488, 4278, 2922, 1528, 766, 1632, 118, 2874] combine_rows_per_src=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=286736384 dispatch_meta_bytes=337992 combine_payload_bytes=286736384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287074376 ingress_bytes=355942400 link_bytes=355942400 token_bytes=573810760 weight_movement_bytes=69206016 total_nvlink_bytes=643016776 weight_to_token_ratio=0.120608 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.339382595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79998 payload_rows_remote=69964 dup_rows=18306 dup_rows_remote=16540 dedup_ratio=0.186218 router_tokens_per_expert=[0, 1165, 1261, 1054, 946, 2626, 1, 1770, 9, 1224, 2234, 54, 132, 145, 261, 26, 1504, 345, 0, 8, 104, 1059, 744, 10, 1201, 0, 4292, 96, 4346, 91, 220, 0, 135, 0, 0, 227, 205, 0, 4, 729, 41, 680, 0, 1228, 135, 0, 4555, 0, 0, 1349, 133, 206, 55, 5309, 3776, 653, 368, 213, 5, 1118, 809, 76, 98, 117] payload_rows_per_dst=[8420, 11550, 9710, 10034, 12458, 11670, 4686, 11470] dup_rows_per_dst=[3974, 4172, 3090, 1766, 760, 1608, 106, 2830] combine_rows_per_src=[8420, 11550, 9710, 10034, 12458, 11670, 4686, 11470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=63328 dup_rows=21472 dup_rows_remote=13206 dedup_ratio=0.218424 router_tokens_per_expert=[0, 978, 1209, 1344, 902, 2594, 0, 1853, 3, 1283, 1772, 60, 152, 189, 254, 16, 1704, 336, 0, 0, 72, 895, 679, 8, 1027, 0, 4389, 93, 4684, 93, 241, 0, 138, 0, 0, 243, 163, 0, 1, 707, 65, 816, 0, 1178, 147, 0, 4607, 0, 0, 1457, 111, 149, 46, 5152, 4008, 671, 320, 204, 5, 1143, 750, 62, 88, 91] payload_rows_per_dst=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] dup_rows_per_dst=[3756, 978, 2966, 1616, 690, 436, 8266, 2764] combine_rows_per_src=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68686 payload_rows_remote=56620 dup_rows=29618 dup_rows_remote=27210 dedup_ratio=0.301290 router_tokens_per_expert=[0, 827, 1162, 1210, 892, 2541, 0, 1851, 1, 1230, 1879, 43, 96, 45, 254, 2, 1496, 337, 0, 1, 7, 1190, 654, 2, 1181, 0, 3923, 108, 5042, 114, 168, 0, 167, 0, 0, 233, 319, 0, 3, 764, 22, 662, 0, 1338, 146, 0, 4540, 0, 0, 1233, 155, 272, 0, 5950, 4301, 596, 331, 152, 5, 872, 677, 36, 7, 115] payload_rows_per_dst=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] dup_rows_per_dst=[3028, 732, 2856, 152, 384, 1196, 18862, 2408] combine_rows_per_src=[7992, 6368, 9600, 3854, 2588, 10986, 15232, 12066] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77914 payload_rows_remote=65708 dup_rows=20390 dup_rows_remote=15806 dedup_ratio=0.207418 router_tokens_per_expert=[0, 908, 1273, 914, 852, 2569, 0, 1810, 0, 1484, 2000, 61, 136, 151, 239, 12, 1673, 348, 0, 3, 4, 1080, 728, 5, 1060, 0, 4036, 88, 4939, 84, 194, 0, 163, 0, 0, 257, 181, 0, 1, 733, 4, 693, 0, 1307, 145, 0, 4620, 0, 0, 1482, 124, 148, 0, 5453, 4312, 601, 276, 216, 6, 958, 647, 72, 6, 96] payload_rows_per_dst=[8112, 12206, 9810, 12682, 12874, 11940, 4608, 5682] dup_rows_per_dst=[3402, 4584, 3010, 6510, 702, 1598, 102, 482] combine_rows_per_src=[8112, 12206, 9810, 12682, 12874, 11940, 4608, 5682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76880 payload_rows_remote=64978 dup_rows=21424 dup_rows_remote=17258 dedup_ratio=0.217936 router_tokens_per_expert=[4, 73, 1260, 1, 465, 1043, 936, 693, 3559, 1831, 238, 118, 693, 13, 4982, 274, 2347, 22, 2344, 342, 573, 264, 0, 308, 841, 2000, 455, 360, 311, 1614, 897, 238, 49, 214, 91, 59, 716, 192, 598, 245, 31, 3065, 81, 430, 112, 28, 1, 5186, 0, 130, 94, 1211, 169, 1854, 0, 140, 38, 0, 2274, 57, 1686, 959, 69, 274] payload_rows_per_dst=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] dup_rows_per_dst=[4166, 4674, 3088, 2536, 392, 4032, 742, 1794] combine_rows_per_src=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=283557888 dispatch_meta_bytes=341448 combine_payload_bytes=283557888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=283899336 ingress_bytes=352763904 link_bytes=352763904 token_bytes=567457224 weight_movement_bytes=69206016 total_nvlink_bytes=636663240 weight_to_token_ratio=0.121958 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327422171e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290521088 dispatch_meta_bytes=342448 combine_payload_bytes=290521088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290863536 ingress_bytes=325124096 link_bytes=325124096 token_bytes=581384624 weight_movement_bytes=34603008 total_nvlink_bytes=615987632 weight_to_token_ratio=0.059518 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223415799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76832 payload_rows_remote=63328 dup_rows=21472 dup_rows_remote=13206 dedup_ratio=0.218424 router_tokens_per_expert=[0, 978, 1209, 1344, 902, 2594, 0, 1853, 3, 1283, 1772, 60, 152, 189, 254, 16, 1704, 336, 0, 0, 72, 895, 679, 8, 1027, 0, 4389, 93, 4684, 93, 241, 0, 138, 0, 0, 243, 163, 0, 1, 707, 65, 816, 0, 1178, 147, 0, 4607, 0, 0, 1457, 111, 149, 46, 5152, 4008, 671, 320, 204, 5, 1143, 750, 62, 88, 91] payload_rows_per_dst=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] dup_rows_per_dst=[3756, 978, 2966, 1616, 690, 436, 8266, 2764] combine_rows_per_src=[8816, 6480, 9610, 9816, 11712, 4964, 13504, 11930] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81336 payload_rows_remote=68336 dup_rows=16968 dup_rows_remote=16188 dedup_ratio=0.172607 router_tokens_per_expert=[0, 895, 1219, 1014, 943, 2927, 4, 1834, 5, 1475, 1972, 63, 132, 164, 246, 10, 1697, 343, 0, 1, 44, 1026, 742, 3, 1090, 0, 3874, 96, 4718, 107, 205, 0, 144, 0, 0, 183, 188, 0, 0, 686, 28, 704, 0, 1205, 143, 0, 4477, 0, 0, 1369, 127, 169, 0, 5689, 4147, 662, 341, 189, 4, 924, 707, 64, 42, 111] payload_rows_per_dst=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] dup_rows_per_dst=[3524, 2084, 3322, 1464, 780, 1576, 1834, 2384] combine_rows_per_src=[8294, 8446, 10244, 9280, 13000, 11538, 8718, 11816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279904256 dispatch_meta_bytes=338096 combine_payload_bytes=279904256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280242352 ingress_bytes=314507264 link_bytes=314507264 token_bytes=560146608 weight_movement_bytes=34603008 total_nvlink_bytes=594749616 weight_to_token_ratio=0.061775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183465515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=231915520 dispatch_meta_bytes=335320 combine_payload_bytes=231915520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=232250840 ingress_bytes=266518528 link_bytes=266518528 token_bytes=464166360 weight_movement_bytes=34603008 total_nvlink_bytes=498769368 weight_to_token_ratio=0.074549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.002887764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259391488 dispatch_meta_bytes=306136 combine_payload_bytes=259391488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259697624 ingress_bytes=363200512 link_bytes=363200512 token_bytes=519089112 weight_movement_bytes=103809024 total_nvlink_bytes=622898136 weight_to_token_ratio=0.199983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366694287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=286572544 dispatch_meta_bytes=346016 combine_payload_bytes=286572544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=286918560 ingress_bytes=390381568 link_bytes=390381568 token_bytes=573491104 weight_movement_bytes=103809024 total_nvlink_bytes=677300128 weight_to_token_ratio=0.181012 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468974412e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=231915520 dispatch_meta_bytes=335320 combine_payload_bytes=231915520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=232250840 ingress_bytes=266518528 link_bytes=266518528 token_bytes=464166360 weight_movement_bytes=34603008 total_nvlink_bytes=498769368 weight_to_token_ratio=0.074549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.002887764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=326056 combine_payload_bytes=269139968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269466024 ingress_bytes=303742976 link_bytes=303742976 token_bytes=538605992 weight_movement_bytes=34603008 total_nvlink_bytes=573209000 weight_to_token_ratio=0.064245 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142960365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=328944 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478832 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532628720 weight_movement_bytes=34603008 total_nvlink_bytes=567231728 weight_to_token_ratio=0.064966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73104 payload_rows_remote=62050 dup_rows=25200 dup_rows_remote=21908 dedup_ratio=0.256348 router_tokens_per_expert=[0, 1588, 436, 307, 5, 1179, 514, 195, 282, 412, 1228, 1997, 1220, 604, 2, 297, 1009, 1609, 341, 574, 26, 17, 1813, 875, 1036, 928, 1479, 1536, 44, 103, 705, 28, 332, 1340, 505, 575, 4174, 2133, 1448, 243, 892, 0, 148, 92, 1753, 1868, 117, 483, 382, 4, 725, 1103, 2, 1, 229, 196, 1748, 527, 27, 321, 0, 2425, 21, 2949] payload_rows_per_dst=[11054, 11172, 7904, 10050, 12236, 7650, 4436, 8602] dup_rows_per_dst=[3292, 4538, 998, 3244, 7688, 3056, 848, 1536] combine_rows_per_src=[11054, 11172, 7904, 10050, 12236, 7650, 4436, 8602] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259391488 dispatch_meta_bytes=306136 combine_payload_bytes=259391488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259697624 ingress_bytes=363200512 link_bytes=363200512 token_bytes=519089112 weight_movement_bytes=103809024 total_nvlink_bytes=622898136 weight_to_token_ratio=0.199983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366694287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=279904256 dispatch_meta_bytes=338096 combine_payload_bytes=279904256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280242352 ingress_bytes=314507264 link_bytes=314507264 token_bytes=560146608 weight_movement_bytes=34603008 total_nvlink_bytes=594749616 weight_to_token_ratio=0.061775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183465515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77456 payload_rows_remote=65320 dup_rows=20848 dup_rows_remote=15148 dedup_ratio=0.212077 router_tokens_per_expert=[262, 1284, 353, 1155, 60, 230, 1419, 556, 0, 0, 29, 576, 2870, 0, 0, 858, 304, 663, 6586, 1, 6, 844, 1535, 0, 0, 4598, 1977, 0, 0, 0, 2479, 0, 3465, 404, 60, 0, 70, 0, 376, 153, 0, 0, 0, 342, 9, 122, 3642, 1588, 813, 706, 1046, 1169, 0, 47, 0, 980, 800, 220, 96, 11, 49, 267, 271, 3801] payload_rows_per_dst=[8712, 7748, 14360, 9398, 8742, 12136, 7366, 8994] dup_rows_per_dst=[1926, 918, 4358, 2906, 848, 5700, 2156, 2036] combine_rows_per_src=[8712, 7748, 14360, 9398, 8742, 12136, 7366, 8994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78768 payload_rows_remote=67244 dup_rows=19536 dup_rows_remote=16672 dedup_ratio=0.198730 router_tokens_per_expert=[185, 1535, 430, 987, 44, 364, 1260, 411, 0, 0, 39, 582, 2695, 0, 0, 754, 362, 680, 6569, 3, 15, 927, 1431, 0, 0, 4569, 1887, 0, 0, 0, 2130, 0, 4034, 346, 46, 0, 97, 0, 317, 192, 1, 0, 0, 348, 1, 181, 3792, 1440, 872, 808, 1058, 935, 0, 28, 0, 1049, 932, 209, 82, 3, 39, 315, 379, 3789] payload_rows_per_dst=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] dup_rows_per_dst=[1752, 3042, 2864, 5032, 618, 1994, 2078, 2156] combine_rows_per_src=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80698 payload_rows_remote=70928 dup_rows=17606 dup_rows_remote=14684 dedup_ratio=0.179097 router_tokens_per_expert=[0, 929, 1282, 934, 905, 2609, 0, 1737, 3, 1415, 2016, 90, 203, 57, 264, 12, 1641, 361, 0, 3, 2, 1046, 681, 3, 1073, 0, 4129, 101, 5099, 101, 214, 0, 126, 0, 0, 282, 201, 0, 5, 637, 4, 714, 0, 1235, 182, 0, 4467, 0, 0, 1314, 133, 221, 17, 5506, 4087, 618, 315, 272, 3, 1031, 689, 56, 13, 114] payload_rows_per_dst=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] dup_rows_per_dst=[3488, 4278, 2922, 1528, 766, 1632, 118, 2874] combine_rows_per_src=[8086, 12016, 9770, 9708, 12748, 11572, 4488, 12310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79226 payload_rows_remote=69436 dup_rows=19078 dup_rows_remote=18376 dedup_ratio=0.194071 router_tokens_per_expert=[161, 1517, 344, 772, 69, 114, 1335, 374, 0, 0, 25, 574, 2757, 0, 0, 769, 348, 672, 6575, 0, 11, 764, 1277, 0, 0, 4867, 1484, 0, 0, 0, 2512, 0, 4130, 403, 70, 0, 86, 0, 408, 149, 0, 0, 0, 355, 4, 143, 3707, 1491, 942, 936, 1040, 1004, 0, 192, 0, 993, 958, 244, 75, 11, 56, 286, 431, 3717] payload_rows_per_dst=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] dup_rows_per_dst=[1624, 912, 4144, 5178, 702, 1950, 2354, 2214] combine_rows_per_src=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=254156800 dispatch_meta_bytes=335832 combine_payload_bytes=254156800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254492632 ingress_bytes=288759808 link_bytes=288759808 token_bytes=508649432 weight_movement_bytes=34603008 total_nvlink_bytes=543252440 weight_to_token_ratio=0.068029 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.086579910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79386 payload_rows_remote=66946 dup_rows=18918 dup_rows_remote=13554 dedup_ratio=0.192444 router_tokens_per_expert=[188, 1654, 261, 850, 66, 233, 1490, 388, 0, 0, 50, 604, 2655, 0, 0, 756, 345, 722, 6527, 1, 7, 677, 1122, 0, 0, 4861, 1601, 0, 1, 0, 2439, 0, 3970, 412, 67, 0, 63, 0, 386, 149, 0, 0, 0, 340, 23, 173, 3782, 1402, 865, 977, 1256, 1031, 0, 270, 0, 1038, 769, 225, 123, 90, 59, 295, 451, 3438] payload_rows_per_dst=[8268, 7302, 15108, 12440, 9432, 9550, 8382, 8904] dup_rows_per_dst=[1992, 828, 3694, 5364, 662, 1890, 2492, 1996] combine_rows_per_src=[8268, 7302, 15108, 12440, 9432, 9550, 8382, 8904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267550720 dispatch_meta_bytes=321872 combine_payload_bytes=267550720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267872592 ingress_bytes=302153728 link_bytes=302153728 token_bytes=535423312 weight_movement_bytes=34603008 total_nvlink_bytes=570026320 weight_to_token_ratio=0.064627 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136980153e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275431424 dispatch_meta_bytes=335664 combine_payload_bytes=275431424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=275767088 ingress_bytes=413843456 link_bytes=413843456 token_bytes=551198512 weight_movement_bytes=138412032 total_nvlink_bytes=689610544 weight_to_token_ratio=0.251111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.557259608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77250 payload_rows_remote=64350 dup_rows=21054 dup_rows_remote=14474 dedup_ratio=0.214172 router_tokens_per_expert=[279, 1299, 124, 1129, 72, 143, 1208, 578, 0, 0, 24, 482, 2915, 0, 0, 809, 337, 617, 6519, 5, 13, 925, 1709, 0, 0, 4609, 1872, 0, 0, 0, 2084, 0, 3822, 330, 72, 0, 46, 0, 481, 179, 0, 0, 0, 282, 22, 155, 3653, 1492, 920, 753, 1166, 1014, 0, 338, 0, 968, 841, 181, 97, 2, 47, 296, 370, 3873] payload_rows_per_dst=[7864, 7750, 6922, 7158, 11558, 9318, 13780, 12900] dup_rows_per_dst=[1800, 710, 290, 754, 3274, 1890, 5756, 6580] combine_rows_per_src=[7864, 7750, 6922, 7158, 11558, 9318, 13780, 12900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79386 payload_rows_remote=66946 dup_rows=18918 dup_rows_remote=13554 dedup_ratio=0.192444 router_tokens_per_expert=[188, 1654, 261, 850, 66, 233, 1490, 388, 0, 0, 50, 604, 2655, 0, 0, 756, 345, 722, 6527, 1, 7, 677, 1122, 0, 0, 4861, 1601, 0, 1, 0, 2439, 0, 3970, 412, 67, 0, 63, 0, 386, 149, 0, 0, 0, 340, 23, 173, 3782, 1402, 865, 977, 1256, 1031, 0, 270, 0, 1038, 769, 225, 123, 90, 59, 295, 451, 3438] payload_rows_per_dst=[8268, 7302, 15108, 12440, 9432, 9550, 8382, 8904] dup_rows_per_dst=[1992, 828, 3694, 5364, 662, 1890, 2492, 1996] combine_rows_per_src=[8268, 7302, 15108, 12440, 9432, 9550, 8382, 8904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76726 payload_rows_remote=63150 dup_rows=21578 dup_rows_remote=15598 dedup_ratio=0.219503 router_tokens_per_expert=[136, 1451, 362, 816, 75, 213, 1462, 363, 0, 0, 16, 538, 2885, 0, 0, 735, 336, 651, 6640, 0, 3, 823, 1333, 0, 0, 5025, 1629, 0, 0, 0, 2194, 0, 4100, 392, 76, 3, 69, 0, 369, 133, 1, 0, 0, 395, 17, 206, 3860, 1550, 715, 778, 1247, 911, 0, 109, 0, 996, 811, 234, 65, 15, 34, 231, 424, 3725] payload_rows_per_dst=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] dup_rows_per_dst=[1598, 912, 244, 638, 8060, 2216, 5980, 1930] combine_rows_per_src=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77508 payload_rows_remote=62518 dup_rows=20796 dup_rows_remote=13892 dedup_ratio=0.211548 router_tokens_per_expert=[162, 1525, 411, 927, 92, 186, 1183, 422, 0, 0, 11, 588, 3044, 0, 0, 794, 349, 700, 6510, 3, 0, 857, 1676, 0, 0, 4717, 1641, 0, 0, 0, 2364, 0, 3867, 432, 93, 1, 65, 0, 427, 174, 0, 0, 0, 323, 1, 183, 3600, 1507, 843, 852, 1068, 980, 0, 39, 0, 922, 951, 228, 82, 5, 44, 239, 339, 3725] payload_rows_per_dst=[12402, 14990, 6824, 8308, 9366, 9346, 7268, 9004] dup_rows_per_dst=[4854, 6904, 346, 1696, 752, 1882, 2140, 2222] combine_rows_per_src=[12402, 14990, 6824, 8308, 9366, 9346, 7268, 9004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76958 payload_rows_remote=66836 dup_rows=21346 dup_rows_remote=15432 dedup_ratio=0.217143 router_tokens_per_expert=[0, 922, 1186, 1192, 909, 2422, 1, 1702, 3, 1490, 2066, 54, 123, 131, 249, 11, 1738, 342, 1, 1, 32, 1097, 699, 3, 1297, 0, 3953, 126, 4365, 108, 195, 1, 162, 0, 1, 203, 319, 0, 10, 811, 43, 827, 1, 1276, 149, 0, 4594, 1, 0, 1309, 144, 249, 4, 5404, 4009, 631, 375, 205, 12, 946, 763, 119, 38, 128] payload_rows_per_dst=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] dup_rows_per_dst=[5914, 4230, 786, 7196, 1050, 1736, 116, 318] combine_rows_per_src=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290521088 dispatch_meta_bytes=342448 combine_payload_bytes=290521088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290863536 ingress_bytes=325124096 link_bytes=325124096 token_bytes=581384624 weight_movement_bytes=34603008 total_nvlink_bytes=615987632 weight_to_token_ratio=0.059518 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223415799e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76726 payload_rows_remote=63150 dup_rows=21578 dup_rows_remote=15598 dedup_ratio=0.219503 router_tokens_per_expert=[136, 1451, 362, 816, 75, 213, 1462, 363, 0, 0, 16, 538, 2885, 0, 0, 735, 336, 651, 6640, 0, 3, 823, 1333, 0, 0, 5025, 1629, 0, 0, 0, 2194, 0, 4100, 392, 76, 3, 69, 0, 369, 133, 1, 0, 0, 395, 17, 206, 3860, 1550, 715, 778, 1247, 911, 0, 109, 0, 996, 811, 234, 65, 15, 34, 231, 424, 3725] payload_rows_per_dst=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] dup_rows_per_dst=[1598, 912, 244, 638, 8060, 2216, 5980, 1930] combine_rows_per_src=[8158, 7436, 6048, 7008, 15504, 9848, 13576, 9148] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284409856 dispatch_meta_bytes=351248 combine_payload_bytes=284409856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284761104 ingress_bytes=319012864 link_bytes=319012864 token_bytes=569170960 weight_movement_bytes=34603008 total_nvlink_bytes=603773968 weight_to_token_ratio=0.060795 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200419725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79226 payload_rows_remote=69436 dup_rows=19078 dup_rows_remote=18376 dedup_ratio=0.194071 router_tokens_per_expert=[161, 1517, 344, 772, 69, 114, 1335, 374, 0, 0, 25, 574, 2757, 0, 0, 769, 348, 672, 6575, 0, 11, 764, 1277, 0, 0, 4867, 1484, 0, 0, 0, 2512, 0, 4130, 403, 70, 0, 86, 0, 408, 149, 0, 0, 0, 355, 4, 143, 3707, 1491, 942, 936, 1040, 1004, 0, 192, 0, 993, 958, 244, 75, 11, 56, 286, 431, 3717] payload_rows_per_dst=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] dup_rows_per_dst=[1624, 912, 4144, 5178, 702, 1950, 2354, 2214] combine_rows_per_src=[7748, 7338, 15150, 12548, 9790, 9450, 7860, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263577600 dispatch_meta_bytes=315296 combine_payload_bytes=263577600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263892896 ingress_bytes=298180608 link_bytes=298180608 token_bytes=527470496 weight_movement_bytes=34603008 total_nvlink_bytes=562073504 weight_to_token_ratio=0.065602 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122029622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274210816 dispatch_meta_bytes=322000 combine_payload_bytes=274210816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274532816 ingress_bytes=412622848 link_bytes=412622848 token_bytes=548743632 weight_movement_bytes=138412032 total_nvlink_bytes=687155664 weight_to_token_ratio=0.252234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.552666558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=258662400 dispatch_meta_bytes=314992 combine_payload_bytes=258662400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258977392 ingress_bytes=293265408 link_bytes=293265408 token_bytes=517639792 weight_movement_bytes=34603008 total_nvlink_bytes=552242800 weight_to_token_ratio=0.066848 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.103534121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263577600 dispatch_meta_bytes=315296 combine_payload_bytes=263577600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263892896 ingress_bytes=298180608 link_bytes=298180608 token_bytes=527470496 weight_movement_bytes=34603008 total_nvlink_bytes=562073504 weight_to_token_ratio=0.065602 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122029622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=256073728 dispatch_meta_bytes=305640 combine_payload_bytes=256073728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256379368 ingress_bytes=290676736 link_bytes=290676736 token_bytes=512453096 weight_movement_bytes=34603008 total_nvlink_bytes=547056104 weight_to_token_ratio=0.067524 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093793156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=273760256 dispatch_meta_bytes=329072 combine_payload_bytes=273760256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274089328 ingress_bytes=342966272 link_bytes=342966272 token_bytes=547849584 weight_movement_bytes=69206016 total_nvlink_bytes=617055600 weight_to_token_ratio=0.126323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290554470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=258662400 dispatch_meta_bytes=314992 combine_payload_bytes=258662400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258977392 ingress_bytes=293265408 link_bytes=293265408 token_bytes=517639792 weight_movement_bytes=34603008 total_nvlink_bytes=552242800 weight_to_token_ratio=0.066848 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.103534121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=284409856 dispatch_meta_bytes=351248 combine_payload_bytes=284409856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284761104 ingress_bytes=319012864 link_bytes=319012864 token_bytes=569170960 weight_movement_bytes=34603008 total_nvlink_bytes=603773968 weight_to_token_ratio=0.060795 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.200419725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76880 payload_rows_remote=64978 dup_rows=21424 dup_rows_remote=17258 dedup_ratio=0.217936 router_tokens_per_expert=[4, 73, 1260, 1, 465, 1043, 936, 693, 3559, 1831, 238, 118, 693, 13, 4982, 274, 2347, 22, 2344, 342, 573, 264, 0, 308, 841, 2000, 455, 360, 311, 1614, 897, 238, 49, 214, 91, 59, 716, 192, 598, 245, 31, 3065, 81, 430, 112, 28, 1, 5186, 0, 130, 94, 1211, 169, 1854, 0, 140, 38, 0, 2274, 57, 1686, 959, 69, 274] payload_rows_per_dst=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] dup_rows_per_dst=[4166, 4674, 3088, 2536, 392, 4032, 742, 1794] combine_rows_per_src=[11902, 12860, 9312, 10896, 3936, 12600, 6454, 8920] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86160 payload_rows_remote=78422 dup_rows=12144 dup_rows_remote=11132 dedup_ratio=0.123535 router_tokens_per_expert=[1829, 66, 0, 4255, 0, 0, 44, 0, 0, 142, 227, 9, 837, 369, 31, 91, 0, 0, 126, 0, 0, 936, 393, 408, 0, 60, 179, 0, 161, 90, 0, 0, 8, 6196, 164, 81, 0, 0, 0, 1574, 813, 373, 0, 814, 2962, 90, 603, 5426, 392, 90, 0, 942, 0, 5819, 6781, 0, 3806, 0, 0, 0, 0, 0, 1965, 0] payload_rows_per_dst=[10172, 6580, 12788, 14184, 10300, 7738, 12710, 11688] dup_rows_per_dst=[2216, 690, 1790, 358, 1888, 1012, 1776, 2414] combine_rows_per_src=[10172, 6580, 12788, 14184, 10300, 7738, 12710, 11688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86440 payload_rows_remote=73456 dup_rows=11864 dup_rows_remote=9886 dedup_ratio=0.120687 router_tokens_per_expert=[1701, 90, 0, 4547, 0, 0, 11, 0, 0, 143, 162, 134, 830, 487, 32, 529, 0, 0, 138, 0, 0, 909, 443, 713, 0, 40, 158, 0, 72, 82, 0, 0, 15, 5862, 231, 112, 0, 0, 0, 1590, 768, 549, 0, 725, 3316, 82, 807, 5278, 339, 82, 0, 872, 0, 5265, 6890, 0, 3789, 0, 0, 0, 0, 0, 1359, 0] payload_rows_per_dst=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] dup_rows_per_dst=[2062, 2724, 1978, 316, 1860, 1668, 648, 608] combine_rows_per_src=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78768 payload_rows_remote=67244 dup_rows=19536 dup_rows_remote=16672 dedup_ratio=0.198730 router_tokens_per_expert=[185, 1535, 430, 987, 44, 364, 1260, 411, 0, 0, 39, 582, 2695, 0, 0, 754, 362, 680, 6569, 3, 15, 927, 1431, 0, 0, 4569, 1887, 0, 0, 0, 2130, 0, 4034, 346, 46, 0, 97, 0, 317, 192, 1, 0, 0, 348, 1, 181, 3792, 1440, 872, 808, 1058, 935, 0, 28, 0, 1049, 932, 209, 82, 3, 39, 315, 379, 3789] payload_rows_per_dst=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] dup_rows_per_dst=[1752, 3042, 2864, 5032, 618, 1994, 2078, 2156] combine_rows_per_src=[8680, 10684, 11524, 12140, 9446, 9532, 7422, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86968 payload_rows_remote=83396 dup_rows=11336 dup_rows_remote=10962 dedup_ratio=0.115316 router_tokens_per_expert=[1944, 65, 0, 4549, 0, 0, 155, 0, 0, 129, 127, 131, 855, 453, 0, 365, 0, 0, 183, 0, 0, 690, 418, 600, 0, 69, 150, 0, 64, 75, 0, 0, 10, 6552, 206, 68, 0, 0, 0, 1689, 797, 464, 0, 791, 3141, 75, 720, 5147, 387, 75, 0, 947, 0, 5052, 6906, 0, 3799, 0, 0, 0, 0, 0, 1304, 0] payload_rows_per_dst=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] dup_rows_per_dst=[2730, 2980, 1700, 300, 374, 1340, 1454, 458] combine_rows_per_src=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=321216512 dispatch_meta_bytes=358216 combine_payload_bytes=321216512 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=321574728 ingress_bytes=425025536 link_bytes=425025536 token_bytes=642791240 weight_movement_bytes=103809024 total_nvlink_bytes=746600264 weight_to_token_ratio=0.161497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599336875e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=328944 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478832 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532628720 weight_movement_bytes=34603008 total_nvlink_bytes=567231728 weight_to_token_ratio=0.064966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87196 payload_rows_remote=72952 dup_rows=11108 dup_rows_remote=10832 dedup_ratio=0.112996 router_tokens_per_expert=[1524, 53, 0, 4480, 0, 0, 75, 0, 0, 129, 131, 117, 855, 408, 173, 465, 0, 0, 114, 0, 0, 806, 414, 709, 0, 66, 138, 0, 78, 69, 0, 0, 19, 5924, 247, 76, 0, 0, 0, 1618, 843, 573, 0, 735, 3465, 69, 679, 5247, 402, 69, 0, 1012, 0, 5329, 6909, 0, 3996, 0, 0, 0, 0, 0, 1136, 0] payload_rows_per_dst=[10266, 13862, 12716, 14244, 5870, 11206, 9326, 9706] dup_rows_per_dst=[1998, 2542, 1864, 276, 926, 1522, 1422, 558] combine_rows_per_src=[10266, 13862, 12716, 14244, 5870, 11206, 9326, 9706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=321216512 dispatch_meta_bytes=358216 combine_payload_bytes=321216512 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=321574728 ingress_bytes=425025536 link_bytes=425025536 token_bytes=642791240 weight_movement_bytes=103809024 total_nvlink_bytes=746600264 weight_to_token_ratio=0.161497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.599336875e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300875776 dispatch_meta_bytes=333368 combine_payload_bytes=300875776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301209144 ingress_bytes=335478784 link_bytes=335478784 token_bytes=602084920 weight_movement_bytes=34603008 total_nvlink_bytes=636687928 weight_to_token_ratio=0.057472 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69112 payload_rows_remote=56464 dup_rows=29192 dup_rows_remote=25554 dedup_ratio=0.296956 router_tokens_per_expert=[1704, 45, 0, 4536, 0, 0, 122, 0, 0, 128, 216, 92, 824, 435, 37, 263, 0, 0, 179, 0, 0, 807, 433, 474, 0, 67, 189, 0, 174, 95, 0, 0, 1, 6174, 168, 56, 0, 0, 0, 1588, 759, 473, 0, 791, 2632, 96, 712, 5632, 326, 95, 0, 755, 0, 5663, 6900, 0, 3631, 0, 0, 0, 0, 0, 1880, 0] payload_rows_per_dst=[6320, 3652, 3492, 862, 13406, 13112, 15620, 12648] dup_rows_per_dst=[1034, 338, 294, 188, 2568, 3814, 17318, 3638] combine_rows_per_src=[6320, 3652, 3492, 862, 13406, 13112, 15620, 12648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86252 payload_rows_remote=72102 dup_rows=12052 dup_rows_remote=9400 dedup_ratio=0.122599 router_tokens_per_expert=[1630, 60, 0, 4526, 0, 0, 6, 0, 0, 168, 141, 124, 885, 435, 14, 242, 0, 0, 154, 0, 0, 849, 422, 665, 0, 90, 142, 0, 81, 71, 0, 0, 0, 6392, 264, 76, 0, 0, 0, 1748, 800, 546, 1, 711, 3097, 74, 748, 5095, 421, 71, 0, 890, 0, 5394, 6862, 0, 3824, 0, 0, 0, 0, 0, 1433, 0] payload_rows_per_dst=[10364, 14150, 12528, 14208, 12720, 10474, 1970, 9838] dup_rows_per_dst=[2080, 2652, 1842, 284, 2244, 1480, 794, 676] combine_rows_per_src=[10364, 14150, 12528, 14208, 12720, 10474, 1970, 9838] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80020 payload_rows_remote=66122 dup_rows=18284 dup_rows_remote=12948 dedup_ratio=0.185994 router_tokens_per_expert=[1518, 60, 0, 4556, 0, 0, 37, 0, 0, 159, 108, 109, 825, 403, 14, 222, 0, 0, 430, 0, 0, 811, 401, 610, 1, 73, 126, 0, 48, 65, 0, 0, 9, 6074, 280, 91, 0, 0, 0, 1715, 783, 530, 8, 720, 3386, 65, 703, 5135, 376, 63, 0, 864, 0, 6114, 6912, 0, 3532, 0, 0, 0, 0, 0, 1216, 0] payload_rows_per_dst=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] dup_rows_per_dst=[1916, 92, 1218, 210, 3026, 2554, 5336, 3932] combine_rows_per_src=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77350 payload_rows_remote=63592 dup_rows=20954 dup_rows_remote=13786 dedup_ratio=0.213155 router_tokens_per_expert=[274, 1684, 341, 981, 69, 277, 1419, 475, 1, 1, 24, 545, 2551, 0, 1, 846, 346, 674, 6446, 2, 1, 750, 1441, 0, 1, 4943, 1558, 0, 1, 0, 2308, 0, 3631, 370, 70, 2, 59, 1, 486, 156, 0, 1, 0, 326, 6, 179, 3985, 1304, 933, 868, 930, 925, 0, 201, 0, 990, 939, 233, 92, 15, 45, 316, 383, 3746] payload_rows_per_dst=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] dup_rows_per_dst=[7168, 6084, 260, 614, 616, 1854, 2132, 2226] combine_rows_per_src=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80020 payload_rows_remote=66122 dup_rows=18284 dup_rows_remote=12948 dedup_ratio=0.185994 router_tokens_per_expert=[1518, 60, 0, 4556, 0, 0, 37, 0, 0, 159, 108, 109, 825, 403, 14, 222, 0, 0, 430, 0, 0, 811, 401, 610, 1, 73, 126, 0, 48, 65, 0, 0, 9, 6074, 280, 91, 0, 0, 0, 1715, 783, 530, 8, 720, 3386, 65, 703, 5135, 376, 63, 0, 864, 0, 6114, 6912, 0, 3532, 0, 0, 0, 0, 0, 1216, 0] payload_rows_per_dst=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] dup_rows_per_dst=[1916, 92, 1218, 210, 3026, 2554, 5336, 3932] combine_rows_per_src=[10426, 3588, 6268, 9840, 13312, 10352, 13898, 12336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86968 payload_rows_remote=83396 dup_rows=11336 dup_rows_remote=10962 dedup_ratio=0.115316 router_tokens_per_expert=[1944, 65, 0, 4549, 0, 0, 155, 0, 0, 129, 127, 131, 855, 453, 0, 365, 0, 0, 183, 0, 0, 690, 418, 600, 0, 69, 150, 0, 64, 75, 0, 0, 10, 6552, 206, 68, 0, 0, 0, 1689, 797, 464, 0, 791, 3141, 75, 720, 5147, 387, 75, 0, 947, 0, 5052, 6906, 0, 3799, 0, 0, 0, 0, 0, 1304, 0] payload_rows_per_dst=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] dup_rows_per_dst=[2730, 2980, 1700, 300, 374, 1340, 1454, 458] combine_rows_per_src=[10696, 14244, 12376, 14228, 3572, 10636, 11468, 9748] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=275431424 dispatch_meta_bytes=335664 combine_payload_bytes=275431424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=275767088 ingress_bytes=413843456 link_bytes=413843456 token_bytes=551198512 weight_movement_bytes=138412032 total_nvlink_bytes=689610544 weight_to_token_ratio=0.251111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.557259608e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=341590016 dispatch_meta_bytes=377432 combine_payload_bytes=341590016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341967448 ingress_bytes=410796032 link_bytes=410796032 token_bytes=683557464 weight_movement_bytes=69206016 total_nvlink_bytes=752763480 weight_to_token_ratio=0.101244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545792397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=298811392 dispatch_meta_bytes=335136 combine_payload_bytes=298811392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299146528 ingress_bytes=333414400 link_bytes=333414400 token_bytes=597957920 weight_movement_bytes=34603008 total_nvlink_bytes=632560928 weight_to_token_ratio=0.057869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254611545e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=231276544 dispatch_meta_bytes=328072 combine_payload_bytes=231276544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=231604616 ingress_bytes=265879552 link_bytes=265879552 token_bytes=462881160 weight_movement_bytes=34603008 total_nvlink_bytes=497484168 weight_to_token_ratio=0.074756 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000483349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295329792 dispatch_meta_bytes=326008 combine_payload_bytes=295329792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295655800 ingress_bytes=329932800 link_bytes=329932800 token_bytes=590985592 weight_movement_bytes=34603008 total_nvlink_bytes=625588600 weight_to_token_ratio=0.058551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241510565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270835712 dispatch_meta_bytes=316280 combine_payload_bytes=270835712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271151992 ingress_bytes=374644736 link_bytes=374644736 token_bytes=541987704 weight_movement_bytes=103809024 total_nvlink_bytes=645796728 weight_to_token_ratio=0.191534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409757980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=260472832 dispatch_meta_bytes=309512 combine_payload_bytes=260472832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260782344 ingress_bytes=295075840 link_bytes=295075840 token_bytes=521255176 weight_movement_bytes=34603008 total_nvlink_bytes=555858184 weight_to_token_ratio=0.066384 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110346630e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=341590016 dispatch_meta_bytes=377432 combine_payload_bytes=341590016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341967448 ingress_bytes=410796032 link_bytes=410796032 token_bytes=683557464 weight_movement_bytes=69206016 total_nvlink_bytes=752763480 weight_to_token_ratio=0.101244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545792397e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72188 dup_rows=12006 dup_rows_remote=8562 dedup_ratio=0.122131 router_tokens_per_expert=[621, 7485, 0, 0, 394, 0, 90, 0, 0, 297, 90, 3529, 0, 113, 589, 1887, 112, 1415, 0, 0, 298, 90, 0, 341, 6521, 4036, 201, 555, 1, 0, 0, 40, 0, 226, 187, 0, 87, 1711, 476, 0, 433, 0, 0, 44, 0, 4460, 0, 0, 5278, 0, 0, 3882, 82, 33, 0, 0, 0, 0, 0, 0, 0, 3121, 229, 198] payload_rows_per_dst=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] dup_rows_per_dst=[1866, 2082, 3444, 544, 2964, 866, 164, 76] combine_rows_per_src=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77144 payload_rows_remote=66840 dup_rows=21160 dup_rows_remote=19502 dedup_ratio=0.215251 router_tokens_per_expert=[595, 7467, 0, 0, 349, 0, 66, 0, 0, 299, 66, 3470, 0, 87, 530, 1566, 87, 1669, 0, 0, 248, 66, 0, 399, 6725, 3744, 215, 602, 0, 0, 0, 23, 0, 201, 191, 0, 94, 1710, 497, 0, 464, 0, 0, 60, 0, 4732, 0, 0, 5447, 0, 0, 3757, 91, 0, 0, 0, 0, 0, 0, 0, 0, 2967, 357, 311] payload_rows_per_dst=[4984, 10258, 3974, 15208, 6456, 10304, 12692, 13268] dup_rows_per_dst=[806, 1778, 964, 5960, 826, 1658, 4002, 5166] combine_rows_per_src=[4984, 10258, 3974, 15208, 6456, 10304, 12692, 13268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76958 payload_rows_remote=66836 dup_rows=21346 dup_rows_remote=15432 dedup_ratio=0.217143 router_tokens_per_expert=[0, 922, 1186, 1192, 909, 2422, 1, 1702, 3, 1490, 2066, 54, 123, 131, 249, 11, 1738, 342, 1, 1, 32, 1097, 699, 3, 1297, 0, 3953, 126, 4365, 108, 195, 1, 162, 0, 1, 203, 319, 0, 10, 811, 43, 827, 1, 1276, 149, 0, 4594, 1, 0, 1309, 144, 249, 4, 5404, 4009, 631, 375, 205, 12, 946, 763, 119, 38, 128] payload_rows_per_dst=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] dup_rows_per_dst=[5914, 4230, 786, 7196, 1050, 1736, 116, 318] combine_rows_per_src=[10122, 12042, 7672, 12894, 12770, 12046, 4558, 4854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273776640 dispatch_meta_bytes=345368 combine_payload_bytes=273776640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274122008 ingress_bytes=308379648 link_bytes=308379648 token_bytes=547898648 weight_movement_bytes=34603008 total_nvlink_bytes=582501656 weight_to_token_ratio=0.063156 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160407789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86230 payload_rows_remote=73076 dup_rows=12074 dup_rows_remote=9176 dedup_ratio=0.122823 router_tokens_per_expert=[555, 7523, 0, 0, 327, 0, 63, 0, 0, 366, 64, 3624, 0, 69, 507, 1838, 69, 1409, 0, 0, 270, 63, 0, 394, 6586, 4143, 255, 479, 0, 0, 0, 42, 0, 239, 218, 0, 82, 1581, 462, 0, 407, 0, 0, 69, 0, 4486, 0, 0, 5444, 0, 0, 4080, 75, 28, 0, 0, 0, 0, 0, 0, 0, 2925, 202, 208] payload_rows_per_dst=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] dup_rows_per_dst=[1614, 1994, 3294, 1234, 2898, 814, 150, 76] combine_rows_per_src=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86440 payload_rows_remote=73456 dup_rows=11864 dup_rows_remote=9886 dedup_ratio=0.120687 router_tokens_per_expert=[1701, 90, 0, 4547, 0, 0, 11, 0, 0, 143, 162, 134, 830, 487, 32, 529, 0, 0, 138, 0, 0, 909, 443, 713, 0, 40, 158, 0, 72, 82, 0, 0, 15, 5862, 231, 112, 0, 0, 0, 1590, 768, 549, 0, 725, 3316, 82, 807, 5278, 339, 82, 0, 872, 0, 5265, 6890, 0, 3789, 0, 0, 0, 0, 0, 1359, 0] payload_rows_per_dst=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] dup_rows_per_dst=[2062, 2724, 1978, 316, 1860, 1668, 648, 608] combine_rows_per_src=[10636, 13634, 12984, 14168, 12566, 10826, 1938, 9688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86626 payload_rows_remote=76496 dup_rows=11678 dup_rows_remote=11126 dedup_ratio=0.118795 router_tokens_per_expert=[656, 7520, 0, 0, 405, 0, 53, 0, 0, 244, 53, 3356, 0, 79, 511, 1816, 79, 1264, 0, 0, 265, 53, 2, 404, 6561, 4549, 217, 546, 0, 0, 0, 29, 0, 214, 188, 0, 69, 1439, 476, 0, 425, 0, 0, 66, 0, 4748, 0, 0, 5720, 0, 0, 3861, 70, 47, 0, 0, 0, 0, 0, 0, 0, 2749, 205, 213] payload_rows_per_dst=[15284, 10184, 13954, 10130, 13354, 9628, 7818, 6274] dup_rows_per_dst=[1984, 1934, 3302, 552, 2858, 850, 138, 60] combine_rows_per_src=[15284, 10184, 13954, 10130, 13354, 9628, 7818, 6274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=58990 dup_rows=23774 dup_rows_remote=16282 dedup_ratio=0.241842 router_tokens_per_expert=[690, 7575, 0, 0, 370, 0, 42, 0, 0, 297, 42, 3710, 0, 57, 537, 1567, 57, 1400, 0, 0, 198, 42, 0, 336, 6600, 3973, 202, 515, 0, 0, 0, 44, 0, 242, 151, 0, 99, 1607, 486, 0, 511, 0, 0, 72, 0, 4367, 0, 0, 4935, 0, 0, 4382, 95, 10, 0, 0, 0, 0, 0, 0, 0, 3319, 281, 341] payload_rows_per_dst=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] dup_rows_per_dst=[508, 1618, 840, 964, 414, 4746, 7192, 7492] combine_rows_per_src=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=295682048 dispatch_meta_bytes=323000 combine_payload_bytes=295682048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296005048 ingress_bytes=330285056 link_bytes=330285056 token_bytes=591687096 weight_movement_bytes=34603008 total_nvlink_bytes=626290104 weight_to_token_ratio=0.058482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242836076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273776640 dispatch_meta_bytes=345368 combine_payload_bytes=273776640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274122008 ingress_bytes=308379648 link_bytes=308379648 token_bytes=547898648 weight_movement_bytes=34603008 total_nvlink_bytes=582501656 weight_to_token_ratio=0.063156 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160407789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86386 payload_rows_remote=75400 dup_rows=11918 dup_rows_remote=9860 dedup_ratio=0.121236 router_tokens_per_expert=[626, 7490, 0, 0, 319, 0, 60, 0, 0, 372, 60, 3649, 0, 76, 538, 1827, 76, 1344, 0, 0, 237, 60, 0, 423, 6280, 4132, 211, 453, 0, 0, 0, 50, 0, 219, 197, 0, 80, 1903, 429, 0, 429, 0, 0, 92, 0, 4288, 0, 0, 5403, 0, 0, 4038, 111, 5, 0, 0, 0, 0, 0, 0, 0, 3183, 245, 247] payload_rows_per_dst=[15248, 10986, 13476, 9248, 13252, 8760, 8166, 7250] dup_rows_per_dst=[1742, 2058, 3364, 444, 3210, 858, 142, 100] combine_rows_per_src=[15248, 10986, 13476, 9248, 13252, 8760, 8166, 7250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76530 payload_rows_remote=62420 dup_rows=21774 dup_rows_remote=16574 dedup_ratio=0.221497 router_tokens_per_expert=[575, 7535, 0, 0, 330, 0, 60, 0, 0, 398, 60, 3284, 0, 92, 526, 1696, 92, 1282, 0, 0, 229, 60, 0, 378, 6414, 4338, 236, 551, 0, 0, 0, 51, 0, 216, 188, 0, 65, 1711, 358, 0, 386, 0, 0, 75, 0, 4626, 0, 0, 5676, 0, 0, 3871, 82, 26, 0, 0, 0, 0, 0, 0, 0, 3223, 274, 188] payload_rows_per_dst=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] dup_rows_per_dst=[508, 1820, 878, 1014, 354, 5066, 5200, 6934] combine_rows_per_src=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85378 payload_rows_remote=73840 dup_rows=12926 dup_rows_remote=9076 dedup_ratio=0.131490 router_tokens_per_expert=[1881, 54, 0, 4777, 0, 1, 161, 0, 1, 131, 194, 132, 824, 470, 73, 296, 0, 0, 132, 1, 0, 782, 434, 736, 0, 70, 185, 0, 141, 91, 0, 0, 8, 6167, 259, 69, 1, 1, 0, 1621, 784, 562, 18, 965, 3067, 94, 615, 5148, 360, 91, 1, 860, 0, 4877, 6832, 1, 4092, 0, 0, 0, 2, 0, 1090, 0] payload_rows_per_dst=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] dup_rows_per_dst=[3850, 2658, 1838, 368, 1832, 1276, 612, 492] combine_rows_per_src=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=58990 dup_rows=23774 dup_rows_remote=16282 dedup_ratio=0.241842 router_tokens_per_expert=[690, 7575, 0, 0, 370, 0, 42, 0, 0, 297, 42, 3710, 0, 57, 537, 1567, 57, 1400, 0, 0, 198, 42, 0, 336, 6600, 3973, 202, 515, 0, 0, 0, 44, 0, 242, 151, 0, 99, 1607, 486, 0, 511, 0, 0, 72, 0, 4367, 0, 0, 4935, 0, 0, 4382, 95, 10, 0, 0, 0, 0, 0, 0, 0, 3319, 281, 341] payload_rows_per_dst=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] dup_rows_per_dst=[508, 1618, 840, 964, 414, 4746, 7192, 7492] combine_rows_per_src=[1696, 8354, 3226, 13824, 4756, 13100, 14034, 15540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=273760256 dispatch_meta_bytes=329072 combine_payload_bytes=273760256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274089328 ingress_bytes=342966272 link_bytes=342966272 token_bytes=547849584 weight_movement_bytes=69206016 total_nvlink_bytes=617055600 weight_to_token_ratio=0.126323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290554470e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=299319296 dispatch_meta_bytes=329008 combine_payload_bytes=299319296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299648304 ingress_bytes=333922304 link_bytes=333922304 token_bytes=598967600 weight_movement_bytes=34603008 total_nvlink_bytes=633570608 weight_to_token_ratio=0.057771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256522747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76530 payload_rows_remote=62420 dup_rows=21774 dup_rows_remote=16574 dedup_ratio=0.221497 router_tokens_per_expert=[575, 7535, 0, 0, 330, 0, 60, 0, 0, 398, 60, 3284, 0, 92, 526, 1696, 92, 1282, 0, 0, 229, 60, 0, 378, 6414, 4338, 236, 551, 0, 0, 0, 51, 0, 216, 188, 0, 65, 1711, 358, 0, 386, 0, 0, 75, 0, 4626, 0, 0, 5676, 0, 0, 3871, 82, 26, 0, 0, 0, 0, 0, 0, 0, 3223, 274, 188] payload_rows_per_dst=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] dup_rows_per_dst=[508, 1820, 878, 1014, 354, 5066, 5200, 6934] combine_rows_per_src=[1422, 10292, 3204, 13490, 4722, 13784, 14110, 15506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86230 payload_rows_remote=73076 dup_rows=12074 dup_rows_remote=9176 dedup_ratio=0.122823 router_tokens_per_expert=[555, 7523, 0, 0, 327, 0, 63, 0, 0, 366, 64, 3624, 0, 69, 507, 1838, 69, 1409, 0, 0, 270, 63, 0, 394, 6586, 4143, 255, 479, 0, 0, 0, 42, 0, 239, 218, 0, 82, 1581, 462, 0, 407, 0, 0, 69, 0, 4486, 0, 0, 5444, 0, 0, 4080, 75, 28, 0, 0, 0, 0, 0, 0, 0, 2925, 202, 208] payload_rows_per_dst=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] dup_rows_per_dst=[1614, 1994, 3294, 1234, 2898, 814, 150, 76] combine_rows_per_src=[15322, 10942, 12920, 9972, 13154, 9110, 8216, 6594] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300875776 dispatch_meta_bytes=333368 combine_payload_bytes=300875776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301209144 ingress_bytes=335478784 link_bytes=335478784 token_bytes=602084920 weight_movement_bytes=34603008 total_nvlink_bytes=636687928 weight_to_token_ratio=0.057472 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=313327616 dispatch_meta_bytes=350488 combine_payload_bytes=313327616 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=313678104 ingress_bytes=451739648 link_bytes=451739648 token_bytes=627005720 weight_movement_bytes=138412032 total_nvlink_bytes=765417752 weight_to_token_ratio=0.220751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.699859927e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=241623040 dispatch_meta_bytes=301088 combine_payload_bytes=241623040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241924128 ingress_bytes=276226048 link_bytes=276226048 token_bytes=483547168 weight_movement_bytes=34603008 total_nvlink_bytes=518150176 weight_to_token_ratio=0.071561 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.039416381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85054 payload_rows_remote=69560 dup_rows=13250 dup_rows_remote=12760 dedup_ratio=0.134786 router_tokens_per_expert=[790, 90, 6, 117, 6771, 0, 7322, 4663, 0, 166, 0, 494, 0, 338, 0, 3301, 105, 303, 2803, 0, 282, 0, 392, 87, 2, 464, 90, 0, 0, 157, 0, 176, 5901, 1, 0, 90, 204, 1286, 2772, 0, 423, 0, 0, 0, 0, 0, 0, 247, 1075, 0, 0, 0, 0, 627, 63, 0, 4725, 0, 0, 1185, 1473, 161, 0, 0] payload_rows_per_dst=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] dup_rows_per_dst=[0, 5110, 344, 1062, 1480, 490, 1686, 3078] combine_rows_per_src=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=308838400 dispatch_meta_bytes=341040 combine_payload_bytes=308838400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=309179440 ingress_bytes=378044416 link_bytes=378044416 token_bytes=618017840 weight_movement_bytes=69206016 total_nvlink_bytes=687223856 weight_to_token_ratio=0.111981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422550702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=241623040 dispatch_meta_bytes=301088 combine_payload_bytes=241623040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241924128 ingress_bytes=276226048 link_bytes=276226048 token_bytes=483547168 weight_movement_bytes=34603008 total_nvlink_bytes=518150176 weight_to_token_ratio=0.071561 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.039416381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=255672320 dispatch_meta_bytes=315976 combine_payload_bytes=255672320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255988296 ingress_bytes=324878336 link_bytes=324878336 token_bytes=511660616 weight_movement_bytes=69206016 total_nvlink_bytes=580866632 weight_to_token_ratio=0.135258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222491024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=302448640 dispatch_meta_bytes=331664 combine_payload_bytes=302448640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302780304 ingress_bytes=371654656 link_bytes=371654656 token_bytes=605228944 weight_movement_bytes=69206016 total_nvlink_bytes=674434960 weight_to_token_ratio=0.114347 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398506550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=299319296 dispatch_meta_bytes=329008 combine_payload_bytes=299319296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299648304 ingress_bytes=333922304 link_bytes=333922304 token_bytes=598967600 weight_movement_bytes=34603008 total_nvlink_bytes=633570608 weight_to_token_ratio=0.057771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256522747e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85482 payload_rows_remote=71938 dup_rows=12822 dup_rows_remote=8218 dedup_ratio=0.130432 router_tokens_per_expert=[813, 82, 8, 172, 6966, 0, 7369, 4826, 0, 164, 0, 417, 0, 267, 0, 3129, 252, 361, 2691, 0, 595, 0, 340, 113, 0, 181, 82, 0, 0, 172, 0, 173, 5845, 0, 0, 82, 224, 1346, 2787, 0, 459, 0, 0, 0, 0, 0, 0, 253, 1066, 0, 0, 0, 0, 622, 55, 0, 4722, 0, 0, 1015, 1431, 72, 0, 0] payload_rows_per_dst=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] dup_rows_per_dst=[1186, 1632, 4604, 540, 2342, 506, 1502, 510] combine_rows_per_src=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85054 payload_rows_remote=69560 dup_rows=13250 dup_rows_remote=12760 dedup_ratio=0.134786 router_tokens_per_expert=[790, 90, 6, 117, 6771, 0, 7322, 4663, 0, 166, 0, 494, 0, 338, 0, 3301, 105, 303, 2803, 0, 282, 0, 392, 87, 2, 464, 90, 0, 0, 157, 0, 176, 5901, 1, 0, 90, 204, 1286, 2772, 0, 423, 0, 0, 0, 0, 0, 0, 247, 1075, 0, 0, 0, 0, 627, 63, 0, 4725, 0, 0, 1185, 1473, 161, 0, 0] payload_rows_per_dst=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] dup_rows_per_dst=[0, 5110, 344, 1062, 1480, 490, 1686, 3078] combine_rows_per_src=[2006, 12814, 7600, 14258, 7226, 15494, 13646, 12010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77350 payload_rows_remote=63592 dup_rows=20954 dup_rows_remote=13786 dedup_ratio=0.213155 router_tokens_per_expert=[274, 1684, 341, 981, 69, 277, 1419, 475, 1, 1, 24, 545, 2551, 0, 1, 846, 346, 674, 6446, 2, 1, 750, 1441, 0, 1, 4943, 1558, 0, 1, 0, 2308, 0, 3631, 370, 70, 2, 59, 1, 486, 156, 0, 1, 0, 326, 6, 179, 3985, 1304, 933, 868, 930, 925, 0, 201, 0, 990, 939, 233, 92, 15, 45, 316, 383, 3746] payload_rows_per_dst=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] dup_rows_per_dst=[7168, 6084, 260, 614, 616, 1854, 2132, 2226] combine_rows_per_src=[13758, 14746, 6168, 7122, 8934, 9748, 7562, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284917760 dispatch_meta_bytes=329280 combine_payload_bytes=284917760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285247040 ingress_bytes=319520768 link_bytes=319520768 token_bytes=570164800 weight_movement_bytes=34603008 total_nvlink_bytes=604767808 weight_to_token_ratio=0.060689 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202330926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85854 payload_rows_remote=79082 dup_rows=12450 dup_rows_remote=10978 dedup_ratio=0.126648 router_tokens_per_expert=[821, 75, 9, 179, 7125, 0, 7445, 4976, 0, 150, 0, 365, 0, 204, 0, 3070, 142, 344, 3037, 0, 357, 0, 387, 69, 0, 428, 75, 0, 0, 139, 0, 190, 5864, 0, 0, 75, 194, 1425, 2428, 0, 389, 0, 0, 0, 0, 0, 0, 242, 1109, 0, 0, 0, 0, 753, 72, 0, 4652, 0, 0, 916, 1380, 66, 0, 0] payload_rows_per_dst=[2168, 12876, 8326, 14930, 6772, 15698, 13710, 11374] dup_rows_per_dst=[0, 4654, 346, 984, 1472, 454, 1886, 2654] combine_rows_per_src=[2168, 12876, 8326, 14930, 6772, 15698, 13710, 11374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86114 payload_rows_remote=71602 dup_rows=12190 dup_rows_remote=11870 dedup_ratio=0.124003 router_tokens_per_expert=[758, 69, 22, 154, 6937, 0, 7428, 5099, 0, 138, 0, 408, 0, 226, 0, 2755, 262, 345, 3079, 0, 712, 0, 402, 79, 0, 107, 69, 0, 0, 123, 0, 180, 5880, 0, 0, 69, 207, 1183, 2845, 0, 407, 0, 0, 0, 0, 0, 0, 223, 1178, 0, 0, 0, 0, 693, 70, 0, 4888, 0, 0, 812, 1267, 78, 0, 0] payload_rows_per_dst=[2006, 12972, 9272, 14512, 7126, 15720, 13764, 10742] dup_rows_per_dst=[0, 4280, 1574, 320, 1482, 396, 1878, 2260] combine_rows_per_src=[2006, 12972, 9272, 14512, 7126, 15720, 13764, 10742] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=294658048 dispatch_meta_bytes=320624 combine_payload_bytes=294658048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294978672 ingress_bytes=329261056 link_bytes=329261056 token_bytes=589636720 weight_movement_bytes=34603008 total_nvlink_bytes=624239728 weight_to_token_ratio=0.058685 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238982846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284917760 dispatch_meta_bytes=329280 combine_payload_bytes=284917760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285247040 ingress_bytes=319520768 link_bytes=319520768 token_bytes=570164800 weight_movement_bytes=34603008 total_nvlink_bytes=604767808 weight_to_token_ratio=0.060689 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202330926e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64610 payload_rows_remote=56818 dup_rows=33694 dup_rows_remote=33162 dedup_ratio=0.342753 router_tokens_per_expert=[813, 71, 8, 157, 6912, 0, 7409, 4817, 0, 145, 0, 390, 0, 228, 0, 3399, 168, 348, 2618, 0, 397, 0, 421, 76, 0, 432, 74, 0, 0, 134, 0, 200, 6013, 0, 0, 74, 165, 1372, 2462, 0, 383, 0, 0, 0, 0, 0, 0, 229, 1034, 0, 0, 0, 0, 570, 97, 0, 4952, 0, 0, 1000, 1503, 81, 0, 0] payload_rows_per_dst=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] dup_rows_per_dst=[21034, 532, 4480, 326, 6296, 450, 8, 568] combine_rows_per_src=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82256 payload_rows_remote=67946 dup_rows=16048 dup_rows_remote=5742 dedup_ratio=0.163249 router_tokens_per_expert=[965, 95, 7, 159, 6805, 0, 7301, 4833, 0, 190, 0, 523, 0, 326, 0, 3028, 257, 337, 2734, 0, 306, 0, 326, 57, 2, 408, 95, 0, 0, 138, 0, 207, 5811, 0, 0, 96, 200, 1284, 2797, 0, 405, 0, 0, 0, 0, 0, 0, 237, 1060, 0, 0, 0, 0, 610, 78, 0, 4679, 0, 0, 1106, 1516, 174, 0, 0] payload_rows_per_dst=[2452, 7242, 7788, 14408, 7130, 15422, 13504, 14310] dup_rows_per_dst=[0, 892, 246, 902, 1624, 464, 1614, 10306] combine_rows_per_src=[2452, 7242, 7788, 14408, 7130, 15422, 13504, 14310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86298 payload_rows_remote=72188 dup_rows=12006 dup_rows_remote=8562 dedup_ratio=0.122131 router_tokens_per_expert=[621, 7485, 0, 0, 394, 0, 90, 0, 0, 297, 90, 3529, 0, 113, 589, 1887, 112, 1415, 0, 0, 298, 90, 0, 341, 6521, 4036, 201, 555, 1, 0, 0, 40, 0, 226, 187, 0, 87, 1711, 476, 0, 433, 0, 0, 44, 0, 4460, 0, 0, 5278, 0, 0, 3882, 82, 33, 0, 0, 0, 0, 0, 0, 0, 3121, 229, 198] payload_rows_per_dst=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] dup_rows_per_dst=[1866, 2082, 3444, 544, 2964, 866, 164, 76] combine_rows_per_src=[15314, 10928, 14110, 9122, 12966, 9008, 7830, 7020] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64610 payload_rows_remote=56818 dup_rows=33694 dup_rows_remote=33162 dedup_ratio=0.342753 router_tokens_per_expert=[813, 71, 8, 157, 6912, 0, 7409, 4817, 0, 145, 0, 390, 0, 228, 0, 3399, 168, 348, 2618, 0, 397, 0, 421, 76, 0, 432, 74, 0, 0, 134, 0, 200, 6013, 0, 0, 74, 165, 1372, 2462, 0, 383, 0, 0, 0, 0, 0, 0, 229, 1034, 0, 0, 0, 0, 570, 97, 0, 4952, 0, 0, 1000, 1503, 81, 0, 0] payload_rows_per_dst=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] dup_rows_per_dst=[21034, 532, 4480, 326, 6296, 450, 8, 568] combine_rows_per_src=[16232, 7792, 13480, 4462, 13876, 774, 3394, 4600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83832 payload_rows_remote=68462 dup_rows=14472 dup_rows_remote=9836 dedup_ratio=0.147217 router_tokens_per_expert=[706, 7523, 0, 1, 367, 0, 57, 0, 0, 332, 54, 3721, 1, 70, 526, 1834, 69, 1287, 1, 2, 299, 54, 4, 392, 6569, 3871, 210, 523, 0, 0, 0, 44, 1, 259, 181, 0, 96, 1483, 469, 0, 485, 0, 0, 69, 2, 4319, 0, 1, 5571, 1, 0, 4046, 96, 82, 0, 0, 0, 1, 2, 0, 0, 2954, 217, 300] payload_rows_per_dst=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] dup_rows_per_dst=[4636, 1996, 3354, 338, 2914, 966, 182, 86] combine_rows_per_src=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85476 payload_rows_remote=71554 dup_rows=12828 dup_rows_remote=10912 dedup_ratio=0.130493 router_tokens_per_expert=[777, 63, 13, 152, 6946, 0, 7481, 5040, 0, 128, 0, 351, 0, 203, 0, 2834, 247, 336, 3079, 0, 445, 2, 376, 92, 0, 314, 65, 0, 0, 125, 0, 166, 5992, 0, 0, 65, 157, 1242, 2929, 0, 409, 0, 0, 0, 0, 0, 0, 185, 1197, 0, 0, 0, 0, 650, 80, 0, 4681, 0, 0, 906, 1356, 68, 0, 0] payload_rows_per_dst=[2010, 8422, 8488, 14498, 10106, 15780, 13922, 12250] dup_rows_per_dst=[0, 1376, 666, 734, 3212, 370, 1916, 4554] combine_rows_per_src=[2010, 8422, 8488, 14498, 10106, 15780, 13922, 12250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=360240 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324280112 ingress_bytes=393125888 link_bytes=393125888 token_bytes=648199984 weight_movement_bytes=69206016 total_nvlink_bytes=717406000 weight_to_token_ratio=0.106766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479301067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85854 payload_rows_remote=79082 dup_rows=12450 dup_rows_remote=10978 dedup_ratio=0.126648 router_tokens_per_expert=[821, 75, 9, 179, 7125, 0, 7445, 4976, 0, 150, 0, 365, 0, 204, 0, 3070, 142, 344, 3037, 0, 357, 0, 387, 69, 0, 428, 75, 0, 0, 139, 0, 190, 5864, 0, 0, 75, 194, 1425, 2428, 0, 389, 0, 0, 0, 0, 0, 0, 242, 1109, 0, 0, 0, 0, 753, 72, 0, 4652, 0, 0, 916, 1380, 66, 0, 0] payload_rows_per_dst=[2168, 12876, 8326, 14930, 6772, 15698, 13710, 11374] dup_rows_per_dst=[0, 4654, 346, 984, 1472, 454, 1886, 2654] combine_rows_per_src=[2168, 12876, 8326, 14930, 6772, 15698, 13710, 11374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=260472832 dispatch_meta_bytes=309512 combine_payload_bytes=260472832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260782344 ingress_bytes=295075840 link_bytes=295075840 token_bytes=521255176 weight_movement_bytes=34603008 total_nvlink_bytes=555858184 weight_to_token_ratio=0.066384 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110346630e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87694 payload_rows_remote=72078 dup_rows=10610 dup_rows_remote=9598 dedup_ratio=0.107931 router_tokens_per_expert=[0, 8102, 4820, 0, 0, 66, 0, 154, 0, 0, 0, 0, 0, 1244, 1084, 156, 0, 90, 0, 723, 0, 0, 0, 1380, 161, 247, 0, 61, 0, 66, 251, 0, 246, 7808, 90, 0, 3831, 0, 6627, 0, 112, 0, 0, 0, 394, 0, 0, 0, 571, 0, 153, 0, 0, 0, 0, 0, 0, 0, 7179, 2981, 90, 90, 0, 375] payload_rows_per_dst=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] dup_rows_per_dst=[0, 742, 776, 1392, 94, 1012, 352, 6242] combine_rows_per_src=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293281792 dispatch_meta_bytes=333888 combine_payload_bytes=293281792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293615680 ingress_bytes=327884800 link_bytes=327884800 token_bytes=586897472 weight_movement_bytes=34603008 total_nvlink_bytes=621500480 weight_to_token_ratio=0.058959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233804106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278306816 dispatch_meta_bytes=294752 combine_payload_bytes=278306816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278601568 ingress_bytes=347512832 link_bytes=347512832 token_bytes=556908384 weight_movement_bytes=69206016 total_nvlink_bytes=626114400 weight_to_token_ratio=0.124268 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307662810e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=295682048 dispatch_meta_bytes=323000 combine_payload_bytes=295682048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296005048 ingress_bytes=330285056 link_bytes=330285056 token_bytes=591687096 weight_movement_bytes=34603008 total_nvlink_bytes=626290104 weight_to_token_ratio=0.058482 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.242836076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=232726528 dispatch_meta_bytes=359920 combine_payload_bytes=232726528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=233086448 ingress_bytes=267329536 link_bytes=267329536 token_bytes=465812976 weight_movement_bytes=34603008 total_nvlink_bytes=500415984 weight_to_token_ratio=0.074285 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005939522e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280420352 dispatch_meta_bytes=313192 combine_payload_bytes=280420352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280733544 ingress_bytes=349626368 link_bytes=349626368 token_bytes=561153896 weight_movement_bytes=69206016 total_nvlink_bytes=630359912 weight_to_token_ratio=0.123328 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315615876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278306816 dispatch_meta_bytes=294752 combine_payload_bytes=278306816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=278601568 ingress_bytes=347512832 link_bytes=347512832 token_bytes=556908384 weight_movement_bytes=69206016 total_nvlink_bytes=626114400 weight_to_token_ratio=0.124268 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307662810e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293085184 dispatch_meta_bytes=329864 combine_payload_bytes=293085184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293415048 ingress_bytes=327688192 link_bytes=327688192 token_bytes=586500232 weight_movement_bytes=34603008 total_nvlink_bytes=621103240 weight_to_token_ratio=0.058999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233064285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=323919872 dispatch_meta_bytes=360240 combine_payload_bytes=323919872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324280112 ingress_bytes=393125888 link_bytes=393125888 token_bytes=648199984 weight_movement_bytes=69206016 total_nvlink_bytes=717406000 weight_to_token_ratio=0.106766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479301067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86176 payload_rows_remote=70570 dup_rows=12128 dup_rows_remote=8926 dedup_ratio=0.123372 router_tokens_per_expert=[0, 8110, 4637, 0, 0, 90, 0, 156, 0, 0, 0, 0, 0, 1237, 1046, 172, 0, 82, 0, 713, 0, 0, 0, 1486, 72, 264, 0, 46, 0, 90, 154, 0, 222, 7827, 82, 0, 4246, 0, 6789, 0, 365, 0, 0, 0, 356, 0, 0, 0, 491, 0, 96, 0, 0, 0, 0, 0, 0, 0, 7123, 2652, 82, 83, 0, 383] payload_rows_per_dst=[13726, 12532, 15606, 7086, 6404, 10158, 14432, 6232] dup_rows_per_dst=[5978, 870, 3202, 448, 172, 970, 320, 168] combine_rows_per_src=[13726, 12532, 15606, 7086, 6404, 10158, 14432, 6232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87694 payload_rows_remote=72078 dup_rows=10610 dup_rows_remote=9598 dedup_ratio=0.107931 router_tokens_per_expert=[0, 8102, 4820, 0, 0, 66, 0, 154, 0, 0, 0, 0, 0, 1244, 1084, 156, 0, 90, 0, 723, 0, 0, 0, 1380, 161, 247, 0, 61, 0, 66, 251, 0, 246, 7808, 90, 0, 3831, 0, 6627, 0, 112, 0, 0, 0, 394, 0, 0, 0, 571, 0, 153, 0, 0, 0, 0, 0, 0, 0, 7179, 2981, 90, 90, 0, 375] payload_rows_per_dst=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] dup_rows_per_dst=[0, 742, 776, 1392, 94, 1012, 352, 6242] combine_rows_per_src=[440, 11888, 3610, 16384, 10218, 15616, 14350, 15188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=289054720 dispatch_meta_bytes=317984 combine_payload_bytes=289054720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289372704 ingress_bytes=323657728 link_bytes=323657728 token_bytes=578427424 weight_movement_bytes=34603008 total_nvlink_bytes=613030432 weight_to_token_ratio=0.059823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217897974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85378 payload_rows_remote=73840 dup_rows=12926 dup_rows_remote=9076 dedup_ratio=0.131490 router_tokens_per_expert=[1881, 54, 0, 4777, 0, 1, 161, 0, 1, 131, 194, 132, 824, 470, 73, 296, 0, 0, 132, 1, 0, 782, 434, 736, 0, 70, 185, 0, 141, 91, 0, 0, 8, 6167, 259, 69, 1, 1, 0, 1621, 784, 562, 18, 965, 3067, 94, 615, 5148, 360, 91, 1, 860, 0, 4877, 6832, 1, 4092, 0, 0, 0, 2, 0, 1090, 0] payload_rows_per_dst=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] dup_rows_per_dst=[3850, 2658, 1838, 368, 1832, 1276, 612, 492] combine_rows_per_src=[11538, 13918, 12628, 14270, 11840, 9294, 2014, 9876] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=295231488 dispatch_meta_bytes=326704 combine_payload_bytes=295231488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295558192 ingress_bytes=329834496 link_bytes=329834496 token_bytes=590789680 weight_movement_bytes=34603008 total_nvlink_bytes=625392688 weight_to_token_ratio=0.058571 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.241140655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90812 payload_rows_remote=86072 dup_rows=7492 dup_rows_remote=7458 dedup_ratio=0.076213 router_tokens_per_expert=[0, 8117, 4708, 0, 0, 63, 0, 175, 0, 0, 0, 0, 0, 1189, 1231, 138, 0, 75, 0, 696, 0, 0, 0, 1491, 64, 246, 0, 74, 0, 63, 139, 0, 255, 7909, 75, 0, 3898, 0, 6730, 0, 366, 0, 0, 0, 447, 0, 0, 0, 537, 0, 87, 0, 0, 0, 0, 0, 0, 0, 7179, 2629, 75, 81, 0, 415] payload_rows_per_dst=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] dup_rows_per_dst=[72, 876, 2984, 1022, 34, 1556, 312, 636] combine_rows_per_src=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90912 payload_rows_remote=74528 dup_rows=7392 dup_rows_remote=6250 dedup_ratio=0.075195 router_tokens_per_expert=[0, 8123, 4685, 0, 0, 53, 0, 312, 0, 0, 0, 0, 0, 1204, 1107, 122, 0, 69, 0, 718, 0, 0, 0, 1625, 78, 261, 0, 101, 0, 53, 147, 0, 217, 7911, 69, 0, 3923, 0, 6698, 0, 118, 0, 0, 0, 382, 0, 0, 0, 525, 0, 79, 0, 0, 0, 0, 0, 0, 0, 7288, 2753, 69, 77, 0, 385] payload_rows_per_dst=[9800, 11870, 15680, 16384, 572, 15822, 14358, 6426] dup_rows_per_dst=[300, 842, 3720, 1142, 0, 1000, 246, 142] combine_rows_per_src=[9800, 11870, 15680, 16384, 572, 15822, 14358, 6426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78672 payload_rows_remote=66296 dup_rows=19632 dup_rows_remote=18776 dedup_ratio=0.199707 router_tokens_per_expert=[0, 8121, 4633, 0, 0, 60, 0, 194, 0, 0, 0, 0, 0, 1268, 993, 135, 0, 74, 0, 720, 0, 0, 0, 1529, 81, 239, 0, 97, 0, 60, 152, 0, 229, 7856, 74, 0, 4220, 0, 6561, 0, 164, 0, 0, 0, 405, 0, 0, 0, 553, 0, 117, 0, 0, 0, 0, 0, 0, 0, 7280, 2789, 71, 73, 0, 404] payload_rows_per_dst=[16242, 12376, 15768, 1096, 15992, 1138, 9528, 6532] dup_rows_per_dst=[9774, 856, 3438, 162, 5036, 0, 224, 142] combine_rows_per_src=[16242, 12376, 15768, 1096, 15992, 1138, 9528, 6532] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83894 payload_rows_remote=68750 dup_rows=14410 dup_rows_remote=7324 dedup_ratio=0.146586 router_tokens_per_expert=[0, 8097, 4462, 0, 0, 42, 0, 177, 0, 0, 0, 0, 0, 1262, 879, 138, 0, 96, 0, 639, 0, 0, 0, 1393, 174, 253, 0, 72, 0, 42, 269, 0, 195, 7822, 96, 0, 4212, 0, 6453, 0, 228, 0, 0, 0, 417, 0, 0, 0, 504, 0, 115, 0, 0, 0, 0, 0, 0, 0, 7152, 3330, 94, 95, 0, 444] payload_rows_per_dst=[438, 7038, 3604, 16384, 11848, 15646, 13792, 15144] dup_rows_per_dst=[0, 268, 652, 1430, 3334, 1288, 352, 7086] combine_rows_per_src=[438, 7038, 3604, 16384, 11848, 15646, 13792, 15144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=352550912 dispatch_meta_bytes=374120 combine_payload_bytes=352550912 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352925032 ingress_bytes=490962944 link_bytes=490962944 token_bytes=705475944 weight_movement_bytes=138412032 total_nvlink_bytes=843887976 weight_to_token_ratio=0.196197 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.847454033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78672 payload_rows_remote=66296 dup_rows=19632 dup_rows_remote=18776 dedup_ratio=0.199707 router_tokens_per_expert=[0, 8121, 4633, 0, 0, 60, 0, 194, 0, 0, 0, 0, 0, 1268, 993, 135, 0, 74, 0, 720, 0, 0, 0, 1529, 81, 239, 0, 97, 0, 60, 152, 0, 229, 7856, 74, 0, 4220, 0, 6561, 0, 164, 0, 0, 0, 405, 0, 0, 0, 553, 0, 117, 0, 0, 0, 0, 0, 0, 0, 7280, 2789, 71, 73, 0, 404] payload_rows_per_dst=[16242, 12376, 15768, 1096, 15992, 1138, 9528, 6532] dup_rows_per_dst=[9774, 856, 3438, 162, 5036, 0, 224, 142] combine_rows_per_src=[16242, 12376, 15768, 1096, 15992, 1138, 9528, 6532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61574 payload_rows_remote=45332 dup_rows=36730 dup_rows_remote=12290 dedup_ratio=0.373637 router_tokens_per_expert=[816, 91, 27, 154, 6989, 1, 7299, 4964, 1, 182, 0, 466, 3, 298, 0, 3372, 96, 341, 2881, 0, 379, 3, 361, 69, 0, 446, 93, 0, 0, 147, 1, 193, 6059, 2, 0, 91, 216, 1260, 2420, 2, 409, 0, 1, 2, 0, 3, 0, 228, 1060, 3, 1, 0, 0, 722, 75, 1, 4643, 0, 0, 926, 1212, 143, 0, 0] payload_rows_per_dst=[16242, 7914, 13334, 1574, 14018, 854, 3696, 3942] dup_rows_per_dst=[24440, 730, 4212, 186, 6082, 432, 28, 620] combine_rows_per_src=[16242, 7914, 13334, 1574, 14018, 854, 3696, 3942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85482 payload_rows_remote=71938 dup_rows=12822 dup_rows_remote=8218 dedup_ratio=0.130432 router_tokens_per_expert=[813, 82, 8, 172, 6966, 0, 7369, 4826, 0, 164, 0, 417, 0, 267, 0, 3129, 252, 361, 2691, 0, 595, 0, 340, 113, 0, 181, 82, 0, 0, 172, 0, 173, 5845, 0, 0, 82, 224, 1346, 2787, 0, 459, 0, 0, 0, 0, 0, 0, 253, 1066, 0, 0, 0, 0, 622, 55, 0, 4722, 0, 0, 1015, 1431, 72, 0, 0] payload_rows_per_dst=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] dup_rows_per_dst=[1186, 1632, 4604, 540, 2342, 506, 1502, 510] combine_rows_per_src=[8528, 9066, 13544, 14608, 8060, 15000, 12150, 4526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88420 payload_rows_remote=74264 dup_rows=9884 dup_rows_remote=9686 dedup_ratio=0.100545 router_tokens_per_expert=[0, 8129, 4849, 0, 0, 60, 0, 235, 0, 0, 0, 0, 0, 1233, 961, 125, 0, 65, 0, 713, 0, 0, 0, 1515, 48, 254, 0, 102, 0, 60, 111, 0, 185, 7869, 65, 0, 4200, 0, 6633, 0, 86, 0, 0, 0, 401, 0, 0, 0, 477, 0, 67, 0, 0, 0, 0, 0, 0, 4, 7416, 2771, 63, 73, 0, 382] payload_rows_per_dst=[590, 12262, 3854, 16384, 10110, 15738, 14156, 15326] dup_rows_per_dst=[0, 776, 732, 1024, 88, 974, 198, 6092] combine_rows_per_src=[590, 12262, 3854, 16384, 10110, 15738, 14156, 15326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=305266688 dispatch_meta_bytes=323112 combine_payload_bytes=305266688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305589800 ingress_bytes=339869696 link_bytes=339869696 token_bytes=610856488 weight_movement_bytes=34603008 total_nvlink_bytes=645459496 weight_to_token_ratio=0.056647 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278902304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93618 payload_rows_remote=84414 dup_rows=4686 dup_rows_remote=3410 dedup_ratio=0.047668 router_tokens_per_expert=[0, 0, 0, 192, 0, 0, 0, 0, 323, 66, 3, 0, 823, 71, 0, 23, 0, 0, 0, 0, 0, 87, 0, 0, 64, 7895, 0, 1394, 52, 3232, 0, 0, 180, 4593, 0, 0, 0, 7273, 0, 359, 2473, 355, 66, 0, 408, 2354, 3085, 0, 0, 161, 0, 90, 0, 0, 0, 0, 19, 0, 6779, 6041, 601, 0, 0, 90] payload_rows_per_dst=[14546, 7808, 13732, 9044, 10068, 9204, 15970, 13246] dup_rows_per_dst=[384, 1812, 0, 440, 196, 1276, 322, 256] combine_rows_per_src=[14546, 7808, 13732, 9044, 10068, 9204, 15970, 13246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90812 payload_rows_remote=86072 dup_rows=7492 dup_rows_remote=7458 dedup_ratio=0.076213 router_tokens_per_expert=[0, 8117, 4708, 0, 0, 63, 0, 175, 0, 0, 0, 0, 0, 1189, 1231, 138, 0, 75, 0, 696, 0, 0, 0, 1491, 64, 246, 0, 74, 0, 63, 139, 0, 255, 7909, 75, 0, 3898, 0, 6730, 0, 366, 0, 0, 0, 447, 0, 0, 0, 537, 0, 87, 0, 0, 0, 0, 0, 0, 0, 7179, 2629, 75, 81, 0, 415] payload_rows_per_dst=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] dup_rows_per_dst=[72, 876, 2984, 1022, 34, 1556, 312, 636] combine_rows_per_src=[5706, 12036, 14826, 16384, 4740, 15888, 14396, 6836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=302448640 dispatch_meta_bytes=331664 combine_payload_bytes=302448640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302780304 ingress_bytes=371654656 link_bytes=371654656 token_bytes=605228944 weight_movement_bytes=69206016 total_nvlink_bytes=674434960 weight_to_token_ratio=0.114347 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398506550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=305266688 dispatch_meta_bytes=323112 combine_payload_bytes=305266688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305589800 ingress_bytes=339869696 link_bytes=339869696 token_bytes=610856488 weight_movement_bytes=34603008 total_nvlink_bytes=645459496 weight_to_token_ratio=0.056647 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278902304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281600000 dispatch_meta_bytes=304296 combine_payload_bytes=281600000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281904296 ingress_bytes=350806016 link_bytes=350806016 token_bytes=563504296 weight_movement_bytes=69206016 total_nvlink_bytes=632710312 weight_to_token_ratio=0.122814 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320054796e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=340288 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271888704 ingress_bytes=306151424 link_bytes=306151424 token_bytes=543437120 weight_movement_bytes=34603008 total_nvlink_bytes=578040128 weight_to_token_ratio=0.063674 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152023161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=185679872 dispatch_meta_bytes=230488 combine_payload_bytes=185679872 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=185910360 ingress_bytes=358694912 link_bytes=358694912 token_bytes=371590232 weight_movement_bytes=173015040 total_nvlink_bytes=544605272 weight_to_token_ratio=0.465607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349740077e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=294658048 dispatch_meta_bytes=320624 combine_payload_bytes=294658048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294978672 ingress_bytes=329261056 link_bytes=329261056 token_bytes=589636720 weight_movement_bytes=34603008 total_nvlink_bytes=624239728 weight_to_token_ratio=0.058685 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238982846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281600000 dispatch_meta_bytes=304296 combine_payload_bytes=281600000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=281904296 ingress_bytes=350806016 link_bytes=350806016 token_bytes=563504296 weight_movement_bytes=69206016 total_nvlink_bytes=632710312 weight_to_token_ratio=0.122814 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320054796e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=304185344 dispatch_meta_bytes=335800 combine_payload_bytes=304185344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304521144 ingress_bytes=338788352 link_bytes=338788352 token_bytes=608706488 weight_movement_bytes=34603008 total_nvlink_bytes=643309496 weight_to_token_ratio=0.056847 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274833294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=77258 dup_rows=7202 dup_rows_remote=7202 dedup_ratio=0.073263 router_tokens_per_expert=[0, 0, 0, 257, 0, 0, 0, 0, 447, 90, 3, 0, 924, 73, 0, 33, 0, 0, 0, 0, 0, 113, 0, 0, 4, 7883, 0, 1154, 88, 3394, 0, 0, 164, 4180, 0, 0, 0, 7241, 0, 443, 2326, 319, 91, 0, 340, 2277, 3361, 0, 0, 72, 0, 82, 0, 0, 0, 0, 1, 0, 6809, 6243, 658, 0, 0, 82] payload_rows_per_dst=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] dup_rows_per_dst=[514, 3146, 0, 278, 188, 2802, 144, 130] combine_rows_per_src=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=352550912 dispatch_meta_bytes=374120 combine_payload_bytes=352550912 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352925032 ingress_bytes=490962944 link_bytes=490962944 token_bytes=705475944 weight_movement_bytes=138412032 total_nvlink_bytes=843887976 weight_to_token_ratio=0.196197 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.847454033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93618 payload_rows_remote=84414 dup_rows=4686 dup_rows_remote=3410 dedup_ratio=0.047668 router_tokens_per_expert=[0, 0, 0, 192, 0, 0, 0, 0, 323, 66, 3, 0, 823, 71, 0, 23, 0, 0, 0, 0, 0, 87, 0, 0, 64, 7895, 0, 1394, 52, 3232, 0, 0, 180, 4593, 0, 0, 0, 7273, 0, 359, 2473, 355, 66, 0, 408, 2354, 3085, 0, 0, 161, 0, 90, 0, 0, 0, 0, 19, 0, 6779, 6041, 601, 0, 0, 90] payload_rows_per_dst=[14546, 7808, 13732, 9044, 10068, 9204, 15970, 13246] dup_rows_per_dst=[384, 1812, 0, 440, 196, 1276, 322, 256] combine_rows_per_src=[14546, 7808, 13732, 9044, 10068, 9204, 15970, 13246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316448768 dispatch_meta_bytes=337840 combine_payload_bytes=316448768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316786608 ingress_bytes=351051776 link_bytes=351051776 token_bytes=633235376 weight_movement_bytes=34603008 total_nvlink_bytes=667838384 weight_to_token_ratio=0.054645 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320979571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345759744 dispatch_meta_bytes=351296 combine_payload_bytes=345759744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=346111040 ingress_bytes=449568768 link_bytes=449568768 token_bytes=691870784 weight_movement_bytes=103809024 total_nvlink_bytes=795679808 weight_to_token_ratio=0.150041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.691691081e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93818 payload_rows_remote=84332 dup_rows=4486 dup_rows_remote=4334 dedup_ratio=0.045634 router_tokens_per_expert=[0, 0, 0, 317, 0, 0, 0, 0, 366, 63, 3, 0, 1014, 73, 0, 29, 0, 0, 0, 0, 0, 69, 0, 0, 9, 7927, 0, 1109, 61, 3564, 0, 0, 150, 4214, 0, 0, 0, 7370, 0, 455, 2496, 262, 73, 0, 387, 2026, 3027, 0, 0, 64, 0, 75, 0, 0, 0, 0, 1, 0, 6957, 6323, 593, 0, 0, 75] payload_rows_per_dst=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] dup_rows_per_dst=[634, 2888, 0, 460, 152, 10, 128, 214] combine_rows_per_src=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83832 payload_rows_remote=68462 dup_rows=14472 dup_rows_remote=9836 dedup_ratio=0.147217 router_tokens_per_expert=[706, 7523, 0, 1, 367, 0, 57, 0, 0, 332, 54, 3721, 1, 70, 526, 1834, 69, 1287, 1, 2, 299, 54, 4, 392, 6569, 3871, 210, 523, 0, 0, 0, 44, 1, 259, 181, 0, 96, 1483, 469, 0, 485, 0, 0, 69, 2, 4319, 0, 1, 5571, 1, 0, 4046, 96, 82, 0, 0, 0, 1, 2, 0, 0, 2954, 217, 300] payload_rows_per_dst=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] dup_rows_per_dst=[4636, 1996, 3354, 338, 2914, 966, 182, 86] combine_rows_per_src=[15370, 11080, 14000, 6260, 13206, 8786, 8268, 6862] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93602 payload_rows_remote=84826 dup_rows=4702 dup_rows_remote=4402 dedup_ratio=0.047831 router_tokens_per_expert=[0, 0, 0, 294, 0, 0, 0, 0, 386, 54, 8, 0, 1119, 69, 0, 6, 0, 0, 0, 0, 0, 79, 0, 0, 3, 7932, 0, 965, 51, 3519, 0, 0, 138, 4077, 0, 0, 0, 7262, 0, 443, 2412, 274, 76, 0, 402, 2082, 3223, 0, 0, 78, 0, 70, 0, 0, 0, 0, 0, 0, 6989, 6412, 660, 0, 0, 69] payload_rows_per_dst=[14524, 11432, 14136, 8776, 9178, 5638, 16004, 13914] dup_rows_per_dst=[588, 3122, 0, 300, 138, 30, 156, 368] combine_rows_per_src=[14524, 11432, 14136, 8776, 9178, 5638, 16004, 13914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345423872 dispatch_meta_bytes=354664 combine_payload_bytes=345423872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=345778536 ingress_bytes=414629888 link_bytes=414629888 token_bytes=691202408 weight_movement_bytes=69206016 total_nvlink_bytes=760408424 weight_to_token_ratio=0.100124 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560218888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68810 payload_rows_remote=52838 dup_rows=29494 dup_rows_remote=12260 dedup_ratio=0.300028 router_tokens_per_expert=[0, 0, 0, 186, 0, 0, 0, 0, 252, 42, 3, 0, 994, 88, 0, 46, 0, 0, 0, 0, 0, 57, 0, 0, 34, 7911, 0, 1306, 40, 3109, 0, 0, 191, 4322, 0, 0, 0, 7258, 0, 368, 2336, 312, 42, 0, 326, 2763, 3236, 0, 0, 174, 0, 96, 0, 0, 0, 0, 5, 0, 6776, 6188, 596, 0, 0, 95] payload_rows_per_dst=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] dup_rows_per_dst=[0, 1752, 0, 0, 8502, 1658, 348, 17234] combine_rows_per_src=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=71086 dup_rows=16266 dup_rows_remote=13576 dedup_ratio=0.165466 router_tokens_per_expert=[0, 0, 0, 254, 0, 0, 0, 0, 331, 60, 4, 0, 909, 67, 0, 14, 0, 0, 0, 0, 0, 76, 0, 0, 10, 7953, 0, 1056, 42, 3670, 0, 0, 145, 4097, 0, 0, 0, 7420, 0, 428, 2513, 280, 60, 0, 420, 2513, 2923, 0, 0, 81, 0, 74, 0, 0, 0, 0, 1, 0, 6892, 6172, 616, 0, 0, 71] payload_rows_per_dst=[14840, 10952, 13936, 16044, 9198, 15144, 670, 1254] dup_rows_per_dst=[508, 2690, 0, 9026, 142, 3746, 32, 122] combine_rows_per_src=[14840, 10952, 13936, 16044, 9198, 15144, 670, 1254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=347447296 dispatch_meta_bytes=356912 combine_payload_bytes=347447296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=347804208 ingress_bytes=451256320 link_bytes=451256320 token_bytes=695251504 weight_movement_bytes=103809024 total_nvlink_bytes=799060528 weight_to_token_ratio=0.149311 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.698041203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62626 payload_rows_remote=46430 dup_rows=35678 dup_rows_remote=16636 dedup_ratio=0.362935 router_tokens_per_expert=[0, 8096, 4818, 0, 0, 54, 0, 233, 0, 0, 4, 0, 0, 1109, 1223, 145, 0, 91, 0, 663, 0, 3, 0, 1427, 140, 249, 0, 116, 0, 54, 231, 1, 168, 7891, 91, 1, 4055, 2, 6677, 1, 244, 1, 4, 0, 496, 2, 0, 2, 507, 1, 133, 0, 0, 0, 0, 3, 0, 1, 7160, 2473, 91, 99, 2, 390] payload_rows_per_dst=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] dup_rows_per_dst=[19042, 922, 1640, 280, 13418, 10, 184, 182] combine_rows_per_src=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68810 payload_rows_remote=52838 dup_rows=29494 dup_rows_remote=12260 dedup_ratio=0.300028 router_tokens_per_expert=[0, 0, 0, 186, 0, 0, 0, 0, 252, 42, 3, 0, 994, 88, 0, 46, 0, 0, 0, 0, 0, 57, 0, 0, 34, 7911, 0, 1306, 40, 3109, 0, 0, 191, 4322, 0, 0, 0, 7258, 0, 368, 2336, 312, 42, 0, 326, 2763, 3236, 0, 0, 174, 0, 96, 0, 0, 0, 0, 5, 0, 6776, 6188, 596, 0, 0, 95] payload_rows_per_dst=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] dup_rows_per_dst=[0, 1752, 0, 0, 8502, 1658, 348, 17234] combine_rows_per_src=[372, 7570, 446, 2760, 15776, 9900, 16014, 15972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87016 payload_rows_remote=71002 dup_rows=11288 dup_rows_remote=11192 dedup_ratio=0.114827 router_tokens_per_expert=[0, 0, 0, 250, 0, 0, 0, 0, 400, 60, 10, 0, 765, 63, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 1, 7942, 0, 1144, 60, 3429, 0, 0, 128, 4284, 0, 0, 0, 7313, 0, 439, 2918, 244, 60, 0, 373, 2126, 2874, 0, 0, 48, 0, 65, 0, 0, 0, 0, 0, 0, 6902, 6414, 685, 0, 0, 63] payload_rows_per_dst=[6896, 6592, 13988, 6246, 13146, 9626, 16014, 14508] dup_rows_per_dst=[196, 1752, 0, 274, 4590, 1816, 96, 2564] combine_rows_per_src=[6896, 6592, 13988, 6246, 13146, 9626, 16014, 14508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=81742 dup_rows=9674 dup_rows_remote=9674 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 247, 87, 0, 7396, 0, 393, 0, 0, 2, 0, 90, 0, 0, 0, 1701, 0, 0, 0, 0, 3368, 0, 4862, 0, 0, 37, 7945, 0, 0, 0, 1, 852, 0, 4, 337, 0, 0, 177, 148, 243, 0, 0, 3201, 0, 0, 0, 0, 6662, 0, 1465, 0, 6008, 0, 0, 0, 0, 161, 0, 551, 718, 2406, 0, 90] payload_rows_per_dst=[5620, 13504, 9014, 15890, 10994, 6888, 13264, 13456] dup_rows_per_dst=[204, 4, 1124, 76, 1766, 0, 1682, 4818] combine_rows_per_src=[5620, 13504, 9014, 15890, 10994, 6888, 13264, 13456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86176 payload_rows_remote=70570 dup_rows=12128 dup_rows_remote=8926 dedup_ratio=0.123372 router_tokens_per_expert=[0, 8110, 4637, 0, 0, 90, 0, 156, 0, 0, 0, 0, 0, 1237, 1046, 172, 0, 82, 0, 713, 0, 0, 0, 1486, 72, 264, 0, 46, 0, 90, 154, 0, 222, 7827, 82, 0, 4246, 0, 6789, 0, 365, 0, 0, 0, 356, 0, 0, 0, 491, 0, 96, 0, 0, 0, 0, 0, 0, 0, 7123, 2652, 82, 83, 0, 383] payload_rows_per_dst=[13726, 12532, 15606, 7086, 6404, 10158, 14432, 6232] dup_rows_per_dst=[5978, 870, 3202, 448, 172, 970, 320, 168] combine_rows_per_src=[13726, 12532, 15606, 7086, 6404, 10158, 14432, 6232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93818 payload_rows_remote=84332 dup_rows=4486 dup_rows_remote=4334 dedup_ratio=0.045634 router_tokens_per_expert=[0, 0, 0, 317, 0, 0, 0, 0, 366, 63, 3, 0, 1014, 73, 0, 29, 0, 0, 0, 0, 0, 69, 0, 0, 9, 7927, 0, 1109, 61, 3564, 0, 0, 150, 4214, 0, 0, 0, 7370, 0, 455, 2496, 262, 73, 0, 387, 2026, 3027, 0, 0, 64, 0, 75, 0, 0, 0, 0, 1, 0, 6957, 6323, 593, 0, 0, 75] payload_rows_per_dst=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] dup_rows_per_dst=[634, 2888, 0, 460, 152, 10, 128, 214] combine_rows_per_src=[14740, 11254, 14052, 9026, 9486, 5486, 16004, 13770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=280420352 dispatch_meta_bytes=313192 combine_payload_bytes=280420352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280733544 ingress_bytes=349626368 link_bytes=349626368 token_bytes=561153896 weight_movement_bytes=69206016 total_nvlink_bytes=630359912 weight_to_token_ratio=0.123328 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315615876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=347447296 dispatch_meta_bytes=356912 combine_payload_bytes=347447296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=347804208 ingress_bytes=451256320 link_bytes=451256320 token_bytes=695251504 weight_movement_bytes=103809024 total_nvlink_bytes=799060528 weight_to_token_ratio=0.149311 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.698041203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216424448 dispatch_meta_bytes=260392 combine_payload_bytes=216424448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=216684840 ingress_bytes=320233472 link_bytes=320233472 token_bytes=433109288 weight_movement_bytes=103809024 total_nvlink_bytes=536918312 weight_to_token_ratio=0.239683 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205012774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89116 payload_rows_remote=79388 dup_rows=9188 dup_rows_remote=7680 dedup_ratio=0.093465 router_tokens_per_expert=[0, 0, 264, 113, 0, 7409, 0, 443, 0, 0, 0, 0, 82, 0, 0, 0, 2037, 0, 0, 0, 0, 3581, 0, 4605, 0, 0, 46, 7928, 0, 0, 0, 9, 839, 0, 0, 267, 0, 0, 195, 331, 284, 0, 0, 3096, 0, 0, 0, 0, 6665, 0, 1607, 0, 6048, 0, 0, 0, 0, 72, 0, 550, 842, 1757, 0, 82] payload_rows_per_dst=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] dup_rows_per_dst=[722, 0, 1508, 110, 1530, 4482, 0, 836] combine_rows_per_src=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291168256 dispatch_meta_bytes=338648 combine_payload_bytes=291168256 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=291506904 ingress_bytes=360374272 link_bytes=360374272 token_bytes=582675160 weight_movement_bytes=69206016 total_nvlink_bytes=651881176 weight_to_token_ratio=0.118773 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356059373e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=190177280 dispatch_meta_bytes=252264 combine_payload_bytes=190177280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=190429544 ingress_bytes=293986304 link_bytes=293986304 token_bytes=380606824 weight_movement_bytes=103809024 total_nvlink_bytes=484415848 weight_to_token_ratio=0.272746 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106246794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=216424448 dispatch_meta_bytes=260392 combine_payload_bytes=216424448 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=216684840 ingress_bytes=320233472 link_bytes=320233472 token_bytes=433109288 weight_movement_bytes=103809024 total_nvlink_bytes=536918312 weight_to_token_ratio=0.239683 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205012774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88630 payload_rows_remote=81742 dup_rows=9674 dup_rows_remote=9674 dedup_ratio=0.098409 router_tokens_per_expert=[0, 0, 247, 87, 0, 7396, 0, 393, 0, 0, 2, 0, 90, 0, 0, 0, 1701, 0, 0, 0, 0, 3368, 0, 4862, 0, 0, 37, 7945, 0, 0, 0, 1, 852, 0, 4, 337, 0, 0, 177, 148, 243, 0, 0, 3201, 0, 0, 0, 0, 6662, 0, 1465, 0, 6008, 0, 0, 0, 0, 161, 0, 551, 718, 2406, 0, 90] payload_rows_per_dst=[5620, 13504, 9014, 15890, 10994, 6888, 13264, 13456] dup_rows_per_dst=[204, 4, 1124, 76, 1766, 0, 1682, 4818] combine_rows_per_src=[5620, 13504, 9014, 15890, 10994, 6888, 13264, 13456] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=290824192 dispatch_meta_bytes=328776 combine_payload_bytes=290824192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291152968 ingress_bytes=325427200 link_bytes=325427200 token_bytes=581977160 weight_movement_bytes=34603008 total_nvlink_bytes=616580168 weight_to_token_ratio=0.059458 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224556355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=345423872 dispatch_meta_bytes=354664 combine_payload_bytes=345423872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=345778536 ingress_bytes=414629888 link_bytes=414629888 token_bytes=691202408 weight_movement_bytes=69206016 total_nvlink_bytes=760408424 weight_to_token_ratio=0.100124 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.560218888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325173248 dispatch_meta_bytes=348272 combine_payload_bytes=325173248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325521520 ingress_bytes=394379264 link_bytes=394379264 token_bytes=650694768 weight_movement_bytes=69206016 total_nvlink_bytes=719900784 weight_to_token_ratio=0.106357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484017420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=334815232 dispatch_meta_bytes=365664 combine_payload_bytes=334815232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=335180896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=669996128 weight_movement_bytes=34603008 total_nvlink_bytes=704599136 weight_to_token_ratio=0.051647 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=289054720 dispatch_meta_bytes=317984 combine_payload_bytes=289054720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289372704 ingress_bytes=323657728 link_bytes=323657728 token_bytes=578427424 weight_movement_bytes=34603008 total_nvlink_bytes=613030432 weight_to_token_ratio=0.059823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217897974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92454 payload_rows_remote=81182 dup_rows=5850 dup_rows_remote=4338 dedup_ratio=0.059509 router_tokens_per_expert=[0, 0, 246, 70, 0, 7471, 0, 419, 0, 0, 4, 0, 75, 0, 0, 0, 1826, 0, 0, 0, 0, 3175, 0, 4711, 0, 0, 35, 7946, 0, 0, 0, 1, 933, 0, 0, 208, 0, 0, 144, 396, 207, 0, 0, 3077, 0, 0, 0, 0, 6758, 0, 1513, 0, 6110, 0, 0, 0, 0, 64, 0, 533, 781, 2374, 0, 75] payload_rows_per_dst=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] dup_rows_per_dst=[686, 8, 1106, 72, 1512, 0, 1696, 770] combine_rows_per_src=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=61574 payload_rows_remote=45332 dup_rows=36730 dup_rows_remote=12290 dedup_ratio=0.373637 router_tokens_per_expert=[816, 91, 27, 154, 6989, 1, 7299, 4964, 1, 182, 0, 466, 3, 298, 0, 3372, 96, 341, 2881, 0, 379, 3, 361, 69, 0, 446, 93, 0, 0, 147, 1, 193, 6059, 2, 0, 91, 216, 1260, 2420, 2, 409, 0, 1, 2, 0, 3, 0, 228, 1060, 3, 1, 0, 0, 722, 75, 1, 4643, 0, 0, 926, 1212, 143, 0, 0] payload_rows_per_dst=[16242, 7914, 13334, 1574, 14018, 854, 3696, 3942] dup_rows_per_dst=[24440, 730, 4212, 186, 6082, 432, 28, 620] combine_rows_per_src=[16242, 7914, 13334, 1574, 14018, 854, 3696, 3942] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90508 payload_rows_remote=74646 dup_rows=7796 dup_rows_remote=7730 dedup_ratio=0.079305 router_tokens_per_expert=[0, 0, 261, 79, 0, 7497, 0, 414, 0, 0, 6, 0, 69, 0, 0, 0, 1730, 0, 0, 0, 0, 3481, 0, 4627, 0, 0, 30, 7931, 0, 0, 0, 3, 972, 0, 0, 226, 0, 0, 148, 532, 201, 0, 0, 2727, 0, 0, 0, 0, 6687, 0, 1656, 0, 6163, 1, 0, 0, 0, 78, 1, 512, 772, 2279, 0, 69] payload_rows_per_dst=[15812, 13512, 9152, 15862, 11404, 12476, 5684, 6606] dup_rows_per_dst=[690, 12, 1270, 66, 1606, 2864, 472, 816] combine_rows_per_src=[15812, 13512, 9152, 15862, 11404, 12476, 5684, 6606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332521472 dispatch_meta_bytes=342080 combine_payload_bytes=332521472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332863552 ingress_bytes=367124480 link_bytes=367124480 token_bytes=665385024 weight_movement_bytes=34603008 total_nvlink_bytes=699988032 weight_to_token_ratio=0.052004 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381459862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68898 payload_rows_remote=52788 dup_rows=29406 dup_rows_remote=23938 dedup_ratio=0.299133 router_tokens_per_expert=[0, 0, 254, 58, 0, 7378, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 1862, 0, 0, 0, 0, 3693, 0, 5067, 0, 0, 33, 7938, 0, 0, 0, 0, 783, 0, 3, 327, 0, 0, 152, 167, 194, 0, 0, 3185, 0, 0, 0, 0, 6673, 0, 1351, 0, 6095, 0, 0, 0, 0, 174, 0, 601, 761, 1779, 0, 96] payload_rows_per_dst=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] dup_rows_per_dst=[2, 0, 1292, 18, 1566, 0, 21060, 5468] combine_rows_per_src=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88550 payload_rows_remote=75004 dup_rows=9754 dup_rows_remote=9752 dedup_ratio=0.099223 router_tokens_per_expert=[0, 0, 239, 76, 0, 7497, 0, 419, 0, 0, 1, 0, 74, 0, 0, 0, 1848, 0, 0, 0, 0, 3509, 0, 4967, 0, 0, 33, 7953, 0, 0, 0, 6, 865, 0, 0, 230, 0, 0, 147, 271, 207, 0, 0, 3027, 0, 0, 0, 0, 6699, 0, 1596, 0, 6153, 0, 0, 0, 0, 82, 0, 512, 765, 1902, 0, 74] payload_rows_per_dst=[15766, 13546, 10364, 13962, 11310, 14528, 3192, 5882] dup_rows_per_dst=[696, 2, 2324, 48, 1650, 4246, 0, 788] combine_rows_per_src=[15766, 13546, 10364, 13962, 11310, 14528, 3192, 5882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305750016 dispatch_meta_bytes=329504 combine_payload_bytes=305750016 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=306079520 ingress_bytes=409559040 link_bytes=409559040 token_bytes=611829536 weight_movement_bytes=103809024 total_nvlink_bytes=715638560 weight_to_token_ratio=0.169670 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.541137695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81016 payload_rows_remote=66530 dup_rows=17288 dup_rows_remote=16694 dedup_ratio=0.175863 router_tokens_per_expert=[0, 0, 0, 297, 3, 0, 0, 4, 372, 55, 2, 3, 917, 90, 3, 70, 0, 0, 0, 0, 4, 69, 0, 0, 3, 7885, 0, 1016, 57, 3664, 2, 2, 182, 4156, 0, 0, 0, 7236, 1, 425, 2482, 339, 122, 0, 362, 2061, 3089, 2, 0, 140, 2, 91, 0, 3, 0, 0, 2, 3, 7012, 6256, 577, 0, 0, 91] payload_rows_per_dst=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] dup_rows_per_dst=[594, 2926, 4, 10410, 152, 3156, 2, 44] combine_rows_per_src=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68898 payload_rows_remote=52788 dup_rows=29406 dup_rows_remote=23938 dedup_ratio=0.299133 router_tokens_per_expert=[0, 0, 254, 58, 0, 7378, 0, 432, 0, 0, 0, 0, 96, 0, 0, 0, 1862, 0, 0, 0, 0, 3693, 0, 5067, 0, 0, 33, 7938, 0, 0, 0, 0, 783, 0, 3, 327, 0, 0, 152, 167, 194, 0, 0, 3185, 0, 0, 0, 0, 6673, 0, 1351, 0, 6095, 0, 0, 0, 0, 174, 0, 601, 761, 1779, 0, 96] payload_rows_per_dst=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] dup_rows_per_dst=[2, 0, 1292, 18, 1566, 0, 21060, 5468] combine_rows_per_src=[1486, 3288, 9818, 4388, 11432, 6758, 15618, 16110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87644 payload_rows_remote=73868 dup_rows=10660 dup_rows_remote=8812 dedup_ratio=0.108439 router_tokens_per_expert=[0, 0, 254, 92, 0, 7503, 0, 399, 0, 0, 2, 0, 65, 0, 0, 0, 1653, 0, 0, 0, 0, 3256, 0, 5117, 0, 0, 34, 7938, 0, 0, 0, 26, 891, 0, 0, 204, 0, 0, 155, 314, 215, 0, 0, 2885, 0, 0, 0, 0, 6790, 0, 1568, 0, 6244, 0, 0, 0, 0, 49, 0, 495, 728, 2210, 0, 65] payload_rows_per_dst=[1490, 13710, 8818, 15876, 11792, 6200, 13776, 15982] dup_rows_per_dst=[0, 4, 1000, 120, 1570, 0, 1848, 6118] combine_rows_per_src=[1490, 13710, 8818, 15876, 11792, 6200, 13776, 15982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87870 payload_rows_remote=73066 dup_rows=10434 dup_rows_remote=5134 dedup_ratio=0.106140 router_tokens_per_expert=[6104, 0, 0, 0, 66, 0, 0, 0, 0, 379, 0, 0, 0, 0, 606, 6560, 60, 7236, 0, 90, 0, 0, 60, 0, 7427, 0, 8, 247, 0, 6246, 90, 6754, 2230, 156, 0, 0, 0, 0, 0, 0, 89, 0, 519, 0, 0, 2890, 247, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 29, 570, 0, 0, 0, 0, 0] payload_rows_per_dst=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] dup_rows_per_dst=[0, 1500, 164, 0, 2902, 5300, 484, 84] combine_rows_per_src=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=185679872 dispatch_meta_bytes=230488 combine_payload_bytes=185679872 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=185910360 ingress_bytes=358694912 link_bytes=358694912 token_bytes=371590232 weight_movement_bytes=173015040 total_nvlink_bytes=544605272 weight_to_token_ratio=0.465607 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349740077e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92454 payload_rows_remote=81182 dup_rows=5850 dup_rows_remote=4338 dedup_ratio=0.059509 router_tokens_per_expert=[0, 0, 246, 70, 0, 7471, 0, 419, 0, 0, 4, 0, 75, 0, 0, 0, 1826, 0, 0, 0, 0, 3175, 0, 4711, 0, 0, 35, 7946, 0, 0, 0, 1, 933, 0, 0, 208, 0, 0, 144, 396, 207, 0, 0, 3077, 0, 0, 0, 0, 6758, 0, 1513, 0, 6110, 0, 0, 0, 0, 64, 0, 533, 781, 2374, 0, 75] payload_rows_per_dst=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] dup_rows_per_dst=[686, 8, 1106, 72, 1512, 0, 1696, 770] combine_rows_per_src=[15726, 13666, 8896, 15892, 11272, 6568, 13550, 6884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94156 payload_rows_remote=79468 dup_rows=4148 dup_rows_remote=3808 dedup_ratio=0.042196 router_tokens_per_expert=[5995, 0, 0, 0, 90, 0, 0, 0, 0, 435, 0, 0, 0, 0, 526, 6258, 44, 7344, 0, 82, 0, 0, 44, 0, 7460, 0, 9, 264, 0, 6353, 82, 6768, 2814, 172, 0, 0, 0, 0, 0, 0, 82, 0, 452, 0, 0, 2671, 264, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 116, 444, 0, 0, 0, 0, 0] payload_rows_per_dst=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] dup_rows_per_dst=[0, 1512, 340, 198, 344, 1008, 456, 290] combine_rows_per_src=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=305750016 dispatch_meta_bytes=329504 combine_payload_bytes=305750016 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=306079520 ingress_bytes=409559040 link_bytes=409559040 token_bytes=611829536 weight_movement_bytes=103809024 total_nvlink_bytes=715638560 weight_to_token_ratio=0.169670 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.541137695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=307216384 dispatch_meta_bytes=339024 combine_payload_bytes=307216384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307555408 ingress_bytes=341819392 link_bytes=341819392 token_bytes=614771792 weight_movement_bytes=34603008 total_nvlink_bytes=649374800 weight_to_token_ratio=0.056286 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286238853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91102 payload_rows_remote=77258 dup_rows=7202 dup_rows_remote=7202 dedup_ratio=0.073263 router_tokens_per_expert=[0, 0, 0, 257, 0, 0, 0, 0, 447, 90, 3, 0, 924, 73, 0, 33, 0, 0, 0, 0, 0, 113, 0, 0, 4, 7883, 0, 1154, 88, 3394, 0, 0, 164, 4180, 0, 0, 0, 7241, 0, 443, 2326, 319, 91, 0, 340, 2277, 3361, 0, 0, 72, 0, 82, 0, 0, 0, 0, 1, 0, 6809, 6243, 658, 0, 0, 82] payload_rows_per_dst=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] dup_rows_per_dst=[514, 3146, 0, 278, 188, 2802, 144, 130] combine_rows_per_src=[14482, 11368, 13844, 9002, 9386, 13684, 15930, 3406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=216219648 dispatch_meta_bytes=306904 combine_payload_bytes=216219648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=216526552 ingress_bytes=250822656 link_bytes=250822656 token_bytes=432746200 weight_movement_bytes=34603008 total_nvlink_bytes=467349208 weight_to_token_ratio=0.079961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.438254616e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=307216384 dispatch_meta_bytes=339024 combine_payload_bytes=307216384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307555408 ingress_bytes=341819392 link_bytes=341819392 token_bytes=614771792 weight_movement_bytes=34603008 total_nvlink_bytes=649374800 weight_to_token_ratio=0.056286 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286238853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272506880 dispatch_meta_bytes=332896 combine_payload_bytes=272506880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272839776 ingress_bytes=307109888 link_bytes=307109888 token_bytes=545346656 weight_movement_bytes=34603008 total_nvlink_bytes=579949664 weight_to_token_ratio=0.063451 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155629784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87870 payload_rows_remote=73066 dup_rows=10434 dup_rows_remote=5134 dedup_ratio=0.106140 router_tokens_per_expert=[6104, 0, 0, 0, 66, 0, 0, 0, 0, 379, 0, 0, 0, 0, 606, 6560, 60, 7236, 0, 90, 0, 0, 60, 0, 7427, 0, 8, 247, 0, 6246, 90, 6754, 2230, 156, 0, 0, 0, 0, 0, 0, 89, 0, 519, 0, 0, 2890, 247, 0, 0, 0, 0, 489, 0, 0, 0, 0, 0, 29, 570, 0, 0, 0, 0, 0] payload_rows_per_dst=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] dup_rows_per_dst=[0, 1500, 164, 0, 2902, 5300, 484, 84] combine_rows_per_src=[12340, 13590, 2114, 690, 14362, 14804, 15348, 14622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=302563328 dispatch_meta_bytes=330720 combine_payload_bytes=302563328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=302894048 ingress_bytes=406372352 link_bytes=406372352 token_bytes=605457376 weight_movement_bytes=103809024 total_nvlink_bytes=709266400 weight_to_token_ratio=0.171456 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529146445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332521472 dispatch_meta_bytes=342080 combine_payload_bytes=332521472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332863552 ingress_bytes=367124480 link_bytes=367124480 token_bytes=665385024 weight_movement_bytes=34603008 total_nvlink_bytes=699988032 weight_to_token_ratio=0.052004 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381459862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325500928 dispatch_meta_bytes=333104 combine_payload_bytes=325500928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325834032 ingress_bytes=394706944 link_bytes=394706944 token_bytes=651334960 weight_movement_bytes=69206016 total_nvlink_bytes=720540976 weight_to_token_ratio=0.106253 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485250454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299278336 dispatch_meta_bytes=312800 combine_payload_bytes=299278336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299591136 ingress_bytes=333881344 link_bytes=333881344 token_bytes=598869472 weight_movement_bytes=34603008 total_nvlink_bytes=633472480 weight_to_token_ratio=0.057781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91514 payload_rows_remote=77688 dup_rows=6790 dup_rows_remote=3624 dedup_ratio=0.069071 router_tokens_per_expert=[5625, 0, 0, 0, 63, 0, 0, 0, 0, 412, 0, 0, 17, 0, 599, 6421, 69, 7357, 0, 75, 0, 0, 69, 0, 7510, 0, 1, 246, 0, 6464, 75, 6746, 2809, 138, 0, 0, 0, 0, 0, 0, 75, 0, 407, 0, 0, 2811, 246, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 114, 482, 0, 0, 0, 0, 0] payload_rows_per_dst=[11376, 13332, 14714, 2438, 13826, 6134, 15282, 14412] dup_rows_per_dst=[0, 1566, 426, 36, 3166, 944, 380, 272] combine_rows_per_src=[11376, 13332, 14714, 2438, 13826, 6134, 15282, 14412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316448768 dispatch_meta_bytes=337840 combine_payload_bytes=316448768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316786608 ingress_bytes=351051776 link_bytes=351051776 token_bytes=633235376 weight_movement_bytes=34603008 total_nvlink_bytes=667838384 weight_to_token_ratio=0.054645 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.320979571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=318210048 dispatch_meta_bytes=325248 combine_payload_bytes=318210048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318535296 ingress_bytes=352813056 link_bytes=352813056 token_bytes=636745344 weight_movement_bytes=34603008 total_nvlink_bytes=671348352 weight_to_token_ratio=0.054344 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327607126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=62626 payload_rows_remote=46430 dup_rows=35678 dup_rows_remote=16636 dedup_ratio=0.362935 router_tokens_per_expert=[0, 8096, 4818, 0, 0, 54, 0, 233, 0, 0, 4, 0, 0, 1109, 1223, 145, 0, 91, 0, 663, 0, 3, 0, 1427, 140, 249, 0, 116, 0, 54, 231, 1, 168, 7891, 91, 1, 4055, 2, 6677, 1, 244, 1, 4, 0, 496, 2, 0, 2, 507, 1, 133, 0, 0, 0, 0, 3, 0, 1, 7160, 2473, 91, 99, 2, 390] payload_rows_per_dst=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] dup_rows_per_dst=[19042, 922, 1640, 280, 13418, 10, 184, 182] combine_rows_per_src=[16196, 12150, 8212, 1302, 16244, 1488, 1104, 5930] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94200 payload_rows_remote=80874 dup_rows=4104 dup_rows_remote=3900 dedup_ratio=0.041748 router_tokens_per_expert=[5323, 0, 0, 0, 53, 0, 0, 0, 0, 495, 0, 0, 38, 0, 591, 5964, 66, 7329, 0, 69, 0, 0, 66, 0, 7504, 0, 1, 261, 0, 6434, 69, 6654, 3909, 122, 0, 0, 0, 0, 0, 0, 69, 0, 426, 0, 0, 2513, 261, 0, 0, 0, 0, 334, 0, 0, 0, 0, 0, 112, 489, 0, 0, 0, 0, 0] payload_rows_per_dst=[10752, 12472, 14658, 13326, 7818, 5618, 15316, 14240] dup_rows_per_dst=[0, 1704, 402, 204, 244, 920, 360, 270] combine_rows_per_src=[10752, 12472, 14658, 13326, 7818, 5618, 15316, 14240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79424 payload_rows_remote=66114 dup_rows=18880 dup_rows_remote=17212 dedup_ratio=0.192057 router_tokens_per_expert=[6074, 0, 0, 0, 60, 0, 0, 0, 0, 428, 0, 0, 2, 0, 626, 6433, 92, 7320, 0, 74, 0, 0, 92, 0, 7466, 0, 2, 239, 0, 6303, 71, 6584, 2448, 131, 0, 0, 0, 0, 0, 0, 72, 0, 398, 0, 0, 2976, 239, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 121, 531, 0, 0, 0, 0, 0] payload_rows_per_dst=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] dup_rows_per_dst=[0, 1668, 516, 15286, 262, 916, 74, 158] combine_rows_per_src=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88020 payload_rows_remote=74106 dup_rows=10284 dup_rows_remote=10050 dedup_ratio=0.104614 router_tokens_per_expert=[6223, 0, 0, 0, 42, 0, 0, 0, 0, 416, 0, 0, 2, 0, 486, 6457, 72, 7301, 0, 96, 0, 0, 72, 0, 7419, 0, 8, 254, 0, 6399, 95, 6425, 2515, 137, 0, 0, 0, 0, 0, 0, 94, 0, 457, 0, 0, 2839, 254, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 86, 563, 0, 0, 0, 0, 0] payload_rows_per_dst=[12530, 1808, 13106, 714, 14620, 16034, 15294, 13914] dup_rows_per_dst=[0, 0, 288, 0, 3482, 5856, 424, 234] combine_rows_per_src=[12530, 1808, 13106, 714, 14620, 16034, 15294, 13914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86774 payload_rows_remote=71458 dup_rows=11530 dup_rows_remote=11032 dedup_ratio=0.117289 router_tokens_per_expert=[5564, 0, 0, 0, 60, 0, 0, 0, 0, 408, 0, 0, 4, 0, 561, 6185, 75, 7391, 0, 65, 0, 0, 75, 0, 7539, 0, 2, 254, 0, 6472, 63, 6736, 3188, 123, 0, 0, 0, 0, 0, 0, 65, 0, 389, 0, 0, 2753, 254, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 105, 453, 0, 0, 0, 0, 0] payload_rows_per_dst=[11248, 6838, 6968, 638, 15146, 16286, 15316, 14334] dup_rows_per_dst=[0, 708, 232, 0, 4420, 5418, 498, 254] combine_rows_per_src=[11248, 6838, 6968, 638, 15146, 16286, 15316, 14334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79424 payload_rows_remote=66114 dup_rows=18880 dup_rows_remote=17212 dedup_ratio=0.192057 router_tokens_per_expert=[6074, 0, 0, 0, 60, 0, 0, 0, 0, 428, 0, 0, 2, 0, 626, 6433, 92, 7320, 0, 74, 0, 0, 92, 0, 7466, 0, 2, 239, 0, 6303, 71, 6584, 2448, 131, 0, 0, 0, 0, 0, 0, 72, 0, 398, 0, 0, 2976, 239, 0, 0, 0, 0, 370, 0, 0, 0, 0, 0, 121, 531, 0, 0, 0, 0, 0] payload_rows_per_dst=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] dup_rows_per_dst=[0, 1668, 516, 15286, 262, 916, 74, 158] combine_rows_per_src=[12268, 13310, 14640, 15264, 4896, 6454, 3062, 9530] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=73086 dup_rows=10020 dup_rows_remote=5334 dedup_ratio=0.101929 router_tokens_per_expert=[0, 0, 30, 154, 0, 0, 3103, 0, 0, 1721, 7404, 0, 0, 0, 759, 0, 963, 361, 0, 87, 4158, 0, 0, 0, 785, 0, 0, 171, 0, 0, 0, 0, 0, 2, 0, 0, 7430, 377, 0, 44, 896, 0, 1642, 0, 0, 0, 0, 0, 1068, 0, 90, 0, 7865, 0, 76, 0, 2336, 0, 0, 0, 6661, 494, 475, 0] payload_rows_per_dst=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] dup_rows_per_dst=[142, 0, 1088, 948, 662, 4686, 1956, 538] combine_rows_per_src=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=60116 dup_rows=22258 dup_rows_remote=14520 dedup_ratio=0.226420 router_tokens_per_expert=[1, 0, 249, 71, 0, 7364, 2, 434, 0, 0, 7, 0, 92, 4, 0, 0, 1715, 0, 0, 0, 0, 3038, 2, 4605, 0, 0, 22, 7936, 0, 2, 0, 10, 871, 9, 0, 300, 0, 2, 160, 498, 214, 3, 7, 3034, 0, 0, 4, 0, 6674, 0, 1623, 0, 6198, 0, 0, 1, 0, 141, 0, 538, 764, 2463, 1, 93] payload_rows_per_dst=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] dup_rows_per_dst=[7738, 16, 7694, 2, 1448, 4400, 0, 960] combine_rows_per_src=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=190177280 dispatch_meta_bytes=252264 combine_payload_bytes=190177280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=190429544 ingress_bytes=293986304 link_bytes=293986304 token_bytes=380606824 weight_movement_bytes=103809024 total_nvlink_bytes=484415848 weight_to_token_ratio=0.272746 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.106246794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91514 payload_rows_remote=77688 dup_rows=6790 dup_rows_remote=3624 dedup_ratio=0.069071 router_tokens_per_expert=[5625, 0, 0, 0, 63, 0, 0, 0, 0, 412, 0, 0, 17, 0, 599, 6421, 69, 7357, 0, 75, 0, 0, 69, 0, 7510, 0, 1, 246, 0, 6464, 75, 6746, 2809, 138, 0, 0, 0, 0, 0, 0, 75, 0, 407, 0, 0, 2811, 246, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 114, 482, 0, 0, 0, 0, 0] payload_rows_per_dst=[11376, 13332, 14714, 2438, 13826, 6134, 15282, 14412] dup_rows_per_dst=[0, 1566, 426, 36, 3166, 944, 380, 272] combine_rows_per_src=[11376, 13332, 14714, 2438, 13826, 6134, 15282, 14412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89116 payload_rows_remote=79388 dup_rows=9188 dup_rows_remote=7680 dedup_ratio=0.093465 router_tokens_per_expert=[0, 0, 264, 113, 0, 7409, 0, 443, 0, 0, 0, 0, 82, 0, 0, 0, 2037, 0, 0, 0, 0, 3581, 0, 4605, 0, 0, 46, 7928, 0, 0, 0, 9, 839, 0, 0, 267, 0, 0, 195, 331, 284, 0, 0, 3096, 0, 0, 0, 0, 6665, 0, 1607, 0, 6048, 0, 0, 0, 0, 72, 0, 550, 842, 1757, 0, 82] payload_rows_per_dst=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] dup_rows_per_dst=[722, 0, 1508, 110, 1530, 4482, 0, 836] combine_rows_per_src=[15736, 13494, 9728, 15856, 10944, 14374, 3214, 5770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=72474 dup_rows=16202 dup_rows_remote=15230 dedup_ratio=0.164815 router_tokens_per_expert=[0, 0, 31, 129, 0, 0, 3347, 0, 0, 2087, 7034, 0, 0, 0, 700, 0, 764, 414, 0, 113, 4009, 0, 0, 0, 663, 0, 0, 314, 0, 0, 0, 0, 0, 13, 0, 0, 7537, 316, 0, 78, 652, 0, 1773, 0, 0, 0, 0, 0, 1191, 0, 82, 0, 7871, 0, 111, 0, 2144, 0, 0, 0, 6902, 546, 331, 0] payload_rows_per_dst=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] dup_rows_per_dst=[5330, 4768, 972, 1542, 3338, 252, 0, 0] combine_rows_per_src=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331259904 dispatch_meta_bytes=339096 combine_payload_bytes=331259904 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=331599000 ingress_bytes=504274944 link_bytes=504274944 token_bytes=662858904 weight_movement_bytes=173015040 total_nvlink_bytes=835873944 weight_to_token_ratio=0.261013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.897546017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=333304 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271136248 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541939192 weight_movement_bytes=69206016 total_nvlink_bytes=611145208 weight_to_token_ratio=0.127701 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=73086 dup_rows=10020 dup_rows_remote=5334 dedup_ratio=0.101929 router_tokens_per_expert=[0, 0, 30, 154, 0, 0, 3103, 0, 0, 1721, 7404, 0, 0, 0, 759, 0, 963, 361, 0, 87, 4158, 0, 0, 0, 785, 0, 0, 171, 0, 0, 0, 0, 0, 2, 0, 0, 7430, 377, 0, 44, 896, 0, 1642, 0, 0, 0, 0, 0, 1068, 0, 90, 0, 7865, 0, 76, 0, 2336, 0, 0, 0, 6661, 494, 475, 0] payload_rows_per_dst=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] dup_rows_per_dst=[142, 0, 1088, 948, 662, 4686, 1956, 538] combine_rows_per_src=[6432, 4960, 10050, 12708, 15044, 15198, 16242, 7650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=303538176 dispatch_meta_bytes=336624 combine_payload_bytes=303538176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303874800 ingress_bytes=338141184 link_bytes=338141184 token_bytes=607412976 weight_movement_bytes=34603008 total_nvlink_bytes=642015984 weight_to_token_ratio=0.056968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272398053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292691968 dispatch_meta_bytes=329960 combine_payload_bytes=292691968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293021928 ingress_bytes=327294976 link_bytes=327294976 token_bytes=585713896 weight_movement_bytes=34603008 total_nvlink_bytes=620316904 weight_to_token_ratio=0.059078 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231584645e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=270802944 dispatch_meta_bytes=333304 combine_payload_bytes=270802944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271136248 ingress_bytes=340008960 link_bytes=340008960 token_bytes=541939192 weight_movement_bytes=69206016 total_nvlink_bytes=611145208 weight_to_token_ratio=0.127701 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279426344e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=318210048 dispatch_meta_bytes=325248 combine_payload_bytes=318210048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318535296 ingress_bytes=352813056 link_bytes=352813056 token_bytes=636745344 weight_movement_bytes=34603008 total_nvlink_bytes=671348352 weight_to_token_ratio=0.054344 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327607126e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=296853504 dispatch_meta_bytes=350816 combine_payload_bytes=296853504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297204320 ingress_bytes=331456512 link_bytes=331456512 token_bytes=594057824 weight_movement_bytes=34603008 total_nvlink_bytes=628660832 weight_to_token_ratio=0.058249 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247244170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299360256 dispatch_meta_bytes=313680 combine_payload_bytes=299360256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299673936 ingress_bytes=333963264 link_bytes=333963264 token_bytes=599034192 weight_movement_bytes=34603008 total_nvlink_bytes=633637200 weight_to_token_ratio=0.057765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256676876e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=246235136 dispatch_meta_bytes=298544 combine_payload_bytes=246235136 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=246533680 ingress_bytes=315441152 link_bytes=315441152 token_bytes=492768816 weight_movement_bytes=69206016 total_nvlink_bytes=561974832 weight_to_token_ratio=0.140443 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186979660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=331259904 dispatch_meta_bytes=339096 combine_payload_bytes=331259904 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=331599000 ingress_bytes=504274944 link_bytes=504274944 token_bytes=662858904 weight_movement_bytes=173015040 total_nvlink_bytes=835873944 weight_to_token_ratio=0.261013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.897546017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88294 payload_rows_remote=73126 dup_rows=10010 dup_rows_remote=9238 dedup_ratio=0.101827 router_tokens_per_expert=[0, 0, 28, 178, 0, 1, 3249, 0, 0, 2064, 7247, 0, 0, 0, 743, 0, 754, 347, 0, 70, 3673, 0, 0, 0, 909, 0, 0, 309, 0, 0, 0, 0, 0, 2, 0, 0, 7509, 384, 0, 75, 733, 0, 1711, 0, 0, 0, 0, 0, 1161, 0, 75, 0, 7904, 0, 41, 0, 2175, 0, 0, 0, 6918, 560, 332, 0] payload_rows_per_dst=[6716, 5614, 8956, 14388, 15168, 15124, 16194, 6134] dup_rows_per_dst=[196, 0, 732, 1884, 772, 4258, 2168, 0] combine_rows_per_src=[6716, 5614, 8956, 14388, 15168, 15124, 16194, 6134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325173248 dispatch_meta_bytes=348272 combine_payload_bytes=325173248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325521520 ingress_bytes=394379264 link_bytes=394379264 token_bytes=650694768 weight_movement_bytes=69206016 total_nvlink_bytes=719900784 weight_to_token_ratio=0.106357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484017420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299524096 dispatch_meta_bytes=329456 combine_payload_bytes=299524096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299853552 ingress_bytes=403333120 link_bytes=403333120 token_bytes=599377648 weight_movement_bytes=103809024 total_nvlink_bytes=703186672 weight_to_token_ratio=0.173195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517710060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82852 payload_rows_remote=66614 dup_rows=15452 dup_rows_remote=13298 dedup_ratio=0.157186 router_tokens_per_expert=[0, 0, 18, 139, 0, 1, 2994, 0, 0, 2111, 6934, 0, 0, 0, 704, 0, 760, 364, 0, 92, 4114, 7, 0, 0, 810, 0, 0, 240, 0, 0, 0, 0, 0, 13, 0, 0, 7556, 335, 0, 63, 769, 0, 1780, 0, 0, 0, 0, 0, 1144, 0, 63, 0, 7897, 0, 92, 0, 2263, 0, 0, 0, 7154, 523, 212, 0] payload_rows_per_dst=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] dup_rows_per_dst=[154, 0, 858, 0, 542, 4186, 2154, 7558] combine_rows_per_src=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81016 payload_rows_remote=66530 dup_rows=17288 dup_rows_remote=16694 dedup_ratio=0.175863 router_tokens_per_expert=[0, 0, 0, 297, 3, 0, 0, 4, 372, 55, 2, 3, 917, 90, 3, 70, 0, 0, 0, 0, 4, 69, 0, 0, 3, 7885, 0, 1016, 57, 3664, 2, 2, 182, 4156, 0, 0, 0, 7236, 1, 425, 2482, 339, 122, 0, 362, 2061, 3089, 2, 0, 140, 2, 91, 0, 3, 0, 0, 2, 3, 7012, 6256, 577, 0, 0, 91] payload_rows_per_dst=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] dup_rows_per_dst=[594, 2926, 4, 10410, 152, 3156, 2, 44] combine_rows_per_src=[14486, 11240, 14166, 16218, 8006, 15128, 470, 1302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84684 payload_rows_remote=69994 dup_rows=13620 dup_rows_remote=11652 dedup_ratio=0.138550 router_tokens_per_expert=[0, 0, 51, 152, 0, 3, 3357, 0, 0, 2268, 6850, 0, 0, 0, 761, 0, 670, 381, 0, 81, 3305, 2, 0, 0, 824, 0, 0, 328, 0, 0, 0, 0, 0, 32, 0, 0, 7525, 286, 0, 69, 601, 0, 1945, 0, 0, 0, 0, 0, 1247, 0, 69, 0, 7909, 0, 71, 0, 2334, 0, 0, 0, 7177, 528, 326, 0] payload_rows_per_dst=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] dup_rows_per_dst=[148, 4170, 554, 1968, 4138, 858, 1784, 0] combine_rows_per_src=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83228 payload_rows_remote=67902 dup_rows=15076 dup_rows_remote=10386 dedup_ratio=0.153361 router_tokens_per_expert=[0, 0, 41, 172, 0, 0, 2992, 0, 0, 1893, 7350, 0, 0, 0, 765, 0, 862, 358, 0, 76, 3905, 0, 0, 0, 733, 0, 0, 274, 0, 0, 0, 0, 0, 7, 0, 0, 7506, 339, 0, 62, 837, 0, 1884, 0, 0, 0, 0, 0, 998, 0, 71, 0, 7879, 0, 71, 0, 2352, 0, 0, 0, 6616, 546, 563, 0] payload_rows_per_dst=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] dup_rows_per_dst=[5934, 4690, 2214, 1328, 584, 326, 0, 0] combine_rows_per_src=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70348 payload_rows_remote=54054 dup_rows=27956 dup_rows_remote=9302 dedup_ratio=0.284383 router_tokens_per_expert=[0, 0, 45, 183, 0, 1, 3233, 0, 0, 1765, 7351, 0, 0, 0, 718, 0, 729, 337, 0, 57, 4254, 0, 0, 0, 702, 0, 0, 275, 0, 0, 0, 0, 0, 2, 0, 0, 7433, 430, 0, 48, 787, 0, 1775, 0, 0, 0, 0, 0, 981, 0, 95, 0, 7870, 0, 40, 0, 2426, 0, 0, 0, 6569, 488, 558, 0] payload_rows_per_dst=[6746, 4966, 9944, 1954, 956, 13286, 16202, 16294] dup_rows_per_dst=[178, 0, 810, 0, 4, 3778, 4532, 18654] combine_rows_per_src=[6746, 4966, 9944, 1954, 956, 13286, 16202, 16294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=272506880 dispatch_meta_bytes=332896 combine_payload_bytes=272506880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272839776 ingress_bytes=307109888 link_bytes=307109888 token_bytes=545346656 weight_movement_bytes=34603008 total_nvlink_bytes=579949664 weight_to_token_ratio=0.063451 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155629784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82852 payload_rows_remote=66614 dup_rows=15452 dup_rows_remote=13298 dedup_ratio=0.157186 router_tokens_per_expert=[0, 0, 18, 139, 0, 1, 2994, 0, 0, 2111, 6934, 0, 0, 0, 704, 0, 760, 364, 0, 92, 4114, 7, 0, 0, 810, 0, 0, 240, 0, 0, 0, 0, 0, 13, 0, 0, 7556, 335, 0, 63, 769, 0, 1780, 0, 0, 0, 0, 0, 1144, 0, 63, 0, 7897, 0, 92, 0, 2263, 0, 0, 0, 7154, 523, 212, 0] payload_rows_per_dst=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] dup_rows_per_dst=[154, 0, 858, 0, 542, 4186, 2154, 7558] combine_rows_per_src=[6150, 5630, 9816, 2100, 12246, 14780, 16238, 15892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88294 payload_rows_remote=73126 dup_rows=10010 dup_rows_remote=9238 dedup_ratio=0.101827 router_tokens_per_expert=[0, 0, 28, 178, 0, 1, 3249, 0, 0, 2064, 7247, 0, 0, 0, 743, 0, 754, 347, 0, 70, 3673, 0, 0, 0, 909, 0, 0, 309, 0, 0, 0, 0, 0, 2, 0, 0, 7509, 384, 0, 75, 733, 0, 1711, 0, 0, 0, 0, 0, 1161, 0, 75, 0, 7904, 0, 41, 0, 2175, 0, 0, 0, 6918, 560, 332, 0] payload_rows_per_dst=[6716, 5614, 8956, 14388, 15168, 15124, 16194, 6134] dup_rows_per_dst=[196, 0, 732, 1884, 772, 4258, 2168, 0] combine_rows_per_src=[6716, 5614, 8956, 14388, 15168, 15124, 16194, 6134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286695424 dispatch_meta_bytes=326584 combine_payload_bytes=286695424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287022008 ingress_bytes=321298432 link_bytes=321298432 token_bytes=573717432 weight_movement_bytes=34603008 total_nvlink_bytes=608320440 weight_to_token_ratio=0.060314 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209020133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90708 payload_rows_remote=79046 dup_rows=7596 dup_rows_remote=5802 dedup_ratio=0.077271 router_tokens_per_expert=[591, 0, 0, 82, 346, 5039, 0, 0, 5561, 40, 172, 0, 0, 0, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 130, 0, 0, 0, 2, 82, 0, 0, 0, 0, 72, 0, 0, 0, 575, 264, 0, 4472, 7545, 8102, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 337, 7115, 3404, 0, 0, 0, 659, 2224, 0, 0] payload_rows_per_dst=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] dup_rows_per_dst=[356, 0, 1794, 428, 1534, 0, 674, 2810] combine_rows_per_src=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278126592 dispatch_meta_bytes=313152 combine_payload_bytes=278126592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278439744 ingress_bytes=347332608 link_bytes=347332608 token_bytes=556566336 weight_movement_bytes=69206016 total_nvlink_bytes=625772352 weight_to_token_ratio=0.124345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306984641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87780 payload_rows_remote=78936 dup_rows=10524 dup_rows_remote=7788 dedup_ratio=0.107056 router_tokens_per_expert=[626, 0, 0, 90, 336, 4735, 0, 0, 4788, 1, 156, 0, 0, 0, 0, 0, 0, 0, 2828, 0, 0, 0, 0, 0, 150, 0, 0, 0, 0, 90, 0, 0, 0, 0, 161, 0, 0, 0, 706, 247, 0, 4637, 7608, 8126, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 389, 7111, 3714, 0, 0, 0, 581, 1978, 0, 4] payload_rows_per_dst=[11254, 9890, 12274, 13404, 7660, 8844, 14402, 10052] dup_rows_per_dst=[320, 0, 2656, 426, 1106, 2736, 778, 2502] combine_rows_per_src=[11254, 9890, 12274, 13404, 7660, 8844, 14402, 10052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71472 payload_rows_remote=58512 dup_rows=26832 dup_rows_remote=24172 dedup_ratio=0.272949 router_tokens_per_expert=[5617, 0, 0, 0, 54, 0, 0, 0, 1, 445, 0, 0, 8, 0, 542, 6226, 70, 7291, 1, 91, 0, 0, 71, 0, 7419, 5, 1, 249, 3, 6260, 92, 6719, 3447, 145, 1, 0, 1, 0, 0, 0, 91, 0, 371, 5, 0, 2620, 249, 5, 0, 0, 0, 389, 1, 0, 0, 0, 4, 129, 523, 0, 2, 4, 0, 0] payload_rows_per_dst=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] dup_rows_per_dst=[2660, 2328, 458, 20248, 290, 842, 0, 6] combine_rows_per_src=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83228 payload_rows_remote=67902 dup_rows=15076 dup_rows_remote=10386 dedup_ratio=0.153361 router_tokens_per_expert=[0, 0, 41, 172, 0, 0, 2992, 0, 0, 1893, 7350, 0, 0, 0, 765, 0, 862, 358, 0, 76, 3905, 0, 0, 0, 733, 0, 0, 274, 0, 0, 0, 0, 0, 7, 0, 0, 7506, 339, 0, 62, 837, 0, 1884, 0, 0, 0, 0, 0, 998, 0, 71, 0, 7879, 0, 71, 0, 2352, 0, 0, 0, 6616, 546, 563, 0] payload_rows_per_dst=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] dup_rows_per_dst=[5934, 4690, 2214, 1328, 584, 326, 0, 0] combine_rows_per_src=[16234, 15326, 10450, 13918, 12982, 5116, 2280, 6922] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84684 payload_rows_remote=69994 dup_rows=13620 dup_rows_remote=11652 dedup_ratio=0.138550 router_tokens_per_expert=[0, 0, 51, 152, 0, 3, 3357, 0, 0, 2268, 6850, 0, 0, 0, 761, 0, 670, 381, 0, 81, 3305, 2, 0, 0, 824, 0, 0, 328, 0, 0, 0, 0, 0, 32, 0, 0, 7525, 286, 0, 69, 601, 0, 1945, 0, 0, 0, 0, 0, 1247, 0, 69, 0, 7909, 0, 71, 0, 2334, 0, 0, 0, 7177, 528, 326, 0] payload_rows_per_dst=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] dup_rows_per_dst=[148, 4170, 554, 1968, 4138, 858, 1784, 0] combine_rows_per_src=[6978, 13194, 8324, 14690, 15388, 6628, 13106, 6376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=221405184 dispatch_meta_bytes=253424 combine_payload_bytes=221405184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=221658608 ingress_bytes=290611200 link_bytes=290611200 token_bytes=443063792 weight_movement_bytes=69206016 total_nvlink_bytes=512269808 weight_to_token_ratio=0.156199 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.093546549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272850944 dispatch_meta_bytes=319648 combine_payload_bytes=272850944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273170592 ingress_bytes=376659968 link_bytes=376659968 token_bytes=546021536 weight_movement_bytes=103809024 total_nvlink_bytes=649830560 weight_to_token_ratio=0.190119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417341136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94156 payload_rows_remote=79468 dup_rows=4148 dup_rows_remote=3808 dedup_ratio=0.042196 router_tokens_per_expert=[5995, 0, 0, 0, 90, 0, 0, 0, 0, 435, 0, 0, 0, 0, 526, 6258, 44, 7344, 0, 82, 0, 0, 44, 0, 7460, 0, 9, 264, 0, 6353, 82, 6768, 2814, 172, 0, 0, 0, 0, 0, 0, 82, 0, 452, 0, 0, 2671, 264, 0, 0, 0, 0, 383, 0, 0, 0, 0, 0, 116, 444, 0, 0, 0, 0, 0] payload_rows_per_dst=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] dup_rows_per_dst=[0, 1512, 340, 198, 344, 1008, 456, 290] combine_rows_per_src=[12170, 12926, 14688, 13218, 5628, 5930, 15230, 14366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=239665152 dispatch_meta_bytes=330736 combine_payload_bytes=239665152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239995888 ingress_bytes=274268160 link_bytes=274268160 token_bytes=479661040 weight_movement_bytes=34603008 total_nvlink_bytes=514264048 weight_to_token_ratio=0.072141 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032049006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=299524096 dispatch_meta_bytes=329456 combine_payload_bytes=299524096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=299853552 ingress_bytes=403333120 link_bytes=403333120 token_bytes=599377648 weight_movement_bytes=103809024 total_nvlink_bytes=703186672 weight_to_token_ratio=0.173195 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517710060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=339392 combine_payload_bytes=323772416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324111808 ingress_bytes=358375424 link_bytes=358375424 token_bytes=647884224 weight_movement_bytes=34603008 total_nvlink_bytes=682487232 weight_to_token_ratio=0.053409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348537869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323321856 dispatch_meta_bytes=346896 combine_payload_bytes=323321856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=323668752 ingress_bytes=461733888 link_bytes=461733888 token_bytes=646990608 weight_movement_bytes=138412032 total_nvlink_bytes=785402640 weight_to_token_ratio=0.213932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.737467448e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=278126592 dispatch_meta_bytes=313152 combine_payload_bytes=278126592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=278439744 ingress_bytes=347332608 link_bytes=347332608 token_bytes=556566336 weight_movement_bytes=69206016 total_nvlink_bytes=625772352 weight_to_token_ratio=0.124345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306984641e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286695424 dispatch_meta_bytes=326584 combine_payload_bytes=286695424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287022008 ingress_bytes=321298432 link_bytes=321298432 token_bytes=573717432 weight_movement_bytes=34603008 total_nvlink_bytes=608320440 weight_to_token_ratio=0.060314 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209020133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90658 payload_rows_remote=75326 dup_rows=7646 dup_rows_remote=6046 dedup_ratio=0.077779 router_tokens_per_expert=[528, 0, 0, 75, 320, 4939, 0, 0, 5320, 31, 138, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 0, 1, 0, 0, 149, 0, 0, 0, 0, 75, 0, 0, 0, 0, 65, 0, 0, 0, 630, 246, 0, 4713, 7525, 8129, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 387, 7177, 3554, 0, 0, 0, 628, 2292, 0, 1] payload_rows_per_dst=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] dup_rows_per_dst=[282, 0, 1736, 448, 1600, 0, 774, 2806] combine_rows_per_src=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325500928 dispatch_meta_bytes=333104 combine_payload_bytes=325500928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325834032 ingress_bytes=394706944 link_bytes=394706944 token_bytes=651334960 weight_movement_bytes=69206016 total_nvlink_bytes=720540976 weight_to_token_ratio=0.106253 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485250454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=308535296 dispatch_meta_bytes=325488 combine_payload_bytes=308535296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308860784 ingress_bytes=343138304 link_bytes=343138304 token_bytes=617396080 weight_movement_bytes=34603008 total_nvlink_bytes=651999088 weight_to_token_ratio=0.056047 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291201813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76046 payload_rows_remote=60116 dup_rows=22258 dup_rows_remote=14520 dedup_ratio=0.226420 router_tokens_per_expert=[1, 0, 249, 71, 0, 7364, 2, 434, 0, 0, 7, 0, 92, 4, 0, 0, 1715, 0, 0, 0, 0, 3038, 2, 4605, 0, 0, 22, 7936, 0, 2, 0, 10, 871, 9, 0, 300, 0, 2, 160, 498, 214, 3, 7, 3034, 0, 0, 4, 0, 6674, 0, 1623, 0, 6198, 0, 0, 1, 0, 141, 0, 538, 764, 2463, 1, 93] payload_rows_per_dst=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] dup_rows_per_dst=[7738, 16, 7694, 2, 1448, 4400, 0, 960] combine_rows_per_src=[15930, 13538, 15196, 66, 6508, 14520, 3248, 7040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91062 payload_rows_remote=74784 dup_rows=7242 dup_rows_remote=6830 dedup_ratio=0.073669 router_tokens_per_expert=[503, 0, 0, 69, 330, 5159, 0, 0, 5509, 67, 153, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 2, 0, 0, 137, 0, 0, 0, 0, 69, 0, 0, 0, 0, 78, 0, 0, 0, 749, 261, 0, 4563, 7534, 8139, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 7175, 3239, 0, 0, 0, 606, 2635, 0, 0] payload_rows_per_dst=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] dup_rows_per_dst=[256, 0, 1202, 412, 1860, 0, 804, 2708] combine_rows_per_src=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90592 payload_rows_remote=79378 dup_rows=7712 dup_rows_remote=7712 dedup_ratio=0.078451 router_tokens_per_expert=[523, 0, 0, 74, 311, 4564, 0, 0, 5447, 29, 131, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 74, 0, 0, 0, 0, 81, 0, 0, 0, 660, 239, 0, 4618, 7546, 8132, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 420, 7176, 3705, 0, 0, 0, 621, 2045, 0, 0] payload_rows_per_dst=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] dup_rows_per_dst=[286, 0, 2190, 474, 1708, 0, 416, 2638] combine_rows_per_src=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=246235136 dispatch_meta_bytes=298544 combine_payload_bytes=246235136 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=246533680 ingress_bytes=315441152 link_bytes=315441152 token_bytes=492768816 weight_movement_bytes=69206016 total_nvlink_bytes=561974832 weight_to_token_ratio=0.140443 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186979660e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306315264 dispatch_meta_bytes=326456 combine_payload_bytes=306315264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306641720 ingress_bytes=340918272 link_bytes=340918272 token_bytes=612956984 weight_movement_bytes=34603008 total_nvlink_bytes=647559992 weight_to_token_ratio=0.056453 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282848011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=348360 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325480648 ingress_bytes=359735296 link_bytes=359735296 token_bytes=650612936 weight_movement_bytes=34603008 total_nvlink_bytes=685215944 weight_to_token_ratio=0.053185 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353654958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69910 payload_rows_remote=59160 dup_rows=28394 dup_rows_remote=24908 dedup_ratio=0.288839 router_tokens_per_expert=[562, 0, 0, 96, 349, 4612, 0, 0, 5597, 9, 136, 0, 0, 0, 0, 0, 0, 0, 2539, 0, 0, 0, 0, 0, 167, 0, 0, 0, 0, 96, 0, 0, 0, 0, 174, 0, 0, 0, 665, 254, 0, 4349, 7545, 8151, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 326, 7123, 3561, 0, 0, 0, 545, 2200, 0, 0] payload_rows_per_dst=[10948, 11484, 5078, 526, 1836, 16384, 12904, 10750] dup_rows_per_dst=[290, 0, 0, 0, 350, 23706, 562, 3486] combine_rows_per_src=[10948, 11484, 5078, 526, 1836, 16384, 12904, 10750] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75430 payload_rows_remote=60888 dup_rows=22874 dup_rows_remote=22126 dedup_ratio=0.232686 router_tokens_per_expert=[500, 0, 0, 65, 319, 4793, 0, 0, 5474, 59, 161, 0, 0, 0, 0, 0, 0, 0, 1955, 0, 0, 5, 0, 0, 141, 0, 0, 0, 1, 65, 0, 0, 0, 0, 48, 0, 0, 0, 720, 254, 0, 4546, 7601, 8132, 0, 0, 0, 0, 0, 0, 0, 65, 0, 0, 374, 7206, 3750, 0, 0, 0, 614, 2304, 0, 0] payload_rows_per_dst=[11100, 11388, 9224, 414, 1932, 16304, 14542, 10526] dup_rows_per_dst=[254, 0, 958, 0, 112, 17992, 748, 2810] combine_rows_per_src=[11100, 11388, 9224, 414, 1932, 16304, 14542, 10526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90658 payload_rows_remote=75326 dup_rows=7646 dup_rows_remote=6046 dedup_ratio=0.077779 router_tokens_per_expert=[528, 0, 0, 75, 320, 4939, 0, 0, 5320, 31, 138, 0, 0, 0, 0, 0, 0, 0, 2154, 0, 0, 1, 0, 0, 149, 0, 0, 0, 0, 75, 0, 0, 0, 0, 65, 0, 0, 0, 630, 246, 0, 4713, 7525, 8129, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 387, 7177, 3554, 0, 0, 0, 628, 2292, 0, 1] payload_rows_per_dst=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] dup_rows_per_dst=[282, 0, 1736, 448, 1600, 0, 774, 2806] combine_rows_per_src=[11442, 10978, 12000, 16258, 15332, 0, 14504, 10144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91580 payload_rows_remote=76524 dup_rows=6724 dup_rows_remote=5664 dedup_ratio=0.068400 router_tokens_per_expert=[787, 350, 0, 0, 0, 0, 1339, 0, 0, 272, 0, 253, 0, 0, 262, 0, 0, 525, 0, 0, 0, 626, 0, 0, 0, 0, 7764, 163, 88, 0, 90, 7559, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 6907, 203, 7835, 83, 7414, 0, 5839, 0, 0, 279, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] payload_rows_per_dst=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] dup_rows_per_dst=[4118, 886, 1060, 0, 68, 0, 504, 88] combine_rows_per_src=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76194 payload_rows_remote=60002 dup_rows=22110 dup_rows_remote=15882 dedup_ratio=0.224915 router_tokens_per_expert=[1, 0, 88, 171, 1, 3, 3082, 0, 0, 2469, 6865, 0, 4, 6, 712, 0, 594, 362, 4, 69, 3494, 2, 0, 0, 1039, 0, 0, 339, 0, 1, 0, 1, 0, 34, 3, 2, 7429, 395, 0, 91, 695, 3, 1911, 0, 1, 0, 0, 5, 1182, 1, 91, 0, 7864, 1, 65, 0, 2314, 0, 1, 0, 6950, 520, 287, 0] payload_rows_per_dst=[16192, 14814, 15886, 14496, 982, 4902, 2680, 6242] dup_rows_per_dst=[6228, 5298, 8022, 2164, 68, 328, 0, 2] combine_rows_per_src=[16192, 14814, 15886, 14496, 982, 4902, 2680, 6242] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87144 payload_rows_remote=85286 dup_rows=11160 dup_rows_remote=11134 dedup_ratio=0.113525 router_tokens_per_expert=[790, 445, 0, 0, 0, 0, 1463, 0, 0, 283, 0, 233, 0, 0, 246, 0, 0, 483, 0, 0, 0, 550, 0, 0, 0, 0, 7764, 26, 96, 0, 66, 7478, 0, 0, 0, 0, 0, 0, 0, 277, 0, 0, 6865, 153, 7877, 90, 7445, 0, 5893, 0, 0, 407, 0, 90, 0, 0, 0, 0, 0, 0, 66, 0, 0, 66] payload_rows_per_dst=[3796, 9032, 13506, 15454, 15384, 1858, 12224, 15890] dup_rows_per_dst=[1600, 452, 892, 7446, 60, 26, 556, 128] combine_rows_per_src=[3796, 9032, 13506, 15454, 15384, 1858, 12224, 15890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90592 payload_rows_remote=79378 dup_rows=7712 dup_rows_remote=7712 dedup_ratio=0.078451 router_tokens_per_expert=[523, 0, 0, 74, 311, 4564, 0, 0, 5447, 29, 131, 0, 0, 0, 0, 0, 0, 0, 2519, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 74, 0, 0, 0, 0, 81, 0, 0, 0, 660, 239, 0, 4618, 7546, 8132, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 420, 7176, 3705, 0, 0, 0, 621, 2045, 0, 0] payload_rows_per_dst=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] dup_rows_per_dst=[286, 0, 2190, 474, 1708, 0, 416, 2638] combine_rows_per_src=[10658, 11214, 12084, 16264, 15344, 7520, 7404, 10104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91062 payload_rows_remote=74784 dup_rows=7242 dup_rows_remote=6830 dedup_ratio=0.073669 router_tokens_per_expert=[503, 0, 0, 69, 330, 5159, 0, 0, 5509, 67, 153, 0, 0, 0, 0, 0, 0, 0, 1704, 0, 0, 2, 0, 0, 137, 0, 0, 0, 0, 69, 0, 0, 0, 0, 78, 0, 0, 0, 749, 261, 0, 4563, 7534, 8139, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 402, 7175, 3239, 0, 0, 0, 606, 2635, 0, 0] payload_rows_per_dst=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] dup_rows_per_dst=[256, 0, 1202, 412, 1860, 0, 804, 2708] combine_rows_per_src=[11866, 11458, 11336, 16278, 15384, 0, 14488, 10252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=242319360 dispatch_meta_bytes=336272 combine_payload_bytes=242319360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=242655632 ingress_bytes=311525376 link_bytes=311525376 token_bytes=484974992 weight_movement_bytes=69206016 total_nvlink_bytes=554181008 weight_to_token_ratio=0.142700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172244910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=249397248 dispatch_meta_bytes=332056 combine_payload_bytes=249397248 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=249729304 ingress_bytes=353206272 link_bytes=353206272 token_bytes=499126552 weight_movement_bytes=103809024 total_nvlink_bytes=602935576 weight_to_token_ratio=0.207981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.329086766e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=72474 dup_rows=16202 dup_rows_remote=15230 dedup_ratio=0.164815 router_tokens_per_expert=[0, 0, 31, 129, 0, 0, 3347, 0, 0, 2087, 7034, 0, 0, 0, 700, 0, 764, 414, 0, 113, 4009, 0, 0, 0, 663, 0, 0, 314, 0, 0, 0, 0, 0, 13, 0, 0, 7537, 316, 0, 78, 652, 0, 1773, 0, 0, 0, 0, 0, 1191, 0, 82, 0, 7871, 0, 111, 0, 2144, 0, 0, 0, 6902, 546, 331, 0] payload_rows_per_dst=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] dup_rows_per_dst=[5330, 4768, 972, 1542, 3338, 252, 0, 0] combine_rows_per_src=[14546, 14874, 9628, 14216, 15430, 4598, 2768, 6042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=245768192 dispatch_meta_bytes=303536 combine_payload_bytes=245768192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246071728 ingress_bytes=280371200 link_bytes=280371200 token_bytes=491839920 weight_movement_bytes=34603008 total_nvlink_bytes=526442928 weight_to_token_ratio=0.070354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055014254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=308535296 dispatch_meta_bytes=325488 combine_payload_bytes=308535296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308860784 ingress_bytes=343138304 link_bytes=343138304 token_bytes=617396080 weight_movement_bytes=34603008 total_nvlink_bytes=651999088 weight_to_token_ratio=0.056047 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291201813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=313442304 dispatch_meta_bytes=328752 combine_payload_bytes=313442304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313771056 ingress_bytes=348045312 link_bytes=348045312 token_bytes=627213360 weight_movement_bytes=34603008 total_nvlink_bytes=661816368 weight_to_token_ratio=0.055169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309666489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=349331456 dispatch_meta_bytes=385680 combine_payload_bytes=349331456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=349717136 ingress_bytes=487743488 link_bytes=487743488 token_bytes=699048592 weight_movement_bytes=138412032 total_nvlink_bytes=837460624 weight_to_token_ratio=0.198001 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.835339479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306315264 dispatch_meta_bytes=326456 combine_payload_bytes=306315264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306641720 ingress_bytes=340918272 link_bytes=340918272 token_bytes=612956984 weight_movement_bytes=34603008 total_nvlink_bytes=647559992 weight_to_token_ratio=0.056453 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282848011e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325132288 dispatch_meta_bytes=348360 combine_payload_bytes=325132288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325480648 ingress_bytes=359735296 link_bytes=359735296 token_bytes=650612936 weight_movement_bytes=34603008 total_nvlink_bytes=685215944 weight_to_token_ratio=0.053185 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.353654958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94148 payload_rows_remote=78640 dup_rows=4156 dup_rows_remote=4050 dedup_ratio=0.042277 router_tokens_per_expert=[822, 325, 0, 0, 0, 0, 1327, 0, 0, 304, 0, 260, 0, 0, 246, 0, 0, 495, 0, 0, 0, 635, 0, 0, 0, 0, 7796, 143, 101, 0, 63, 7600, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 6863, 132, 7851, 75, 7504, 0, 5816, 0, 0, 256, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 97] payload_rows_per_dst=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] dup_rows_per_dst=[1394, 978, 1030, 52, 106, 0, 450, 146] combine_rows_per_src=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=296853504 dispatch_meta_bytes=350816 combine_payload_bytes=296853504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297204320 ingress_bytes=331456512 link_bytes=331456512 token_bytes=594057824 weight_movement_bytes=34603008 total_nvlink_bytes=628660832 weight_to_token_ratio=0.058249 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247244170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=322109440 dispatch_meta_bytes=330760 combine_payload_bytes=322109440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322440200 ingress_bytes=356712448 link_bytes=356712448 token_bytes=644549640 weight_movement_bytes=34603008 total_nvlink_bytes=679152648 weight_to_token_ratio=0.053686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342280224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71084 payload_rows_remote=59570 dup_rows=27220 dup_rows_remote=26786 dedup_ratio=0.276896 router_tokens_per_expert=[778, 290, 0, 0, 0, 0, 1342, 0, 0, 286, 0, 286, 0, 0, 254, 0, 0, 464, 0, 0, 0, 626, 0, 0, 0, 0, 7810, 237, 119, 0, 60, 7627, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 6950, 152, 7852, 69, 7510, 0, 5706, 0, 0, 205, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 107] payload_rows_per_dst=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] dup_rows_per_dst=[1308, 0, 8, 15482, 32, 9788, 434, 168] combine_rows_per_src=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68136 payload_rows_remote=52262 dup_rows=30168 dup_rows_remote=30020 dedup_ratio=0.306885 router_tokens_per_expert=[762, 452, 0, 0, 0, 0, 1609, 0, 0, 251, 0, 177, 0, 0, 254, 0, 0, 528, 0, 0, 0, 551, 0, 0, 0, 0, 7747, 45, 104, 0, 42, 7459, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 6844, 99, 7893, 99, 7527, 0, 5822, 0, 0, 396, 0, 95, 0, 0, 0, 0, 0, 0, 42, 0, 0, 76] payload_rows_per_dst=[4138, 1360, 2156, 15776, 556, 16196, 12080, 15874] dup_rows_per_dst=[1508, 4, 2, 5310, 0, 22650, 546, 148] combine_rows_per_src=[4138, 1360, 2156, 15776, 556, 16196, 12080, 15874] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83024 payload_rows_remote=69104 dup_rows=15280 dup_rows_remote=11968 dedup_ratio=0.155436 router_tokens_per_expert=[581, 0, 7, 91, 340, 5045, 1, 0, 5286, 30, 152, 0, 0, 0, 0, 0, 0, 0, 2123, 4, 2, 1, 4, 0, 162, 0, 0, 3, 0, 91, 0, 0, 4, 0, 140, 0, 0, 0, 759, 249, 0, 4921, 7432, 8128, 0, 4, 0, 0, 1, 1, 1, 91, 0, 0, 361, 7110, 3253, 8, 0, 2, 588, 2174, 0, 2] payload_rows_per_dst=[13920, 13650, 12162, 16262, 15162, 4818, 908, 6142] dup_rows_per_dst=[3312, 6696, 1948, 506, 2006, 0, 2, 810] combine_rows_per_src=[13920, 13650, 12162, 16262, 15162, 4818, 908, 6142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93276 payload_rows_remote=78028 dup_rows=5028 dup_rows_remote=5026 dedup_ratio=0.051147 router_tokens_per_expert=[0, 0, 61, 0, 0, 0, 559, 0, 0, 0, 89, 0, 6954, 0, 0, 7161, 90, 0, 0, 0, 0, 2325, 0, 0, 0, 0, 0, 8102, 0, 0, 8190, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 408, 0, 0, 686, 5600, 0, 0] payload_rows_per_dst=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] dup_rows_per_dst=[766, 0, 3576, 0, 180, 2, 494, 10] combine_rows_per_src=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92212 payload_rows_remote=88428 dup_rows=6092 dup_rows_remote=6090 dedup_ratio=0.061971 router_tokens_per_expert=[821, 347, 0, 0, 0, 0, 1332, 0, 0, 266, 0, 260, 0, 0, 260, 0, 0, 483, 0, 0, 0, 585, 0, 0, 0, 0, 7793, 259, 80, 0, 53, 7597, 0, 0, 0, 0, 0, 0, 0, 302, 0, 0, 6847, 132, 7870, 69, 7515, 0, 5945, 0, 0, 142, 0, 69, 0, 0, 0, 0, 0, 0, 53, 0, 0, 72] payload_rows_per_dst=[13586, 16244, 14890, 3784, 15552, 402, 11896, 15858] dup_rows_per_dst=[3606, 914, 940, 2, 82, 0, 416, 132] combine_rows_per_src=[13586, 16244, 14890, 3784, 15552, 402, 11896, 15858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71472 payload_rows_remote=58512 dup_rows=26832 dup_rows_remote=24172 dedup_ratio=0.272949 router_tokens_per_expert=[5617, 0, 0, 0, 54, 0, 0, 0, 1, 445, 0, 0, 8, 0, 542, 6226, 70, 7291, 1, 91, 0, 0, 71, 0, 7419, 5, 1, 249, 3, 6260, 92, 6719, 3447, 145, 1, 0, 1, 0, 0, 0, 91, 0, 371, 5, 0, 2620, 249, 5, 0, 0, 0, 389, 1, 0, 0, 0, 4, 129, 523, 0, 2, 4, 0, 0] payload_rows_per_dst=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] dup_rows_per_dst=[2660, 2328, 458, 20248, 290, 842, 0, 6] combine_rows_per_src=[12960, 13042, 14590, 16044, 6898, 5840, 780, 1318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91386 payload_rows_remote=75146 dup_rows=6918 dup_rows_remote=6084 dedup_ratio=0.070374 router_tokens_per_expert=[848, 332, 0, 0, 0, 0, 1344, 0, 0, 273, 0, 218, 0, 0, 237, 0, 0, 493, 0, 0, 0, 627, 0, 0, 0, 0, 7809, 109, 90, 0, 60, 7575, 0, 0, 0, 0, 0, 0, 0, 265, 0, 0, 6836, 136, 7893, 74, 7526, 0, 5881, 0, 0, 303, 0, 71, 0, 0, 0, 0, 0, 0, 60, 0, 0, 92] payload_rows_per_dst=[15932, 16240, 14910, 518, 15530, 14246, 12030, 1980] dup_rows_per_dst=[4266, 834, 1002, 0, 52, 264, 480, 20] combine_rows_per_src=[15932, 16240, 14910, 518, 15530, 14246, 12030, 1980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=243998720 dispatch_meta_bytes=345424 combine_payload_bytes=243998720 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=244344144 ingress_bytes=278601728 link_bytes=278601728 token_bytes=488342864 weight_movement_bytes=34603008 total_nvlink_bytes=522945872 weight_to_token_ratio=0.070858 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.048355873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=214065152 dispatch_meta_bytes=329128 combine_payload_bytes=214065152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214394280 ingress_bytes=248668160 link_bytes=248668160 token_bytes=428459432 weight_movement_bytes=34603008 total_nvlink_bytes=463062440 weight_to_token_ratio=0.080761 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.357182667e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283049984 dispatch_meta_bytes=324288 combine_payload_bytes=283049984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283374272 ingress_bytes=317652992 link_bytes=317652992 token_bytes=566424256 weight_movement_bytes=34603008 total_nvlink_bytes=601027264 weight_to_token_ratio=0.061090 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.195302636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=319602688 dispatch_meta_bytes=332216 combine_payload_bytes=319602688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319934904 ingress_bytes=354205696 link_bytes=354205696 token_bytes=639537592 weight_movement_bytes=34603008 total_nvlink_bytes=674140600 weight_to_token_ratio=0.054106 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332847518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=362201088 dispatch_meta_bytes=378072 combine_payload_bytes=362201088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362579160 ingress_bytes=500613120 link_bytes=500613120 token_bytes=724780248 weight_movement_bytes=138412032 total_nvlink_bytes=863192280 weight_to_token_ratio=0.190971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.883766868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93198 payload_rows_remote=76944 dup_rows=5106 dup_rows_remote=4956 dedup_ratio=0.051941 router_tokens_per_expert=[0, 0, 69, 0, 0, 0, 519, 0, 0, 0, 75, 0, 7026, 0, 0, 7309, 75, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 8117, 0, 0, 8163, 0, 0, 0, 75, 0, 0, 8127, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 310, 0, 0, 904, 5155, 0, 0] payload_rows_per_dst=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] dup_rows_per_dst=[1176, 0, 3186, 0, 150, 0, 492, 102] combine_rows_per_src=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=307798016 dispatch_meta_bytes=324920 combine_payload_bytes=307798016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308122936 ingress_bytes=342401024 link_bytes=342401024 token_bytes=615920952 weight_movement_bytes=34603008 total_nvlink_bytes=650523960 weight_to_token_ratio=0.056181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.288427488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90708 payload_rows_remote=79046 dup_rows=7596 dup_rows_remote=5802 dedup_ratio=0.077271 router_tokens_per_expert=[591, 0, 0, 82, 346, 5039, 0, 0, 5561, 40, 172, 0, 0, 0, 0, 0, 0, 0, 2256, 0, 0, 0, 0, 0, 130, 0, 0, 0, 2, 82, 0, 0, 0, 0, 72, 0, 0, 0, 575, 264, 0, 4472, 7545, 8102, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 337, 7115, 3404, 0, 0, 0, 659, 2224, 0, 0] payload_rows_per_dst=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] dup_rows_per_dst=[356, 0, 1794, 428, 1534, 0, 674, 2810] combine_rows_per_src=[11760, 11546, 11662, 16204, 15378, 0, 14394, 9764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=315162624 dispatch_meta_bytes=327600 combine_payload_bytes=315162624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=315490224 ingress_bytes=418971648 link_bytes=418971648 token_bytes=630652848 weight_movement_bytes=103809024 total_nvlink_bytes=734461872 weight_to_token_ratio=0.164606 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576556581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=239665152 dispatch_meta_bytes=330736 combine_payload_bytes=239665152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=239995888 ingress_bytes=274268160 link_bytes=274268160 token_bytes=479661040 weight_movement_bytes=34603008 total_nvlink_bytes=514264048 weight_to_token_ratio=0.072141 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032049006e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68136 payload_rows_remote=52262 dup_rows=30168 dup_rows_remote=30020 dedup_ratio=0.306885 router_tokens_per_expert=[762, 452, 0, 0, 0, 0, 1609, 0, 0, 251, 0, 177, 0, 0, 254, 0, 0, 528, 0, 0, 0, 551, 0, 0, 0, 0, 7747, 45, 104, 0, 42, 7459, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 6844, 99, 7893, 99, 7527, 0, 5822, 0, 0, 396, 0, 95, 0, 0, 0, 0, 0, 0, 42, 0, 0, 76] payload_rows_per_dst=[4138, 1360, 2156, 15776, 556, 16196, 12080, 15874] dup_rows_per_dst=[1508, 4, 2, 5310, 0, 22650, 546, 148] combine_rows_per_src=[4138, 1360, 2156, 15776, 556, 16196, 12080, 15874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71084 payload_rows_remote=59570 dup_rows=27220 dup_rows_remote=26786 dedup_ratio=0.276896 router_tokens_per_expert=[778, 290, 0, 0, 0, 0, 1342, 0, 0, 286, 0, 286, 0, 0, 254, 0, 0, 464, 0, 0, 0, 626, 0, 0, 0, 0, 7810, 237, 119, 0, 60, 7627, 0, 0, 0, 0, 0, 0, 0, 299, 0, 0, 6950, 152, 7852, 69, 7510, 0, 5706, 0, 0, 205, 0, 63, 0, 0, 0, 0, 0, 0, 60, 0, 0, 107] payload_rows_per_dst=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] dup_rows_per_dst=[1308, 0, 8, 15482, 32, 9788, 434, 168] combine_rows_per_src=[3512, 1652, 2172, 16224, 4438, 15702, 11514, 15870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94148 payload_rows_remote=78640 dup_rows=4156 dup_rows_remote=4050 dedup_ratio=0.042277 router_tokens_per_expert=[822, 325, 0, 0, 0, 0, 1327, 0, 0, 304, 0, 260, 0, 0, 246, 0, 0, 495, 0, 0, 0, 635, 0, 0, 0, 0, 7796, 143, 101, 0, 63, 7600, 0, 0, 0, 0, 0, 0, 0, 303, 0, 0, 6863, 132, 7851, 75, 7504, 0, 5816, 0, 0, 256, 0, 75, 0, 0, 0, 0, 0, 0, 63, 0, 0, 97] payload_rows_per_dst=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] dup_rows_per_dst=[1394, 978, 1030, 52, 106, 0, 450, 146] combine_rows_per_src=[3554, 16234, 14956, 15762, 15508, 414, 11844, 15876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93804 payload_rows_remote=88060 dup_rows=4500 dup_rows_remote=4500 dedup_ratio=0.045776 router_tokens_per_expert=[0, 0, 44, 0, 0, 0, 495, 0, 0, 0, 82, 0, 7069, 0, 0, 7176, 82, 0, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 8110, 0, 0, 8112, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 336, 0, 0, 861, 5146, 0, 0] payload_rows_per_dst=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] dup_rows_per_dst=[1078, 0, 0, 2850, 136, 0, 420, 16] combine_rows_per_src=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93276 payload_rows_remote=78028 dup_rows=5028 dup_rows_remote=5026 dedup_ratio=0.051147 router_tokens_per_expert=[0, 0, 61, 0, 0, 0, 559, 0, 0, 0, 89, 0, 6954, 0, 0, 7161, 90, 0, 0, 0, 0, 2325, 0, 0, 0, 0, 0, 8102, 0, 0, 8190, 0, 0, 0, 90, 0, 0, 8126, 0, 0, 0, 0, 464, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 247, 0, 0, 0, 408, 0, 0, 686, 5600, 0, 0] payload_rows_per_dst=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] dup_rows_per_dst=[766, 0, 3576, 0, 180, 2, 494, 10] combine_rows_per_src=[7984, 178, 15162, 8694, 16252, 15248, 16380, 13378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=214065152 dispatch_meta_bytes=329128 combine_payload_bytes=214065152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214394280 ingress_bytes=248668160 link_bytes=248668160 token_bytes=428459432 weight_movement_bytes=34603008 total_nvlink_bytes=463062440 weight_to_token_ratio=0.080761 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=48128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.357182667e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=243998720 dispatch_meta_bytes=345424 combine_payload_bytes=243998720 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=244344144 ingress_bytes=278601728 link_bytes=278601728 token_bytes=488342864 weight_movement_bytes=34603008 total_nvlink_bytes=522945872 weight_to_token_ratio=0.070858 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.048355873e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=360693760 dispatch_meta_bytes=370240 combine_payload_bytes=360693760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=361064000 ingress_bytes=395296768 link_bytes=395296768 token_bytes=721757760 weight_movement_bytes=34603008 total_nvlink_bytes=756360768 weight_to_token_ratio=0.047943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487469914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=322109440 dispatch_meta_bytes=330760 combine_payload_bytes=322109440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322440200 ingress_bytes=356712448 link_bytes=356712448 token_bytes=644549640 weight_movement_bytes=34603008 total_nvlink_bytes=679152648 weight_to_token_ratio=0.053686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342280224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96450 payload_rows_remote=96450 dup_rows=1854 dup_rows_remote=1854 dedup_ratio=0.018860 router_tokens_per_expert=[0, 0, 66, 0, 0, 0, 530, 0, 0, 0, 69, 0, 7145, 0, 0, 7196, 69, 0, 0, 0, 0, 3355, 0, 0, 0, 0, 0, 8123, 0, 0, 8113, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 339, 0, 0, 804, 4529, 0, 0] payload_rows_per_dst=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] dup_rows_per_dst=[1192, 0, 0, 0, 138, 0, 522, 2] combine_rows_per_src=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=319602688 dispatch_meta_bytes=332216 combine_payload_bytes=319602688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319934904 ingress_bytes=354205696 link_bytes=354205696 token_bytes=639537592 weight_movement_bytes=34603008 total_nvlink_bytes=674140600 weight_to_token_ratio=0.054106 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.332847518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73260 payload_rows_remote=58030 dup_rows=25044 dup_rows_remote=17528 dedup_ratio=0.254761 router_tokens_per_expert=[0, 0, 92, 0, 0, 0, 529, 0, 0, 0, 71, 0, 6987, 0, 0, 7153, 74, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 8121, 0, 0, 8189, 0, 0, 0, 71, 0, 0, 8132, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 320, 0, 0, 831, 5461, 0, 0] payload_rows_per_dst=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] dup_rows_per_dst=[1242, 7516, 0, 16258, 0, 0, 0, 28] combine_rows_per_src=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76194 payload_rows_remote=60002 dup_rows=22110 dup_rows_remote=15882 dedup_ratio=0.224915 router_tokens_per_expert=[1, 0, 88, 171, 1, 3, 3082, 0, 0, 2469, 6865, 0, 4, 6, 712, 0, 594, 362, 4, 69, 3494, 2, 0, 0, 1039, 0, 0, 339, 0, 1, 0, 1, 0, 34, 3, 2, 7429, 395, 0, 91, 695, 3, 1911, 0, 1, 0, 0, 5, 1182, 1, 91, 0, 7864, 1, 65, 0, 2314, 0, 1, 0, 6950, 520, 287, 0] payload_rows_per_dst=[16192, 14814, 15886, 14496, 982, 4902, 2680, 6242] dup_rows_per_dst=[6228, 5298, 8022, 2164, 68, 328, 0, 2] combine_rows_per_src=[16192, 14814, 15886, 14496, 982, 4902, 2680, 6242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=395059200 dispatch_meta_bytes=393216 combine_payload_bytes=395059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=395452416 ingress_bytes=498868224 link_bytes=498868224 token_bytes=790511616 weight_movement_bytes=103809024 total_nvlink_bytes=894320640 weight_to_token_ratio=0.131319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.877200965e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=237690880 dispatch_meta_bytes=302232 combine_payload_bytes=237690880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237993112 ingress_bytes=341499904 link_bytes=341499904 token_bytes=475683992 weight_movement_bytes=103809024 total_nvlink_bytes=579493016 weight_to_token_ratio=0.218231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285036646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92820 payload_rows_remote=76590 dup_rows=5484 dup_rows_remote=4976 dedup_ratio=0.055786 router_tokens_per_expert=[0, 0, 75, 0, 0, 0, 449, 0, 0, 0, 63, 0, 7180, 0, 0, 7196, 65, 0, 0, 0, 0, 2814, 0, 0, 0, 0, 0, 8129, 0, 0, 8115, 0, 0, 0, 63, 0, 0, 8131, 0, 0, 0, 0, 338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 254, 0, 0, 0, 302, 0, 0, 765, 5213, 0, 0] payload_rows_per_dst=[898, 126, 15422, 16258, 16262, 15068, 16230, 12556] dup_rows_per_dst=[150, 0, 4696, 0, 126, 0, 508, 4] combine_rows_per_src=[898, 126, 15422, 16258, 16262, 15068, 16230, 12556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323772416 dispatch_meta_bytes=339392 combine_payload_bytes=323772416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324111808 ingress_bytes=358375424 link_bytes=358375424 token_bytes=647884224 weight_movement_bytes=34603008 total_nvlink_bytes=682487232 weight_to_token_ratio=0.053409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348537869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=245768192 dispatch_meta_bytes=303536 combine_payload_bytes=245768192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246071728 ingress_bytes=280371200 link_bytes=280371200 token_bytes=491839920 weight_movement_bytes=34603008 total_nvlink_bytes=526442928 weight_to_token_ratio=0.070354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.055014254e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93156 payload_rows_remote=80500 dup_rows=5148 dup_rows_remote=5118 dedup_ratio=0.052368 router_tokens_per_expert=[0, 0, 72, 0, 0, 0, 601, 0, 0, 0, 94, 0, 6902, 0, 0, 7125, 94, 0, 0, 0, 0, 2694, 0, 0, 0, 0, 0, 8097, 0, 0, 8186, 0, 0, 0, 95, 0, 0, 8150, 0, 0, 0, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 256, 0, 0, 0, 428, 0, 0, 829, 5086, 0, 0] payload_rows_per_dst=[1202, 188, 15110, 16194, 16300, 15134, 16372, 12656] dup_rows_per_dst=[144, 0, 4270, 0, 190, 2, 512, 30] combine_rows_per_src=[1202, 188, 15110, 16194, 16300, 15134, 16372, 12656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91302 payload_rows_remote=75360 dup_rows=7002 dup_rows_remote=2910 dedup_ratio=0.071228 router_tokens_per_expert=[801, 430, 0, 0, 0, 0, 1296, 0, 2, 272, 2, 351, 3, 0, 249, 0, 1, 527, 0, 0, 0, 568, 0, 1, 0, 1, 7750, 232, 113, 2, 54, 7490, 7, 3, 5, 0, 6, 0, 0, 283, 0, 4, 6765, 123, 7871, 93, 7486, 2, 5952, 0, 1, 177, 0, 91, 0, 0, 0, 0, 0, 8, 54, 0, 0, 76] payload_rows_per_dst=[15942, 16188, 14782, 1804, 15492, 15870, 10948, 276] dup_rows_per_dst=[4092, 1070, 942, 8, 88, 316, 486, 0] combine_rows_per_src=[15942, 16188, 14782, 1804, 15492, 15870, 10948, 276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89070 payload_rows_remote=72686 dup_rows=9234 dup_rows_remote=7586 dedup_ratio=0.093933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 239, 0, 743, 0, 239, 8034, 8192, 0, 0, 0, 0, 0, 3221, 0, 7659, 0, 90, 0, 7835, 0, 0, 0, 0, 1073, 5977, 82, 0, 0, 0, 0, 297, 888, 0, 0, 0, 2632, 0, 114, 0, 0, 0, 0, 0, 0, 90, 1, 82, 0, 959, 361] payload_rows_per_dst=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] dup_rows_per_dst=[0, 1166, 1648, 4178, 1450, 610, 0, 182] combine_rows_per_src=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93198 payload_rows_remote=76944 dup_rows=5106 dup_rows_remote=4956 dedup_ratio=0.051941 router_tokens_per_expert=[0, 0, 69, 0, 0, 0, 519, 0, 0, 0, 75, 0, 7026, 0, 0, 7309, 75, 0, 0, 0, 0, 2642, 0, 0, 0, 0, 0, 8117, 0, 0, 8163, 0, 0, 0, 75, 0, 0, 8127, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 246, 0, 0, 0, 310, 0, 0, 904, 5155, 0, 0] payload_rows_per_dst=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] dup_rows_per_dst=[1176, 0, 3186, 0, 150, 0, 492, 102] combine_rows_per_src=[16234, 3396, 13054, 0, 16254, 15298, 16326, 12636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88008 payload_rows_remote=72398 dup_rows=10296 dup_rows_remote=8446 dedup_ratio=0.104736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 436, 0, 736, 0, 291, 7934, 8192, 0, 0, 0, 0, 0, 3507, 0, 7733, 0, 66, 0, 7845, 0, 0, 0, 0, 893, 5855, 90, 0, 0, 0, 0, 226, 771, 0, 0, 0, 2518, 0, 108, 0, 0, 0, 0, 0, 0, 66, 3, 90, 0, 1090, 303] payload_rows_per_dst=[16372, 8478, 2066, 7146, 8700, 15610, 13574, 16062] dup_rows_per_dst=[0, 1230, 0, 0, 918, 1850, 3388, 2910] combine_rows_per_src=[16372, 8478, 2066, 7146, 8700, 15610, 13574, 16062] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96450 payload_rows_remote=96450 dup_rows=1854 dup_rows_remote=1854 dedup_ratio=0.018860 router_tokens_per_expert=[0, 0, 66, 0, 0, 0, 530, 0, 0, 0, 69, 0, 7145, 0, 0, 7196, 69, 0, 0, 0, 0, 3355, 0, 0, 0, 0, 0, 8123, 0, 0, 8113, 0, 0, 0, 69, 0, 0, 8139, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 339, 0, 0, 804, 4529, 0, 0] payload_rows_per_dst=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] dup_rows_per_dst=[1192, 0, 0, 0, 138, 0, 522, 2] combine_rows_per_src=[16246, 14428, 6848, 0, 16278, 15082, 16226, 11342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73260 payload_rows_remote=58030 dup_rows=25044 dup_rows_remote=17528 dedup_ratio=0.254761 router_tokens_per_expert=[0, 0, 92, 0, 0, 0, 529, 0, 0, 0, 71, 0, 6987, 0, 0, 7153, 74, 0, 0, 0, 0, 2500, 0, 0, 0, 0, 0, 8121, 0, 0, 8189, 0, 0, 0, 71, 0, 0, 8132, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 239, 0, 0, 0, 320, 0, 0, 831, 5461, 0, 0] payload_rows_per_dst=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] dup_rows_per_dst=[1242, 7516, 0, 16258, 0, 0, 0, 28] combine_rows_per_src=[16242, 15230, 5148, 16384, 142, 6440, 478, 13196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=313712640 dispatch_meta_bytes=326264 combine_payload_bytes=313712640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314038904 ingress_bytes=348315648 link_bytes=348315648 token_bytes=627751544 weight_movement_bytes=34603008 total_nvlink_bytes=662354552 weight_to_token_ratio=0.055122 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310683742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88008 payload_rows_remote=72398 dup_rows=10296 dup_rows_remote=8446 dedup_ratio=0.104736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 436, 0, 736, 0, 291, 7934, 8192, 0, 0, 0, 0, 0, 3507, 0, 7733, 0, 66, 0, 7845, 0, 0, 0, 0, 893, 5855, 90, 0, 0, 0, 0, 226, 771, 0, 0, 0, 2518, 0, 108, 0, 0, 0, 0, 0, 0, 66, 3, 90, 0, 1090, 303] payload_rows_per_dst=[16372, 8478, 2066, 7146, 8700, 15610, 13574, 16062] dup_rows_per_dst=[0, 1230, 0, 0, 918, 1850, 3388, 2910] combine_rows_per_src=[16372, 8478, 2066, 7146, 8700, 15610, 13574, 16062] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=329728000 dispatch_meta_bytes=342472 combine_payload_bytes=329728000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=330070472 ingress_bytes=364331008 link_bytes=364331008 token_bytes=659798472 weight_movement_bytes=34603008 total_nvlink_bytes=694401480 weight_to_token_ratio=0.052445 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=39808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.370948252e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308674560 dispatch_meta_bytes=313080 combine_payload_bytes=308674560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308987640 ingress_bytes=343277568 link_bytes=343277568 token_bytes=617662200 weight_movement_bytes=34603008 total_nvlink_bytes=652265208 weight_to_token_ratio=0.056023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291725852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=297721856 dispatch_meta_bytes=321088 combine_payload_bytes=297721856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=298042944 ingress_bytes=436133888 link_bytes=436133888 token_bytes=595764800 weight_movement_bytes=138412032 total_nvlink_bytes=734176832 weight_to_token_ratio=0.232327 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641136709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=315162624 dispatch_meta_bytes=327600 combine_payload_bytes=315162624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=315490224 ingress_bytes=418971648 link_bytes=418971648 token_bytes=630652848 weight_movement_bytes=103809024 total_nvlink_bytes=734461872 weight_to_token_ratio=0.164606 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.576556581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88728 payload_rows_remote=80574 dup_rows=9576 dup_rows_remote=8478 dedup_ratio=0.097412 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 373, 0, 704, 0, 138, 8042, 8192, 0, 0, 0, 0, 0, 2976, 0, 7725, 0, 53, 0, 7888, 0, 0, 0, 0, 1191, 6295, 69, 0, 0, 0, 0, 165, 694, 0, 0, 0, 2834, 0, 79, 0, 0, 0, 0, 0, 0, 55, 24, 69, 0, 837, 345] payload_rows_per_dst=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] dup_rows_per_dst=[0, 1554, 78, 1098, 994, 1606, 2318, 1928] combine_rows_per_src=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87402 payload_rows_remote=71680 dup_rows=10902 dup_rows_remote=9052 dedup_ratio=0.110901 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 500, 0, 645, 0, 166, 8023, 8192, 0, 0, 0, 0, 0, 3538, 0, 7710, 0, 60, 0, 7861, 0, 0, 0, 0, 796, 6075, 74, 0, 0, 0, 0, 244, 1060, 0, 0, 0, 2355, 0, 76, 0, 0, 0, 0, 0, 0, 60, 0, 71, 0, 873, 348] payload_rows_per_dst=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] dup_rows_per_dst=[0, 1850, 1284, 6380, 1268, 0, 0, 120] combine_rows_per_src=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=296542208 dispatch_meta_bytes=323376 combine_payload_bytes=296542208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296865584 ingress_bytes=331145216 link_bytes=331145216 token_bytes=593407792 weight_movement_bytes=34603008 total_nvlink_bytes=628010800 weight_to_token_ratio=0.058312 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246072788e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88460 payload_rows_remote=86426 dup_rows=9844 dup_rows_remote=9694 dedup_ratio=0.100138 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 255, 0, 770, 0, 151, 8052, 8192, 0, 0, 0, 0, 0, 3320, 0, 7710, 0, 63, 0, 7838, 0, 0, 0, 0, 1017, 6192, 75, 0, 0, 0, 0, 246, 930, 0, 0, 0, 2488, 0, 70, 0, 0, 0, 0, 0, 0, 63, 3, 75, 0, 908, 344] payload_rows_per_dst=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] dup_rows_per_dst=[0, 1290, 0, 0, 150, 2224, 3546, 2634] combine_rows_per_src=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=395059200 dispatch_meta_bytes=393216 combine_payload_bytes=395059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=395452416 ingress_bytes=498868224 link_bytes=498868224 token_bytes=790511616 weight_movement_bytes=103809024 total_nvlink_bytes=894320640 weight_to_token_ratio=0.131319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.877200965e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=237690880 dispatch_meta_bytes=302232 combine_payload_bytes=237690880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237993112 ingress_bytes=341499904 link_bytes=341499904 token_bytes=475683992 weight_movement_bytes=103809024 total_nvlink_bytes=579493016 weight_to_token_ratio=0.218231 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285036646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91580 payload_rows_remote=76524 dup_rows=6724 dup_rows_remote=5664 dedup_ratio=0.068400 router_tokens_per_expert=[787, 350, 0, 0, 0, 0, 1339, 0, 0, 272, 0, 253, 0, 0, 262, 0, 0, 525, 0, 0, 0, 626, 0, 0, 0, 0, 7764, 163, 88, 0, 90, 7559, 0, 0, 0, 0, 0, 0, 0, 298, 0, 0, 6907, 203, 7835, 83, 7414, 0, 5839, 0, 0, 279, 0, 82, 0, 0, 0, 0, 0, 0, 90, 0, 0, 44] payload_rows_per_dst=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] dup_rows_per_dst=[4118, 886, 1060, 0, 68, 0, 504, 88] combine_rows_per_src=[15952, 16216, 15056, 682, 15356, 572, 11896, 15850] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=354000896 dispatch_meta_bytes=384480 combine_payload_bytes=354000896 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=354385376 ingress_bytes=457809920 link_bytes=457809920 token_bytes=708386272 weight_movement_bytes=103809024 total_nvlink_bytes=812195296 weight_to_token_ratio=0.146543 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722701872e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83024 payload_rows_remote=69104 dup_rows=15280 dup_rows_remote=11968 dedup_ratio=0.155436 router_tokens_per_expert=[581, 0, 7, 91, 340, 5045, 1, 0, 5286, 30, 152, 0, 0, 0, 0, 0, 0, 0, 2123, 4, 2, 1, 4, 0, 162, 0, 0, 3, 0, 91, 0, 0, 4, 0, 140, 0, 0, 0, 759, 249, 0, 4921, 7432, 8128, 0, 4, 0, 0, 1, 1, 1, 91, 0, 0, 361, 7110, 3253, 8, 0, 2, 588, 2174, 0, 2] payload_rows_per_dst=[13920, 13650, 12162, 16262, 15162, 4818, 908, 6142] dup_rows_per_dst=[3312, 6696, 1948, 506, 2006, 0, 2, 810] combine_rows_per_src=[13920, 13650, 12162, 16262, 15162, 4818, 908, 6142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=330031104 dispatch_meta_bytes=356208 combine_payload_bytes=330031104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330387312 ingress_bytes=399237120 link_bytes=399237120 token_bytes=660418416 weight_movement_bytes=69206016 total_nvlink_bytes=729624432 weight_to_token_ratio=0.104791 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293601280 dispatch_meta_bytes=322928 combine_payload_bytes=293601280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293924208 ingress_bytes=328204288 link_bytes=328204288 token_bytes=587525488 weight_movement_bytes=34603008 total_nvlink_bytes=622128496 weight_to_token_ratio=0.058896 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235006313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85616 payload_rows_remote=71398 dup_rows=12688 dup_rows_remote=8700 dedup_ratio=0.129069 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 383, 0, 0, 0, 205, 0, 726, 0, 151, 8081, 8192, 0, 0, 0, 0, 0, 3133, 0, 7731, 0, 60, 0, 7881, 0, 0, 0, 0, 1261, 6374, 65, 0, 0, 0, 0, 126, 737, 0, 0, 0, 2637, 0, 92, 0, 0, 0, 0, 0, 0, 60, 11, 63, 0, 847, 336] payload_rows_per_dst=[0, 866, 16384, 6386, 15892, 15582, 14218, 16288] dup_rows_per_dst=[0, 310, 1754, 0, 2522, 1606, 3988, 2508] combine_rows_per_src=[0, 866, 16384, 6386, 15892, 15582, 14218, 16288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=283049984 dispatch_meta_bytes=324288 combine_payload_bytes=283049984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283374272 ingress_bytes=317652992 link_bytes=317652992 token_bytes=566424256 weight_movement_bytes=34603008 total_nvlink_bytes=601027264 weight_to_token_ratio=0.061090 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=38912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.195302636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94742 payload_rows_remote=84704 dup_rows=3562 dup_rows_remote=1788 dedup_ratio=0.036235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 74, 207, 7788, 1303, 0, 0, 7577, 0, 0, 4156, 72, 90, 0, 0, 0, 0, 517, 0, 0, 0, 0, 4724, 0, 7762, 0, 0, 0, 0, 82, 0, 0, 0, 0, 253, 6141, 0, 107, 71, 7599, 82, 0, 0, 0, 82, 0, 0] payload_rows_per_dst=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] dup_rows_per_dst=[0, 930, 1774, 0, 84, 60, 550, 164] combine_rows_per_src=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85930 payload_rows_remote=69894 dup_rows=12374 dup_rows_remote=9820 dedup_ratio=0.125875 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 328, 0, 0, 0, 491, 0, 754, 0, 248, 7921, 8192, 0, 0, 0, 0, 0, 3459, 0, 7717, 0, 42, 0, 7903, 0, 0, 0, 0, 791, 6011, 96, 0, 0, 0, 0, 239, 748, 0, 0, 0, 2782, 0, 56, 0, 0, 0, 0, 0, 0, 42, 13, 95, 0, 887, 337] payload_rows_per_dst=[0, 1032, 16384, 7002, 15998, 15522, 13956, 16036] dup_rows_per_dst=[0, 606, 2004, 0, 1582, 1886, 3742, 2554] combine_rows_per_src=[0, 1032, 16384, 7002, 15998, 15522, 13956, 16036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=313442304 dispatch_meta_bytes=328752 combine_payload_bytes=313442304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313771056 ingress_bytes=348045312 link_bytes=348045312 token_bytes=627213360 weight_movement_bytes=34603008 total_nvlink_bytes=661816368 weight_to_token_ratio=0.055169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309666489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68452 payload_rows_remote=52268 dup_rows=29852 dup_rows_remote=28506 dedup_ratio=0.303670 router_tokens_per_expert=[0, 1, 74, 0, 0, 0, 600, 0, 0, 0, 91, 0, 7018, 0, 8, 7175, 93, 0, 0, 0, 1, 2946, 0, 4, 0, 1, 0, 8090, 0, 7, 8116, 0, 0, 0, 91, 0, 0, 8130, 5, 1, 0, 0, 423, 0, 1, 8, 6, 0, 0, 1, 0, 0, 0, 249, 4, 0, 1, 389, 0, 1, 853, 4763, 1, 0] payload_rows_per_dst=[16184, 16342, 6088, 16264, 2526, 866, 508, 9674] dup_rows_per_dst=[1346, 26598, 0, 1888, 6, 10, 0, 4] combine_rows_per_src=[16184, 16342, 6088, 16264, 2526, 866, 508, 9674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88460 payload_rows_remote=86426 dup_rows=9844 dup_rows_remote=9694 dedup_ratio=0.100138 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 0, 255, 0, 770, 0, 151, 8052, 8192, 0, 0, 0, 0, 0, 3320, 0, 7710, 0, 63, 0, 7838, 0, 0, 0, 0, 1017, 6192, 75, 0, 0, 0, 0, 246, 930, 0, 0, 0, 2488, 0, 70, 0, 0, 0, 0, 0, 0, 63, 3, 75, 0, 908, 344] payload_rows_per_dst=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] dup_rows_per_dst=[0, 1290, 0, 0, 150, 2224, 3546, 2634] combine_rows_per_src=[16384, 15676, 1842, 6766, 2034, 15548, 13954, 16256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=80734 dup_rows=2202 dup_rows_remote=2022 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 164, 238, 7811, 1236, 0, 0, 7644, 0, 0, 4202, 161, 66, 0, 0, 0, 0, 554, 0, 0, 0, 0, 4537, 0, 7684, 0, 0, 0, 0, 90, 0, 0, 0, 0, 286, 5906, 0, 146, 126, 7722, 90, 0, 0, 0, 90, 0, 0] payload_rows_per_dst=[15288, 14040, 4532, 8858, 10102, 15368, 12290, 15624] dup_rows_per_dst=[0, 710, 414, 0, 80, 180, 638, 180] combine_rows_per_src=[15288, 14040, 4532, 8858, 10102, 15368, 12290, 15624] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88728 payload_rows_remote=80574 dup_rows=9576 dup_rows_remote=8478 dedup_ratio=0.097412 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 404, 0, 0, 0, 373, 0, 704, 0, 138, 8042, 8192, 0, 0, 0, 0, 0, 2976, 0, 7725, 0, 53, 0, 7888, 0, 0, 0, 0, 1191, 6295, 69, 0, 0, 0, 0, 165, 694, 0, 0, 0, 2834, 0, 79, 0, 0, 0, 0, 0, 0, 55, 24, 69, 0, 837, 345] payload_rows_per_dst=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] dup_rows_per_dst=[0, 1554, 78, 1098, 994, 1606, 2318, 1928] combine_rows_per_src=[16384, 15776, 2684, 8154, 7230, 15562, 10394, 12544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87402 payload_rows_remote=71680 dup_rows=10902 dup_rows_remote=9052 dedup_ratio=0.110901 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 425, 0, 0, 0, 500, 0, 645, 0, 166, 8023, 8192, 0, 0, 0, 0, 0, 3538, 0, 7710, 0, 60, 0, 7861, 0, 0, 0, 0, 796, 6075, 74, 0, 0, 0, 0, 244, 1060, 0, 0, 0, 2355, 0, 76, 0, 0, 0, 0, 0, 0, 60, 0, 71, 0, 873, 348] payload_rows_per_dst=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] dup_rows_per_dst=[0, 1850, 1284, 6380, 1268, 0, 0, 120] combine_rows_per_src=[16384, 15722, 16384, 16236, 12622, 2608, 4862, 2584] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292446208 dispatch_meta_bytes=320392 combine_payload_bytes=292446208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292766600 ingress_bytes=327049216 link_bytes=327049216 token_bytes=585212808 weight_movement_bytes=34603008 total_nvlink_bytes=619815816 weight_to_token_ratio=0.059129 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.230659870e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=346947584 dispatch_meta_bytes=345968 combine_payload_bytes=346947584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347293552 ingress_bytes=416153600 link_bytes=416153600 token_bytes=694241136 weight_movement_bytes=69206016 total_nvlink_bytes=763447152 weight_to_token_ratio=0.099686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96102 payload_rows_remote=80734 dup_rows=2202 dup_rows_remote=2022 dedup_ratio=0.022400 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 399, 0, 0, 0, 164, 238, 7811, 1236, 0, 0, 7644, 0, 0, 4202, 161, 66, 0, 0, 0, 0, 554, 0, 0, 0, 0, 4537, 0, 7684, 0, 0, 0, 0, 90, 0, 0, 0, 0, 286, 5906, 0, 146, 126, 7722, 90, 0, 0, 0, 90, 0, 0] payload_rows_per_dst=[15288, 14040, 4532, 8858, 10102, 15368, 12290, 15624] dup_rows_per_dst=[0, 710, 414, 0, 80, 180, 638, 180] combine_rows_per_src=[15288, 14040, 4532, 8858, 10102, 15368, 12290, 15624] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=286285824 dispatch_meta_bytes=318856 combine_payload_bytes=286285824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286604680 ingress_bytes=320888832 link_bytes=320888832 token_bytes=572890504 weight_movement_bytes=34603008 total_nvlink_bytes=607493512 weight_to_token_ratio=0.060401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207478841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=323096 combine_payload_bytes=214089728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214412824 ingress_bytes=248692736 link_bytes=248692736 token_bytes=428502552 weight_movement_bytes=34603008 total_nvlink_bytes=463105560 weight_to_token_ratio=0.080753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.358107442e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=354000896 dispatch_meta_bytes=384480 combine_payload_bytes=354000896 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=354385376 ingress_bytes=457809920 link_bytes=457809920 token_bytes=708386272 weight_movement_bytes=103809024 total_nvlink_bytes=812195296 weight_to_token_ratio=0.146543 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.722701872e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96552 payload_rows_remote=86988 dup_rows=1752 dup_rows_remote=1752 dedup_ratio=0.017822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 78, 210, 7868, 1332, 0, 0, 7656, 0, 0, 4651, 78, 53, 0, 0, 0, 0, 492, 0, 0, 0, 0, 4170, 0, 7707, 0, 0, 0, 0, 69, 0, 0, 0, 0, 272, 6329, 0, 74, 78, 7510, 69, 0, 0, 0, 69, 0, 0] payload_rows_per_dst=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] dup_rows_per_dst=[0, 774, 138, 0, 108, 138, 456, 138] combine_rows_per_src=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86502 payload_rows_remote=70870 dup_rows=11802 dup_rows_remote=11062 dedup_ratio=0.120056 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 0, 82, 241, 7816, 1314, 0, 0, 7639, 0, 0, 4339, 81, 60, 0, 0, 0, 0, 502, 0, 0, 0, 0, 4414, 0, 7764, 0, 0, 0, 0, 71, 0, 0, 0, 0, 302, 6113, 0, 179, 71, 7652, 71, 0, 0, 0, 71, 0, 0] payload_rows_per_dst=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] dup_rows_per_dst=[0, 740, 3104, 0, 3704, 0, 4112, 142] combine_rows_per_src=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91302 payload_rows_remote=75360 dup_rows=7002 dup_rows_remote=2910 dedup_ratio=0.071228 router_tokens_per_expert=[801, 430, 0, 0, 0, 0, 1296, 0, 2, 272, 2, 351, 3, 0, 249, 0, 1, 527, 0, 0, 0, 568, 0, 1, 0, 1, 7750, 232, 113, 2, 54, 7490, 7, 3, 5, 0, 6, 0, 0, 283, 0, 4, 6765, 123, 7871, 93, 7486, 2, 5952, 0, 1, 177, 0, 91, 0, 0, 0, 0, 0, 8, 54, 0, 0, 76] payload_rows_per_dst=[15942, 16188, 14782, 1804, 15492, 15870, 10948, 276] dup_rows_per_dst=[4092, 1070, 942, 8, 88, 316, 486, 0] combine_rows_per_src=[15942, 16188, 14782, 1804, 15492, 15870, 10948, 276] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=330686464 dispatch_meta_bytes=331024 combine_payload_bytes=330686464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331017488 ingress_bytes=399892480 link_bytes=399892480 token_bytes=661703952 weight_movement_bytes=69206016 total_nvlink_bytes=730909968 weight_to_token_ratio=0.104588 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.504763208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96312 payload_rows_remote=86946 dup_rows=1992 dup_rows_remote=1872 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 68, 219, 7814, 1263, 0, 0, 7640, 0, 0, 4708, 64, 63, 0, 0, 0, 0, 478, 0, 0, 0, 0, 4265, 0, 7739, 0, 0, 0, 0, 75, 0, 0, 0, 0, 286, 6156, 0, 165, 63, 7558, 75, 0, 0, 0, 75, 0, 0] payload_rows_per_dst=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] dup_rows_per_dst=[0, 756, 150, 0, 120, 150, 666, 150] combine_rows_per_src=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=330031104 dispatch_meta_bytes=356208 combine_payload_bytes=330031104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=330387312 ingress_bytes=399237120 link_bytes=399237120 token_bytes=660418416 weight_movement_bytes=69206016 total_nvlink_bytes=729624432 weight_to_token_ratio=0.104791 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=293601280 dispatch_meta_bytes=322928 combine_payload_bytes=293601280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293924208 ingress_bytes=328204288 link_bytes=328204288 token_bytes=587525488 weight_movement_bytes=34603008 total_nvlink_bytes=622128496 weight_to_token_ratio=0.058896 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235006313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93804 payload_rows_remote=88060 dup_rows=4500 dup_rows_remote=4500 dedup_ratio=0.045776 router_tokens_per_expert=[0, 0, 44, 0, 0, 0, 495, 0, 0, 0, 82, 0, 7069, 0, 0, 7176, 82, 0, 0, 0, 0, 2790, 0, 0, 0, 0, 0, 8110, 0, 0, 8112, 0, 0, 0, 82, 0, 0, 8102, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 0, 0, 336, 0, 0, 861, 5146, 0, 0] payload_rows_per_dst=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] dup_rows_per_dst=[1078, 0, 0, 2850, 136, 0, 420, 16] combine_rows_per_src=[16220, 14302, 5744, 3514, 13364, 15154, 12836, 12670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356130816 dispatch_meta_bytes=355272 combine_payload_bytes=356130816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=356486088 ingress_bytes=390733824 link_bytes=390733824 token_bytes=712616904 weight_movement_bytes=34603008 total_nvlink_bytes=747219912 weight_to_token_ratio=0.048558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.470299923e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356302848 dispatch_meta_bytes=354960 combine_payload_bytes=356302848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=356657808 ingress_bytes=425508864 link_bytes=425508864 token_bytes=712960656 weight_movement_bytes=69206016 total_nvlink_bytes=782166672 weight_to_token_ratio=0.097068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601155599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290283520 dispatch_meta_bytes=327728 combine_payload_bytes=290283520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290611248 ingress_bytes=324886528 link_bytes=324886528 token_bytes=580894768 weight_movement_bytes=34603008 total_nvlink_bytes=615497776 weight_to_token_ratio=0.059568 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222521849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89872 payload_rows_remote=76758 dup_rows=8432 dup_rows_remote=7986 dedup_ratio=0.085775 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 48, 208, 7872, 1288, 0, 0, 7668, 0, 0, 4680, 48, 60, 0, 0, 0, 0, 447, 0, 0, 0, 0, 4082, 0, 7799, 0, 0, 0, 0, 63, 0, 0, 0, 0, 289, 6383, 0, 60, 48, 7582, 63, 0, 0, 0, 63, 0, 0] payload_rows_per_dst=[6766, 802, 16368, 12914, 9020, 15598, 13114, 15290] dup_rows_per_dst=[0, 0, 2464, 5232, 38, 126, 446, 126] combine_rows_per_src=[6766, 802, 16368, 12914, 9020, 15598, 13114, 15290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=308674560 dispatch_meta_bytes=313080 combine_payload_bytes=308674560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308987640 ingress_bytes=343277568 link_bytes=343277568 token_bytes=617662200 weight_movement_bytes=34603008 total_nvlink_bytes=652265208 weight_to_token_ratio=0.056023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291725852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93298 payload_rows_remote=79076 dup_rows=5006 dup_rows_remote=4680 dedup_ratio=0.050924 router_tokens_per_expert=[0, 198, 113, 1405, 7976, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4706, 0, 0, 279, 0, 0, 6995, 0, 0, 0, 0, 90, 8148, 82, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 73, 0, 0, 508, 0, 0, 0, 0, 0, 7925, 82, 2267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] dup_rows_per_dst=[3116, 0, 326, 254, 0, 458, 852, 0] combine_rows_per_src=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85982 payload_rows_remote=70546 dup_rows=12322 dup_rows_remote=12132 dedup_ratio=0.125346 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 352, 0, 0, 0, 179, 263, 7790, 1300, 0, 0, 7624, 0, 0, 4100, 174, 42, 0, 0, 0, 0, 606, 0, 0, 0, 0, 4519, 0, 7677, 0, 0, 0, 0, 95, 0, 0, 0, 0, 322, 6073, 0, 64, 159, 7623, 95, 0, 0, 0, 95, 0, 0] payload_rows_per_dst=[0, 704, 16354, 15332, 10148, 15354, 12654, 15436] dup_rows_per_dst=[0, 0, 2710, 8548, 102, 190, 582, 190] combine_rows_per_src=[0, 704, 16354, 15332, 10148, 15354, 12654, 15436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=360693760 dispatch_meta_bytes=370240 combine_payload_bytes=360693760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=361064000 ingress_bytes=395296768 link_bytes=395296768 token_bytes=721757760 weight_movement_bytes=34603008 total_nvlink_bytes=756360768 weight_to_token_ratio=0.047943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487469914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87000 payload_rows_remote=70622 dup_rows=11304 dup_rows_remote=11286 dedup_ratio=0.114990 router_tokens_per_expert=[0, 0, 8, 0, 7, 0, 1, 3, 0, 0, 371, 1, 0, 0, 311, 0, 912, 0, 245, 7938, 8179, 0, 3, 0, 0, 0, 2906, 0, 7687, 1, 55, 0, 7856, 2, 4, 0, 2, 1242, 6330, 91, 0, 3, 0, 6, 269, 836, 0, 0, 0, 2359, 0, 71, 0, 6, 0, 0, 2, 0, 70, 8, 91, 7, 923, 346] payload_rows_per_dst=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] dup_rows_per_dst=[18, 1362, 3036, 5122, 1596, 6, 0, 164] combine_rows_per_src=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92696 payload_rows_remote=76744 dup_rows=5608 dup_rows_remote=4790 dedup_ratio=0.057048 router_tokens_per_expert=[0, 156, 87, 1359, 7976, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 0, 0, 275, 0, 0, 7221, 0, 0, 0, 0, 66, 8132, 90, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 161, 0, 0, 586, 0, 0, 0, 0, 0, 7854, 90, 2024, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3204, 9592, 14616, 16264, 15702, 15952, 1172, 16194] dup_rows_per_dst=[360, 0, 376, 312, 0, 818, 0, 3742] combine_rows_per_src=[3204, 9592, 14616, 16264, 15702, 15952, 1172, 16194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96312 payload_rows_remote=86946 dup_rows=1992 dup_rows_remote=1872 dedup_ratio=0.020264 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 378, 0, 0, 0, 68, 219, 7814, 1263, 0, 0, 7640, 0, 0, 4708, 64, 63, 0, 0, 0, 0, 478, 0, 0, 0, 0, 4265, 0, 7739, 0, 0, 0, 0, 75, 0, 0, 0, 0, 286, 6156, 0, 165, 63, 7558, 75, 0, 0, 0, 75, 0, 0] payload_rows_per_dst=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] dup_rows_per_dst=[0, 756, 150, 0, 120, 150, 666, 150] combine_rows_per_src=[15280, 15628, 2950, 9670, 9366, 15478, 12674, 15266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96552 payload_rows_remote=86988 dup_rows=1752 dup_rows_remote=1752 dedup_ratio=0.017822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 78, 210, 7868, 1332, 0, 0, 7656, 0, 0, 4651, 78, 53, 0, 0, 0, 0, 492, 0, 0, 0, 0, 4170, 0, 7707, 0, 0, 0, 0, 69, 0, 0, 0, 0, 272, 6329, 0, 74, 78, 7510, 69, 0, 0, 0, 69, 0, 0] payload_rows_per_dst=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] dup_rows_per_dst=[0, 774, 138, 0, 108, 138, 456, 138] combine_rows_per_src=[15312, 15736, 3102, 9564, 9216, 15414, 13050, 15158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86502 payload_rows_remote=70870 dup_rows=11802 dup_rows_remote=11062 dedup_ratio=0.120056 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 370, 0, 0, 0, 82, 241, 7816, 1314, 0, 0, 7639, 0, 0, 4339, 81, 60, 0, 0, 0, 0, 502, 0, 0, 0, 0, 4414, 0, 7764, 0, 0, 0, 0, 71, 0, 0, 0, 0, 302, 6113, 0, 179, 71, 7652, 71, 0, 0, 0, 71, 0, 0] payload_rows_per_dst=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] dup_rows_per_dst=[0, 740, 3104, 0, 3704, 0, 4112, 142] combine_rows_per_src=[15278, 15632, 15698, 8960, 12476, 522, 13242, 4694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=314400768 dispatch_meta_bytes=338976 combine_payload_bytes=314400768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314739744 ingress_bytes=383606784 link_bytes=383606784 token_bytes=629140512 weight_movement_bytes=69206016 total_nvlink_bytes=698346528 weight_to_token_ratio=0.110001 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.443481445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323895296 dispatch_meta_bytes=335024 combine_payload_bytes=323895296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324230320 ingress_bytes=393101312 link_bytes=393101312 token_bytes=648125616 weight_movement_bytes=69206016 total_nvlink_bytes=717331632 weight_to_token_ratio=0.106779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479208590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92696 payload_rows_remote=76744 dup_rows=5608 dup_rows_remote=4790 dedup_ratio=0.057048 router_tokens_per_expert=[0, 156, 87, 1359, 7976, 180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4796, 0, 0, 275, 0, 0, 7221, 0, 0, 0, 0, 66, 8132, 90, 0, 0, 0, 0, 0, 7851, 0, 0, 0, 0, 0, 0, 0, 248, 0, 0, 0, 161, 0, 0, 586, 0, 0, 0, 0, 0, 7854, 90, 2024, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3204, 9592, 14616, 16264, 15702, 15952, 1172, 16194] dup_rows_per_dst=[360, 0, 376, 312, 0, 818, 0, 3742] combine_rows_per_src=[3204, 9592, 14616, 16264, 15702, 15952, 1172, 16194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=288956416 dispatch_meta_bytes=330712 combine_payload_bytes=288956416 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=289287128 ingress_bytes=392765440 link_bytes=392765440 token_bytes=578243544 weight_movement_bytes=103809024 total_nvlink_bytes=682052568 weight_to_token_ratio=0.179525 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.037500 gemm_active_groups=5 gemm_peak_group_rows=47872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.477944731e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=289267712 dispatch_meta_bytes=327632 combine_payload_bytes=289267712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289595344 ingress_bytes=323870720 link_bytes=323870720 token_bytes=578863056 weight_movement_bytes=34603008 total_nvlink_bytes=613466064 weight_to_token_ratio=0.059778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218699446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314343424 dispatch_meta_bytes=326136 combine_payload_bytes=314343424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314669560 ingress_bytes=348946432 link_bytes=348946432 token_bytes=629012984 weight_movement_bytes=34603008 total_nvlink_bytes=663615992 weight_to_token_ratio=0.055012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313057331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93182 payload_rows_remote=77226 dup_rows=5122 dup_rows_remote=5122 dedup_ratio=0.052104 router_tokens_per_expert=[0, 157, 69, 1378, 7985, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4759, 0, 0, 278, 0, 0, 7069, 0, 0, 0, 0, 63, 8123, 75, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 509, 0, 0, 0, 0, 0, 7984, 75, 2303, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] dup_rows_per_dst=[300, 0, 412, 276, 0, 394, 244, 3496] combine_rows_per_src=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356302848 dispatch_meta_bytes=354960 combine_payload_bytes=356302848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=356657808 ingress_bytes=425508864 link_bytes=425508864 token_bytes=712960656 weight_movement_bytes=69206016 total_nvlink_bytes=782166672 weight_to_token_ratio=0.097068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601155599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290283520 dispatch_meta_bytes=327728 combine_payload_bytes=290283520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290611248 ingress_bytes=324886528 link_bytes=324886528 token_bytes=580894768 weight_movement_bytes=34603008 total_nvlink_bytes=615497776 weight_to_token_ratio=0.059568 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.222521849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89070 payload_rows_remote=72686 dup_rows=9234 dup_rows_remote=7586 dedup_ratio=0.093933 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 0, 0, 239, 0, 743, 0, 239, 8034, 8192, 0, 0, 0, 0, 0, 3221, 0, 7659, 0, 90, 0, 7835, 0, 0, 0, 0, 1073, 5977, 82, 0, 0, 0, 0, 297, 888, 0, 0, 0, 2632, 0, 114, 0, 0, 0, 0, 0, 0, 90, 1, 82, 0, 959, 361] payload_rows_per_dst=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] dup_rows_per_dst=[0, 1166, 1648, 4178, 1450, 610, 0, 182] combine_rows_per_src=[16384, 15670, 16384, 13660, 12814, 5862, 5492, 2804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316317696 dispatch_meta_bytes=329392 combine_payload_bytes=316317696 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316647088 ingress_bytes=420126720 link_bytes=420126720 token_bytes=632964784 weight_movement_bytes=103809024 total_nvlink_bytes=736773808 weight_to_token_ratio=0.164004 gemm_rows_defined=49152 gemm_rows_charged=51200 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.040000 gemm_active_groups=2 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580903024e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92226 payload_rows_remote=83368 dup_rows=6078 dup_rows_remote=5862 dedup_ratio=0.061829 router_tokens_per_expert=[0, 143, 92, 1423, 7994, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4604, 0, 0, 298, 0, 0, 7035, 0, 0, 0, 0, 60, 8117, 63, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 451, 0, 0, 0, 0, 0, 7989, 63, 2488, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] dup_rows_per_dst=[256, 0, 104, 246, 0, 376, 216, 4880] combine_rows_per_src=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91754 payload_rows_remote=75486 dup_rows=6550 dup_rows_remote=1764 dedup_ratio=0.066630 router_tokens_per_expert=[0, 138, 57, 1393, 7983, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4532, 0, 0, 234, 0, 0, 7055, 0, 0, 0, 0, 42, 8120, 95, 0, 0, 0, 0, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 174, 0, 0, 553, 0, 0, 0, 0, 0, 7866, 95, 2566, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3176, 9064, 468, 16240, 15654, 15966, 14918, 16268] dup_rows_per_dst=[382, 0, 0, 274, 0, 810, 298, 4786] combine_rows_per_src=[3176, 9064, 468, 16240, 15654, 15966, 14918, 16268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86550 payload_rows_remote=71328 dup_rows=11754 dup_rows_remote=11750 dedup_ratio=0.119568 router_tokens_per_expert=[0, 0, 7, 2, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 369, 9, 0, 0, 140, 275, 7760, 1197, 1, 0, 7598, 5, 0, 4745, 140, 54, 0, 0, 0, 6, 581, 7, 0, 0, 0, 3992, 5, 7641, 1, 7, 0, 0, 91, 0, 1, 1, 0, 280, 6267, 1, 195, 140, 7436, 93, 0, 0, 8, 91, 0, 0] payload_rows_per_dst=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] dup_rows_per_dst=[4, 696, 2894, 390, 7512, 12, 64, 182] combine_rows_per_src=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=297721856 dispatch_meta_bytes=321088 combine_payload_bytes=297721856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=298042944 ingress_bytes=436133888 link_bytes=436133888 token_bytes=595764800 weight_movement_bytes=138412032 total_nvlink_bytes=734176832 weight_to_token_ratio=0.232327 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641136709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95458 payload_rows_remote=80440 dup_rows=2846 dup_rows_remote=2726 dedup_ratio=0.028951 router_tokens_per_expert=[7698, 0, 0, 0, 0, 0, 0, 2740, 419, 0, 4984, 0, 0, 0, 0, 161, 0, 0, 0, 201, 0, 161, 0, 7669, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7406, 0, 0, 163, 0, 0, 0, 0, 247, 0, 7598, 0, 0, 0, 0, 0, 296, 177, 393, 0, 89] payload_rows_per_dst=[5480, 11074, 3816, 15396, 13094, 15018, 15196, 16384] dup_rows_per_dst=[0, 54, 94, 0, 174, 120, 494, 1910] combine_rows_per_src=[5480, 11074, 3816, 15396, 13094, 15018, 15196, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95844 payload_rows_remote=79592 dup_rows=2460 dup_rows_remote=2216 dedup_ratio=0.025024 router_tokens_per_expert=[0, 133, 79, 1461, 8004, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4477, 0, 0, 307, 0, 0, 7057, 0, 0, 0, 0, 53, 8126, 69, 0, 0, 0, 0, 0, 7976, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 78, 0, 0, 601, 0, 0, 0, 0, 0, 7966, 69, 2401, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3346, 8954, 14322, 16252, 15952, 16008, 16070, 4940] dup_rows_per_dst=[276, 0, 406, 244, 0, 470, 1064, 0] combine_rows_per_src=[3346, 8954, 14322, 16252, 15952, 16008, 16070, 4940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=356130816 dispatch_meta_bytes=355272 combine_payload_bytes=356130816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=356486088 ingress_bytes=390733824 link_bytes=390733824 token_bytes=712616904 weight_movement_bytes=34603008 total_nvlink_bytes=747219912 weight_to_token_ratio=0.048558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=34816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.470299923e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69728 payload_rows_remote=57310 dup_rows=28576 dup_rows_remote=23860 dedup_ratio=0.290690 router_tokens_per_expert=[0, 142, 76, 1496, 7969, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4530, 0, 0, 295, 0, 0, 7032, 0, 0, 0, 0, 60, 8100, 71, 0, 0, 0, 0, 0, 7966, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 81, 0, 0, 579, 0, 0, 0, 0, 0, 7964, 71, 2418, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12418, 14582, 1886, 3096, 314, 16070, 4978] dup_rows_per_dst=[19472, 4716, 3202, 8, 0, 162, 1016, 0] combine_rows_per_src=[16384, 12418, 14582, 1886, 3096, 314, 16070, 4978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=341475328 dispatch_meta_bytes=356920 combine_payload_bytes=341475328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341832248 ingress_bytes=410681344 link_bytes=410681344 token_bytes=683307576 weight_movement_bytes=69206016 total_nvlink_bytes=752513592 weight_to_token_ratio=0.101281 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545360835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309190656 dispatch_meta_bytes=309000 combine_payload_bytes=309190656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309499656 ingress_bytes=343793664 link_bytes=343793664 token_bytes=618690312 weight_movement_bytes=34603008 total_nvlink_bytes=653293320 weight_to_token_ratio=0.055929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293667880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292159488 dispatch_meta_bytes=332312 combine_payload_bytes=292159488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292491800 ingress_bytes=326762496 link_bytes=326762496 token_bytes=584651288 weight_movement_bytes=34603008 total_nvlink_bytes=619254296 weight_to_token_ratio=0.059186 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229580966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68452 payload_rows_remote=52268 dup_rows=29852 dup_rows_remote=28506 dedup_ratio=0.303670 router_tokens_per_expert=[0, 1, 74, 0, 0, 0, 600, 0, 0, 0, 91, 0, 7018, 0, 8, 7175, 93, 0, 0, 0, 1, 2946, 0, 4, 0, 1, 0, 8090, 0, 7, 8116, 0, 0, 0, 91, 0, 0, 8130, 5, 1, 0, 0, 423, 0, 1, 8, 6, 0, 0, 1, 0, 0, 0, 249, 4, 0, 1, 389, 0, 1, 853, 4763, 1, 0] payload_rows_per_dst=[16184, 16342, 6088, 16264, 2526, 866, 508, 9674] dup_rows_per_dst=[1346, 26598, 0, 1888, 6, 10, 0, 4] combine_rows_per_src=[16184, 16342, 6088, 16264, 2526, 866, 508, 9674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=314343424 dispatch_meta_bytes=326136 combine_payload_bytes=314343424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314669560 ingress_bytes=348946432 link_bytes=348946432 token_bytes=629012984 weight_movement_bytes=34603008 total_nvlink_bytes=663615992 weight_to_token_ratio=0.055012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.313057331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329482240 dispatch_meta_bytes=332664 combine_payload_bytes=329482240 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329814904 ingress_bytes=433291264 link_bytes=433291264 token_bytes=659297144 weight_movement_bytes=103809024 total_nvlink_bytes=763106168 weight_to_token_ratio=0.157454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630440144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95572 payload_rows_remote=94664 dup_rows=2732 dup_rows_remote=2732 dedup_ratio=0.027791 router_tokens_per_expert=[7845, 0, 0, 0, 0, 0, 0, 2650, 392, 0, 5112, 0, 0, 0, 0, 64, 0, 0, 0, 197, 3, 64, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7417, 0, 0, 194, 0, 0, 0, 0, 246, 0, 7728, 0, 0, 0, 0, 0, 211, 144, 419, 0, 75] payload_rows_per_dst=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] dup_rows_per_dst=[0, 10, 394, 0, 0, 138, 492, 1698] combine_rows_per_src=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=326008832 dispatch_meta_bytes=327232 combine_payload_bytes=326008832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326336064 ingress_bytes=395214848 link_bytes=395214848 token_bytes=652344896 weight_movement_bytes=69206016 total_nvlink_bytes=721550912 weight_to_token_ratio=0.106088 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.487161656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=234741760 dispatch_meta_bytes=324680 combine_payload_bytes=234741760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=235066440 ingress_bytes=269344768 link_bytes=269344768 token_bytes=469808200 weight_movement_bytes=34603008 total_nvlink_bytes=504411208 weight_to_token_ratio=0.073653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.013522678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94742 payload_rows_remote=84704 dup_rows=3562 dup_rows_remote=1788 dedup_ratio=0.036235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 465, 0, 0, 0, 74, 207, 7788, 1303, 0, 0, 7577, 0, 0, 4156, 72, 90, 0, 0, 0, 0, 517, 0, 0, 0, 0, 4724, 0, 7762, 0, 0, 0, 0, 82, 0, 0, 0, 0, 253, 6141, 0, 107, 71, 7599, 82, 0, 0, 0, 82, 0, 0] payload_rows_per_dst=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] dup_rows_per_dst=[0, 930, 1774, 0, 84, 60, 550, 164] combine_rows_per_src=[15154, 15576, 10038, 8636, 10398, 6984, 12594, 15362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=387743744 dispatch_meta_bytes=389584 combine_payload_bytes=387743744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388133328 ingress_bytes=456949760 link_bytes=456949760 token_bytes=775877072 weight_movement_bytes=69206016 total_nvlink_bytes=845083088 weight_to_token_ratio=0.089197 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.719465159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=1118 dedup_ratio=0.031921 router_tokens_per_expert=[7725, 0, 0, 0, 0, 0, 0, 2914, 345, 0, 4800, 0, 0, 0, 0, 173, 0, 0, 0, 160, 10, 174, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7291, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7596, 0, 0, 0, 0, 0, 330, 152, 433, 0, 95] payload_rows_per_dst=[5828, 10598, 688, 15450, 16188, 14838, 15192, 16384] dup_rows_per_dst=[0, 38, 0, 0, 428, 144, 508, 2020] combine_rows_per_src=[5828, 10598, 688, 15450, 16188, 14838, 15192, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95856 payload_rows_remote=80352 dup_rows=2448 dup_rows_remote=1940 dedup_ratio=0.024902 router_tokens_per_expert=[7863, 0, 0, 0, 0, 0, 0, 2787, 388, 0, 5021, 0, 0, 0, 0, 48, 0, 0, 0, 149, 0, 48, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7462, 0, 0, 178, 0, 0, 0, 0, 254, 0, 7752, 0, 0, 0, 0, 0, 197, 155, 401, 0, 65] payload_rows_per_dst=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] dup_rows_per_dst=[0, 24, 0, 0, 130, 150, 508, 1636] combine_rows_per_src=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67868 payload_rows_remote=51500 dup_rows=30436 dup_rows_remote=11172 dedup_ratio=0.309611 router_tokens_per_expert=[0, 159, 71, 1330, 7961, 182, 0, 0, 1, 0, 5, 0, 5, 0, 6, 4837, 8, 0, 270, 0, 5, 7015, 0, 0, 0, 0, 55, 8113, 91, 0, 5, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 209, 0, 4, 0, 142, 5, 2, 493, 4, 0, 0, 8, 0, 7879, 91, 2334, 1, 3, 0, 1, 0, 0] payload_rows_per_dst=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] dup_rows_per_dst=[19264, 9684, 398, 0, 0, 280, 810, 0] combine_rows_per_src=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93888 payload_rows_remote=87266 dup_rows=4416 dup_rows_remote=4304 dedup_ratio=0.044922 router_tokens_per_expert=[0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 1, 0, 0, 303, 0, 0, 0, 0, 66, 0, 436, 0, 0, 0, 0, 0, 0, 6029, 8126, 0, 0, 0, 0, 0, 60, 0, 89, 66, 0, 0, 0, 7845, 8192, 0, 0, 544, 0, 0, 0, 0, 90, 0, 0, 0, 1167, 0, 0, 0, 0, 0] payload_rows_per_dst=[10372, 5700, 10252, 12434, 16206, 6622, 16050, 16252] dup_rows_per_dst=[0, 0, 446, 496, 120, 112, 908, 2334] combine_rows_per_src=[10372, 5700, 10252, 12434, 16206, 6622, 16050, 16252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=346947584 dispatch_meta_bytes=345968 combine_payload_bytes=346947584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347293552 ingress_bytes=416153600 link_bytes=416153600 token_bytes=694241136 weight_movement_bytes=69206016 total_nvlink_bytes=763447152 weight_to_token_ratio=0.099686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96142 payload_rows_remote=80454 dup_rows=2162 dup_rows_remote=1772 dedup_ratio=0.021993 router_tokens_per_expert=[7751, 0, 0, 0, 0, 0, 0, 3130, 345, 0, 4614, 0, 0, 0, 0, 72, 0, 0, 0, 195, 0, 72, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7408, 0, 0, 165, 0, 0, 0, 0, 264, 0, 7793, 0, 0, 0, 0, 0, 199, 195, 443, 0, 82] payload_rows_per_dst=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] dup_rows_per_dst=[0, 10, 390, 0, 810, 88, 528, 336] combine_rows_per_src=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=323096 combine_payload_bytes=214089728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=214412824 ingress_bytes=248692736 link_bytes=248692736 token_bytes=428502552 weight_movement_bytes=34603008 total_nvlink_bytes=463105560 weight_to_token_ratio=0.080753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.358107442e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92226 payload_rows_remote=83368 dup_rows=6078 dup_rows_remote=5862 dedup_ratio=0.061829 router_tokens_per_expert=[0, 143, 92, 1423, 7994, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4604, 0, 0, 298, 0, 0, 7035, 0, 0, 0, 0, 60, 8117, 63, 0, 0, 0, 0, 0, 8016, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 48, 0, 0, 451, 0, 0, 0, 0, 0, 7989, 63, 2488, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] dup_rows_per_dst=[256, 0, 104, 246, 0, 376, 216, 4880] combine_rows_per_src=[3316, 9208, 6390, 16234, 16032, 15988, 8858, 16200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91754 payload_rows_remote=75486 dup_rows=6550 dup_rows_remote=1764 dedup_ratio=0.066630 router_tokens_per_expert=[0, 138, 57, 1393, 7983, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4532, 0, 0, 234, 0, 0, 7055, 0, 0, 0, 0, 42, 8120, 95, 0, 0, 0, 0, 0, 7827, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 174, 0, 0, 553, 0, 0, 0, 0, 0, 7866, 95, 2566, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3176, 9064, 468, 16240, 15654, 15966, 14918, 16268] dup_rows_per_dst=[382, 0, 0, 274, 0, 810, 298, 4786] combine_rows_per_src=[3176, 9064, 468, 16240, 15654, 15966, 14918, 16268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93182 payload_rows_remote=77226 dup_rows=5122 dup_rows_remote=5122 dedup_ratio=0.052104 router_tokens_per_expert=[0, 157, 69, 1378, 7985, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4759, 0, 0, 278, 0, 0, 7069, 0, 0, 0, 0, 63, 8123, 75, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 64, 0, 0, 509, 0, 0, 0, 0, 0, 7984, 75, 2303, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] dup_rows_per_dst=[300, 0, 412, 276, 0, 394, 244, 3496] combine_rows_per_src=[3208, 9518, 14282, 16246, 15956, 15970, 4740, 13262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328904 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329868488 ingress_bytes=398745600 link_bytes=398745600 token_bytes=659408072 weight_movement_bytes=69206016 total_nvlink_bytes=728614088 weight_to_token_ratio=0.104952 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500447591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95458 payload_rows_remote=80440 dup_rows=2846 dup_rows_remote=2726 dedup_ratio=0.028951 router_tokens_per_expert=[7698, 0, 0, 0, 0, 0, 0, 2740, 419, 0, 4984, 0, 0, 0, 0, 161, 0, 0, 0, 201, 0, 161, 0, 7669, 0, 0, 0, 0, 0, 0, 0, 0, 558, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7406, 0, 0, 163, 0, 0, 0, 0, 247, 0, 7598, 0, 0, 0, 0, 0, 296, 177, 393, 0, 89] payload_rows_per_dst=[5480, 11074, 3816, 15396, 13094, 15018, 15196, 16384] dup_rows_per_dst=[0, 54, 94, 0, 174, 120, 494, 1910] combine_rows_per_src=[5480, 11074, 3816, 15396, 13094, 15018, 15196, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=341475328 dispatch_meta_bytes=356920 combine_payload_bytes=341475328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=341832248 ingress_bytes=410681344 link_bytes=410681344 token_bytes=683307576 weight_movement_bytes=69206016 total_nvlink_bytes=752513592 weight_to_token_ratio=0.101281 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.545360835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309190656 dispatch_meta_bytes=309000 combine_payload_bytes=309190656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309499656 ingress_bytes=343793664 link_bytes=343793664 token_bytes=618690312 weight_movement_bytes=34603008 total_nvlink_bytes=653293320 weight_to_token_ratio=0.055929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293667880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95612 payload_rows_remote=79930 dup_rows=2692 dup_rows_remote=2692 dedup_ratio=0.027384 router_tokens_per_expert=[7841, 0, 0, 0, 0, 0, 0, 2959, 407, 0, 4698, 0, 0, 0, 0, 78, 0, 0, 0, 170, 0, 78, 0, 7727, 0, 0, 0, 0, 0, 0, 0, 0, 474, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7418, 0, 0, 162, 0, 0, 0, 0, 261, 0, 7843, 0, 0, 0, 0, 0, 213, 148, 414, 0, 69] payload_rows_per_dst=[5918, 10356, 15610, 15682, 948, 15028, 15686, 16384] dup_rows_per_dst=[0, 10, 340, 0, 0, 132, 522, 1688] combine_rows_per_src=[5918, 10356, 15610, 15682, 948, 15028, 15686, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=316317696 dispatch_meta_bytes=329392 combine_payload_bytes=316317696 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=316647088 ingress_bytes=420126720 link_bytes=420126720 token_bytes=632964784 weight_movement_bytes=103809024 total_nvlink_bytes=736773808 weight_to_token_ratio=0.164004 gemm_rows_defined=49152 gemm_rows_charged=51200 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.040000 gemm_active_groups=2 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.580903024e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69950 payload_rows_remote=59330 dup_rows=28354 dup_rows_remote=28320 dedup_ratio=0.288432 router_tokens_per_expert=[7829, 0, 0, 0, 0, 0, 0, 2946, 438, 0, 4808, 0, 0, 0, 0, 81, 0, 0, 0, 167, 1, 81, 0, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 483, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7329, 0, 0, 207, 0, 0, 0, 0, 239, 0, 7736, 0, 0, 0, 0, 0, 218, 147, 422, 0, 74] payload_rows_per_dst=[13018, 10620, 16118, 4850, 16384, 5328, 2058, 1574] dup_rows_per_dst=[14118, 34, 11412, 0, 966, 1618, 58, 148] combine_rows_per_src=[13018, 10620, 16118, 4850, 16384, 5328, 2058, 1574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=319600 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323010672 ingress_bytes=357294080 link_bytes=357294080 token_bytes=645701744 weight_movement_bytes=34603008 total_nvlink_bytes=680304752 weight_to_token_ratio=0.053590 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344468859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329121792 dispatch_meta_bytes=329168 combine_payload_bytes=329121792 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329450960 ingress_bytes=398327808 link_bytes=398327808 token_bytes=658572752 weight_movement_bytes=69206016 total_nvlink_bytes=727778768 weight_to_token_ratio=0.105085 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498875473e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=357441536 dispatch_meta_bytes=366280 combine_payload_bytes=357441536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357807816 ingress_bytes=461250560 link_bytes=461250560 token_bytes=715249352 weight_movement_bytes=103809024 total_nvlink_bytes=819058376 weight_to_token_ratio=0.145137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735648724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=210944000 dispatch_meta_bytes=250688 combine_payload_bytes=210944000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211194688 ingress_bytes=280150016 link_bytes=280150016 token_bytes=422138688 weight_movement_bytes=69206016 total_nvlink_bytes=491344704 weight_to_token_ratio=0.163941 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054181956e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329482240 dispatch_meta_bytes=332664 combine_payload_bytes=329482240 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=329814904 ingress_bytes=433291264 link_bytes=433291264 token_bytes=659297144 weight_movement_bytes=103809024 total_nvlink_bytes=763106168 weight_to_token_ratio=0.157454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.630440144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87000 payload_rows_remote=70622 dup_rows=11304 dup_rows_remote=11286 dedup_ratio=0.114990 router_tokens_per_expert=[0, 0, 8, 0, 7, 0, 1, 3, 0, 0, 371, 1, 0, 0, 311, 0, 912, 0, 245, 7938, 8179, 0, 3, 0, 0, 0, 2906, 0, 7687, 1, 55, 0, 7856, 2, 4, 0, 2, 1242, 6330, 91, 0, 3, 0, 6, 269, 836, 0, 0, 0, 2359, 0, 71, 0, 6, 0, 0, 2, 0, 70, 8, 91, 7, 923, 346] payload_rows_per_dst=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] dup_rows_per_dst=[18, 1362, 3036, 5122, 1596, 6, 0, 164] combine_rows_per_src=[16378, 15716, 16372, 16176, 12534, 2222, 4872, 2730] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94496 payload_rows_remote=92100 dup_rows=3808 dup_rows_remote=3786 dedup_ratio=0.038737 router_tokens_per_expert=[0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 1, 0, 0, 345, 0, 0, 0, 0, 63, 0, 315, 0, 0, 0, 0, 0, 0, 6247, 8129, 0, 0, 0, 0, 0, 69, 0, 75, 63, 0, 0, 0, 7976, 8192, 0, 0, 536, 0, 0, 0, 0, 75, 0, 0, 0, 890, 0, 0, 0, 0, 0] payload_rows_per_dst=[16118, 13954, 16384, 12858, 2396, 276, 16252, 16258] dup_rows_per_dst=[0, 2, 816, 266, 22, 0, 922, 1780] combine_rows_per_src=[16118, 13954, 16384, 12858, 2396, 276, 16252, 16258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=327393280 dispatch_meta_bytes=330488 combine_payload_bytes=327393280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=327723768 ingress_bytes=361996288 link_bytes=361996288 token_bytes=655117048 weight_movement_bytes=34603008 total_nvlink_bytes=689720056 weight_to_token_ratio=0.052820 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362162889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=243015680 dispatch_meta_bytes=350600 combine_payload_bytes=243015680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=243366280 ingress_bytes=277618688 link_bytes=277618688 token_bytes=486381960 weight_movement_bytes=34603008 total_nvlink_bytes=520984968 weight_to_token_ratio=0.071144 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.044656773e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93298 payload_rows_remote=79076 dup_rows=5006 dup_rows_remote=4680 dedup_ratio=0.050924 router_tokens_per_expert=[0, 198, 113, 1405, 7976, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4706, 0, 0, 279, 0, 0, 6995, 0, 0, 0, 0, 90, 8148, 82, 0, 0, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 73, 0, 0, 508, 0, 0, 0, 0, 0, 7925, 82, 2267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] dup_rows_per_dst=[3116, 0, 326, 254, 0, 458, 852, 0] combine_rows_per_src=[5876, 9412, 14222, 13266, 15912, 13898, 16014, 4698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=377241600 dispatch_meta_bytes=383544 combine_payload_bytes=377241600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=377625144 ingress_bytes=446447616 link_bytes=446447616 token_bytes=754866744 weight_movement_bytes=69206016 total_nvlink_bytes=824072760 weight_to_token_ratio=0.091680 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.679946437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92972 payload_rows_remote=76686 dup_rows=5332 dup_rows_remote=4450 dedup_ratio=0.054240 router_tokens_per_expert=[0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 336, 0, 0, 0, 0, 60, 0, 294, 0, 0, 0, 0, 0, 0, 6475, 8132, 0, 0, 0, 0, 0, 75, 0, 65, 60, 0, 0, 0, 8015, 8192, 0, 0, 504, 0, 0, 0, 0, 65, 0, 0, 0, 712, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 13734, 792, 13258, 16254, 16384, 16286, 16264] dup_rows_per_dst=[0, 0, 0, 280, 150, 2596, 882, 1424] combine_rows_per_src=[0, 13734, 792, 13258, 16254, 16384, 16286, 16264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78398 payload_rows_remote=62098 dup_rows=19906 dup_rows_remote=17914 dedup_ratio=0.202494 router_tokens_per_expert=[0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 338, 0, 0, 0, 0, 42, 0, 382, 0, 0, 0, 0, 0, 0, 6218, 8150, 0, 0, 0, 0, 0, 72, 0, 95, 44, 0, 0, 0, 7822, 8192, 0, 0, 515, 0, 0, 0, 0, 96, 0, 0, 0, 996, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 760, 12738, 16194, 16384, 16022, 16300] dup_rows_per_dst=[0, 0, 0, 462, 144, 16464, 844, 1992] combine_rows_per_src=[0, 0, 760, 12738, 16194, 16384, 16022, 16300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92830 payload_rows_remote=77112 dup_rows=5474 dup_rows_remote=4732 dedup_ratio=0.055684 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 298, 0, 89, 7859, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6995, 0, 0, 0, 0, 0, 0, 1631, 2314, 0, 0, 0, 0, 0, 0, 0, 7111, 0, 8107, 89, 0, 0, 0, 0, 0, 282, 0, 246, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 7502, 0, 0, 0, 20] payload_rows_per_dst=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] dup_rows_per_dst=[456, 0, 0, 528, 3214, 742, 0, 534] combine_rows_per_src=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66164 payload_rows_remote=50066 dup_rows=32140 dup_rows_remote=11430 dedup_ratio=0.326945 router_tokens_per_expert=[7737, 5, 0, 0, 0, 0, 4, 2939, 378, 8, 4673, 0, 0, 0, 0, 139, 0, 2, 1, 169, 14, 140, 0, 7639, 0, 0, 0, 0, 0, 1, 0, 3, 563, 4, 0, 0, 0, 0, 1, 8183, 0, 9, 0, 7409, 4, 7, 195, 0, 0, 0, 0, 250, 4, 7719, 0, 4, 0, 0, 0, 262, 161, 434, 0, 91] payload_rows_per_dst=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] dup_rows_per_dst=[20710, 7988, 2102, 0, 1130, 22, 6, 182] combine_rows_per_src=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95970 payload_rows_remote=79586 dup_rows=2334 dup_rows_remote=1428 dedup_ratio=0.023743 router_tokens_per_expert=[0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 363, 0, 0, 0, 0, 90, 0, 418, 0, 0, 0, 0, 0, 0, 6073, 8102, 0, 0, 0, 0, 0, 44, 0, 82, 90, 0, 0, 0, 7952, 8192, 0, 0, 507, 0, 0, 0, 0, 82, 0, 0, 0, 1061, 0, 0, 0, 0, 0] payload_rows_per_dst=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] dup_rows_per_dst=[0, 0, 906, 370, 74, 94, 552, 338] combine_rows_per_src=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=289267712 dispatch_meta_bytes=327632 combine_payload_bytes=289267712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289595344 ingress_bytes=323870720 link_bytes=323870720 token_bytes=578863056 weight_movement_bytes=34603008 total_nvlink_bytes=613466064 weight_to_token_ratio=0.059778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.218699446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95856 payload_rows_remote=80352 dup_rows=2448 dup_rows_remote=1940 dedup_ratio=0.024902 router_tokens_per_expert=[7863, 0, 0, 0, 0, 0, 0, 2787, 388, 0, 5021, 0, 0, 0, 0, 48, 0, 0, 0, 149, 0, 48, 0, 7752, 0, 0, 0, 0, 0, 0, 0, 0, 440, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7462, 0, 0, 178, 0, 0, 0, 0, 254, 0, 7752, 0, 0, 0, 0, 0, 197, 155, 401, 0, 65] payload_rows_per_dst=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] dup_rows_per_dst=[0, 24, 0, 0, 130, 150, 508, 1636] combine_rows_per_src=[5574, 10890, 394, 15726, 16254, 15130, 15504, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95166 payload_rows_remote=78782 dup_rows=3138 dup_rows_remote=1118 dedup_ratio=0.031921 router_tokens_per_expert=[7725, 0, 0, 0, 0, 0, 0, 2914, 345, 0, 4800, 0, 0, 0, 0, 173, 0, 0, 0, 160, 10, 174, 0, 7715, 0, 0, 0, 0, 0, 0, 0, 0, 593, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7291, 0, 0, 200, 0, 0, 0, 0, 254, 0, 7596, 0, 0, 0, 0, 0, 330, 152, 433, 0, 95] payload_rows_per_dst=[5828, 10598, 688, 15450, 16188, 14838, 15192, 16384] dup_rows_per_dst=[0, 38, 0, 0, 428, 144, 508, 2020] combine_rows_per_src=[5828, 10598, 688, 15450, 16188, 14838, 15192, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95572 payload_rows_remote=94664 dup_rows=2732 dup_rows_remote=2732 dedup_ratio=0.027791 router_tokens_per_expert=[7845, 0, 0, 0, 0, 0, 0, 2650, 392, 0, 5112, 0, 0, 0, 0, 64, 0, 0, 0, 197, 3, 64, 0, 7745, 0, 0, 0, 0, 0, 0, 0, 0, 454, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7417, 0, 0, 194, 0, 0, 0, 0, 246, 0, 7728, 0, 0, 0, 0, 0, 211, 144, 419, 0, 75] payload_rows_per_dst=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] dup_rows_per_dst=[0, 10, 394, 0, 0, 138, 492, 1698] combine_rows_per_src=[5300, 11126, 15624, 15690, 908, 15084, 15456, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325984256 dispatch_meta_bytes=324056 combine_payload_bytes=325984256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326308312 ingress_bytes=360587264 link_bytes=360587264 token_bytes=652292568 weight_movement_bytes=34603008 total_nvlink_bytes=686895576 weight_to_token_ratio=0.053048 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356860845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329121792 dispatch_meta_bytes=329168 combine_payload_bytes=329121792 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329450960 ingress_bytes=398327808 link_bytes=398327808 token_bytes=658572752 weight_movement_bytes=69206016 total_nvlink_bytes=727778768 weight_to_token_ratio=0.105085 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498875473e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=322691072 dispatch_meta_bytes=319600 combine_payload_bytes=322691072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323010672 ingress_bytes=357294080 link_bytes=357294080 token_bytes=645701744 weight_movement_bytes=34603008 total_nvlink_bytes=680304752 weight_to_token_ratio=0.053590 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.344468859e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93888 payload_rows_remote=87266 dup_rows=4416 dup_rows_remote=4304 dedup_ratio=0.044922 router_tokens_per_expert=[0, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 1, 0, 0, 303, 0, 0, 0, 0, 66, 0, 436, 0, 0, 0, 0, 0, 0, 6029, 8126, 0, 0, 0, 0, 0, 60, 0, 89, 66, 0, 0, 0, 7845, 8192, 0, 0, 544, 0, 0, 0, 0, 90, 0, 0, 0, 1167, 0, 0, 0, 0, 0] payload_rows_per_dst=[10372, 5700, 10252, 12434, 16206, 6622, 16050, 16252] dup_rows_per_dst=[0, 0, 446, 496, 120, 112, 908, 2334] combine_rows_per_src=[10372, 5700, 10252, 12434, 16206, 6622, 16050, 16252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94880 payload_rows_remote=81608 dup_rows=3424 dup_rows_remote=3110 dedup_ratio=0.034831 router_tokens_per_expert=[0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 346, 6, 0, 0, 0, 53, 0, 346, 0, 0, 0, 0, 0, 0, 6447, 8139, 0, 0, 0, 0, 0, 65, 0, 69, 47, 0, 0, 0, 7971, 8192, 0, 0, 540, 0, 0, 0, 0, 69, 0, 0, 0, 679, 0, 0, 0, 0, 0] payload_rows_per_dst=[16120, 16246, 16384, 13272, 130, 232, 16218, 16278] dup_rows_per_dst=[0, 0, 810, 314, 0, 0, 942, 1358] combine_rows_per_src=[16120, 16246, 16384, 13272, 130, 232, 16218, 16278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323895296 dispatch_meta_bytes=335024 combine_payload_bytes=323895296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324230320 ingress_bytes=393101312 link_bytes=393101312 token_bytes=648125616 weight_movement_bytes=69206016 total_nvlink_bytes=717331632 weight_to_token_ratio=0.106779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.479208590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=387743744 dispatch_meta_bytes=389584 combine_payload_bytes=387743744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=388133328 ingress_bytes=456949760 link_bytes=456949760 token_bytes=775877072 weight_movement_bytes=69206016 total_nvlink_bytes=845083088 weight_to_token_ratio=0.089197 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.719465159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96718 payload_rows_remote=80482 dup_rows=1586 dup_rows_remote=1586 dedup_ratio=0.016134 router_tokens_per_expert=[0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 350, 0, 0, 0, 0, 60, 0, 310, 0, 0, 0, 0, 0, 0, 6147, 8132, 0, 0, 0, 0, 0, 92, 0, 74, 60, 0, 0, 0, 7964, 8192, 0, 0, 566, 0, 0, 0, 0, 74, 0, 0, 0, 957, 0, 0, 0, 0, 0] payload_rows_per_dst=[16112, 16236, 16384, 12600, 16264, 16076, 1132, 1914] dup_rows_per_dst=[0, 0, 820, 314, 184, 120, 148, 0] combine_rows_per_src=[16112, 16236, 16384, 12600, 16264, 16076, 1132, 1914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=205070336 dispatch_meta_bytes=245984 combine_payload_bytes=205070336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=205316320 ingress_bytes=274276352 link_bytes=274276352 token_bytes=410386656 weight_movement_bytes=69206016 total_nvlink_bytes=479592672 weight_to_token_ratio=0.168636 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032079832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86550 payload_rows_remote=71328 dup_rows=11754 dup_rows_remote=11750 dedup_ratio=0.119568 router_tokens_per_expert=[0, 0, 7, 2, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 369, 9, 0, 0, 140, 275, 7760, 1197, 1, 0, 7598, 5, 0, 4745, 140, 54, 0, 0, 0, 6, 581, 7, 0, 0, 0, 3992, 5, 7641, 1, 7, 0, 0, 91, 0, 1, 1, 0, 280, 6267, 1, 195, 140, 7436, 93, 0, 0, 8, 91, 0, 0] payload_rows_per_dst=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] dup_rows_per_dst=[4, 696, 2894, 390, 7512, 12, 64, 182] combine_rows_per_src=[15222, 15580, 15614, 10200, 15830, 12730, 1172, 202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=357441536 dispatch_meta_bytes=366280 combine_payload_bytes=357441536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=357807816 ingress_bytes=461250560 link_bytes=461250560 token_bytes=715249352 weight_movement_bytes=103809024 total_nvlink_bytes=819058376 weight_to_token_ratio=0.145137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735648724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=314105856 dispatch_meta_bytes=324544 combine_payload_bytes=314105856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314430400 ingress_bytes=348708864 link_bytes=348708864 token_bytes=628536256 weight_movement_bytes=34603008 total_nvlink_bytes=663139264 weight_to_token_ratio=0.055053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312163382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95740 payload_rows_remote=93782 dup_rows=2564 dup_rows_remote=2564 dedup_ratio=0.026082 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 350, 0, 75, 7878, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 1452, 2200, 0, 0, 0, 0, 0, 0, 0, 7232, 0, 8117, 75, 0, 0, 0, 0, 0, 291, 0, 245, 0, 0, 0, 0, 64, 0, 0, 0, 0, 321, 7471, 0, 0, 0, 12] payload_rows_per_dst=[16234, 10340, 14466, 6812, 1958, 15756, 15082, 15092] dup_rows_per_dst=[700, 124, 0, 492, 0, 732, 0, 516] combine_rows_per_src=[16234, 10340, 14466, 6812, 1958, 15756, 15082, 15092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315850752 dispatch_meta_bytes=327376 combine_payload_bytes=315850752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316178128 ingress_bytes=350453760 link_bytes=350453760 token_bytes=632028880 weight_movement_bytes=34603008 total_nvlink_bytes=666631888 weight_to_token_ratio=0.054749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318729285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=254353408 dispatch_meta_bytes=320048 combine_payload_bytes=254353408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254673456 ingress_bytes=288956416 link_bytes=288956416 token_bytes=509026864 weight_movement_bytes=34603008 total_nvlink_bytes=543629872 weight_to_token_ratio=0.067979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087319731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=334266368 dispatch_meta_bytes=338872 combine_payload_bytes=334266368 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=334605240 ingress_bytes=368869376 link_bytes=368869376 token_bytes=668871608 weight_movement_bytes=34603008 total_nvlink_bytes=703474616 weight_to_token_ratio=0.051733 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388025765e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=329654272 dispatch_meta_bytes=328272 combine_payload_bytes=329654272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329982544 ingress_bytes=398860288 link_bytes=398860288 token_bytes=659636816 weight_movement_bytes=69206016 total_nvlink_bytes=728842832 weight_to_token_ratio=0.104915 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=39424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500879153e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14113173328 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=7060492112 ingress_bytes=8506007552 link_bytes=8506007552 total_nvlink_bytes=15566499664 weight_to_token_ratio=0.102977 prefetch_share_of_ingress=0.085429 dedup_ratio=0.120512 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.391627027e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=384131072 dispatch_meta_bytes=385384 combine_payload_bytes=384131072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=384516456 ingress_bytes=487940096 link_bytes=487940096 token_bytes=768647528 weight_movement_bytes=103809024 total_nvlink_bytes=872456552 weight_to_token_ratio=0.135054 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.836079299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=96142 payload_rows_remote=80454 dup_rows=2162 dup_rows_remote=1772 dedup_ratio=0.021993 router_tokens_per_expert=[7751, 0, 0, 0, 0, 0, 0, 3130, 345, 0, 4614, 0, 0, 0, 0, 72, 0, 0, 0, 195, 0, 72, 0, 7772, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7408, 0, 0, 165, 0, 0, 0, 0, 264, 0, 7793, 0, 0, 0, 0, 0, 199, 195, 443, 0, 82] payload_rows_per_dst=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] dup_rows_per_dst=[0, 10, 390, 0, 810, 88, 528, 336] combine_rows_per_src=[6260, 10052, 15688, 15502, 14518, 15058, 15586, 3478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14785478488 prefetch_bytes=726663168 reduce_bytes=865075200 weight_movement_bytes=1591738368 egress_bytes=7396769624 ingress_bytes=8980447232 link_bytes=8980447232 total_nvlink_bytes=16377216856 weight_to_token_ratio=0.107656 prefetch_share_of_ingress=0.080916 dedup_ratio=0.104343 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=25 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.469247824e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=73206 dup_rows=8936 dup_rows_remote=1488 dedup_ratio=0.090902 router_tokens_per_expert=[0, 0, 8050, 0, 1, 0, 0, 1, 9, 0, 0, 0, 8088, 0, 9, 0, 0, 345, 1, 0, 0, 0, 54, 0, 411, 3, 0, 0, 0, 0, 0, 6354, 8121, 0, 0, 0, 8, 0, 69, 0, 96, 54, 0, 1, 8, 7859, 8183, 0, 9, 536, 0, 0, 1, 0, 91, 0, 0, 9, 773, 8, 0, 0, 0, 0] payload_rows_per_dst=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] dup_rows_per_dst=[7448, 16, 800, 216, 138, 118, 184, 16] combine_rows_per_src=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84090 payload_rows_remote=68920 dup_rows=14214 dup_rows_remote=14214 dedup_ratio=0.144592 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 304, 0, 63, 7846, 6346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7081, 0, 0, 0, 0, 0, 0, 1473, 2156, 0, 0, 0, 0, 0, 0, 0, 7283, 0, 8129, 65, 0, 0, 0, 0, 0, 249, 0, 254, 0, 0, 0, 0, 48, 0, 0, 0, 0, 319, 7535, 0, 0, 0, 1] payload_rows_per_dst=[608, 126, 14162, 6748, 16384, 15692, 15170, 15200] dup_rows_per_dst=[0, 0, 0, 510, 12566, 628, 0, 510] combine_rows_per_src=[608, 126, 14162, 6748, 16384, 15692, 15170, 15200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91178 payload_rows_remote=76892 dup_rows=7126 dup_rows_remote=7126 dedup_ratio=0.072489 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 322, 0, 81, 7816, 6234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 1616, 2116, 0, 0, 0, 0, 0, 0, 0, 7218, 0, 8111, 82, 0, 0, 0, 0, 0, 267, 0, 263, 0, 0, 0, 0, 72, 0, 0, 0, 0, 346, 7464, 0, 0, 0, 1] payload_rows_per_dst=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] dup_rows_per_dst=[644, 1366, 0, 4294, 0, 612, 0, 210] combine_rows_per_src=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292159488 dispatch_meta_bytes=332312 combine_payload_bytes=292159488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292491800 ingress_bytes=326762496 link_bytes=326762496 token_bytes=584651288 weight_movement_bytes=34603008 total_nvlink_bytes=619254296 weight_to_token_ratio=0.059186 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.229580966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92972 payload_rows_remote=76686 dup_rows=5332 dup_rows_remote=4450 dedup_ratio=0.054240 router_tokens_per_expert=[0, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 336, 0, 0, 0, 0, 60, 0, 294, 0, 0, 0, 0, 0, 0, 6475, 8132, 0, 0, 0, 0, 0, 75, 0, 65, 60, 0, 0, 0, 8015, 8192, 0, 0, 504, 0, 0, 0, 0, 65, 0, 0, 0, 712, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 13734, 792, 13258, 16254, 16384, 16286, 16264] dup_rows_per_dst=[0, 0, 0, 280, 150, 2596, 882, 1424] combine_rows_per_src=[0, 13734, 792, 13258, 16254, 16384, 16286, 16264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78398 payload_rows_remote=62098 dup_rows=19906 dup_rows_remote=17914 dedup_ratio=0.202494 router_tokens_per_expert=[0, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 338, 0, 0, 0, 0, 42, 0, 382, 0, 0, 0, 0, 0, 0, 6218, 8150, 0, 0, 0, 0, 0, 72, 0, 95, 44, 0, 0, 0, 7822, 8192, 0, 0, 515, 0, 0, 0, 0, 96, 0, 0, 0, 996, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 0, 760, 12738, 16194, 16384, 16022, 16300] dup_rows_per_dst=[0, 0, 0, 462, 144, 16464, 844, 1992] combine_rows_per_src=[0, 0, 760, 12738, 16194, 16384, 16022, 16300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94496 payload_rows_remote=92100 dup_rows=3808 dup_rows_remote=3786 dedup_ratio=0.038737 router_tokens_per_expert=[0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8117, 0, 1, 0, 0, 345, 0, 0, 0, 0, 63, 0, 315, 0, 0, 0, 0, 0, 0, 6247, 8129, 0, 0, 0, 0, 0, 69, 0, 75, 63, 0, 0, 0, 7976, 8192, 0, 0, 536, 0, 0, 0, 0, 75, 0, 0, 0, 890, 0, 0, 0, 0, 0] payload_rows_per_dst=[16118, 13954, 16384, 12858, 2396, 276, 16252, 16258] dup_rows_per_dst=[0, 2, 816, 266, 22, 0, 922, 1780] combine_rows_per_src=[16118, 13954, 16384, 12858, 2396, 276, 16252, 16258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=314949632 dispatch_meta_bytes=336072 combine_payload_bytes=314949632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=315285704 ingress_bytes=384155648 link_bytes=384155648 token_bytes=630235336 weight_movement_bytes=69206016 total_nvlink_bytes=699441352 weight_to_token_ratio=0.109810 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445546776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=314105856 dispatch_meta_bytes=324544 combine_payload_bytes=314105856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=314430400 ingress_bytes=348708864 link_bytes=348708864 token_bytes=628536256 weight_movement_bytes=34603008 total_nvlink_bytes=663139264 weight_to_token_ratio=0.055053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.312163382e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=254353408 dispatch_meta_bytes=320048 combine_payload_bytes=254353408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254673456 ingress_bytes=288956416 link_bytes=288956416 token_bytes=509026864 weight_movement_bytes=34603008 total_nvlink_bytes=543629872 weight_to_token_ratio=0.067979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087319731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92830 payload_rows_remote=77112 dup_rows=5474 dup_rows_remote=4732 dedup_ratio=0.055684 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 298, 0, 89, 7859, 6111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6995, 0, 0, 0, 0, 0, 0, 1631, 2314, 0, 0, 0, 0, 0, 0, 0, 7111, 0, 8107, 89, 0, 0, 0, 0, 0, 282, 0, 246, 0, 0, 0, 0, 161, 0, 0, 0, 0, 337, 7502, 0, 0, 0, 20] payload_rows_per_dst=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] dup_rows_per_dst=[456, 0, 0, 528, 3214, 742, 0, 534] combine_rows_per_src=[12214, 178, 13990, 7362, 13148, 15718, 15036, 15184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95774 payload_rows_remote=88730 dup_rows=2530 dup_rows_remote=2008 dedup_ratio=0.025736 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 325, 0, 69, 7853, 6105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 0, 0, 0, 1521, 2262, 0, 0, 0, 0, 0, 0, 0, 7216, 0, 8123, 69, 0, 0, 0, 0, 0, 252, 0, 260, 0, 0, 0, 0, 78, 0, 0, 0, 0, 330, 7523, 0, 0, 0, 28] payload_rows_per_dst=[16246, 12210, 14276, 7044, 0, 15706, 15108, 15184] dup_rows_per_dst=[650, 138, 0, 522, 0, 642, 0, 578] combine_rows_per_src=[16246, 12210, 14276, 7044, 0, 15706, 15108, 15184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72894 payload_rows_remote=56708 dup_rows=25410 dup_rows_remote=13564 dedup_ratio=0.258484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 282, 0, 93, 7882, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 0, 1476, 2058, 0, 0, 0, 0, 0, 0, 0, 7107, 0, 8099, 96, 0, 0, 0, 0, 0, 247, 0, 253, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 7450, 0, 0, 0, 0] payload_rows_per_dst=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] dup_rows_per_dst=[0, 0, 0, 520, 12358, 686, 0, 11846] combine_rows_per_src=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=377241600 dispatch_meta_bytes=383544 combine_payload_bytes=377241600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=377625144 ingress_bytes=446447616 link_bytes=446447616 token_bytes=754866744 weight_movement_bytes=69206016 total_nvlink_bytes=824072760 weight_to_token_ratio=0.091680 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.679946437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79206 payload_rows_remote=63300 dup_rows=19098 dup_rows_remote=6412 dedup_ratio=0.194275 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 329, 0, 71, 7877, 6348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7414, 0, 0, 0, 0, 0, 0, 1560, 1844, 0, 0, 0, 0, 0, 0, 0, 7144, 0, 8121, 74, 0, 0, 0, 0, 0, 269, 0, 239, 0, 0, 0, 0, 81, 0, 0, 0, 0, 313, 7467, 0, 0, 0, 1] payload_rows_per_dst=[16242, 15906, 14828, 15988, 7974, 686, 6954, 628] dup_rows_per_dst=[658, 12686, 0, 5754, 0, 0, 0, 0] combine_rows_per_src=[16242, 15906, 14828, 15988, 7974, 686, 6954, 628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=329539584 dispatch_meta_bytes=328904 combine_payload_bytes=329539584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329868488 ingress_bytes=398745600 link_bytes=398745600 token_bytes=659408072 weight_movement_bytes=69206016 total_nvlink_bytes=728614088 weight_to_token_ratio=0.104952 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500447591e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14100577032 prefetch_bytes=657457152 reduce_bytes=692060160 weight_movement_bytes=1349517312 egress_bytes=7054162696 ingress_bytes=8395931648 link_bytes=8395931648 total_nvlink_bytes=15450094344 weight_to_token_ratio=0.095707 prefetch_share_of_ingress=0.078307 dedup_ratio=0.113706 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=20 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21336 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.373618038e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=299851776 dispatch_meta_bytes=298776 combine_payload_bytes=299851776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300150552 ingress_bytes=403660800 link_bytes=403660800 token_bytes=600002328 weight_movement_bytes=103809024 total_nvlink_bytes=703811352 weight_to_token_ratio=0.173014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282296320 dispatch_meta_bytes=332536 combine_payload_bytes=282296320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282628856 ingress_bytes=316899328 link_bytes=316899328 token_bytes=564925176 weight_movement_bytes=34603008 total_nvlink_bytes=599528184 weight_to_token_ratio=0.061252 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192466659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67868 payload_rows_remote=51500 dup_rows=30436 dup_rows_remote=11172 dedup_ratio=0.309611 router_tokens_per_expert=[0, 159, 71, 1330, 7961, 182, 0, 0, 1, 0, 5, 0, 5, 0, 6, 4837, 8, 0, 270, 0, 5, 7015, 0, 0, 0, 0, 55, 8113, 91, 0, 5, 0, 0, 0, 7857, 0, 0, 0, 0, 0, 0, 0, 209, 0, 4, 0, 142, 5, 2, 493, 4, 0, 0, 8, 0, 7879, 91, 2334, 1, 3, 0, 1, 0, 0] payload_rows_per_dst=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] dup_rows_per_dst=[19264, 9684, 398, 0, 0, 280, 810, 0] combine_rows_per_src=[16368, 15738, 14198, 302, 0, 440, 15962, 4860] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315850752 dispatch_meta_bytes=327376 combine_payload_bytes=315850752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316178128 ingress_bytes=350453760 link_bytes=350453760 token_bytes=632028880 weight_movement_bytes=34603008 total_nvlink_bytes=666631888 weight_to_token_ratio=0.054749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318729285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=363438080 dispatch_meta_bytes=362952 combine_payload_bytes=363438080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=363801032 ingress_bytes=398041088 link_bytes=398041088 token_bytes=727239112 weight_movement_bytes=34603008 total_nvlink_bytes=761842120 weight_to_token_ratio=0.047581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.497796569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232275968 dispatch_meta_bytes=281088 combine_payload_bytes=232275968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=232557056 ingress_bytes=301481984 link_bytes=301481984 token_bytes=464833024 weight_movement_bytes=69206016 total_nvlink_bytes=534039040 weight_to_token_ratio=0.148884 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=259276800 dispatch_meta_bytes=278848 combine_payload_bytes=259276800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=259555648 ingress_bytes=363085824 link_bytes=363085824 token_bytes=518832448 weight_movement_bytes=103809024 total_nvlink_bytes=622641472 weight_to_token_ratio=0.200082 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=30080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.366262725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13108263848 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6557924264 ingress_bytes=7865253888 link_bytes=7865253888 total_nvlink_bytes=14423178152 weight_to_token_ratio=0.100312 prefetch_share_of_ingress=0.087990 dedup_ratio=0.153995 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=20701 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.286796399e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14113173328 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=7060492112 ingress_bytes=8506007552 link_bytes=8506007552 total_nvlink_bytes=15566499664 weight_to_token_ratio=0.102977 prefetch_share_of_ingress=0.085429 dedup_ratio=0.120512 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=22606 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.391627027e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=210944000 dispatch_meta_bytes=250688 combine_payload_bytes=210944000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=211194688 ingress_bytes=280150016 link_bytes=280150016 token_bytes=422138688 weight_movement_bytes=69206016 total_nvlink_bytes=491344704 weight_to_token_ratio=0.163941 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054181956e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84090 payload_rows_remote=68920 dup_rows=14214 dup_rows_remote=14214 dedup_ratio=0.144592 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 304, 0, 63, 7846, 6346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7081, 0, 0, 0, 0, 0, 0, 1473, 2156, 0, 0, 0, 0, 0, 0, 0, 7283, 0, 8129, 65, 0, 0, 0, 0, 0, 249, 0, 254, 0, 0, 0, 0, 48, 0, 0, 0, 0, 319, 7535, 0, 0, 0, 1] payload_rows_per_dst=[608, 126, 14162, 6748, 16384, 15692, 15170, 15200] dup_rows_per_dst=[0, 0, 0, 510, 12566, 628, 0, 510] combine_rows_per_src=[608, 126, 14162, 6748, 16384, 15692, 15170, 15200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72894 payload_rows_remote=56708 dup_rows=25410 dup_rows_remote=13564 dedup_ratio=0.258484 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 282, 0, 93, 7882, 6272, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7314, 0, 0, 0, 0, 0, 0, 1476, 2058, 0, 0, 0, 0, 0, 0, 0, 7107, 0, 8099, 96, 0, 0, 0, 0, 0, 247, 0, 253, 0, 0, 0, 0, 174, 0, 0, 0, 0, 349, 7450, 0, 0, 0, 0] payload_rows_per_dst=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] dup_rows_per_dst=[0, 0, 0, 520, 12358, 686, 0, 11846] combine_rows_per_src=[564, 186, 2194, 6548, 16384, 15764, 15068, 16186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14477019568 prefetch_bytes=692060160 reduce_bytes=1003487232 weight_movement_bytes=1695547392 egress_bytes=7242459568 ingress_bytes=8930107392 link_bytes=8930107392 total_nvlink_bytes=16172566960 weight_to_token_ratio=0.117120 prefetch_share_of_ingress=0.077497 dedup_ratio=0.103558 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=29 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030870 gemm_inner_pad_upper_bound=21590 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.461011964e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12035948800 prefetch_bytes=692060160 reduce_bytes=484442112 weight_movement_bytes=1176502272 egress_bytes=6021587200 ingress_bytes=7190863872 link_bytes=7190863872 total_nvlink_bytes=13212451072 weight_to_token_ratio=0.097749 prefetch_share_of_ingress=0.096242 dedup_ratio=0.205356 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=14 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.176462689e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95740 payload_rows_remote=93782 dup_rows=2564 dup_rows_remote=2564 dedup_ratio=0.026082 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 350, 0, 75, 7878, 6136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7233, 0, 0, 0, 0, 0, 0, 1452, 2200, 0, 0, 0, 0, 0, 0, 0, 7232, 0, 8117, 75, 0, 0, 0, 0, 0, 291, 0, 245, 0, 0, 0, 0, 64, 0, 0, 0, 0, 321, 7471, 0, 0, 0, 12] payload_rows_per_dst=[16234, 10340, 14466, 6812, 1958, 15756, 15082, 15092] dup_rows_per_dst=[700, 124, 0, 492, 0, 732, 0, 516] combine_rows_per_src=[16234, 10340, 14466, 6812, 1958, 15756, 15082, 15092] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12767808408 prefetch_bytes=761266176 reduce_bytes=553648128 weight_movement_bytes=1314914304 egress_bytes=6387698584 ingress_bytes=7695024128 link_bytes=7695024128 total_nvlink_bytes=14082722712 weight_to_token_ratio=0.102987 prefetch_share_of_ingress=0.098930 dedup_ratio=0.180763 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=16 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030978 gemm_inner_pad_upper_bound=21844 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.258945926e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95970 payload_rows_remote=79586 dup_rows=2334 dup_rows_remote=1428 dedup_ratio=0.023743 router_tokens_per_expert=[0, 0, 7986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 363, 0, 0, 0, 0, 90, 0, 418, 0, 0, 0, 0, 0, 0, 6073, 8102, 0, 0, 0, 0, 0, 44, 0, 82, 90, 0, 0, 0, 7952, 8192, 0, 0, 507, 0, 0, 0, 0, 82, 0, 0, 0, 1061, 0, 0, 0, 0, 0] payload_rows_per_dst=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] dup_rows_per_dst=[0, 0, 906, 370, 74, 94, 552, 338] combine_rows_per_src=[15972, 16220, 16384, 12612, 13770, 6908, 9872, 4232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=282296320 dispatch_meta_bytes=332536 combine_payload_bytes=282296320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282628856 ingress_bytes=316899328 link_bytes=316899328 token_bytes=564925176 weight_movement_bytes=34603008 total_nvlink_bytes=599528184 weight_to_token_ratio=0.061252 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192466659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=232275968 dispatch_meta_bytes=281088 combine_payload_bytes=232275968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=232557056 ingress_bytes=301481984 link_bytes=301481984 token_bytes=464833024 weight_movement_bytes=69206016 total_nvlink_bytes=534039040 weight_to_token_ratio=0.148884 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=384131072 dispatch_meta_bytes=385384 combine_payload_bytes=384131072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=384516456 ingress_bytes=487940096 link_bytes=487940096 token_bytes=768647528 weight_movement_bytes=103809024 total_nvlink_bytes=872456552 weight_to_token_ratio=0.135054 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.836079299e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13108263848 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6557924264 ingress_bytes=7865253888 link_bytes=7865253888 total_nvlink_bytes=14423178152 weight_to_token_ratio=0.100312 prefetch_share_of_ingress=0.087990 dedup_ratio=0.153995 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=20701 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.286796399e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12035948800 prefetch_bytes=692060160 reduce_bytes=484442112 weight_movement_bytes=1176502272 egress_bytes=6021587200 ingress_bytes=7190863872 link_bytes=7190863872 total_nvlink_bytes=13212451072 weight_to_token_ratio=0.097749 prefetch_share_of_ingress=0.096242 dedup_ratio=0.205356 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=14 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22098 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.176462689e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66164 payload_rows_remote=50066 dup_rows=32140 dup_rows_remote=11430 dedup_ratio=0.326945 router_tokens_per_expert=[7737, 5, 0, 0, 0, 0, 4, 2939, 378, 8, 4673, 0, 0, 0, 0, 139, 0, 2, 1, 169, 14, 140, 0, 7639, 0, 0, 0, 0, 0, 1, 0, 3, 563, 4, 0, 0, 0, 0, 1, 8183, 0, 9, 0, 7409, 4, 7, 195, 0, 0, 0, 0, 250, 4, 7719, 0, 4, 0, 0, 0, 262, 161, 434, 0, 91] payload_rows_per_dst=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] dup_rows_per_dst=[20710, 7988, 2102, 0, 1130, 22, 6, 182] combine_rows_per_src=[16098, 15420, 15634, 8, 16372, 408, 510, 1714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14785478488 prefetch_bytes=726663168 reduce_bytes=865075200 weight_movement_bytes=1591738368 egress_bytes=7396769624 ingress_bytes=8980447232 link_bytes=8980447232 total_nvlink_bytes=16377216856 weight_to_token_ratio=0.107656 prefetch_share_of_ingress=0.080916 dedup_ratio=0.104343 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=25 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.469247824e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=205070336 dispatch_meta_bytes=245984 combine_payload_bytes=205070336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=205316320 ingress_bytes=274276352 link_bytes=274276352 token_bytes=410386656 weight_movement_bytes=69206016 total_nvlink_bytes=479592672 weight_to_token_ratio=0.168636 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032079832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=73206 dup_rows=8936 dup_rows_remote=1488 dedup_ratio=0.090902 router_tokens_per_expert=[0, 0, 8050, 0, 1, 0, 0, 1, 9, 0, 0, 0, 8088, 0, 9, 0, 0, 345, 1, 0, 0, 0, 54, 0, 411, 3, 0, 0, 0, 0, 0, 6354, 8121, 0, 0, 0, 8, 0, 69, 0, 96, 54, 0, 1, 8, 7859, 8183, 0, 9, 536, 0, 0, 1, 0, 91, 0, 0, 9, 773, 8, 0, 0, 0, 0] payload_rows_per_dst=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] dup_rows_per_dst=[7448, 16, 800, 216, 138, 118, 184, 16] combine_rows_per_src=[16162, 16196, 16366, 5814, 16258, 15918, 1090, 1564] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=299851776 dispatch_meta_bytes=298776 combine_payload_bytes=299851776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=300150552 ingress_bytes=403660800 link_bytes=403660800 token_bytes=600002328 weight_movement_bytes=103809024 total_nvlink_bytes=703811352 weight_to_token_ratio=0.173014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=45440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74760 payload_rows_remote=58558 dup_rows=23544 dup_rows_remote=22880 dedup_ratio=0.239502 router_tokens_per_expert=[0, 0, 5, 6, 9, 5, 316, 0, 91, 7863, 6187, 0, 0, 0, 1, 0, 0, 1, 0, 0, 8, 7011, 5, 0, 0, 3, 0, 0, 1548, 2236, 6, 0, 1, 9, 0, 0, 8, 7141, 0, 8092, 91, 0, 0, 0, 1, 0, 293, 0, 250, 0, 0, 0, 0, 140, 0, 0, 0, 0, 341, 7464, 0, 2, 0, 18] payload_rows_per_dst=[16202, 15868, 10092, 16012, 14316, 768, 780, 722] dup_rows_per_dst=[664, 16364, 10, 6502, 2, 2, 0, 0] combine_rows_per_src=[16202, 15868, 10092, 16012, 14316, 768, 780, 722] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325984256 dispatch_meta_bytes=324056 combine_payload_bytes=325984256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326308312 ingress_bytes=360587264 link_bytes=360587264 token_bytes=652292568 weight_movement_bytes=34603008 total_nvlink_bytes=686895576 weight_to_token_ratio=0.053048 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356860845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=239853568 dispatch_meta_bytes=325752 combine_payload_bytes=239853568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=240179320 ingress_bytes=274456576 link_bytes=274456576 token_bytes=480032888 weight_movement_bytes=34603008 total_nvlink_bytes=514635896 weight_to_token_ratio=0.072085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.032758000e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11788090256 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=5897599888 ingress_bytes=7274610688 link_bytes=7274610688 total_nvlink_bytes=13172210576 weight_to_token_ratio=0.117417 prefetch_share_of_ingress=0.095134 dedup_ratio=0.218534 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030543 gemm_inner_pad_upper_bound=21082 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.190164103e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91178 payload_rows_remote=76892 dup_rows=7126 dup_rows_remote=7126 dedup_ratio=0.072489 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 322, 0, 81, 7816, 6234, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 1616, 2116, 0, 0, 0, 0, 0, 0, 0, 7218, 0, 8111, 82, 0, 0, 0, 0, 0, 267, 0, 263, 0, 0, 0, 0, 72, 0, 0, 0, 0, 346, 7464, 0, 0, 0, 1] payload_rows_per_dst=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] dup_rows_per_dst=[644, 1366, 0, 4294, 0, 612, 0, 210] combine_rows_per_src=[16222, 12812, 14286, 12668, 0, 14170, 15106, 5914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=314949632 dispatch_meta_bytes=336072 combine_payload_bytes=314949632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=315285704 ingress_bytes=384155648 link_bytes=384155648 token_bytes=630235336 weight_movement_bytes=69206016 total_nvlink_bytes=699441352 weight_to_token_ratio=0.109810 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.445546776e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14100577032 prefetch_bytes=657457152 reduce_bytes=692060160 weight_movement_bytes=1349517312 egress_bytes=7054162696 ingress_bytes=8395931648 link_bytes=8395931648 total_nvlink_bytes=15450094344 weight_to_token_ratio=0.095707 prefetch_share_of_ingress=0.078307 dedup_ratio=0.113706 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=20 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=21336 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.373618038e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:55:27] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 803.2 | throughput per GPU (TFLOP/s/GPU): 158.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787875E+00 | loss scale: 1.0 | grad norm: 4.876 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=67308 dup_rows=22472 dup_rows_remote=20216 dedup_ratio=0.228597 router_tokens_per_expert=[1083, 359, 855, 1228, 621, 914, 826, 883, 1387, 295, 954, 312, 272, 1507, 909, 1038, 744, 1048, 851, 666, 570, 729, 352, 1421, 485, 706, 766, 347, 807, 132, 814, 533, 1066, 784, 1315, 871, 452, 199, 555, 864, 518, 642, 654, 686, 1080, 622, 670, 518, 707, 578, 799, 430, 984, 1425, 810, 679, 1274, 358, 1421, 616, 212, 815, 1175, 959] payload_rows_per_dst=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] dup_rows_per_dst=[2810, 3430, 2536, 2566, 3054, 2256, 3024, 2796] combine_rows_per_src=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] + [2026-08-26 18:07:34] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 799.6 | throughput per GPU (TFLOP/s/GPU): 159.6 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.787875E+00 | loss scale: 1.0 | grad norm: 4.876 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75472 payload_rows_remote=63572 dup_rows=22832 dup_rows_remote=18828 dedup_ratio=0.232259 router_tokens_per_expert=[1202, 381, 959, 1184, 619, 872, 778, 840, 1402, 419, 1042, 340, 298, 1488, 873, 1018, 705, 1065, 903, 595, 640, 646, 360, 1381, 439, 733, 812, 339, 804, 116, 833, 617, 1086, 914, 1184, 857, 407, 233, 479, 796, 561, 608, 556, 632, 1092, 642, 602, 628, 666, 580, 896, 551, 1003, 1320, 801, 723, 1227, 306, 1338, 624, 212, 833, 1178, 914] payload_rows_per_dst=[8414, 9484, 9070, 9522, 9658, 8366, 9058, 11900] dup_rows_per_dst=[2888, 3668, 1642, 2840, 3742, 2276, 1772, 4004] combine_rows_per_src=[8414, 9484, 9070, 9522, 9658, 8366, 9058, 11900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=64456 dup_rows=23770 dup_rows_remote=21168 dedup_ratio=0.241801 router_tokens_per_expert=[1166, 381, 801, 1217, 581, 1000, 781, 734, 1434, 496, 949, 301, 225, 1652, 842, 1076, 793, 1178, 876, 608, 528, 551, 441, 1365, 494, 636, 774, 350, 864, 116, 854, 609, 1123, 909, 1190, 895, 456, 212, 504, 869, 650, 611, 613, 626, 1095, 661, 655, 559, 618, 573, 805, 489, 984, 1401, 675, 764, 1262, 329, 1342, 535, 161, 794, 1138, 981] payload_rows_per_dst=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] dup_rows_per_dst=[3666, 2968, 2602, 2998, 3046, 3766, 2762, 1962] combine_rows_per_src=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75250 payload_rows_remote=66014 dup_rows=23054 dup_rows_remote=20124 dedup_ratio=0.234517 router_tokens_per_expert=[1094, 371, 765, 1264, 657, 782, 823, 867, 1427, 441, 1047, 353, 302, 1577, 969, 1148, 697, 995, 912, 661, 581, 739, 413, 1194, 485, 618, 749, 377, 791, 118, 882, 643, 984, 890, 1198, 893, 505, 202, 558, 853, 521, 645, 546, 667, 1112, 600, 635, 529, 723, 491, 780, 546, 1011, 1371, 732, 751, 1282, 327, 1335, 599, 176, 905, 1096, 947] payload_rows_per_dst=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] dup_rows_per_dst=[2646, 3978, 2624, 2750, 2930, 2964, 2900, 2262] combine_rows_per_src=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75556 payload_rows_remote=66002 dup_rows=22748 dup_rows_remote=20184 dedup_ratio=0.231405 router_tokens_per_expert=[1153, 355, 816, 1253, 672, 928, 817, 788, 1356, 507, 984, 353, 257, 1466, 915, 1033, 764, 1089, 891, 650, 469, 656, 378, 1320, 549, 678, 822, 404, 804, 111, 871, 491, 1090, 801, 1332, 880, 354, 205, 569, 841, 494, 527, 558, 627, 1134, 691, 709, 567, 757, 559, 785, 474, 944, 1370, 715, 709, 1271, 434, 1430, 688, 217, 865, 1067, 888] payload_rows_per_dst=[8230, 9494, 9762, 9782, 9140, 8312, 9554, 11282] dup_rows_per_dst=[2828, 3822, 2672, 2610, 3004, 2302, 2564, 2946] combine_rows_per_src=[8230, 9494, 9762, 9782, 9140, 8312, 9554, 11282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=66988 dup_rows=23334 dup_rows_remote=20268 dedup_ratio=0.237366 router_tokens_per_expert=[1180, 330, 832, 1125, 592, 824, 862, 798, 1430, 318, 1037, 299, 263, 1666, 1012, 1165, 720, 1094, 856, 602, 659, 769, 356, 1370, 435, 732, 887, 355, 818, 114, 778, 548, 1012, 861, 1215, 885, 358, 228, 554, 805, 547, 599, 543, 696, 1102, 629, 614, 455, 716, 587, 787, 473, 980, 1403, 862, 636, 1375, 290, 1380, 647, 215, 817, 1207, 848] payload_rows_per_dst=[9638, 7982, 10268, 9822, 8926, 9658, 9898, 8778] dup_rows_per_dst=[3448, 3066, 2584, 2844, 2910, 3472, 2990, 2020] combine_rows_per_src=[9638, 7982, 10268, 9822, 8926, 9658, 9898, 8778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=67308 dup_rows=22472 dup_rows_remote=20216 dedup_ratio=0.228597 router_tokens_per_expert=[1083, 359, 855, 1228, 621, 914, 826, 883, 1387, 295, 954, 312, 272, 1507, 909, 1038, 744, 1048, 851, 666, 570, 729, 352, 1421, 485, 706, 766, 347, 807, 132, 814, 533, 1066, 784, 1315, 871, 452, 199, 555, 864, 518, 642, 654, 686, 1080, 622, 670, 518, 707, 578, 799, 430, 984, 1425, 810, 679, 1274, 358, 1421, 616, 212, 815, 1175, 959] payload_rows_per_dst=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] dup_rows_per_dst=[2810, 3430, 2536, 2566, 3054, 2256, 3024, 2796] combine_rows_per_src=[8272, 9360, 10226, 9628, 9158, 8524, 9800, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=64456 dup_rows=23770 dup_rows_remote=21168 dedup_ratio=0.241801 router_tokens_per_expert=[1166, 381, 801, 1217, 581, 1000, 781, 734, 1434, 496, 949, 301, 225, 1652, 842, 1076, 793, 1178, 876, 608, 528, 551, 441, 1365, 494, 636, 774, 350, 864, 116, 854, 609, 1123, 909, 1190, 895, 456, 212, 504, 869, 650, 611, 613, 626, 1095, 661, 655, 559, 618, 573, 805, 489, 984, 1401, 675, 764, 1262, 329, 1342, 535, 161, 794, 1138, 981] payload_rows_per_dst=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] dup_rows_per_dst=[3666, 2968, 2602, 2998, 3046, 3766, 2762, 1962] combine_rows_per_src=[9656, 7678, 10078, 9700, 9270, 9858, 9856, 8438] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=64926 dup_rows=23574 dup_rows_remote=20530 dedup_ratio=0.239807 router_tokens_per_expert=[1145, 432, 852, 1215, 575, 868, 802, 840, 1352, 435, 1055, 310, 289, 1646, 828, 1059, 733, 1108, 825, 573, 543, 645, 360, 1320, 439, 728, 840, 358, 807, 93, 877, 636, 1044, 828, 1351, 859, 457, 194, 522, 732, 599, 698, 620, 608, 1211, 634, 634, 490, 661, 586, 778, 487, 957, 1418, 733, 693, 1417, 295, 1309, 590, 159, 847, 1211, 942] payload_rows_per_dst=[9714, 7902, 9814, 9804, 9146, 9776, 9736, 8838] dup_rows_per_dst=[3472, 3026, 2400, 3044, 2828, 3830, 2890, 2084] combine_rows_per_src=[9714, 7902, 9814, 9804, 9146, 9776, 9736, 8838] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75250 payload_rows_remote=66014 dup_rows=23054 dup_rows_remote=20124 dedup_ratio=0.234517 router_tokens_per_expert=[1094, 371, 765, 1264, 657, 782, 823, 867, 1427, 441, 1047, 353, 302, 1577, 969, 1148, 697, 995, 912, 661, 581, 739, 413, 1194, 485, 618, 749, 377, 791, 118, 882, 643, 984, 890, 1198, 893, 505, 202, 558, 853, 521, 645, 546, 667, 1112, 600, 635, 529, 723, 491, 780, 546, 1011, 1371, 732, 751, 1282, 327, 1335, 599, 176, 905, 1096, 947] payload_rows_per_dst=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] dup_rows_per_dst=[2646, 3978, 2624, 2750, 2930, 2964, 2900, 2262] combine_rows_per_src=[8072, 9924, 9760, 9730, 9236, 9022, 9910, 9596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=66988 dup_rows=23334 dup_rows_remote=20268 dedup_ratio=0.237366 router_tokens_per_expert=[1180, 330, 832, 1125, 592, 824, 862, 798, 1430, 318, 1037, 299, 263, 1666, 1012, 1165, 720, 1094, 856, 602, 659, 769, 356, 1370, 435, 732, 887, 355, 818, 114, 778, 548, 1012, 861, 1215, 885, 358, 228, 554, 805, 547, 599, 543, 696, 1102, 629, 614, 455, 716, 587, 787, 473, 980, 1403, 862, 636, 1375, 290, 1380, 647, 215, 817, 1207, 848] payload_rows_per_dst=[9638, 7982, 10268, 9822, 8926, 9658, 9898, 8778] dup_rows_per_dst=[3448, 3066, 2584, 2844, 2910, 3472, 2990, 2020] combine_rows_per_src=[9638, 7982, 10268, 9822, 8926, 9658, 9898, 8778] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275693568 dispatch_meta_bytes=350096 combine_payload_bytes=275693568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276043664 ingress_bytes=310296576 link_bytes=310296576 token_bytes=551737232 weight_movement_bytes=34603008 total_nvlink_bytes=586340240 weight_to_token_ratio=0.062716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167621035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=260390912 dispatch_meta_bytes=329600 combine_payload_bytes=260390912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260720512 ingress_bytes=329596928 link_bytes=329596928 token_bytes=521111424 weight_movement_bytes=69206016 total_nvlink_bytes=590317440 weight_to_token_ratio=0.132805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240246705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=342496 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264354272 ingress_bytes=333217792 link_bytes=333217792 token_bytes=528366048 weight_movement_bytes=69206016 total_nvlink_bytes=597572064 weight_to_token_ratio=0.130981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253871725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270393344 dispatch_meta_bytes=344552 combine_payload_bytes=270393344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270737896 ingress_bytes=304996352 link_bytes=304996352 token_bytes=541131240 weight_movement_bytes=34603008 total_nvlink_bytes=575734248 weight_to_token_ratio=0.063946 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147676718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270344192 dispatch_meta_bytes=344744 combine_payload_bytes=270344192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270688936 ingress_bytes=374153216 link_bytes=374153216 token_bytes=541033128 weight_movement_bytes=103809024 total_nvlink_bytes=644842152 weight_to_token_ratio=0.191872 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274382848 dispatch_meta_bytes=349024 combine_payload_bytes=274382848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274731872 ingress_bytes=343588864 link_bytes=343588864 token_bytes=549114720 weight_movement_bytes=69206016 total_nvlink_bytes=618320736 weight_to_token_ratio=0.126032 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292897234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=64302 dup_rows=23570 dup_rows_remote=19452 dedup_ratio=0.239766 router_tokens_per_expert=[1174, 409, 833, 1264, 651, 819, 804, 840, 1428, 377, 926, 355, 327, 1586, 877, 1067, 726, 1122, 807, 564, 680, 643, 391, 1200, 457, 708, 820, 312, 808, 105, 874, 566, 1071, 792, 1329, 856, 489, 201, 565, 825, 535, 601, 668, 716, 1071, 638, 666, 599, 674, 578, 790, 488, 992, 1328, 756, 670, 1327, 267, 1467, 549, 187, 781, 1113, 1043] payload_rows_per_dst=[10432, 8876, 9840, 8542, 9290, 9932, 9298, 8524] dup_rows_per_dst=[4118, 3646, 2426, 2122, 2966, 3990, 2292, 2010] combine_rows_per_src=[10432, 8876, 9840, 8542, 9290, 9932, 9298, 8524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275693568 dispatch_meta_bytes=350096 combine_payload_bytes=275693568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276043664 ingress_bytes=310296576 link_bytes=310296576 token_bytes=551737232 weight_movement_bytes=34603008 total_nvlink_bytes=586340240 weight_to_token_ratio=0.062716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167621035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=342496 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=264354272 ingress_bytes=333217792 link_bytes=333217792 token_bytes=528366048 weight_movement_bytes=69206016 total_nvlink_bytes=597572064 weight_to_token_ratio=0.130981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253871725e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=341824 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266278720 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532215616 weight_movement_bytes=34603008 total_nvlink_bytes=566818624 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270393344 dispatch_meta_bytes=344552 combine_payload_bytes=270393344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270737896 ingress_bytes=304996352 link_bytes=304996352 token_bytes=541131240 weight_movement_bytes=34603008 total_nvlink_bytes=575734248 weight_to_token_ratio=0.063946 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147676718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263380992 dispatch_meta_bytes=335016 combine_payload_bytes=263380992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263716008 ingress_bytes=332587008 link_bytes=332587008 token_bytes=527097000 weight_movement_bytes=69206016 total_nvlink_bytes=596303016 weight_to_token_ratio=0.131297 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251498136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73692 payload_rows_remote=64436 dup_rows=24612 dup_rows_remote=20290 dedup_ratio=0.250366 router_tokens_per_expert=[629, 1050, 761, 412, 969, 333, 339, 1112, 41, 849, 1651, 293, 615, 1034, 784, 1507, 880, 1289, 731, 418, 1669, 132, 1929, 1156, 717, 674, 751, 902, 103, 319, 166, 110, 413, 193, 434, 241, 892, 1468, 1037, 1175, 2064, 1432, 988, 1446, 296, 1429, 14, 252, 939, 971, 947, 14, 137, 1643, 190, 1487, 812, 356, 869, 498, 1167, 334, 330, 359] payload_rows_per_dst=[8812, 6446, 7884, 10618, 11022, 8810, 10844, 9256] dup_rows_per_dst=[2398, 1624, 1328, 4062, 3986, 3922, 2970, 4322] combine_rows_per_src=[8812, 6446, 7884, 10618, 11022, 8810, 10844, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274382848 dispatch_meta_bytes=349024 combine_payload_bytes=274382848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274731872 ingress_bytes=343588864 link_bytes=343588864 token_bytes=549114720 weight_movement_bytes=69206016 total_nvlink_bytes=618320736 weight_to_token_ratio=0.126032 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292897234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=64740 dup_rows=23794 dup_rows_remote=21376 dedup_ratio=0.242045 router_tokens_per_expert=[604, 1024, 815, 426, 1017, 242, 304, 1179, 80, 907, 1613, 310, 596, 940, 842, 1482, 955, 1245, 817, 377, 1513, 175, 1754, 1212, 639, 684, 726, 914, 95, 299, 162, 102, 411, 249, 434, 221, 1055, 1395, 1087, 1075, 2086, 1520, 943, 1350, 315, 1563, 13, 208, 1042, 1045, 1058, 4, 113, 1561, 214, 1369, 891, 340, 914, 523, 1062, 346, 313, 382] payload_rows_per_dst=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] dup_rows_per_dst=[2440, 3366, 2704, 2412, 2418, 3528, 2624, 4302] combine_rows_per_src=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=64216 dup_rows=23806 dup_rows_remote=21122 dedup_ratio=0.242167 router_tokens_per_expert=[661, 845, 879, 553, 1114, 177, 324, 1155, 67, 976, 1680, 314, 641, 984, 870, 1493, 956, 1133, 685, 366, 1523, 197, 1716, 1464, 733, 654, 785, 931, 186, 361, 197, 107, 352, 195, 373, 149, 894, 1209, 1147, 1010, 2099, 1406, 890, 1228, 277, 1570, 10, 250, 1179, 998, 1177, 4, 115, 1494, 164, 1352, 921, 421, 963, 475, 1077, 323, 308, 395] payload_rows_per_dst=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] dup_rows_per_dst=[2408, 2130, 1478, 4018, 3272, 3264, 2684, 4552] combine_rows_per_src=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=66306 dup_rows=23906 dup_rows_remote=20660 dedup_ratio=0.243184 router_tokens_per_expert=[570, 893, 956, 398, 1180, 241, 355, 1111, 51, 912, 1695, 362, 638, 881, 756, 1561, 844, 1188, 774, 413, 1483, 214, 1902, 1331, 732, 760, 889, 888, 113, 336, 178, 112, 321, 251, 374, 169, 961, 1173, 880, 1017, 2166, 1357, 860, 1293, 293, 1568, 22, 276, 1227, 1010, 1113, 5, 120, 1369, 187, 1469, 927, 353, 944, 555, 1071, 353, 351, 400] payload_rows_per_dst=[8906, 8240, 8834, 9964, 10416, 8092, 10324, 9622] dup_rows_per_dst=[2502, 2082, 2008, 3508, 3266, 3246, 2676, 4618] combine_rows_per_src=[8906, 8240, 8834, 9964, 10416, 8092, 10324, 9622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73692 payload_rows_remote=64436 dup_rows=24612 dup_rows_remote=20290 dedup_ratio=0.250366 router_tokens_per_expert=[629, 1050, 761, 412, 969, 333, 339, 1112, 41, 849, 1651, 293, 615, 1034, 784, 1507, 880, 1289, 731, 418, 1669, 132, 1929, 1156, 717, 674, 751, 902, 103, 319, 166, 110, 413, 193, 434, 241, 892, 1468, 1037, 1175, 2064, 1432, 988, 1446, 296, 1429, 14, 252, 939, 971, 947, 14, 137, 1643, 190, 1487, 812, 356, 869, 498, 1167, 334, 330, 359] payload_rows_per_dst=[8812, 6446, 7884, 10618, 11022, 8810, 10844, 9256] dup_rows_per_dst=[2398, 1624, 1328, 4062, 3986, 3922, 2970, 4322] combine_rows_per_src=[8812, 6446, 7884, 10618, 11022, 8810, 10844, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=64302 dup_rows=23570 dup_rows_remote=19452 dedup_ratio=0.239766 router_tokens_per_expert=[1174, 409, 833, 1264, 651, 819, 804, 840, 1428, 377, 926, 355, 327, 1586, 877, 1067, 726, 1122, 807, 564, 680, 643, 391, 1200, 457, 708, 820, 312, 808, 105, 874, 566, 1071, 792, 1329, 856, 489, 201, 565, 825, 535, 601, 668, 716, 1071, 638, 666, 599, 674, 578, 790, 488, 992, 1328, 756, 670, 1327, 267, 1467, 549, 187, 781, 1113, 1043] payload_rows_per_dst=[10432, 8876, 9840, 8542, 9290, 9932, 9298, 8524] dup_rows_per_dst=[4118, 3646, 2426, 2122, 2966, 3990, 2292, 2010] combine_rows_per_src=[10432, 8876, 9840, 8542, 9290, 9932, 9298, 8524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75332 payload_rows_remote=65116 dup_rows=22972 dup_rows_remote=19804 dedup_ratio=0.233683 router_tokens_per_expert=[501, 1145, 677, 383, 1168, 295, 501, 1136, 52, 950, 1876, 340, 567, 958, 826, 1570, 890, 1190, 727, 325, 1540, 209, 1748, 1387, 776, 768, 820, 893, 96, 260, 157, 108, 308, 177, 411, 181, 1019, 1050, 999, 1134, 1784, 1422, 894, 1294, 269, 1502, 18, 268, 1226, 1106, 1315, 2, 94, 1513, 190, 1433, 830, 328, 867, 514, 1037, 351, 386, 391] payload_rows_per_dst=[9268, 10452, 10216, 8402, 8994, 9730, 10412, 7858] dup_rows_per_dst=[3000, 3826, 3168, 2850, 1564, 3956, 2498, 2110] combine_rows_per_src=[9268, 10452, 10216, 8402, 8994, 9730, 10412, 7858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=64216 dup_rows=23806 dup_rows_remote=21122 dedup_ratio=0.242167 router_tokens_per_expert=[661, 845, 879, 553, 1114, 177, 324, 1155, 67, 976, 1680, 314, 641, 984, 870, 1493, 956, 1133, 685, 366, 1523, 197, 1716, 1464, 733, 654, 785, 931, 186, 361, 197, 107, 352, 195, 373, 149, 894, 1209, 1147, 1010, 2099, 1406, 890, 1228, 277, 1570, 10, 250, 1179, 998, 1177, 4, 115, 1494, 164, 1352, 921, 421, 963, 475, 1077, 323, 308, 395] payload_rows_per_dst=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] dup_rows_per_dst=[2408, 2130, 1478, 4018, 3272, 3264, 2684, 4552] combine_rows_per_src=[9008, 8560, 8124, 10368, 10746, 7998, 10282, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=66190 dup_rows=23680 dup_rows_remote=21090 dedup_ratio=0.240885 router_tokens_per_expert=[645, 944, 712, 449, 1198, 286, 517, 1214, 90, 951, 1715, 349, 716, 1023, 748, 1590, 782, 1142, 688, 245, 1584, 177, 1773, 1226, 747, 615, 830, 933, 94, 211, 186, 123, 414, 178, 401, 163, 974, 1138, 970, 1054, 1863, 1399, 833, 1374, 319, 1586, 28, 283, 1224, 1099, 1183, 8, 87, 1584, 213, 1363, 935, 358, 917, 548, 1084, 354, 334, 381] payload_rows_per_dst=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] dup_rows_per_dst=[2606, 3874, 2276, 2590, 1626, 3408, 2928, 4372] combine_rows_per_src=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75900 payload_rows_remote=65984 dup_rows=22404 dup_rows_remote=19026 dedup_ratio=0.227905 router_tokens_per_expert=[572, 961, 959, 365, 963, 289, 251, 1090, 44, 935, 1544, 332, 611, 944, 771, 1466, 871, 1166, 727, 456, 1575, 215, 1905, 1252, 763, 736, 761, 910, 97, 327, 156, 118, 308, 245, 410, 218, 979, 1416, 1043, 1047, 2242, 1533, 950, 1275, 355, 1624, 14, 267, 967, 1014, 1042, 7, 134, 1595, 204, 1435, 896, 407, 821, 454, 1064, 324, 381, 349] payload_rows_per_dst=[10474, 9916, 11410, 8820, 9336, 9890, 8334, 7720] dup_rows_per_dst=[3674, 3378, 4304, 2726, 1996, 3382, 1272, 1672] combine_rows_per_src=[10474, 9916, 11410, 8820, 9336, 9890, 8334, 7720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=344464 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265519504 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530694544 weight_movement_bytes=34603008 total_nvlink_bytes=565297552 weight_to_token_ratio=0.065203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=341352 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263370088 ingress_bytes=332234752 link_bytes=332234752 token_bytes=526398824 weight_movement_bytes=69206016 total_nvlink_bytes=595604840 weight_to_token_ratio=0.131471 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250172625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=66190 dup_rows=23680 dup_rows_remote=21090 dedup_ratio=0.240885 router_tokens_per_expert=[645, 944, 712, 449, 1198, 286, 517, 1214, 90, 951, 1715, 349, 716, 1023, 748, 1590, 782, 1142, 688, 245, 1584, 177, 1773, 1226, 747, 615, 830, 933, 94, 211, 186, 123, 414, 178, 401, 163, 974, 1138, 970, 1054, 1863, 1399, 833, 1374, 319, 1586, 28, 283, 1224, 1099, 1183, 8, 87, 1584, 213, 1363, 935, 358, 917, 548, 1084, 354, 334, 381] payload_rows_per_dst=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] dup_rows_per_dst=[2606, 3874, 2276, 2590, 1626, 3408, 2928, 4372] combine_rows_per_src=[9324, 10490, 9412, 8434, 8958, 8236, 10594, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=64740 dup_rows=23794 dup_rows_remote=21376 dedup_ratio=0.242045 router_tokens_per_expert=[604, 1024, 815, 426, 1017, 242, 304, 1179, 80, 907, 1613, 310, 596, 940, 842, 1482, 955, 1245, 817, 377, 1513, 175, 1754, 1212, 639, 684, 726, 914, 95, 299, 162, 102, 411, 249, 434, 221, 1055, 1395, 1087, 1075, 2086, 1520, 943, 1350, 315, 1563, 13, 208, 1042, 1045, 1058, 4, 113, 1561, 214, 1369, 891, 340, 914, 523, 1062, 346, 313, 382] payload_rows_per_dst=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] dup_rows_per_dst=[2440, 3366, 2704, 2412, 2418, 3528, 2624, 4302] combine_rows_per_src=[8782, 9840, 9884, 8338, 9770, 8296, 10188, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271589376 dispatch_meta_bytes=347864 combine_payload_bytes=271589376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271937240 ingress_bytes=375398400 link_bytes=375398400 token_bytes=543526616 weight_movement_bytes=103809024 total_nvlink_bytes=647335640 weight_to_token_ratio=0.190992 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412593957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263929856 dispatch_meta_bytes=338904 combine_payload_bytes=263929856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264268760 ingress_bytes=298532864 link_bytes=298532864 token_bytes=528198616 weight_movement_bytes=34603008 total_nvlink_bytes=562801624 weight_to_token_ratio=0.065511 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123355133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271589376 dispatch_meta_bytes=347864 combine_payload_bytes=271589376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=271937240 ingress_bytes=375398400 link_bytes=375398400 token_bytes=543526616 weight_movement_bytes=103809024 total_nvlink_bytes=647335640 weight_to_token_ratio=0.190992 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412593957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=339680 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267054816 ingress_bytes=370524160 link_bytes=370524160 token_bytes=533769952 weight_movement_bytes=103809024 total_nvlink_bytes=637578976 weight_to_token_ratio=0.194483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394252584e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=341352 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263370088 ingress_bytes=332234752 link_bytes=332234752 token_bytes=526398824 weight_movement_bytes=69206016 total_nvlink_bytes=595604840 weight_to_token_ratio=0.131471 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.250172625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270270464 dispatch_meta_bytes=340040 combine_payload_bytes=270270464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270610504 ingress_bytes=374079488 link_bytes=374079488 token_bytes=540880968 weight_movement_bytes=103809024 total_nvlink_bytes=644689992 weight_to_token_ratio=0.191926 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407630997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271114240 dispatch_meta_bytes=349120 combine_payload_bytes=271114240 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=271463360 ingress_bytes=340320256 link_bytes=340320256 token_bytes=542577600 weight_movement_bytes=69206016 total_nvlink_bytes=611783616 weight_to_token_ratio=0.127550 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280597725e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=344464 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265519504 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530694544 weight_movement_bytes=34603008 total_nvlink_bytes=565297552 weight_to_token_ratio=0.065203 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75876 payload_rows_remote=65224 dup_rows=22428 dup_rows_remote=18378 dedup_ratio=0.228149 router_tokens_per_expert=[701, 950, 850, 376, 1172, 273, 384, 1051, 67, 857, 1636, 354, 605, 1038, 882, 1601, 802, 1141, 726, 372, 1635, 182, 1876, 1304, 685, 674, 732, 924, 127, 254, 199, 126, 358, 162, 393, 173, 941, 1402, 893, 1145, 2104, 1338, 908, 1328, 285, 1594, 39, 271, 1161, 983, 917, 8, 119, 1475, 235, 1414, 936, 404, 898, 588, 1025, 375, 315, 409] payload_rows_per_dst=[10652, 10372, 11316, 8492, 9072, 9464, 8484, 8024] dup_rows_per_dst=[4050, 3708, 3958, 2702, 1862, 3082, 1190, 1876] combine_rows_per_src=[10652, 10372, 11316, 8492, 9072, 9464, 8484, 8024] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=63200 dup_rows=24200 dup_rows_remote=20076 dedup_ratio=0.246175 router_tokens_per_expert=[81, 1019, 486, 291, 49, 1016, 837, 251, 542, 324, 844, 1433, 1572, 1174, 269, 1760, 903, 1599, 283, 711, 75, 145, 1238, 987, 483, 885, 942, 662, 187, 593, 908, 300, 433, 1091, 734, 645, 2220, 1929, 1861, 527, 553, 126, 345, 205, 1706, 1691, 134, 1264, 837, 125, 1089, 963, 49, 189, 297, 661, 1117, 393, 108, 533, 106, 1594, 442, 2336] payload_rows_per_dst=[8918, 9430, 9718, 8550, 8466, 9254, 8864, 10904] dup_rows_per_dst=[2286, 3262, 2164, 1370, 2692, 4306, 3996, 4124] combine_rows_per_src=[8918, 9430, 9718, 8550, 8466, 9254, 8864, 10904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=66876 dup_rows=22786 dup_rows_remote=19080 dedup_ratio=0.231791 router_tokens_per_expert=[18, 1122, 423, 349, 17, 1216, 688, 211, 532, 397, 1096, 1706, 1492, 899, 119, 1139, 981, 1538, 342, 634, 78, 84, 1438, 1125, 570, 882, 1028, 834, 157, 471, 967, 164, 447, 1330, 798, 569, 2882, 2017, 1608, 517, 803, 96, 247, 253, 1475, 1790, 135, 1375, 777, 92, 835, 992, 15, 78, 282, 532, 1046, 487, 83, 492, 10, 1835, 140, 2397] payload_rows_per_dst=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] dup_rows_per_dst=[1902, 2602, 2630, 1556, 4178, 3706, 3626, 2586] combine_rows_per_src=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270270464 dispatch_meta_bytes=340040 combine_payload_bytes=270270464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270610504 ingress_bytes=374079488 link_bytes=374079488 token_bytes=540880968 weight_movement_bytes=103809024 total_nvlink_bytes=644689992 weight_to_token_ratio=0.191926 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407630997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=263380992 dispatch_meta_bytes=335016 combine_payload_bytes=263380992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263716008 ingress_bytes=332587008 link_bytes=332587008 token_bytes=527097000 weight_movement_bytes=69206016 total_nvlink_bytes=596303016 weight_to_token_ratio=0.131297 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251498136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74994 payload_rows_remote=64488 dup_rows=23310 dup_rows_remote=19106 dedup_ratio=0.237122 router_tokens_per_expert=[39, 1061, 468, 399, 46, 1115, 793, 256, 469, 250, 936, 1572, 1588, 1132, 278, 1782, 903, 1325, 214, 647, 88, 198, 1315, 943, 539, 862, 954, 813, 156, 584, 896, 213, 417, 1152, 721, 665, 2111, 1913, 1992, 495, 587, 114, 272, 156, 1785, 1758, 159, 1234, 988, 114, 844, 957, 51, 140, 308, 694, 1153, 410, 69, 487, 93, 1700, 423, 2356] payload_rows_per_dst=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] dup_rows_per_dst=[2448, 3282, 1810, 1338, 4204, 3574, 3732, 2922] combine_rows_per_src=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75130 payload_rows_remote=66214 dup_rows=23174 dup_rows_remote=19648 dedup_ratio=0.235738 router_tokens_per_expert=[33, 1515, 503, 369, 26, 1133, 810, 229, 549, 360, 1068, 1517, 1792, 1156, 178, 941, 911, 1603, 308, 667, 69, 116, 1227, 1005, 662, 960, 975, 728, 191, 527, 1077, 190, 388, 1117, 548, 605, 2501, 1994, 1899, 484, 837, 138, 336, 219, 1559, 1750, 185, 1253, 914, 141, 712, 956, 27, 150, 341, 479, 1158, 400, 66, 466, 28, 1524, 193, 2389] payload_rows_per_dst=[10022, 8948, 9466, 9018, 10160, 8584, 8916, 10016] dup_rows_per_dst=[2798, 2590, 2346, 1602, 3910, 3970, 3526, 2432] combine_rows_per_src=[10022, 8948, 9466, 9018, 10160, 8584, 8916, 10016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74968 payload_rows_remote=65012 dup_rows=23336 dup_rows_remote=20952 dedup_ratio=0.237386 router_tokens_per_expert=[12, 1111, 486, 358, 12, 1043, 660, 237, 570, 410, 1155, 1452, 1721, 984, 73, 991, 927, 1734, 270, 678, 177, 106, 1308, 970, 802, 927, 1085, 671, 104, 492, 847, 167, 433, 1025, 631, 595, 2826, 2478, 1725, 388, 716, 67, 383, 270, 1628, 1900, 123, 935, 850, 93, 777, 1098, 17, 83, 232, 443, 1325, 549, 81, 397, 2, 1839, 179, 2524] payload_rows_per_dst=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] dup_rows_per_dst=[2268, 2758, 2384, 3548, 2032, 3530, 3830, 2986] combine_rows_per_src=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=334408 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267491912 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534649416 weight_movement_bytes=34603008 total_nvlink_bytes=569252424 weight_to_token_ratio=0.064721 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73756 payload_rows_remote=63934 dup_rows=24548 dup_rows_remote=20476 dedup_ratio=0.249715 router_tokens_per_expert=[24, 904, 486, 331, 33, 1251, 837, 233, 471, 427, 1084, 1562, 1537, 1025, 116, 1708, 923, 1594, 320, 653, 92, 90, 1294, 1222, 554, 819, 992, 843, 211, 566, 1094, 190, 417, 1111, 714, 567, 2394, 2119, 1656, 456, 746, 89, 312, 316, 1434, 1539, 131, 1181, 910, 117, 944, 1070, 18, 73, 262, 565, 1151, 394, 76, 520, 12, 1718, 310, 2344] payload_rows_per_dst=[10844, 9822, 9844, 11148, 7648, 8066, 8938, 7446] dup_rows_per_dst=[3552, 4072, 2532, 3628, 2194, 3430, 3768, 1372] combine_rows_per_src=[10844, 9822, 9844, 11148, 7648, 8066, 8938, 7446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75518 payload_rows_remote=66876 dup_rows=22786 dup_rows_remote=19080 dedup_ratio=0.231791 router_tokens_per_expert=[18, 1122, 423, 349, 17, 1216, 688, 211, 532, 397, 1096, 1706, 1492, 899, 119, 1139, 981, 1538, 342, 634, 78, 84, 1438, 1125, 570, 882, 1028, 834, 157, 471, 967, 164, 447, 1330, 798, 569, 2882, 2017, 1608, 517, 803, 96, 247, 253, 1475, 1790, 135, 1375, 777, 92, 835, 992, 15, 78, 282, 532, 1046, 487, 83, 492, 10, 1835, 140, 2397] payload_rows_per_dst=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] dup_rows_per_dst=[1902, 2602, 2630, 1556, 4178, 3706, 3626, 2586] combine_rows_per_src=[9170, 9174, 9810, 8590, 10394, 8642, 9344, 10394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=63200 dup_rows=24200 dup_rows_remote=20076 dedup_ratio=0.246175 router_tokens_per_expert=[81, 1019, 486, 291, 49, 1016, 837, 251, 542, 324, 844, 1433, 1572, 1174, 269, 1760, 903, 1599, 283, 711, 75, 145, 1238, 987, 483, 885, 942, 662, 187, 593, 908, 300, 433, 1091, 734, 645, 2220, 1929, 1861, 527, 553, 126, 345, 205, 1706, 1691, 134, 1264, 837, 125, 1089, 963, 49, 189, 297, 661, 1117, 393, 108, 533, 106, 1594, 442, 2336] payload_rows_per_dst=[8918, 9430, 9718, 8550, 8466, 9254, 8864, 10904] dup_rows_per_dst=[2286, 3262, 2164, 1370, 2692, 4306, 3996, 4124] combine_rows_per_src=[8918, 9430, 9718, 8550, 8466, 9254, 8864, 10904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264142848 dispatch_meta_bytes=334376 combine_payload_bytes=264142848 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=264477224 ingress_bytes=437157888 link_bytes=437157888 token_bytes=528620072 weight_movement_bytes=173015040 total_nvlink_bytes=701635112 weight_to_token_ratio=0.327296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.644989938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75104 payload_rows_remote=64612 dup_rows=23200 dup_rows_remote=20082 dedup_ratio=0.236003 router_tokens_per_expert=[16, 1247, 553, 350, 12, 1121, 721, 224, 530, 445, 985, 1505, 1557, 842, 54, 971, 957, 1704, 276, 712, 94, 78, 1202, 1062, 742, 977, 1005, 798, 181, 491, 821, 139, 429, 1145, 740, 657, 2815, 2449, 1725, 469, 714, 58, 314, 238, 1553, 1800, 130, 1180, 655, 87, 965, 1234, 20, 34, 189, 480, 1244, 557, 52, 441, 3, 1950, 125, 2328] payload_rows_per_dst=[9342, 8272, 9870, 10492, 8814, 8506, 9178, 10630] dup_rows_per_dst=[2260, 2392, 2300, 3118, 3112, 3468, 3780, 2770] combine_rows_per_src=[9342, 8272, 9870, 10492, 8814, 8506, 9178, 10630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74994 payload_rows_remote=64488 dup_rows=23310 dup_rows_remote=19106 dedup_ratio=0.237122 router_tokens_per_expert=[39, 1061, 468, 399, 46, 1115, 793, 256, 469, 250, 936, 1572, 1588, 1132, 278, 1782, 903, 1325, 214, 647, 88, 198, 1315, 943, 539, 862, 954, 813, 156, 584, 896, 213, 417, 1152, 721, 665, 2111, 1913, 1992, 495, 587, 114, 272, 156, 1785, 1758, 159, 1234, 988, 114, 844, 957, 51, 140, 308, 694, 1153, 410, 69, 487, 93, 1700, 423, 2356] payload_rows_per_dst=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] dup_rows_per_dst=[2448, 3282, 1810, 1338, 4204, 3574, 3732, 2922] combine_rows_per_src=[9082, 9556, 9456, 8696, 10506, 8556, 8682, 10460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258867200 dispatch_meta_bytes=333104 combine_payload_bytes=258867200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259200304 ingress_bytes=328073216 link_bytes=328073216 token_bytes=518067504 weight_movement_bytes=69206016 total_nvlink_bytes=587273520 weight_to_token_ratio=0.133585 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234513100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73756 payload_rows_remote=63934 dup_rows=24548 dup_rows_remote=20476 dedup_ratio=0.249715 router_tokens_per_expert=[24, 904, 486, 331, 33, 1251, 837, 233, 471, 427, 1084, 1562, 1537, 1025, 116, 1708, 923, 1594, 320, 653, 92, 90, 1294, 1222, 554, 819, 992, 843, 211, 566, 1094, 190, 417, 1111, 714, 567, 2394, 2119, 1656, 456, 746, 89, 312, 316, 1434, 1539, 131, 1181, 910, 117, 944, 1070, 18, 73, 262, 565, 1151, 394, 76, 520, 12, 1718, 310, 2344] payload_rows_per_dst=[10844, 9822, 9844, 11148, 7648, 8066, 8938, 7446] dup_rows_per_dst=[3552, 4072, 2532, 3628, 2194, 3430, 3768, 1372] combine_rows_per_src=[10844, 9822, 9844, 11148, 7648, 8066, 8938, 7446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74968 payload_rows_remote=65012 dup_rows=23336 dup_rows_remote=20952 dedup_ratio=0.237386 router_tokens_per_expert=[12, 1111, 486, 358, 12, 1043, 660, 237, 570, 410, 1155, 1452, 1721, 984, 73, 991, 927, 1734, 270, 678, 177, 106, 1308, 970, 802, 927, 1085, 671, 104, 492, 847, 167, 433, 1025, 631, 595, 2826, 2478, 1725, 388, 716, 67, 383, 270, 1628, 1900, 123, 935, 850, 93, 777, 1098, 17, 83, 232, 443, 1325, 549, 81, 397, 2, 1839, 179, 2524] payload_rows_per_dst=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] dup_rows_per_dst=[2268, 2758, 2384, 3548, 2032, 3530, 3830, 2986] combine_rows_per_src=[9074, 8450, 9956, 11598, 7562, 8514, 9008, 10806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=343448 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271555992 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542768536 weight_movement_bytes=34603008 total_nvlink_bytes=577371544 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273924096 dispatch_meta_bytes=343824 combine_payload_bytes=273924096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274267920 ingress_bytes=308527104 link_bytes=308527104 token_bytes=548192016 weight_movement_bytes=34603008 total_nvlink_bytes=582795024 weight_to_token_ratio=0.063122 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160962654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258867200 dispatch_meta_bytes=333104 combine_payload_bytes=258867200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259200304 ingress_bytes=328073216 link_bytes=328073216 token_bytes=518067504 weight_movement_bytes=69206016 total_nvlink_bytes=587273520 weight_to_token_ratio=0.133585 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234513100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=343448 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271555992 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542768536 weight_movement_bytes=34603008 total_nvlink_bytes=577371544 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343856 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633008 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922160 weight_movement_bytes=34603008 total_nvlink_bytes=567525168 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75876 payload_rows_remote=65224 dup_rows=22428 dup_rows_remote=18378 dedup_ratio=0.228149 router_tokens_per_expert=[701, 950, 850, 376, 1172, 273, 384, 1051, 67, 857, 1636, 354, 605, 1038, 882, 1601, 802, 1141, 726, 372, 1635, 182, 1876, 1304, 685, 674, 732, 924, 127, 254, 199, 126, 358, 162, 393, 173, 941, 1402, 893, 1145, 2104, 1338, 908, 1328, 285, 1594, 39, 271, 1161, 983, 917, 8, 119, 1475, 235, 1414, 936, 404, 898, 588, 1025, 375, 315, 409] payload_rows_per_dst=[10652, 10372, 11316, 8492, 9072, 9464, 8484, 8024] dup_rows_per_dst=[4050, 3708, 3958, 2702, 1862, 3082, 1190, 1876] combine_rows_per_src=[10652, 10372, 11316, 8492, 9072, 9464, 8484, 8024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=261873664 dispatch_meta_bytes=337640 combine_payload_bytes=261873664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=262211304 ingress_bytes=365682688 link_bytes=365682688 token_bytes=524084968 weight_movement_bytes=103809024 total_nvlink_bytes=627893992 weight_to_token_ratio=0.198077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376034515e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=338776 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264989528 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529640280 weight_movement_bytes=34603008 total_nvlink_bytes=564243288 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264142848 dispatch_meta_bytes=334376 combine_payload_bytes=264142848 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=264477224 ingress_bytes=437157888 link_bytes=437157888 token_bytes=528620072 weight_movement_bytes=173015040 total_nvlink_bytes=701635112 weight_to_token_ratio=0.327296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.644989938e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=63286 dup_rows=24146 dup_rows_remote=20680 dedup_ratio=0.245626 router_tokens_per_expert=[36, 1095, 436, 351, 25, 1103, 776, 244, 554, 422, 890, 1560, 1543, 1039, 182, 1346, 898, 1578, 282, 685, 94, 88, 1291, 978, 600, 928, 898, 795, 156, 585, 997, 162, 408, 1297, 774, 543, 2678, 1905, 1833, 448, 673, 101, 310, 224, 1618, 1778, 153, 1242, 773, 86, 927, 1072, 24, 196, 234, 596, 1150, 524, 54, 541, 33, 1707, 219, 2414] payload_rows_per_dst=[10872, 9804, 10872, 10992, 9066, 8474, 6770, 7308] dup_rows_per_dst=[3466, 3890, 3146, 3060, 3366, 3724, 2346, 1148] combine_rows_per_src=[10872, 9804, 10872, 10992, 9066, 8474, 6770, 7308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75986 payload_rows_remote=65166 dup_rows=22318 dup_rows_remote=19406 dedup_ratio=0.227030 router_tokens_per_expert=[279, 532, 1325, 97, 420, 403, 1093, 819, 1856, 2920, 246, 372, 1191, 353, 1589, 466, 1746, 48, 2367, 417, 716, 271, 5, 612, 1141, 1170, 660, 373, 417, 1211, 1461, 618, 512, 539, 146, 190, 1104, 592, 1405, 522, 325, 2513, 90, 485, 316, 434, 350, 3879, 9, 194, 184, 1103, 510, 1244, 151, 151, 115, 145, 2002, 623, 1022, 768, 117, 218] payload_rows_per_dst=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] dup_rows_per_dst=[1594, 1056, 956, 5580, 1686, 5364, 3170, 2912] combine_rows_per_src=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343856 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633008 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922160 weight_movement_bytes=34603008 total_nvlink_bytes=567525168 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=334408 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267491912 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534649416 weight_movement_bytes=34603008 total_nvlink_bytes=569252424 weight_to_token_ratio=0.064721 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76156 payload_rows_remote=68024 dup_rows=22148 dup_rows_remote=20312 dedup_ratio=0.225301 router_tokens_per_expert=[287, 618, 1236, 149, 405, 465, 1190, 819, 1834, 3078, 223, 387, 1044, 351, 1709, 445, 1778, 56, 2504, 468, 558, 348, 6, 572, 1081, 1040, 621, 441, 423, 1146, 1574, 693, 436, 748, 160, 174, 1151, 446, 1374, 495, 378, 2297, 118, 462, 288, 459, 339, 3626, 28, 223, 212, 1073, 552, 1222, 180, 214, 190, 130, 2042, 576, 876, 781, 136, 217] payload_rows_per_dst=[10460, 7298, 9406, 8998, 8132, 11200, 10090, 10572] dup_rows_per_dst=[3026, 1020, 3174, 1892, 1836, 4734, 3474, 2992] combine_rows_per_src=[10460, 7298, 9406, 8998, 8132, 11200, 10090, 10572] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77712 payload_rows_remote=66602 dup_rows=20592 dup_rows_remote=16418 dedup_ratio=0.209473 router_tokens_per_expert=[168, 263, 1403, 76, 494, 602, 1146, 858, 2651, 3007, 342, 355, 1024, 434, 2646, 343, 2076, 35, 2275, 413, 829, 419, 1, 522, 535, 1539, 563, 396, 467, 1164, 1395, 603, 522, 509, 104, 134, 1188, 385, 1164, 506, 164, 2665, 134, 433, 236, 281, 345, 3384, 1, 172, 145, 1081, 564, 1020, 41, 130, 133, 66, 1978, 573, 955, 812, 74, 209] payload_rows_per_dst=[9452, 9006, 9560, 9964, 7480, 11110, 9782, 11358] dup_rows_per_dst=[2322, 1282, 3580, 1606, 1544, 4174, 2540, 3544] combine_rows_per_src=[9452, 9006, 9560, 9964, 7480, 11110, 9782, 11358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75986 payload_rows_remote=65166 dup_rows=22318 dup_rows_remote=19406 dedup_ratio=0.227030 router_tokens_per_expert=[279, 532, 1325, 97, 420, 403, 1093, 819, 1856, 2920, 246, 372, 1191, 353, 1589, 466, 1746, 48, 2367, 417, 716, 271, 5, 612, 1141, 1170, 660, 373, 417, 1211, 1461, 618, 512, 539, 146, 190, 1104, 592, 1405, 522, 325, 2513, 90, 485, 316, 434, 350, 3879, 9, 194, 184, 1103, 510, 1244, 151, 151, 115, 145, 2002, 623, 1022, 768, 117, 218] payload_rows_per_dst=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] dup_rows_per_dst=[1594, 1056, 956, 5580, 1686, 5364, 3170, 2912] combine_rows_per_src=[8342, 7378, 6680, 12600, 8334, 12070, 9762, 10820] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77316 payload_rows_remote=67592 dup_rows=20988 dup_rows_remote=18400 dedup_ratio=0.213501 router_tokens_per_expert=[182, 416, 1265, 144, 488, 710, 1116, 841, 2564, 2981, 262, 298, 982, 424, 2338, 361, 2016, 34, 2224, 397, 591, 350, 2, 434, 765, 1281, 614, 419, 425, 1196, 1508, 850, 455, 518, 179, 201, 1254, 443, 1166, 546, 238, 2479, 120, 478, 200, 357, 377, 3699, 4, 224, 178, 1027, 409, 1082, 80, 171, 162, 77, 2010, 621, 728, 859, 68, 264] payload_rows_per_dst=[8524, 8304, 9034, 11176, 7878, 11358, 9724, 11318] dup_rows_per_dst=[1800, 1026, 3062, 2940, 1646, 4538, 2588, 3388] combine_rows_per_src=[8524, 8304, 9034, 11176, 7878, 11358, 9724, 11318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278626304 dispatch_meta_bytes=353344 combine_payload_bytes=278626304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278979648 ingress_bytes=313229312 link_bytes=313229312 token_bytes=557605952 weight_movement_bytes=34603008 total_nvlink_bytes=592208960 weight_to_token_ratio=0.062056 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178656684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76148 payload_rows_remote=65142 dup_rows=22156 dup_rows_remote=18390 dedup_ratio=0.225382 router_tokens_per_expert=[172, 325, 1358, 109, 461, 525, 1218, 816, 2260, 2900, 234, 384, 1126, 330, 2153, 419, 2028, 45, 2492, 436, 757, 353, 0, 544, 933, 1416, 652, 355, 495, 1266, 1478, 616, 460, 586, 121, 127, 1351, 519, 1254, 531, 181, 2323, 111, 471, 239, 316, 301, 3753, 6, 233, 117, 1161, 503, 1098, 72, 119, 145, 100, 1786, 621, 853, 743, 105, 190] payload_rows_per_dst=[10060, 11006, 9606, 9620, 12656, 5774, 9670, 7756] dup_rows_per_dst=[2864, 3766, 3704, 1846, 4748, 1150, 2748, 1330] combine_rows_per_src=[10060, 11006, 9606, 9620, 12656, 5774, 9670, 7756] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272801792 dispatch_meta_bytes=332080 combine_payload_bytes=272801792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273133872 ingress_bytes=376610816 link_bytes=376610816 token_bytes=545935664 weight_movement_bytes=103809024 total_nvlink_bytes=649744688 weight_to_token_ratio=0.190149 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417156181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77224 payload_rows_remote=67748 dup_rows=21080 dup_rows_remote=19706 dedup_ratio=0.214437 router_tokens_per_expert=[194, 233, 1433, 67, 505, 836, 1070, 916, 2578, 3013, 357, 372, 1342, 393, 2326, 344, 1852, 32, 2484, 413, 924, 368, 1, 530, 634, 1650, 613, 358, 307, 1424, 1211, 439, 447, 527, 97, 179, 1217, 333, 1002, 420, 111, 2751, 118, 466, 137, 273, 146, 3918, 2, 181, 97, 1216, 405, 1047, 8, 58, 113, 49, 2009, 590, 991, 755, 83, 187] payload_rows_per_dst=[9834, 9674, 9580, 9476, 7652, 10794, 9644, 10570] dup_rows_per_dst=[3096, 1888, 3628, 1374, 1568, 4270, 2410, 2846] combine_rows_per_src=[9834, 9674, 9580, 9476, 7652, 10794, 9644, 10570] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76634 payload_rows_remote=66962 dup_rows=21670 dup_rows_remote=18064 dedup_ratio=0.220439 router_tokens_per_expert=[167, 273, 1592, 50, 527, 694, 1212, 916, 2282, 2930, 243, 391, 1245, 289, 2425, 509, 1960, 18, 2484, 417, 885, 327, 1, 547, 557, 1548, 635, 315, 276, 1332, 1531, 503, 429, 912, 129, 209, 1244, 381, 945, 588, 177, 2622, 123, 506, 145, 396, 79, 3620, 3, 190, 128, 1109, 318, 1171, 51, 91, 156, 46, 2026, 577, 844, 576, 92, 188] payload_rows_per_dst=[10306, 11402, 9672, 9148, 12196, 6534, 9528, 7848] dup_rows_per_dst=[3618, 3366, 3606, 1184, 4718, 1562, 2454, 1162] combine_rows_per_src=[10306, 11402, 9672, 9148, 12196, 6534, 9528, 7848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259219456 dispatch_meta_bytes=335864 combine_payload_bytes=259219456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259555320 ingress_bytes=328425472 link_bytes=328425472 token_bytes=518774776 weight_movement_bytes=69206016 total_nvlink_bytes=587980792 weight_to_token_ratio=0.133403 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235838611e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=338288 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267258224 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534178160 weight_movement_bytes=34603008 total_nvlink_bytes=568781168 weight_to_token_ratio=0.064778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77224 payload_rows_remote=67748 dup_rows=21080 dup_rows_remote=19706 dedup_ratio=0.214437 router_tokens_per_expert=[194, 233, 1433, 67, 505, 836, 1070, 916, 2578, 3013, 357, 372, 1342, 393, 2326, 344, 1852, 32, 2484, 413, 924, 368, 1, 530, 634, 1650, 613, 358, 307, 1424, 1211, 439, 447, 527, 97, 179, 1217, 333, 1002, 420, 111, 2751, 118, 466, 137, 273, 146, 3918, 2, 181, 97, 1216, 405, 1047, 8, 58, 113, 49, 2009, 590, 991, 755, 83, 187] payload_rows_per_dst=[9834, 9674, 9580, 9476, 7652, 10794, 9644, 10570] dup_rows_per_dst=[3096, 1888, 3628, 1374, 1568, 4270, 2410, 2846] combine_rows_per_src=[9834, 9674, 9580, 9476, 7652, 10794, 9644, 10570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272801792 dispatch_meta_bytes=332080 combine_payload_bytes=272801792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273133872 ingress_bytes=376610816 link_bytes=376610816 token_bytes=545935664 weight_movement_bytes=103809024 total_nvlink_bytes=649744688 weight_to_token_ratio=0.190149 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417156181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76156 payload_rows_remote=68024 dup_rows=22148 dup_rows_remote=20312 dedup_ratio=0.225301 router_tokens_per_expert=[287, 618, 1236, 149, 405, 465, 1190, 819, 1834, 3078, 223, 387, 1044, 351, 1709, 445, 1778, 56, 2504, 468, 558, 348, 6, 572, 1081, 1040, 621, 441, 423, 1146, 1574, 693, 436, 748, 160, 174, 1151, 446, 1374, 495, 378, 2297, 118, 462, 288, 459, 339, 3626, 28, 223, 212, 1073, 552, 1222, 180, 214, 190, 130, 2042, 576, 876, 781, 136, 217] payload_rows_per_dst=[10460, 7298, 9406, 8998, 8132, 11200, 10090, 10572] dup_rows_per_dst=[3026, 1020, 3174, 1892, 1836, 4734, 3474, 2992] combine_rows_per_src=[10460, 7298, 9406, 8998, 8132, 11200, 10090, 10572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276856832 dispatch_meta_bytes=343968 combine_payload_bytes=276856832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277200800 ingress_bytes=311459840 link_bytes=311459840 token_bytes=554057632 weight_movement_bytes=34603008 total_nvlink_bytes=588660640 weight_to_token_ratio=0.062454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171998303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=276856832 dispatch_meta_bytes=343968 combine_payload_bytes=276856832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277200800 ingress_bytes=311459840 link_bytes=311459840 token_bytes=554057632 weight_movement_bytes=34603008 total_nvlink_bytes=588660640 weight_to_token_ratio=0.062454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.171998303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=63286 dup_rows=24146 dup_rows_remote=20680 dedup_ratio=0.245626 router_tokens_per_expert=[36, 1095, 436, 351, 25, 1103, 776, 244, 554, 422, 890, 1560, 1543, 1039, 182, 1346, 898, 1578, 282, 685, 94, 88, 1291, 978, 600, 928, 898, 795, 156, 585, 997, 162, 408, 1297, 774, 543, 2678, 1905, 1833, 448, 673, 101, 310, 224, 1618, 1778, 153, 1242, 773, 86, 927, 1072, 24, 196, 234, 596, 1150, 524, 54, 541, 33, 1707, 219, 2414] payload_rows_per_dst=[10872, 9804, 10872, 10992, 9066, 8474, 6770, 7308] dup_rows_per_dst=[3466, 3890, 3146, 3060, 3366, 3724, 2346, 1148] combine_rows_per_src=[10872, 9804, 10872, 10992, 9066, 8474, 6770, 7308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=334128 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267155760 ingress_bytes=370630656 link_bytes=370630656 token_bytes=533977392 weight_movement_bytes=103809024 total_nvlink_bytes=637786416 weight_to_token_ratio=0.194407 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394653320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=340104 combine_payload_bytes=274276352 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274616456 ingress_bytes=343482368 link_bytes=343482368 token_bytes=548892808 weight_movement_bytes=69206016 total_nvlink_bytes=618098824 weight_to_token_ratio=0.126083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292496498e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277495808 dispatch_meta_bytes=349816 combine_payload_bytes=277495808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277845624 ingress_bytes=346701824 link_bytes=346701824 token_bytes=555341432 weight_movement_bytes=69206016 total_nvlink_bytes=624547448 weight_to_token_ratio=0.124619 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304611052e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=278626304 dispatch_meta_bytes=353344 combine_payload_bytes=278626304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278979648 ingress_bytes=313229312 link_bytes=313229312 token_bytes=557605952 weight_movement_bytes=34603008 total_nvlink_bytes=592208960 weight_to_token_ratio=0.062056 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178656684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74998 payload_rows_remote=64756 dup_rows=23306 dup_rows_remote=20542 dedup_ratio=0.237081 router_tokens_per_expert=[254, 411, 1313, 126, 477, 521, 1175, 813, 2220, 2903, 330, 352, 999, 430, 2337, 403, 2059, 61, 2175, 454, 803, 397, 1, 578, 707, 1409, 633, 420, 423, 1247, 1413, 611, 512, 494, 126, 171, 1158, 430, 1275, 454, 245, 2688, 131, 505, 275, 319, 361, 3495, 6, 225, 146, 1150, 391, 984, 99, 113, 130, 107, 2050, 643, 909, 797, 92, 216] payload_rows_per_dst=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] dup_rows_per_dst=[2764, 7752, 3690, 1322, 4026, 172, 2088, 1492] combine_rows_per_src=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72910 payload_rows_remote=62012 dup_rows=25394 dup_rows_remote=19250 dedup_ratio=0.258321 router_tokens_per_expert=[0, 1647, 1468, 1057, 1138, 1769, 670, 1060, 186, 1210, 2575, 59, 152, 280, 245, 920, 1935, 354, 0, 56, 125, 939, 280, 185, 962, 0, 2366, 460, 2721, 690, 670, 1, 184, 0, 3, 498, 1134, 50, 89, 1025, 70, 258, 22, 1009, 518, 0, 3132, 0, 30, 956, 301, 631, 359, 1921, 1152, 1109, 929, 594, 65, 1699, 1166, 1530, 1434, 1104] payload_rows_per_dst=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] dup_rows_per_dst=[3316, 2178, 676, 5134, 3674, 814, 3458, 6144] combine_rows_per_src=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=340104 combine_payload_bytes=274276352 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=274616456 ingress_bytes=343482368 link_bytes=343482368 token_bytes=548892808 weight_movement_bytes=69206016 total_nvlink_bytes=618098824 weight_to_token_ratio=0.126083 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292496498e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259219456 dispatch_meta_bytes=335864 combine_payload_bytes=259219456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259555320 ingress_bytes=328425472 link_bytes=328425472 token_bytes=518774776 weight_movement_bytes=69206016 total_nvlink_bytes=587980792 weight_to_token_ratio=0.133403 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235838611e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73826 payload_rows_remote=65074 dup_rows=24478 dup_rows_remote=21594 dedup_ratio=0.249003 router_tokens_per_expert=[1, 1690, 1413, 1136, 1266, 1763, 663, 1083, 260, 1381, 2697, 75, 176, 282, 254, 827, 1849, 353, 0, 101, 138, 846, 286, 131, 963, 0, 1951, 391, 2720, 792, 575, 0, 216, 0, 4, 503, 1077, 34, 64, 1074, 84, 283, 26, 1038, 406, 0, 3021, 0, 58, 903, 312, 765, 385, 2043, 1010, 1135, 989, 523, 76, 1612, 1253, 1655, 1288, 1252] payload_rows_per_dst=[7930, 10038, 10456, 7258, 8752, 9654, 9868, 9870] dup_rows_per_dst=[4408, 3110, 2392, 2086, 2884, 1300, 3354, 4944] combine_rows_per_src=[7930, 10038, 10456, 7258, 8752, 9654, 9868, 9870] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64268 dup_rows=23906 dup_rows_remote=23176 dedup_ratio=0.243184 router_tokens_per_expert=[0, 1224, 1303, 1223, 1098, 1779, 430, 1454, 148, 1238, 2484, 51, 118, 271, 256, 623, 2003, 374, 0, 8, 103, 959, 413, 119, 826, 0, 3227, 273, 3742, 462, 393, 0, 172, 0, 0, 546, 706, 5, 45, 981, 49, 239, 0, 1007, 272, 0, 3863, 0, 0, 1109, 226, 597, 332, 2371, 1394, 1064, 605, 565, 33, 1692, 1182, 1323, 1049, 1123] payload_rows_per_dst=[7400, 8518, 8098, 11112, 8412, 10130, 10374, 10354] dup_rows_per_dst=[3156, 1860, 940, 5654, 2964, 730, 3812, 4790] combine_rows_per_src=[7400, 8518, 8098, 11112, 8412, 10130, 10374, 10354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72910 payload_rows_remote=62012 dup_rows=25394 dup_rows_remote=19250 dedup_ratio=0.258321 router_tokens_per_expert=[0, 1647, 1468, 1057, 1138, 1769, 670, 1060, 186, 1210, 2575, 59, 152, 280, 245, 920, 1935, 354, 0, 56, 125, 939, 280, 185, 962, 0, 2366, 460, 2721, 690, 670, 1, 184, 0, 3, 498, 1134, 50, 89, 1025, 70, 258, 22, 1009, 518, 0, 3132, 0, 30, 956, 301, 631, 359, 1921, 1152, 1109, 929, 594, 65, 1699, 1166, 1530, 1434, 1104] payload_rows_per_dst=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] dup_rows_per_dst=[3316, 2178, 676, 5134, 3674, 814, 3458, 6144] combine_rows_per_src=[7238, 9076, 7072, 10606, 9356, 9204, 9460, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=346672 combine_payload_bytes=266543104 prefetch_bytes=103809024 reduce_bytes=0 egress_bytes=266889776 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533432880 weight_movement_bytes=103809024 total_nvlink_bytes=637241904 weight_to_token_ratio=0.194606 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020000 gemm_active_groups=12 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=3.906250000e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73890 payload_rows_remote=63790 dup_rows=24414 dup_rows_remote=20946 dedup_ratio=0.248352 router_tokens_per_expert=[0, 1384, 1370, 1038, 1144, 2077, 652, 1551, 241, 1174, 2471, 42, 94, 268, 249, 548, 2201, 365, 8, 44, 147, 876, 265, 132, 1039, 4, 2383, 323, 3110, 647, 410, 1, 211, 2, 2, 620, 881, 7, 51, 1163, 56, 372, 33, 1139, 395, 1, 3489, 1, 13, 911, 305, 660, 423, 2193, 1339, 940, 783, 495, 43, 1462, 1184, 1330, 1194, 1196] payload_rows_per_dst=[7458, 8238, 7376, 10664, 9618, 10080, 10100, 10356] dup_rows_per_dst=[3718, 1936, 700, 5170, 3512, 892, 3468, 5018] combine_rows_per_src=[7458, 8238, 7376, 10664, 9618, 10080, 10100, 10356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263241728 dispatch_meta_bytes=349776 combine_payload_bytes=263241728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263591504 ingress_bytes=297844736 link_bytes=297844736 token_bytes=526833232 weight_movement_bytes=34603008 total_nvlink_bytes=561436240 weight_to_token_ratio=0.065681 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120765763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74884 payload_rows_remote=64142 dup_rows=23420 dup_rows_remote=19498 dedup_ratio=0.238241 router_tokens_per_expert=[0, 1487, 1314, 1100, 1308, 1848, 659, 1329, 179, 1353, 2784, 62, 156, 293, 262, 751, 1992, 307, 0, 56, 111, 756, 291, 116, 943, 0, 2825, 343, 3215, 542, 526, 0, 167, 0, 0, 636, 882, 16, 68, 1066, 51, 314, 9, 1018, 312, 0, 3401, 0, 3, 907, 242, 597, 297, 1949, 1169, 1024, 622, 615, 55, 1875, 1264, 1492, 1237, 956] payload_rows_per_dst=[9986, 10742, 11012, 8268, 8868, 11608, 7132, 7268] dup_rows_per_dst=[5648, 3922, 2676, 2090, 3156, 2352, 1346, 2230] combine_rows_per_src=[9986, 10742, 11012, 8268, 8868, 11608, 7132, 7268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=341192 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265581768 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530822344 weight_movement_bytes=34603008 total_nvlink_bytes=565425352 weight_to_token_ratio=0.065188 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75022 payload_rows_remote=66786 dup_rows=23282 dup_rows_remote=21614 dedup_ratio=0.236837 router_tokens_per_expert=[0, 1199, 1313, 1392, 1179, 1637, 258, 1604, 115, 1435, 2893, 50, 133, 266, 245, 322, 2319, 336, 0, 1, 91, 1178, 555, 170, 1105, 0, 2974, 198, 3998, 294, 381, 0, 154, 0, 0, 752, 672, 1, 36, 845, 38, 345, 3, 959, 296, 0, 3718, 0, 4, 1345, 168, 442, 285, 2554, 1491, 833, 523, 596, 19, 1496, 1120, 884, 881, 1051] payload_rows_per_dst=[7658, 9604, 12790, 8236, 8134, 11598, 10372, 6630] dup_rows_per_dst=[3024, 3082, 4506, 1668, 3268, 2112, 3872, 1750] combine_rows_per_src=[7658, 9604, 12790, 8236, 8134, 11598, 10372, 6630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=254001152 dispatch_meta_bytes=325048 combine_payload_bytes=254001152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=254326200 ingress_bytes=357810176 link_bytes=357810176 token_bytes=508327352 weight_movement_bytes=103809024 total_nvlink_bytes=612136376 weight_to_token_ratio=0.204217 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.346410886e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=62738 dup_rows=23920 dup_rows_remote=20032 dedup_ratio=0.243327 router_tokens_per_expert=[0, 1192, 1375, 1497, 1157, 1886, 426, 2079, 140, 1632, 2385, 51, 138, 265, 245, 417, 2445, 301, 0, 6, 103, 1138, 247, 111, 1117, 0, 2192, 280, 3416, 372, 462, 0, 189, 0, 0, 645, 770, 0, 52, 914, 48, 341, 1, 1001, 375, 0, 3735, 0, 2, 1179, 224, 551, 219, 2408, 1539, 981, 457, 586, 26, 1610, 1003, 1316, 876, 1029] payload_rows_per_dst=[8100, 9890, 11646, 8178, 9184, 11812, 9154, 6420] dup_rows_per_dst=[3650, 3288, 3888, 2242, 3886, 2410, 3022, 1534] combine_rows_per_src=[8100, 9890, 11646, 8178, 9184, 11812, 9154, 6420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263241728 dispatch_meta_bytes=349776 combine_payload_bytes=263241728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263591504 ingress_bytes=297844736 link_bytes=297844736 token_bytes=526833232 weight_movement_bytes=34603008 total_nvlink_bytes=561436240 weight_to_token_ratio=0.065681 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120765763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75022 payload_rows_remote=66786 dup_rows=23282 dup_rows_remote=21614 dedup_ratio=0.236837 router_tokens_per_expert=[0, 1199, 1313, 1392, 1179, 1637, 258, 1604, 115, 1435, 2893, 50, 133, 266, 245, 322, 2319, 336, 0, 1, 91, 1178, 555, 170, 1105, 0, 2974, 198, 3998, 294, 381, 0, 154, 0, 0, 752, 672, 1, 36, 845, 38, 345, 3, 959, 296, 0, 3718, 0, 4, 1345, 168, 442, 285, 2554, 1491, 833, 523, 596, 19, 1496, 1120, 884, 881, 1051] payload_rows_per_dst=[7658, 9604, 12790, 8236, 8134, 11598, 10372, 6630] dup_rows_per_dst=[3024, 3082, 4506, 1668, 3268, 2112, 3872, 1750] combine_rows_per_src=[7658, 9604, 12790, 8236, 8134, 11598, 10372, 6630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261283840 dispatch_meta_bytes=338944 combine_payload_bytes=261283840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261622784 ingress_bytes=365092864 link_bytes=365092864 token_bytes=522906624 weight_movement_bytes=103809024 total_nvlink_bytes=626715648 weight_to_token_ratio=0.198523 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373815055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73826 payload_rows_remote=65074 dup_rows=24478 dup_rows_remote=21594 dedup_ratio=0.249003 router_tokens_per_expert=[1, 1690, 1413, 1136, 1266, 1763, 663, 1083, 260, 1381, 2697, 75, 176, 282, 254, 827, 1849, 353, 0, 101, 138, 846, 286, 131, 963, 0, 1951, 391, 2720, 792, 575, 0, 216, 0, 4, 503, 1077, 34, 64, 1074, 84, 283, 26, 1038, 406, 0, 3021, 0, 58, 903, 312, 765, 385, 2043, 1010, 1135, 989, 523, 76, 1612, 1253, 1655, 1288, 1252] payload_rows_per_dst=[7930, 10038, 10456, 7258, 8752, 9654, 9868, 9870] dup_rows_per_dst=[4408, 3110, 2392, 2086, 2884, 1300, 3354, 4944] combine_rows_per_src=[7930, 10038, 10456, 7258, 8752, 9654, 9868, 9870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74998 payload_rows_remote=64756 dup_rows=23306 dup_rows_remote=20542 dedup_ratio=0.237081 router_tokens_per_expert=[254, 411, 1313, 126, 477, 521, 1175, 813, 2220, 2903, 330, 352, 999, 430, 2337, 403, 2059, 61, 2175, 454, 803, 397, 1, 578, 707, 1409, 633, 420, 423, 1247, 1413, 611, 512, 494, 126, 171, 1158, 430, 1275, 454, 245, 2688, 131, 505, 275, 319, 361, 3495, 6, 225, 146, 1150, 391, 984, 99, 113, 130, 107, 2050, 643, 909, 797, 92, 216] payload_rows_per_dst=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] dup_rows_per_dst=[2764, 7752, 3690, 1322, 4026, 172, 2088, 1492] combine_rows_per_src=[10242, 13080, 9366, 9578, 12204, 3500, 8632, 8396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262725632 dispatch_meta_bytes=334560 combine_payload_bytes=262725632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263060192 ingress_bytes=297328640 link_bytes=297328640 token_bytes=525785824 weight_movement_bytes=34603008 total_nvlink_bytes=560388832 weight_to_token_ratio=0.065812 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118823735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=6 unused_prefetch_slots=10 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=3 remote_demand_peak=3 slot_utilization=0.375000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[6, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=331080 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257305928 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514280776 weight_movement_bytes=34603008 total_nvlink_bytes=548883784 weight_to_token_ratio=0.067284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=273555456 dispatch_meta_bytes=353600 combine_payload_bytes=273555456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273909056 ingress_bytes=342761472 link_bytes=342761472 token_bytes=547464512 weight_movement_bytes=69206016 total_nvlink_bytes=616670528 weight_to_token_ratio=0.126412 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289783825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=346672 combine_payload_bytes=266543104 prefetch_bytes=103809024 reduce_bytes=0 egress_bytes=266889776 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533432880 weight_movement_bytes=103809024 total_nvlink_bytes=637241904 weight_to_token_ratio=0.194606 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.020000 gemm_active_groups=12 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=3.906250000e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74934 payload_rows_remote=60912 dup_rows=23370 dup_rows_remote=17710 dedup_ratio=0.237732 router_tokens_per_expert=[614, 1783, 556, 322, 69, 381, 2885, 432, 1, 0, 297, 649, 2188, 0, 0, 842, 452, 1035, 5252, 28, 267, 759, 424, 0, 100, 3511, 957, 0, 78, 0, 3541, 125, 2729, 447, 70, 459, 310, 210, 958, 517, 36, 0, 7, 1332, 63, 498, 2638, 546, 1035, 1258, 1122, 1112, 28, 523, 0, 1117, 1032, 273, 189, 481, 225, 457, 605, 1327] payload_rows_per_dst=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] dup_rows_per_dst=[1466, 918, 504, 4854, 2098, 5068, 2802, 5660] combine_rows_per_src=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74982 payload_rows_remote=64786 dup_rows=23322 dup_rows_remote=17904 dedup_ratio=0.237244 router_tokens_per_expert=[0, 1385, 1447, 1109, 1202, 2096, 646, 1498, 221, 1000, 2428, 61, 147, 335, 251, 624, 1937, 366, 0, 32, 122, 780, 345, 108, 913, 2, 2706, 328, 3348, 536, 484, 0, 177, 0, 1, 603, 863, 16, 55, 1009, 56, 268, 8, 1054, 394, 0, 3558, 0, 11, 1056, 256, 557, 431, 2018, 1217, 1029, 642, 693, 49, 1466, 1357, 1344, 1277, 1230] payload_rows_per_dst=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] dup_rows_per_dst=[5418, 3124, 2968, 2000, 3498, 2124, 1626, 2564] combine_rows_per_src=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=256974848 dispatch_meta_bytes=331080 combine_payload_bytes=256974848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257305928 ingress_bytes=291577856 link_bytes=291577856 token_bytes=514280776 weight_movement_bytes=34603008 total_nvlink_bytes=548883784 weight_to_token_ratio=0.067284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097183998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76234 payload_rows_remote=62770 dup_rows=22070 dup_rows_remote=15362 dedup_ratio=0.224508 router_tokens_per_expert=[673, 1804, 632, 384, 77, 170, 2887, 555, 1, 0, 279, 632, 2027, 0, 0, 691, 531, 962, 5424, 26, 163, 763, 434, 0, 134, 3347, 942, 0, 63, 1, 3687, 93, 2756, 454, 86, 637, 296, 147, 1013, 389, 31, 0, 4, 1217, 79, 365, 2519, 621, 1018, 1246, 1115, 1103, 48, 489, 0, 1212, 953, 383, 394, 476, 254, 499, 661, 1305] payload_rows_per_dst=[10688, 9108, 6914, 10438, 13464, 7770, 9616, 8236] dup_rows_per_dst=[3676, 1676, 1076, 2572, 6708, 1902, 2846, 1614] combine_rows_per_src=[10688, 9108, 6914, 10438, 13464, 7770, 9616, 8236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=341192 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265581768 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530822344 weight_movement_bytes=34603008 total_nvlink_bytes=565425352 weight_to_token_ratio=0.065188 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75644 payload_rows_remote=65272 dup_rows=22660 dup_rows_remote=18614 dedup_ratio=0.230509 router_tokens_per_expert=[408, 1968, 506, 381, 59, 335, 2493, 563, 0, 0, 192, 565, 2441, 0, 0, 770, 404, 823, 5652, 17, 56, 1117, 583, 0, 49, 3983, 1206, 0, 36, 0, 3308, 66, 2742, 385, 73, 307, 217, 27, 906, 313, 20, 0, 1, 1015, 61, 598, 2658, 1045, 1041, 1100, 1520, 1196, 9, 525, 0, 1086, 1053, 265, 78, 336, 166, 338, 559, 1531] payload_rows_per_dst=[8136, 7062, 5600, 12076, 11032, 10372, 10064, 11302] dup_rows_per_dst=[1668, 874, 400, 5220, 3488, 4046, 2890, 4074] combine_rows_per_src=[8136, 7062, 5600, 12076, 11032, 10372, 10064, 11302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75638 payload_rows_remote=65834 dup_rows=22666 dup_rows_remote=19762 dedup_ratio=0.230570 router_tokens_per_expert=[621, 1814, 640, 435, 79, 324, 2755, 531, 1, 0, 258, 580, 2217, 0, 1, 887, 579, 874, 5289, 29, 91, 869, 561, 0, 129, 3645, 1045, 5, 100, 0, 3043, 102, 2857, 456, 84, 652, 234, 197, 1092, 350, 37, 1, 3, 992, 72, 627, 2384, 758, 1048, 1072, 1344, 1179, 23, 513, 1, 1174, 1011, 295, 214, 369, 209, 389, 625, 1386] payload_rows_per_dst=[7332, 7018, 5488, 11386, 9914, 10658, 9804, 14038] dup_rows_per_dst=[1556, 870, 518, 4752, 1930, 4600, 2904, 5536] combine_rows_per_src=[7332, 7018, 5488, 11386, 9914, 10658, 9804, 14038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78770 payload_rows_remote=66644 dup_rows=19534 dup_rows_remote=16066 dedup_ratio=0.198710 router_tokens_per_expert=[528, 2178, 512, 362, 62, 353, 2681, 454, 0, 0, 222, 657, 2312, 0, 0, 522, 395, 893, 5469, 26, 141, 961, 452, 0, 66, 4084, 1000, 0, 41, 0, 3574, 72, 2566, 420, 78, 388, 202, 166, 880, 296, 20, 0, 1, 1013, 85, 606, 2658, 740, 987, 1322, 1571, 1084, 19, 458, 0, 1147, 1015, 261, 445, 358, 181, 447, 613, 1108] payload_rows_per_dst=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] dup_rows_per_dst=[3678, 3468, 3808, 624, 1502, 2074, 2980, 1400] combine_rows_per_src=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=249495552 dispatch_meta_bytes=314488 combine_payload_bytes=249495552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249810040 ingress_bytes=284098560 link_bytes=284098560 token_bytes=499305592 weight_movement_bytes=34603008 total_nvlink_bytes=533908600 weight_to_token_ratio=0.069302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069040009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78828 payload_rows_remote=64754 dup_rows=19476 dup_rows_remote=15612 dedup_ratio=0.198120 router_tokens_per_expert=[316, 2097, 435, 228, 88, 542, 2276, 576, 0, 0, 206, 635, 2765, 0, 0, 769, 367, 894, 5783, 16, 45, 1412, 452, 0, 78, 3321, 1015, 1, 38, 0, 3562, 43, 3241, 421, 116, 313, 127, 29, 622, 371, 21, 0, 0, 864, 53, 617, 2933, 977, 1026, 1361, 1286, 1229, 20, 450, 0, 1106, 1101, 260, 123, 282, 168, 315, 529, 1231] payload_rows_per_dst=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] dup_rows_per_dst=[2826, 3830, 3864, 638, 1546, 2262, 3058, 1452] combine_rows_per_src=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=330760 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265694216 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531057672 weight_movement_bytes=138412032 total_nvlink_bytes=669469704 weight_to_token_ratio=0.260635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78602 payload_rows_remote=69780 dup_rows=19702 dup_rows_remote=18964 dedup_ratio=0.200419 router_tokens_per_expert=[277, 1839, 419, 471, 71, 258, 2341, 474, 0, 0, 224, 604, 2925, 0, 0, 601, 378, 786, 5642, 19, 53, 1239, 533, 0, 6, 4191, 1182, 0, 32, 0, 3536, 24, 2924, 398, 84, 172, 200, 39, 720, 329, 2, 0, 0, 938, 59, 663, 3091, 885, 1122, 1161, 1358, 1247, 8, 471, 0, 1080, 1067, 252, 135, 258, 187, 313, 549, 1315] payload_rows_per_dst=[9602, 12436, 13812, 8822, 8396, 9068, 9780, 6686] dup_rows_per_dst=[2698, 4654, 3488, 738, 1336, 2208, 3114, 1466] combine_rows_per_src=[9602, 12436, 13812, 8822, 8396, 9068, 9780, 6686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74934 payload_rows_remote=60912 dup_rows=23370 dup_rows_remote=17710 dedup_ratio=0.237732 router_tokens_per_expert=[614, 1783, 556, 322, 69, 381, 2885, 432, 1, 0, 297, 649, 2188, 0, 0, 842, 452, 1035, 5252, 28, 267, 759, 424, 0, 100, 3511, 957, 0, 78, 0, 3541, 125, 2729, 447, 70, 459, 310, 210, 958, 517, 36, 0, 7, 1332, 63, 498, 2638, 546, 1035, 1258, 1122, 1112, 28, 523, 0, 1117, 1032, 273, 189, 481, 225, 457, 605, 1327] payload_rows_per_dst=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] dup_rows_per_dst=[1466, 918, 504, 4854, 2098, 5068, 2802, 5660] combine_rows_per_src=[6848, 7036, 5426, 11770, 9302, 10942, 9588, 14022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=312528 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257418448 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514524368 weight_movement_bytes=34603008 total_nvlink_bytes=549127376 weight_to_token_ratio=0.067252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75638 payload_rows_remote=65834 dup_rows=22666 dup_rows_remote=19762 dedup_ratio=0.230570 router_tokens_per_expert=[621, 1814, 640, 435, 79, 324, 2755, 531, 1, 0, 258, 580, 2217, 0, 1, 887, 579, 874, 5289, 29, 91, 869, 561, 0, 129, 3645, 1045, 5, 100, 0, 3043, 102, 2857, 456, 84, 652, 234, 197, 1092, 350, 37, 1, 3, 992, 72, 627, 2384, 758, 1048, 1072, 1344, 1179, 23, 513, 1, 1174, 1011, 295, 214, 369, 209, 389, 625, 1386] payload_rows_per_dst=[7332, 7018, 5488, 11386, 9914, 10658, 9804, 14038] dup_rows_per_dst=[1556, 870, 518, 4752, 1930, 4600, 2904, 5536] combine_rows_per_src=[7332, 7018, 5488, 11386, 9914, 10658, 9804, 14038] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=335544 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267689656 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535043768 weight_movement_bytes=34603008 total_nvlink_bytes=569646776 weight_to_token_ratio=0.064673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78770 payload_rows_remote=66644 dup_rows=19534 dup_rows_remote=16066 dedup_ratio=0.198710 router_tokens_per_expert=[528, 2178, 512, 362, 62, 353, 2681, 454, 0, 0, 222, 657, 2312, 0, 0, 522, 395, 893, 5469, 26, 141, 961, 452, 0, 66, 4084, 1000, 0, 41, 0, 3574, 72, 2566, 420, 78, 388, 202, 166, 880, 296, 20, 0, 1, 1013, 85, 606, 2658, 740, 987, 1322, 1571, 1084, 19, 458, 0, 1147, 1015, 261, 445, 358, 181, 447, 613, 1108] payload_rows_per_dst=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] dup_rows_per_dst=[3678, 3468, 3808, 624, 1502, 2074, 2980, 1400] combine_rows_per_src=[10582, 12126, 13566, 8882, 8490, 8172, 9496, 7456] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78602 payload_rows_remote=69780 dup_rows=19702 dup_rows_remote=18964 dedup_ratio=0.200419 router_tokens_per_expert=[277, 1839, 419, 471, 71, 258, 2341, 474, 0, 0, 224, 604, 2925, 0, 0, 601, 378, 786, 5642, 19, 53, 1239, 533, 0, 6, 4191, 1182, 0, 32, 0, 3536, 24, 2924, 398, 84, 172, 200, 39, 720, 329, 2, 0, 0, 938, 59, 663, 3091, 885, 1122, 1161, 1358, 1247, 8, 471, 0, 1080, 1067, 252, 135, 258, 187, 313, 549, 1315] payload_rows_per_dst=[9602, 12436, 13812, 8822, 8396, 9068, 9780, 6686] dup_rows_per_dst=[2698, 4654, 3488, 738, 1336, 2208, 3114, 1466] combine_rows_per_src=[9602, 12436, 13812, 8822, 8396, 9068, 9780, 6686] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=249495552 dispatch_meta_bytes=314488 combine_payload_bytes=249495552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249810040 ingress_bytes=284098560 link_bytes=284098560 token_bytes=499305592 weight_movement_bytes=34603008 total_nvlink_bytes=533908600 weight_to_token_ratio=0.069302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069040009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76234 payload_rows_remote=62770 dup_rows=22070 dup_rows_remote=15362 dedup_ratio=0.224508 router_tokens_per_expert=[673, 1804, 632, 384, 77, 170, 2887, 555, 1, 0, 279, 632, 2027, 0, 0, 691, 531, 962, 5424, 26, 163, 763, 434, 0, 134, 3347, 942, 0, 63, 1, 3687, 93, 2756, 454, 86, 637, 296, 147, 1013, 389, 31, 0, 4, 1217, 79, 365, 2519, 621, 1018, 1246, 1115, 1103, 48, 489, 0, 1212, 953, 383, 394, 476, 254, 499, 661, 1305] payload_rows_per_dst=[10688, 9108, 6914, 10438, 13464, 7770, 9616, 8236] dup_rows_per_dst=[3676, 1676, 1076, 2572, 6708, 1902, 2846, 1614] combine_rows_per_src=[10688, 9108, 6914, 10438, 13464, 7770, 9616, 8236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78828 payload_rows_remote=64754 dup_rows=19476 dup_rows_remote=15612 dedup_ratio=0.198120 router_tokens_per_expert=[316, 2097, 435, 228, 88, 542, 2276, 576, 0, 0, 206, 635, 2765, 0, 0, 769, 367, 894, 5783, 16, 45, 1412, 452, 0, 78, 3321, 1015, 1, 38, 0, 3562, 43, 3241, 421, 116, 313, 127, 29, 622, 371, 21, 0, 0, 864, 53, 617, 2933, 977, 1026, 1361, 1286, 1229, 20, 450, 0, 1106, 1101, 260, 123, 282, 168, 315, 529, 1231] payload_rows_per_dst=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] dup_rows_per_dst=[2826, 3830, 3864, 638, 1546, 2262, 3058, 1452] combine_rows_per_src=[10290, 11562, 14074, 8836, 8934, 8668, 9898, 6566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269656064 dispatch_meta_bytes=342384 combine_payload_bytes=269656064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269998448 ingress_bytes=338862080 link_bytes=338862080 token_bytes=539654512 weight_movement_bytes=69206016 total_nvlink_bytes=608860528 weight_to_token_ratio=0.128241 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275110726e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74982 payload_rows_remote=64786 dup_rows=23322 dup_rows_remote=17904 dedup_ratio=0.237244 router_tokens_per_expert=[0, 1385, 1447, 1109, 1202, 2096, 646, 1498, 221, 1000, 2428, 61, 147, 335, 251, 624, 1937, 366, 0, 32, 122, 780, 345, 108, 913, 2, 2706, 328, 3348, 536, 484, 0, 177, 0, 1, 603, 863, 16, 55, 1009, 56, 268, 8, 1054, 394, 0, 3558, 0, 11, 1056, 256, 557, 431, 2018, 1217, 1029, 642, 693, 49, 1466, 1357, 1344, 1277, 1230] payload_rows_per_dst=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] dup_rows_per_dst=[5418, 3124, 2968, 2000, 3498, 2124, 1626, 2564] combine_rows_per_src=[10196, 9698, 11108, 7938, 9138, 11484, 7886, 7534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272973824 dispatch_meta_bytes=330840 combine_payload_bytes=272973824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273304664 ingress_bytes=307576832 link_bytes=307576832 token_bytes=546278488 weight_movement_bytes=34603008 total_nvlink_bytes=580881496 weight_to_token_ratio=0.063343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157386857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=321464 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265553848 ingress_bytes=369041408 link_bytes=369041408 token_bytes=530786232 weight_movement_bytes=103809024 total_nvlink_bytes=634595256 weight_to_token_ratio=0.195576 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388673108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285818880 dispatch_meta_bytes=354976 combine_payload_bytes=285818880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=286173856 ingress_bytes=389627904 link_bytes=389627904 token_bytes=571992736 weight_movement_bytes=103809024 total_nvlink_bytes=675801760 weight_to_token_ratio=0.181487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.466138435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=257105920 dispatch_meta_bytes=312528 combine_payload_bytes=257105920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257418448 ingress_bytes=291708928 link_bytes=291708928 token_bytes=514524368 weight_movement_bytes=34603008 total_nvlink_bytes=549127376 weight_to_token_ratio=0.067252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.097677212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=62556 dup_rows=21028 dup_rows_remote=12462 dedup_ratio=0.213908 router_tokens_per_expert=[766, 98, 0, 5005, 84, 0, 74, 88, 0, 161, 240, 139, 900, 633, 429, 969, 0, 0, 425, 0, 0, 458, 416, 2242, 104, 50, 334, 0, 102, 469, 0, 0, 88, 2539, 313, 3, 0, 0, 0, 2302, 1012, 134, 202, 2088, 5418, 185, 1421, 3786, 422, 86, 0, 1425, 2, 1897, 6421, 0, 4816, 0, 0, 0, 0, 0, 349, 57] payload_rows_per_dst=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] dup_rows_per_dst=[1068, 930, 666, 330, 1882, 6462, 1124, 8566] combine_rows_per_src=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75010 payload_rows_remote=63474 dup_rows=23294 dup_rows_remote=17572 dedup_ratio=0.236959 router_tokens_per_expert=[468, 1744, 517, 323, 73, 325, 2769, 624, 0, 0, 277, 655, 2198, 0, 0, 787, 457, 930, 5249, 26, 75, 1023, 525, 0, 86, 3916, 1220, 0, 38, 0, 3406, 91, 2536, 486, 84, 333, 296, 156, 831, 360, 41, 0, 5, 1196, 53, 588, 2668, 696, 1021, 1112, 1598, 1219, 9, 499, 0, 1165, 1014, 250, 147, 358, 214, 399, 591, 1425] payload_rows_per_dst=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] dup_rows_per_dst=[5722, 3934, 5768, 474, 1528, 2160, 2068, 1640] combine_rows_per_src=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 3, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83352 payload_rows_remote=74924 dup_rows=14952 dup_rows_remote=12952 dedup_ratio=0.152100 router_tokens_per_expert=[759, 92, 0, 4942, 91, 0, 43, 90, 0, 177, 329, 264, 878, 573, 426, 779, 0, 0, 458, 0, 0, 456, 408, 2027, 122, 54, 320, 0, 134, 418, 0, 0, 94, 2382, 337, 0, 0, 1, 0, 2400, 1013, 155, 146, 2122, 5264, 223, 1635, 4041, 397, 93, 0, 1555, 43, 1683, 6435, 1, 4861, 0, 0, 0, 0, 0, 367, 64] payload_rows_per_dst=[11076, 11750, 5966, 12252, 8428, 8564, 14890, 10426] dup_rows_per_dst=[958, 3184, 732, 372, 2000, 2024, 5524, 158] combine_rows_per_src=[11076, 11750, 5966, 12252, 8428, 8564, 14890, 10426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=321464 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265553848 ingress_bytes=369041408 link_bytes=369041408 token_bytes=530786232 weight_movement_bytes=103809024 total_nvlink_bytes=634595256 weight_to_token_ratio=0.195576 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388673108e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81094 payload_rows_remote=72166 dup_rows=17210 dup_rows_remote=14086 dedup_ratio=0.175069 router_tokens_per_expert=[804, 80, 0, 5227, 58, 0, 0, 31, 0, 126, 207, 151, 849, 417, 373, 622, 0, 0, 514, 0, 0, 537, 445, 1336, 107, 59, 180, 0, 112, 77, 0, 0, 65, 2485, 235, 8, 0, 0, 0, 3148, 1041, 223, 198, 1706, 5550, 190, 1231, 4347, 365, 77, 0, 1492, 0, 2918, 6565, 0, 4572, 0, 0, 0, 0, 0, 399, 25] payload_rows_per_dst=[11386, 9486, 4770, 11892, 9770, 8928, 15080, 9782] dup_rows_per_dst=[1014, 1824, 894, 278, 2112, 3124, 7754, 210] combine_rows_per_src=[11386, 9486, 4770, 11892, 9770, 8928, 15080, 9782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=330760 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265694216 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531057672 weight_movement_bytes=138412032 total_nvlink_bytes=669469704 weight_to_token_ratio=0.260635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77276 payload_rows_remote=62556 dup_rows=21028 dup_rows_remote=12462 dedup_ratio=0.213908 router_tokens_per_expert=[766, 98, 0, 5005, 84, 0, 74, 88, 0, 161, 240, 139, 900, 633, 429, 969, 0, 0, 425, 0, 0, 458, 416, 2242, 104, 50, 334, 0, 102, 469, 0, 0, 88, 2539, 313, 3, 0, 0, 0, 2302, 1012, 134, 202, 2088, 5418, 185, 1421, 3786, 422, 86, 0, 1425, 2, 1897, 6421, 0, 4816, 0, 0, 0, 0, 0, 349, 57] payload_rows_per_dst=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] dup_rows_per_dst=[1068, 930, 666, 330, 1882, 6462, 1124, 8566] combine_rows_per_src=[10988, 6012, 6416, 9434, 8608, 12706, 8392, 14720] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306888704 dispatch_meta_bytes=351504 combine_payload_bytes=306888704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307240208 ingress_bytes=341491712 link_bytes=341491712 token_bytes=614128912 weight_movement_bytes=34603008 total_nvlink_bytes=648731920 weight_to_token_ratio=0.056345 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285005820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295591936 dispatch_meta_bytes=345008 combine_payload_bytes=295591936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=295936944 ingress_bytes=434003968 link_bytes=434003968 token_bytes=591528880 weight_movement_bytes=138412032 total_nvlink_bytes=729940912 weight_to_token_ratio=0.233990 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633121991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79754 payload_rows_remote=67686 dup_rows=18550 dup_rows_remote=14494 dedup_ratio=0.188700 router_tokens_per_expert=[819, 61, 0, 4770, 108, 0, 10, 49, 0, 202, 302, 325, 951, 524, 420, 698, 0, 0, 451, 1, 0, 543, 470, 1508, 96, 66, 339, 0, 154, 268, 0, 1, 49, 2385, 249, 1, 0, 1, 0, 2762, 1092, 217, 192, 2107, 5291, 247, 1410, 4205, 405, 110, 0, 1292, 78, 2252, 6467, 1, 4712, 0, 0, 1, 4, 0, 449, 37] payload_rows_per_dst=[10634, 5786, 5102, 12052, 9114, 12886, 12068, 12112] dup_rows_per_dst=[1000, 1058, 844, 378, 1780, 6054, 4056, 3380] combine_rows_per_src=[10634, 5786, 5102, 12052, 9114, 12886, 12068, 12112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=256229376 dispatch_meta_bytes=300072 combine_payload_bytes=256229376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256529448 ingress_bytes=290832384 link_bytes=290832384 token_bytes=512758824 weight_movement_bytes=34603008 total_nvlink_bytes=547361832 weight_to_token_ratio=0.067484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094378847e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82828 payload_rows_remote=70698 dup_rows=15476 dup_rows_remote=12442 dedup_ratio=0.157430 router_tokens_per_expert=[795, 97, 0, 5290, 72, 0, 0, 94, 0, 180, 247, 147, 849, 528, 427, 863, 0, 0, 453, 0, 0, 518, 375, 1714, 135, 53, 210, 0, 81, 379, 0, 0, 95, 2460, 299, 5, 0, 0, 0, 2657, 1082, 214, 208, 1996, 5559, 152, 1277, 4341, 410, 72, 0, 1526, 0, 2010, 6423, 0, 4492, 0, 0, 0, 0, 0, 308, 59] payload_rows_per_dst=[11728, 12130, 14304, 12500, 10894, 7866, 3842, 9564] dup_rows_per_dst=[968, 3034, 4662, 334, 4158, 1992, 174, 154] combine_rows_per_src=[11728, 12130, 14304, 12500, 10894, 7866, 3842, 9564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=295591936 dispatch_meta_bytes=345008 combine_payload_bytes=295591936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=295936944 ingress_bytes=434003968 link_bytes=434003968 token_bytes=591528880 weight_movement_bytes=138412032 total_nvlink_bytes=729940912 weight_to_token_ratio=0.233990 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633121991e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=324184 combine_payload_bytes=259989504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260313688 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520303192 weight_movement_bytes=69206016 total_nvlink_bytes=589509208 weight_to_token_ratio=0.133011 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84748 payload_rows_remote=70264 dup_rows=13556 dup_rows_remote=9516 dedup_ratio=0.137899 router_tokens_per_expert=[784, 85, 0, 5275, 58, 0, 0, 47, 0, 178, 169, 138, 828, 507, 382, 614, 0, 0, 206, 1, 0, 474, 372, 1730, 119, 88, 389, 0, 82, 73, 0, 0, 86, 2429, 295, 0, 0, 0, 0, 2758, 1040, 182, 176, 1804, 5622, 152, 1381, 4219, 415, 73, 0, 1617, 0, 2936, 6479, 0, 4483, 0, 0, 0, 0, 0, 389, 17] payload_rows_per_dst=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] dup_rows_per_dst=[984, 2298, 4040, 370, 2058, 1930, 1666, 210] combine_rows_per_src=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84682 payload_rows_remote=72374 dup_rows=13622 dup_rows_remote=13328 dedup_ratio=0.138570 router_tokens_per_expert=[807, 75, 0, 5465, 27, 0, 0, 22, 0, 150, 132, 129, 825, 528, 397, 632, 0, 0, 300, 0, 0, 548, 400, 1488, 123, 71, 334, 0, 52, 64, 0, 0, 79, 2323, 269, 0, 0, 0, 0, 2939, 1020, 190, 188, 1817, 5657, 109, 1105, 4420, 391, 64, 0, 1606, 8, 2895, 6630, 0, 4537, 0, 0, 0, 0, 0, 328, 8] payload_rows_per_dst=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] dup_rows_per_dst=[982, 2450, 3490, 294, 1906, 1648, 2684, 168] combine_rows_per_src=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84682 payload_rows_remote=72374 dup_rows=13622 dup_rows_remote=13328 dedup_ratio=0.138570 router_tokens_per_expert=[807, 75, 0, 5465, 27, 0, 0, 22, 0, 150, 132, 129, 825, 528, 397, 632, 0, 0, 300, 0, 0, 548, 400, 1488, 123, 71, 334, 0, 52, 64, 0, 0, 79, 2323, 269, 0, 0, 0, 0, 2939, 1020, 190, 188, 1817, 5657, 109, 1105, 4420, 391, 64, 0, 1606, 8, 2895, 6630, 0, 4537, 0, 0, 0, 0, 0, 328, 8] payload_rows_per_dst=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] dup_rows_per_dst=[982, 2450, 3490, 294, 1906, 1648, 2684, 168] combine_rows_per_src=[11810, 11976, 12944, 12308, 9314, 7210, 9542, 9578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=256229376 dispatch_meta_bytes=300072 combine_payload_bytes=256229376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256529448 ingress_bytes=290832384 link_bytes=290832384 token_bytes=512758824 weight_movement_bytes=34603008 total_nvlink_bytes=547361832 weight_to_token_ratio=0.067484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094378847e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84748 payload_rows_remote=70264 dup_rows=13556 dup_rows_remote=9516 dedup_ratio=0.137899 router_tokens_per_expert=[784, 85, 0, 5275, 58, 0, 0, 47, 0, 178, 169, 138, 828, 507, 382, 614, 0, 0, 206, 1, 0, 474, 372, 1730, 119, 88, 389, 0, 82, 73, 0, 0, 86, 2429, 295, 0, 0, 0, 0, 2758, 1040, 182, 176, 1804, 5622, 152, 1381, 4219, 415, 73, 0, 1617, 0, 2936, 6479, 0, 4483, 0, 0, 0, 0, 0, 389, 17] payload_rows_per_dst=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] dup_rows_per_dst=[984, 2298, 4040, 370, 2058, 1930, 1666, 210] combine_rows_per_src=[11514, 11772, 14484, 12376, 9214, 7540, 8280, 9568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75010 payload_rows_remote=63474 dup_rows=23294 dup_rows_remote=17572 dedup_ratio=0.236959 router_tokens_per_expert=[468, 1744, 517, 323, 73, 325, 2769, 624, 0, 0, 277, 655, 2198, 0, 0, 787, 457, 930, 5249, 26, 75, 1023, 525, 0, 86, 3916, 1220, 0, 38, 0, 3406, 91, 2536, 486, 84, 333, 296, 156, 831, 360, 41, 0, 5, 1196, 53, 588, 2668, 696, 1021, 1112, 1598, 1219, 9, 499, 0, 1165, 1014, 250, 147, 358, 214, 399, 591, 1425] payload_rows_per_dst=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] dup_rows_per_dst=[5722, 3934, 5768, 474, 1528, 2160, 2068, 1640] combine_rows_per_src=[11536, 11732, 13998, 6222, 8636, 8334, 7396, 7156] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83352 payload_rows_remote=74924 dup_rows=14952 dup_rows_remote=12952 dedup_ratio=0.152100 router_tokens_per_expert=[759, 92, 0, 4942, 91, 0, 43, 90, 0, 177, 329, 264, 878, 573, 426, 779, 0, 0, 458, 0, 0, 456, 408, 2027, 122, 54, 320, 0, 134, 418, 0, 0, 94, 2382, 337, 0, 0, 1, 0, 2400, 1013, 155, 146, 2122, 5264, 223, 1635, 4041, 397, 93, 0, 1555, 43, 1683, 6435, 1, 4861, 0, 0, 0, 0, 0, 367, 64] payload_rows_per_dst=[11076, 11750, 5966, 12252, 8428, 8564, 14890, 10426] dup_rows_per_dst=[958, 3184, 732, 372, 2000, 2024, 5524, 158] combine_rows_per_src=[11076, 11750, 5966, 12252, 8428, 8564, 14890, 10426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=328720 combine_payload_bytes=277241856 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=277570576 ingress_bytes=415653888 link_bytes=415653888 token_bytes=554812432 weight_movement_bytes=138412032 total_nvlink_bytes=693224464 weight_to_token_ratio=0.249475 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.564072118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81394 payload_rows_remote=74818 dup_rows=16910 dup_rows_remote=16364 dedup_ratio=0.172017 router_tokens_per_expert=[1919, 7201, 1, 117, 556, 3, 66, 0, 0, 120, 109, 814, 2, 327, 409, 1019, 392, 858, 0, 0, 272, 84, 807, 619, 4320, 4615, 950, 529, 0, 0, 91, 299, 0, 203, 614, 2, 93, 1911, 710, 28, 611, 2, 0, 54, 804, 6798, 7, 0, 6073, 0, 211, 1246, 483, 94, 305, 49, 0, 4, 0, 0, 0, 1923, 147, 281] payload_rows_per_dst=[4800, 15184, 11648, 11228, 6576, 14028, 4388, 13542] dup_rows_per_dst=[524, 4818, 3056, 1740, 546, 2524, 388, 3314] combine_rows_per_src=[4800, 15184, 11648, 11228, 6576, 14028, 4388, 13542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289579008 dispatch_meta_bytes=332560 combine_payload_bytes=289579008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289911568 ingress_bytes=324182016 link_bytes=324182016 token_bytes=579490576 weight_movement_bytes=34603008 total_nvlink_bytes=614093584 weight_to_token_ratio=0.059713 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219870827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287801344 dispatch_meta_bytes=319120 combine_payload_bytes=287801344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288120464 ingress_bytes=322404352 link_bytes=322404352 token_bytes=575921808 weight_movement_bytes=34603008 total_nvlink_bytes=610524816 weight_to_token_ratio=0.060083 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213181621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82730 payload_rows_remote=69244 dup_rows=15574 dup_rows_remote=13594 dedup_ratio=0.158427 router_tokens_per_expert=[2286, 7460, 0, 60, 566, 0, 50, 0, 0, 149, 50, 590, 0, 176, 480, 907, 143, 1257, 0, 0, 265, 50, 154, 499, 5555, 4413, 786, 405, 0, 0, 0, 313, 0, 284, 335, 0, 79, 2910, 666, 4, 662, 3, 0, 59, 472, 6537, 0, 0, 5705, 0, 35, 1375, 296, 77, 0, 0, 0, 0, 0, 0, 0, 2672, 118, 249] payload_rows_per_dst=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] dup_rows_per_dst=[512, 4120, 2030, 1070, 562, 1980, 1004, 4296] combine_rows_per_src=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=296443904 dispatch_meta_bytes=342808 combine_payload_bytes=296443904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296786712 ingress_bytes=331046912 link_bytes=331046912 token_bytes=593230616 weight_movement_bytes=34603008 total_nvlink_bytes=627833624 weight_to_token_ratio=0.058330 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245702878e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=306888704 dispatch_meta_bytes=351504 combine_payload_bytes=306888704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307240208 ingress_bytes=341491712 link_bytes=341491712 token_bytes=614128912 weight_movement_bytes=34603008 total_nvlink_bytes=648731920 weight_to_token_ratio=0.056345 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285005820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287801344 dispatch_meta_bytes=319120 combine_payload_bytes=287801344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288120464 ingress_bytes=322404352 link_bytes=322404352 token_bytes=575921808 weight_movement_bytes=34603008 total_nvlink_bytes=610524816 weight_to_token_ratio=0.060083 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213181621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259989504 dispatch_meta_bytes=324184 combine_payload_bytes=259989504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260313688 ingress_bytes=329195520 link_bytes=329195520 token_bytes=520303192 weight_movement_bytes=69206016 total_nvlink_bytes=589509208 weight_to_token_ratio=0.133011 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=364728 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306819256 ingress_bytes=341057536 link_bytes=341057536 token_bytes=613273784 weight_movement_bytes=34603008 total_nvlink_bytes=647876792 weight_to_token_ratio=0.056423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283372051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76466 payload_rows_remote=62908 dup_rows=21838 dup_rows_remote=18404 dedup_ratio=0.222148 router_tokens_per_expert=[1870, 7238, 0, 120, 579, 0, 57, 1, 0, 136, 82, 619, 0, 328, 422, 1129, 244, 869, 0, 0, 291, 94, 759, 646, 4380, 4921, 927, 555, 0, 0, 83, 325, 0, 230, 701, 0, 86, 1893, 615, 18, 585, 10, 0, 50, 769, 6735, 4, 0, 6077, 0, 209, 1285, 486, 91, 167, 47, 0, 0, 0, 0, 0, 2016, 165, 238] payload_rows_per_dst=[4750, 8608, 4398, 10894, 6580, 15064, 12614, 13558] dup_rows_per_dst=[504, 2458, 1408, 2202, 506, 9528, 1798, 3434] combine_rows_per_src=[4750, 8608, 4398, 10894, 6580, 15064, 12614, 13558] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82730 payload_rows_remote=69244 dup_rows=15574 dup_rows_remote=13594 dedup_ratio=0.158427 router_tokens_per_expert=[2286, 7460, 0, 60, 566, 0, 50, 0, 0, 149, 50, 590, 0, 176, 480, 907, 143, 1257, 0, 0, 265, 50, 154, 499, 5555, 4413, 786, 405, 0, 0, 0, 313, 0, 284, 335, 0, 79, 2910, 666, 4, 662, 3, 0, 59, 472, 6537, 0, 0, 5705, 0, 35, 1375, 296, 77, 0, 0, 0, 0, 0, 0, 0, 2672, 118, 249] payload_rows_per_dst=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] dup_rows_per_dst=[512, 4120, 2030, 1070, 562, 1980, 1004, 4296] combine_rows_per_src=[5412, 15504, 9428, 8372, 7994, 13486, 9342, 13192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82262 payload_rows_remote=70926 dup_rows=16042 dup_rows_remote=15184 dedup_ratio=0.163188 router_tokens_per_expert=[732, 91, 0, 5112, 75, 0, 6, 81, 0, 170, 220, 115, 917, 511, 450, 887, 0, 0, 476, 0, 0, 468, 441, 1633, 121, 66, 230, 0, 87, 489, 0, 0, 82, 2489, 233, 4, 0, 0, 0, 3109, 1068, 205, 195, 1724, 5574, 163, 1330, 4037, 431, 78, 0, 1105, 0, 2616, 6477, 0, 4444, 0, 0, 0, 0, 0, 356, 54] payload_rows_per_dst=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] dup_rows_per_dst=[858, 2866, 4664, 344, 5210, 1888, 16, 196] combine_rows_per_src=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283623424 dispatch_meta_bytes=331352 combine_payload_bytes=283623424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283954776 ingress_bytes=352829440 link_bytes=352829440 token_bytes=567578200 weight_movement_bytes=69206016 total_nvlink_bytes=636784216 weight_to_token_ratio=0.121932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327668778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80602 payload_rows_remote=76114 dup_rows=17702 dup_rows_remote=17032 dedup_ratio=0.180074 router_tokens_per_expert=[2214, 7341, 0, 118, 580, 0, 59, 0, 0, 131, 60, 785, 0, 338, 375, 890, 166, 1061, 0, 0, 279, 81, 343, 564, 4974, 4939, 762, 440, 0, 0, 44, 306, 0, 219, 503, 0, 73, 2447, 537, 2, 556, 2, 0, 53, 714, 6587, 6, 0, 6093, 0, 60, 1386, 406, 72, 0, 4, 0, 0, 0, 0, 0, 2234, 142, 206] payload_rows_per_dst=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] dup_rows_per_dst=[5256, 670, 2494, 1462, 5738, 112, 1952, 18] combine_rows_per_src=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81844 payload_rows_remote=70154 dup_rows=16460 dup_rows_remote=14968 dedup_ratio=0.167440 router_tokens_per_expert=[2130, 7290, 1, 58, 623, 0, 37, 1, 0, 122, 92, 742, 22, 293, 489, 943, 186, 1221, 1, 0, 297, 37, 462, 577, 5044, 4417, 793, 410, 1, 1, 11, 305, 0, 276, 479, 0, 107, 2514, 680, 13, 711, 3, 0, 66, 776, 6280, 7, 1, 5851, 1, 138, 1409, 511, 108, 5, 2, 0, 0, 1, 0, 1, 2088, 191, 327] payload_rows_per_dst=[5178, 15294, 4352, 11472, 7472, 13194, 11690, 13192] dup_rows_per_dst=[522, 4692, 1210, 1658, 666, 2494, 1492, 3726] combine_rows_per_src=[5178, 15294, 4352, 11472, 7472, 13194, 11690, 13192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=257671168 dispatch_meta_bytes=325248 combine_payload_bytes=257671168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257996416 ingress_bytes=292274176 link_bytes=292274176 token_bytes=515667584 weight_movement_bytes=34603008 total_nvlink_bytes=550270592 weight_to_token_ratio=0.067103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099804194e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283623424 dispatch_meta_bytes=331352 combine_payload_bytes=283623424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283954776 ingress_bytes=352829440 link_bytes=352829440 token_bytes=567578200 weight_movement_bytes=69206016 total_nvlink_bytes=636784216 weight_to_token_ratio=0.121932 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327668778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83474 payload_rows_remote=72800 dup_rows=14830 dup_rows_remote=13616 dedup_ratio=0.150859 router_tokens_per_expert=[2643, 7507, 0, 43, 523, 0, 50, 0, 0, 129, 50, 824, 0, 153, 429, 1012, 143, 1180, 0, 0, 187, 50, 190, 513, 5540, 4418, 843, 379, 0, 0, 1, 303, 0, 271, 301, 0, 65, 2713, 679, 6, 572, 1, 0, 71, 326, 6590, 0, 0, 5666, 0, 18, 1338, 212, 64, 0, 0, 0, 0, 0, 0, 0, 2776, 185, 188] payload_rows_per_dst=[11030, 9778, 13080, 10674, 7520, 13496, 7562, 10334] dup_rows_per_dst=[3194, 2724, 2526, 1214, 550, 1624, 656, 2342] combine_rows_per_src=[11030, 9778, 13080, 10674, 7520, 13496, 7562, 10334] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290512896 dispatch_meta_bytes=344440 combine_payload_bytes=290512896 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=290857336 ingress_bytes=325115904 link_bytes=325115904 token_bytes=581370232 weight_movement_bytes=34603008 total_nvlink_bytes=615973240 weight_to_token_ratio=0.059520 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223384973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80602 payload_rows_remote=76114 dup_rows=17702 dup_rows_remote=17032 dedup_ratio=0.180074 router_tokens_per_expert=[2214, 7341, 0, 118, 580, 0, 59, 0, 0, 131, 60, 785, 0, 338, 375, 890, 166, 1061, 0, 0, 279, 81, 343, 564, 4974, 4939, 762, 440, 0, 0, 44, 306, 0, 219, 503, 0, 73, 2447, 537, 2, 556, 2, 0, 53, 714, 6587, 6, 0, 6093, 0, 60, 1386, 406, 72, 0, 4, 0, 0, 0, 0, 0, 2234, 142, 206] payload_rows_per_dst=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] dup_rows_per_dst=[5256, 670, 2494, 1462, 5738, 112, 1952, 18] combine_rows_per_src=[15368, 4488, 12442, 11520, 14998, 2550, 14090, 5146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83474 payload_rows_remote=72800 dup_rows=14830 dup_rows_remote=13616 dedup_ratio=0.150859 router_tokens_per_expert=[2643, 7507, 0, 43, 523, 0, 50, 0, 0, 129, 50, 824, 0, 153, 429, 1012, 143, 1180, 0, 0, 187, 50, 190, 513, 5540, 4418, 843, 379, 0, 0, 1, 303, 0, 271, 301, 0, 65, 2713, 679, 6, 572, 1, 0, 71, 326, 6590, 0, 0, 5666, 0, 18, 1338, 212, 64, 0, 0, 0, 0, 0, 0, 0, 2776, 185, 188] payload_rows_per_dst=[11030, 9778, 13080, 10674, 7520, 13496, 7562, 10334] dup_rows_per_dst=[3194, 2724, 2526, 1214, 550, 1624, 656, 2342] combine_rows_per_src=[11030, 9778, 13080, 10674, 7520, 13496, 7562, 10334] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81930 payload_rows_remote=69276 dup_rows=16374 dup_rows_remote=14050 dedup_ratio=0.166565 router_tokens_per_expert=[2532, 7444, 0, 76, 518, 0, 51, 0, 0, 138, 52, 769, 0, 278, 373, 774, 159, 953, 0, 0, 215, 51, 278, 550, 5283, 4702, 835, 406, 0, 0, 0, 339, 0, 266, 315, 0, 74, 2675, 504, 0, 520, 0, 0, 88, 443, 6610, 2, 0, 5936, 0, 15, 1466, 172, 76, 0, 0, 0, 0, 0, 0, 0, 2829, 141, 244] payload_rows_per_dst=[15934, 4110, 12654, 11072, 13024, 4916, 13832, 6388] dup_rows_per_dst=[5308, 658, 2324, 1492, 4494, 560, 1498, 40] combine_rows_per_src=[15934, 4110, 12654, 11072, 13024, 4916, 13832, 6388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82262 payload_rows_remote=70926 dup_rows=16042 dup_rows_remote=15184 dedup_ratio=0.163188 router_tokens_per_expert=[732, 91, 0, 5112, 75, 0, 6, 81, 0, 170, 220, 115, 917, 511, 450, 887, 0, 0, 476, 0, 0, 468, 441, 1633, 121, 66, 230, 0, 87, 489, 0, 0, 82, 2489, 233, 4, 0, 0, 0, 3109, 1068, 205, 195, 1724, 5574, 163, 1330, 4037, 431, 78, 0, 1105, 0, 2616, 6477, 0, 4444, 0, 0, 0, 0, 0, 356, 54] payload_rows_per_dst=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] dup_rows_per_dst=[858, 2866, 4664, 344, 5210, 1888, 16, 196] combine_rows_per_src=[11336, 11748, 14326, 12790, 11856, 7482, 3212, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287350784 dispatch_meta_bytes=340488 combine_payload_bytes=287350784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287691272 ingress_bytes=356556800 link_bytes=356556800 token_bytes=575042056 weight_movement_bytes=69206016 total_nvlink_bytes=644248072 weight_to_token_ratio=0.120349 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341694533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81394 payload_rows_remote=74818 dup_rows=16910 dup_rows_remote=16364 dedup_ratio=0.172017 router_tokens_per_expert=[1919, 7201, 1, 117, 556, 3, 66, 0, 0, 120, 109, 814, 2, 327, 409, 1019, 392, 858, 0, 0, 272, 84, 807, 619, 4320, 4615, 950, 529, 0, 0, 91, 299, 0, 203, 614, 2, 93, 1911, 710, 28, 611, 2, 0, 54, 804, 6798, 7, 0, 6073, 0, 211, 1246, 483, 94, 305, 49, 0, 4, 0, 0, 0, 1923, 147, 281] payload_rows_per_dst=[4800, 15184, 11648, 11228, 6576, 14028, 4388, 13542] dup_rows_per_dst=[524, 4818, 3056, 1740, 546, 2524, 388, 3314] combine_rows_per_src=[4800, 15184, 11648, 11228, 6576, 14028, 4388, 13542] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=287350784 dispatch_meta_bytes=340488 combine_payload_bytes=287350784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287691272 ingress_bytes=356556800 link_bytes=356556800 token_bytes=575042056 weight_movement_bytes=69206016 total_nvlink_bytes=644248072 weight_to_token_ratio=0.120349 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341694533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86886 payload_rows_remote=73432 dup_rows=11418 dup_rows_remote=8768 dedup_ratio=0.116150 router_tokens_per_expert=[603, 91, 4269, 1022, 3570, 0, 6658, 5713, 1, 184, 0, 479, 0, 836, 1, 412, 311, 317, 3423, 90, 934, 50, 392, 148, 0, 91, 488, 0, 0, 157, 0, 475, 6023, 236, 0, 91, 216, 967, 715, 0, 437, 0, 0, 0, 0, 0, 0, 250, 2444, 0, 4, 67, 0, 1291, 75, 0, 2666, 0, 4, 1580, 1236, 135, 0, 0] payload_rows_per_dst=[9314, 10350, 9872, 13332, 13454, 14196, 7458, 8910] dup_rows_per_dst=[1258, 2014, 1458, 516, 2650, 494, 696, 2332] combine_rows_per_src=[9314, 10350, 9872, 13332, 13454, 14196, 7458, 8910] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82444 payload_rows_remote=67672 dup_rows=15860 dup_rows_remote=15358 dedup_ratio=0.161336 router_tokens_per_expert=[609, 77, 3340, 1082, 3256, 0, 6992, 5447, 0, 155, 0, 442, 0, 767, 0, 532, 312, 362, 2854, 77, 807, 57, 365, 88, 1, 77, 435, 0, 0, 128, 0, 560, 6251, 249, 0, 78, 205, 1825, 796, 0, 394, 0, 0, 0, 0, 0, 0, 251, 2410, 0, 0, 59, 1, 1157, 153, 0, 2569, 0, 1, 2087, 1711, 133, 0, 0] payload_rows_per_dst=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] dup_rows_per_dst=[500, 3684, 1036, 434, 1226, 502, 5226, 3252] combine_rows_per_src=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=311762944 dispatch_meta_bytes=372584 combine_payload_bytes=311762944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312135528 ingress_bytes=346365952 link_bytes=346365952 token_bytes=623898472 weight_movement_bytes=34603008 total_nvlink_bytes=658501480 weight_to_token_ratio=0.055463 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303347193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298188800 dispatch_meta_bytes=345664 combine_payload_bytes=298188800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=298534464 ingress_bytes=401997824 link_bytes=401997824 token_bytes=596723264 weight_movement_bytes=103809024 total_nvlink_bytes=700532288 weight_to_token_ratio=0.173965 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.512685448e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=283754496 dispatch_meta_bytes=333304 combine_payload_bytes=283754496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284087800 ingress_bytes=318357504 link_bytes=318357504 token_bytes=567842296 weight_movement_bytes=34603008 total_nvlink_bytes=602445304 weight_to_token_ratio=0.060938 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197953658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=306454528 dispatch_meta_bytes=364728 combine_payload_bytes=306454528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306819256 ingress_bytes=341057536 link_bytes=341057536 token_bytes=613273784 weight_movement_bytes=34603008 total_nvlink_bytes=647876792 weight_to_token_ratio=0.056423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283372051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=290512896 dispatch_meta_bytes=344440 combine_payload_bytes=290512896 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=290857336 ingress_bytes=325115904 link_bytes=325115904 token_bytes=581370232 weight_movement_bytes=34603008 total_nvlink_bytes=615973240 weight_to_token_ratio=0.059520 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.223384973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84000 payload_rows_remote=74762 dup_rows=14304 dup_rows_remote=12016 dedup_ratio=0.145508 router_tokens_per_expert=[570, 84, 4048, 1109, 3547, 0, 6879, 5686, 0, 169, 0, 425, 0, 783, 0, 398, 291, 331, 3032, 84, 938, 59, 414, 140, 0, 84, 520, 0, 0, 142, 0, 493, 6125, 313, 0, 85, 197, 1189, 740, 0, 412, 0, 0, 0, 0, 0, 0, 224, 2481, 0, 0, 58, 0, 1259, 80, 0, 2694, 0, 1, 1774, 1193, 101, 0, 0] payload_rows_per_dst=[3526, 10260, 9280, 13342, 9126, 14584, 14644, 9238] dup_rows_per_dst=[0, 2324, 1298, 508, 2078, 446, 5362, 2288] combine_rows_per_src=[3526, 10260, 9280, 13342, 9126, 14584, 14644, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82444 payload_rows_remote=67672 dup_rows=15860 dup_rows_remote=15358 dedup_ratio=0.161336 router_tokens_per_expert=[609, 77, 3340, 1082, 3256, 0, 6992, 5447, 0, 155, 0, 442, 0, 767, 0, 532, 312, 362, 2854, 77, 807, 57, 365, 88, 1, 77, 435, 0, 0, 128, 0, 560, 6251, 249, 0, 78, 205, 1825, 796, 0, 394, 0, 0, 0, 0, 0, 0, 251, 2410, 0, 0, 59, 1, 1157, 153, 0, 2569, 0, 1, 2087, 1711, 133, 0, 0] payload_rows_per_dst=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] dup_rows_per_dst=[500, 3684, 1036, 434, 1226, 502, 5226, 3252] combine_rows_per_src=[5456, 10880, 8808, 12862, 5080, 14772, 14836, 9750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=300777472 dispatch_meta_bytes=328800 combine_payload_bytes=300777472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301106272 ingress_bytes=404586496 link_bytes=404586496 token_bytes=601883744 weight_movement_bytes=103809024 total_nvlink_bytes=705692768 weight_to_token_ratio=0.172474 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522426413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277184512 dispatch_meta_bytes=332120 combine_payload_bytes=277184512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277516632 ingress_bytes=311787520 link_bytes=311787520 token_bytes=554701144 weight_movement_bytes=34603008 total_nvlink_bytes=589304152 weight_to_token_ratio=0.062381 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173231337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=64940 dup_rows=18000 dup_rows_remote=13352 dedup_ratio=0.183105 router_tokens_per_expert=[1936, 7369, 0, 84, 558, 0, 59, 0, 0, 115, 60, 561, 0, 318, 461, 1084, 159, 1253, 0, 0, 269, 60, 334, 587, 4965, 4523, 831, 454, 0, 0, 9, 300, 0, 240, 415, 0, 77, 2549, 709, 4, 631, 1, 0, 66, 674, 6545, 2, 0, 5945, 0, 109, 1434, 431, 80, 0, 0, 0, 0, 0, 0, 0, 2534, 127, 230] payload_rows_per_dst=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] dup_rows_per_dst=[4648, 810, 2702, 1510, 6028, 132, 2144, 26] combine_rows_per_src=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306225152 dispatch_meta_bytes=347112 combine_payload_bytes=306225152 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=306572264 ingress_bytes=340828160 link_bytes=340828160 token_bytes=612797416 weight_movement_bytes=34603008 total_nvlink_bytes=647400424 weight_to_token_ratio=0.056467 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282508927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81594 payload_rows_remote=66950 dup_rows=16710 dup_rows_remote=11976 dedup_ratio=0.169983 router_tokens_per_expert=[618, 108, 3860, 1106, 2950, 1, 6791, 5596, 1, 214, 0, 504, 1, 823, 1, 360, 315, 362, 2683, 107, 851, 47, 408, 97, 2, 107, 526, 0, 0, 144, 1, 546, 6153, 319, 0, 107, 207, 1633, 784, 0, 401, 2, 0, 0, 1, 1, 2, 245, 2300, 0, 3, 66, 0, 1057, 110, 0, 2761, 0, 12, 1978, 1728, 152, 0, 0] payload_rows_per_dst=[3662, 12514, 8620, 13306, 4712, 14388, 14644, 9748] dup_rows_per_dst=[4, 4914, 1120, 538, 1388, 498, 4734, 3514] combine_rows_per_src=[3662, 12514, 8620, 13306, 4712, 14388, 14644, 9748] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79986 payload_rows_remote=70558 dup_rows=18318 dup_rows_remote=16462 dedup_ratio=0.186340 router_tokens_per_expert=[553, 72, 3811, 1125, 3272, 0, 6909, 5659, 0, 144, 0, 410, 0, 772, 0, 505, 313, 300, 2946, 72, 855, 58, 410, 150, 0, 72, 485, 0, 0, 131, 0, 481, 6248, 271, 0, 72, 208, 1366, 727, 0, 425, 0, 0, 0, 0, 0, 0, 245, 2521, 0, 0, 57, 0, 1214, 92, 0, 2565, 0, 2, 1861, 1693, 80, 0, 0] payload_rows_per_dst=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] dup_rows_per_dst=[6158, 1856, 3188, 428, 2608, 490, 602, 2988] combine_rows_per_src=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277184512 dispatch_meta_bytes=332120 combine_payload_bytes=277184512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277516632 ingress_bytes=311787520 link_bytes=311787520 token_bytes=554701144 weight_movement_bytes=34603008 total_nvlink_bytes=589304152 weight_to_token_ratio=0.062381 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173231337e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=306225152 dispatch_meta_bytes=347112 combine_payload_bytes=306225152 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=306572264 ingress_bytes=340828160 link_bytes=340828160 token_bytes=612797416 weight_movement_bytes=34603008 total_nvlink_bytes=647400424 weight_to_token_ratio=0.056467 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282508927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79986 payload_rows_remote=70558 dup_rows=18318 dup_rows_remote=16462 dedup_ratio=0.186340 router_tokens_per_expert=[553, 72, 3811, 1125, 3272, 0, 6909, 5659, 0, 144, 0, 410, 0, 772, 0, 505, 313, 300, 2946, 72, 855, 58, 410, 150, 0, 72, 485, 0, 0, 131, 0, 481, 6248, 271, 0, 72, 208, 1366, 727, 0, 425, 0, 0, 0, 0, 0, 0, 245, 2521, 0, 0, 57, 0, 1214, 92, 0, 2565, 0, 2, 1861, 1693, 80, 0, 0] payload_rows_per_dst=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] dup_rows_per_dst=[6158, 1856, 3188, 428, 2608, 490, 602, 2988] combine_rows_per_src=[12704, 9428, 11112, 13228, 11084, 5850, 7166, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83766 payload_rows_remote=71466 dup_rows=14538 dup_rows_remote=14146 dedup_ratio=0.147888 router_tokens_per_expert=[637, 64, 2881, 1044, 3607, 0, 7143, 5208, 0, 126, 0, 360, 0, 693, 0, 671, 317, 314, 2449, 64, 719, 51, 391, 84, 5, 64, 451, 0, 0, 115, 0, 503, 6359, 334, 0, 65, 204, 2241, 780, 0, 387, 0, 0, 0, 0, 0, 0, 251, 1965, 0, 0, 71, 0, 1149, 256, 0, 3049, 0, 0, 2349, 1676, 55, 0, 0] payload_rows_per_dst=[9360, 7990, 7668, 12300, 14438, 15074, 6356, 10580] dup_rows_per_dst=[1344, 1472, 1110, 392, 5528, 488, 526, 3678] combine_rows_per_src=[9360, 7990, 7668, 12300, 14438, 15074, 6356, 10580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84578 payload_rows_remote=75300 dup_rows=13726 dup_rows_remote=12258 dedup_ratio=0.139628 router_tokens_per_expert=[617, 73, 3066, 787, 3535, 0, 6962, 5224, 0, 147, 0, 400, 0, 763, 0, 431, 329, 296, 3326, 73, 775, 50, 416, 108, 1, 73, 449, 0, 0, 125, 0, 443, 6115, 289, 0, 74, 195, 1977, 764, 0, 403, 0, 0, 0, 0, 0, 0, 248, 2426, 0, 0, 88, 0, 1558, 157, 0, 2575, 0, 0, 1916, 1816, 82, 0, 0] payload_rows_per_dst=[8898, 8034, 9278, 12220, 13952, 14738, 7648, 9810] dup_rows_per_dst=[1126, 1580, 1468, 410, 4876, 488, 810, 2968] combine_rows_per_src=[8898, 8034, 9278, 12220, 13952, 14738, 7648, 9810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86886 payload_rows_remote=73432 dup_rows=11418 dup_rows_remote=8768 dedup_ratio=0.116150 router_tokens_per_expert=[603, 91, 4269, 1022, 3570, 0, 6658, 5713, 1, 184, 0, 479, 0, 836, 1, 412, 311, 317, 3423, 90, 934, 50, 392, 148, 0, 91, 488, 0, 0, 157, 0, 475, 6023, 236, 0, 91, 216, 967, 715, 0, 437, 0, 0, 0, 0, 0, 0, 250, 2444, 0, 4, 67, 0, 1291, 75, 0, 2666, 0, 4, 1580, 1236, 135, 0, 0] payload_rows_per_dst=[9314, 10350, 9872, 13332, 13454, 14196, 7458, 8910] dup_rows_per_dst=[1258, 2014, 1458, 516, 2650, 494, 696, 2332] combine_rows_per_src=[9314, 10350, 9872, 13332, 13454, 14196, 7458, 8910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=313168 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266307408 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532301648 weight_movement_bytes=103809024 total_nvlink_bytes=636110672 weight_to_token_ratio=0.195019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80304 payload_rows_remote=64940 dup_rows=18000 dup_rows_remote=13352 dedup_ratio=0.183105 router_tokens_per_expert=[1936, 7369, 0, 84, 558, 0, 59, 0, 0, 115, 60, 561, 0, 318, 461, 1084, 159, 1253, 0, 0, 269, 60, 334, 587, 4965, 4523, 831, 454, 0, 0, 9, 300, 0, 240, 415, 0, 77, 2549, 709, 4, 631, 1, 0, 66, 674, 6545, 2, 0, 5945, 0, 109, 1434, 431, 80, 0, 0, 0, 0, 0, 0, 0, 2534, 127, 230] payload_rows_per_dst=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] dup_rows_per_dst=[4648, 810, 2702, 1510, 6028, 132, 2144, 26] combine_rows_per_src=[15364, 4388, 12552, 10724, 15050, 2616, 13854, 5756] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=274227200 dispatch_meta_bytes=315704 combine_payload_bytes=274227200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274542904 ingress_bytes=308830208 link_bytes=308830208 token_bytes=548770104 weight_movement_bytes=34603008 total_nvlink_bytes=583373112 weight_to_token_ratio=0.063056 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162103210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289005568 dispatch_meta_bytes=348080 combine_payload_bytes=289005568 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=289353648 ingress_bytes=358211584 link_bytes=358211584 token_bytes=578359216 weight_movement_bytes=69206016 total_nvlink_bytes=647565232 weight_to_token_ratio=0.119659 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347921352e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88644 payload_rows_remote=81194 dup_rows=9660 dup_rows_remote=9620 dedup_ratio=0.098267 router_tokens_per_expert=[0, 7779, 2847, 0, 0, 66, 27, 3942, 0, 1, 0, 0, 0, 694, 2825, 157, 0, 91, 131, 363, 0, 0, 0, 1037, 420, 252, 0, 174, 0, 66, 234, 0, 54, 7803, 91, 0, 2843, 0, 3600, 0, 860, 0, 1, 0, 2058, 245, 0, 0, 91, 0, 286, 325, 0, 67, 1, 0, 0, 371, 6531, 685, 98, 1269, 0, 767] payload_rows_per_dst=[5880, 12904, 14924, 15744, 7450, 10062, 15920, 5760] dup_rows_per_dst=[0, 2334, 1382, 2106, 40, 1952, 1226, 620] combine_rows_per_src=[5880, 12904, 14924, 15744, 7450, 10062, 15920, 5760] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86018 payload_rows_remote=74758 dup_rows=12286 dup_rows_remote=9336 dedup_ratio=0.124980 router_tokens_per_expert=[0, 7754, 3038, 0, 0, 50, 275, 2886, 0, 0, 0, 0, 0, 1115, 3145, 128, 0, 78, 5, 317, 1, 0, 2, 1017, 493, 251, 0, 122, 0, 50, 231, 0, 264, 7822, 78, 0, 4483, 0, 1925, 0, 303, 0, 0, 0, 2076, 243, 0, 0, 78, 0, 226, 371, 0, 50, 0, 0, 0, 359, 6823, 1554, 127, 645, 0, 767] payload_rows_per_dst=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] dup_rows_per_dst=[0, 2492, 644, 2140, 56, 2950, 1192, 2812] combine_rows_per_src=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289005568 dispatch_meta_bytes=348080 combine_payload_bytes=289005568 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=289353648 ingress_bytes=358211584 link_bytes=358211584 token_bytes=578359216 weight_movement_bytes=69206016 total_nvlink_bytes=647565232 weight_to_token_ratio=0.119659 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347921352e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=292724736 dispatch_meta_bytes=342448 combine_payload_bytes=292724736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293067184 ingress_bytes=327327744 link_bytes=327327744 token_bytes=585791920 weight_movement_bytes=34603008 total_nvlink_bytes=620394928 weight_to_token_ratio=0.059070 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231707949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=308428800 dispatch_meta_bytes=350232 combine_payload_bytes=308428800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308779032 ingress_bytes=343031808 link_bytes=343031808 token_bytes=617207832 weight_movement_bytes=34603008 total_nvlink_bytes=651810840 weight_to_token_ratio=0.056064 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.290801077e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=300777472 dispatch_meta_bytes=328800 combine_payload_bytes=300777472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=301106272 ingress_bytes=404586496 link_bytes=404586496 token_bytes=601883744 weight_movement_bytes=103809024 total_nvlink_bytes=705692768 weight_to_token_ratio=0.172474 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522426413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=313168 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266307408 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532301648 weight_movement_bytes=103809024 total_nvlink_bytes=636110672 weight_to_token_ratio=0.195019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79460 payload_rows_remote=63944 dup_rows=18844 dup_rows_remote=8774 dedup_ratio=0.191691 router_tokens_per_expert=[0, 7793, 2478, 0, 0, 57, 43, 3858, 0, 0, 0, 0, 0, 636, 2685, 145, 0, 85, 94, 338, 1, 0, 0, 1105, 342, 241, 0, 160, 0, 57, 184, 0, 80, 7823, 85, 0, 3055, 0, 3271, 0, 908, 0, 0, 0, 2358, 236, 0, 0, 85, 0, 244, 333, 0, 57, 0, 0, 0, 419, 6673, 717, 85, 1548, 0, 873] payload_rows_per_dst=[200, 12724, 3244, 11078, 10022, 10746, 15930, 15516] dup_rows_per_dst=[0, 1924, 2, 1290, 2036, 2368, 1154, 10070] combine_rows_per_src=[200, 12724, 3244, 11078, 10022, 10746, 15930, 15516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86018 payload_rows_remote=74758 dup_rows=12286 dup_rows_remote=9336 dedup_ratio=0.124980 router_tokens_per_expert=[0, 7754, 3038, 0, 0, 50, 275, 2886, 0, 0, 0, 0, 0, 1115, 3145, 128, 0, 78, 5, 317, 1, 0, 2, 1017, 493, 251, 0, 122, 0, 50, 231, 0, 264, 7822, 78, 0, 4483, 0, 1925, 0, 303, 0, 0, 0, 2076, 243, 0, 0, 78, 0, 226, 371, 0, 50, 0, 0, 0, 359, 6823, 1554, 127, 645, 0, 767] payload_rows_per_dst=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] dup_rows_per_dst=[0, 2492, 644, 2140, 56, 2950, 1192, 2812] combine_rows_per_src=[6726, 12056, 8826, 15662, 4478, 11260, 15902, 11108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=332570624 dispatch_meta_bytes=363256 combine_payload_bytes=332570624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=332933880 ingress_bytes=436379648 link_bytes=436379648 token_bytes=665504504 weight_movement_bytes=103809024 total_nvlink_bytes=769313528 weight_to_token_ratio=0.155985 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642061484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=306208768 dispatch_meta_bytes=336376 combine_payload_bytes=306208768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306545144 ingress_bytes=340811776 link_bytes=340811776 token_bytes=612753912 weight_movement_bytes=34603008 total_nvlink_bytes=647356920 weight_to_token_ratio=0.056471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282447275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64670 payload_rows_remote=49490 dup_rows=33634 dup_rows_remote=13820 dedup_ratio=0.342143 router_tokens_per_expert=[596, 77, 3641, 967, 3117, 0, 6786, 5733, 0, 154, 0, 409, 0, 849, 0, 449, 322, 364, 2941, 77, 973, 44, 431, 137, 3, 77, 526, 0, 0, 136, 0, 553, 6205, 298, 0, 77, 224, 1629, 824, 0, 405, 0, 0, 0, 0, 0, 0, 243, 2501, 0, 0, 72, 0, 1075, 150, 0, 2591, 0, 2, 1908, 1499, 87, 0, 0] payload_rows_per_dst=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] dup_rows_per_dst=[19814, 1870, 7462, 226, 1356, 486, 556, 1864] combine_rows_per_src=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79034 payload_rows_remote=63114 dup_rows=19270 dup_rows_remote=17976 dedup_ratio=0.196025 router_tokens_per_expert=[0, 7718, 3050, 0, 0, 37, 347, 3145, 0, 2, 0, 0, 1, 1136, 3349, 147, 0, 108, 152, 294, 3, 1, 5, 1097, 478, 245, 0, 162, 1, 37, 301, 0, 238, 7814, 107, 1, 3856, 0, 1909, 0, 306, 0, 1, 1, 2056, 245, 0, 1, 107, 1, 274, 366, 0, 44, 0, 1, 0, 405, 6676, 1029, 123, 891, 2, 882] payload_rows_per_dst=[1116, 12582, 3302, 15656, 4464, 10468, 15920, 15526] dup_rows_per_dst=[0, 2978, 18, 2228, 46, 2464, 1294, 10242] combine_rows_per_src=[1116, 12582, 3302, 15656, 4464, 10468, 15920, 15526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=261914624 dispatch_meta_bytes=290872 combine_payload_bytes=261914624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262205496 ingress_bytes=331120640 link_bytes=331120640 token_bytes=524120120 weight_movement_bytes=69206016 total_nvlink_bytes=593326136 weight_to_token_ratio=0.132042 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245980311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=306208768 dispatch_meta_bytes=336376 combine_payload_bytes=306208768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306545144 ingress_bytes=340811776 link_bytes=340811776 token_bytes=612753912 weight_movement_bytes=34603008 total_nvlink_bytes=647356920 weight_to_token_ratio=0.056471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282447275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79034 payload_rows_remote=63114 dup_rows=19270 dup_rows_remote=17976 dedup_ratio=0.196025 router_tokens_per_expert=[0, 7718, 3050, 0, 0, 37, 347, 3145, 0, 2, 0, 0, 1, 1136, 3349, 147, 0, 108, 152, 294, 3, 1, 5, 1097, 478, 245, 0, 162, 1, 37, 301, 0, 238, 7814, 107, 1, 3856, 0, 1909, 0, 306, 0, 1, 1, 2056, 245, 0, 1, 107, 1, 274, 366, 0, 44, 0, 1, 0, 405, 6676, 1029, 123, 891, 2, 882] payload_rows_per_dst=[1116, 12582, 3302, 15656, 4464, 10468, 15920, 15526] dup_rows_per_dst=[0, 2978, 18, 2228, 46, 2464, 1294, 10242] combine_rows_per_src=[1116, 12582, 3302, 15656, 4464, 10468, 15920, 15526] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73264 payload_rows_remote=65574 dup_rows=25040 dup_rows_remote=24526 dedup_ratio=0.254720 router_tokens_per_expert=[0, 7817, 2809, 0, 0, 59, 247, 3471, 0, 0, 0, 0, 0, 949, 3022, 131, 0, 77, 66, 353, 2, 0, 0, 1076, 341, 258, 0, 127, 0, 59, 154, 0, 154, 7829, 72, 0, 3851, 0, 2510, 0, 570, 0, 0, 0, 2222, 241, 0, 0, 72, 0, 176, 304, 0, 59, 0, 0, 0, 401, 6762, 1161, 87, 925, 0, 738] payload_rows_per_dst=[16132, 7690, 15250, 1574, 16134, 9306, 1078, 6100] dup_rows_per_dst=[12674, 514, 1422, 304, 7850, 1608, 144, 524] combine_rows_per_src=[16132, 7690, 15250, 1574, 16134, 9306, 1078, 6100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88130 payload_rows_remote=72062 dup_rows=10174 dup_rows_remote=8524 dedup_ratio=0.103495 router_tokens_per_expert=[0, 7970, 2559, 0, 0, 50, 143, 2641, 0, 0, 0, 0, 0, 1143, 3729, 115, 0, 91, 0, 323, 0, 0, 0, 912, 284, 244, 0, 143, 0, 50, 168, 0, 253, 7901, 65, 0, 4777, 0, 1922, 0, 395, 0, 0, 0, 2084, 238, 0, 0, 65, 0, 226, 336, 0, 50, 0, 0, 0, 350, 6908, 1555, 107, 611, 0, 744] payload_rows_per_dst=[7828, 10382, 15230, 16068, 4430, 11764, 16032, 6396] dup_rows_per_dst=[944, 1606, 1238, 1650, 50, 3224, 1124, 338] combine_rows_per_src=[7828, 10382, 15230, 16068, 4430, 11764, 16032, 6396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83764 payload_rows_remote=68630 dup_rows=14540 dup_rows_remote=13388 dedup_ratio=0.147909 router_tokens_per_expert=[0, 7886, 2665, 0, 0, 51, 202, 2558, 0, 0, 0, 0, 0, 1111, 3635, 125, 0, 74, 0, 331, 0, 0, 0, 938, 359, 244, 0, 168, 0, 51, 178, 0, 261, 7843, 74, 0, 4670, 0, 2452, 0, 392, 0, 0, 0, 1976, 242, 0, 0, 74, 0, 201, 300, 0, 51, 0, 0, 0, 365, 6800, 1414, 73, 670, 0, 718] payload_rows_per_dst=[12620, 8908, 15134, 9618, 10378, 11634, 9254, 6218] dup_rows_per_dst=[5284, 834, 1152, 1202, 2274, 2926, 606, 262] combine_rows_per_src=[12620, 8908, 15134, 9618, 10378, 11634, 9254, 6218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88644 payload_rows_remote=81194 dup_rows=9660 dup_rows_remote=9620 dedup_ratio=0.098267 router_tokens_per_expert=[0, 7779, 2847, 0, 0, 66, 27, 3942, 0, 1, 0, 0, 0, 694, 2825, 157, 0, 91, 131, 363, 0, 0, 0, 1037, 420, 252, 0, 174, 0, 66, 234, 0, 54, 7803, 91, 0, 2843, 0, 3600, 0, 860, 0, 1, 0, 2058, 245, 0, 0, 91, 0, 286, 325, 0, 67, 1, 0, 0, 371, 6531, 685, 98, 1269, 0, 767] payload_rows_per_dst=[5880, 12904, 14924, 15744, 7450, 10062, 15920, 5760] dup_rows_per_dst=[0, 2334, 1382, 2106, 40, 1952, 1226, 620] combine_rows_per_src=[5880, 12904, 14924, 15744, 7450, 10062, 15920, 5760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64670 payload_rows_remote=49490 dup_rows=33634 dup_rows_remote=13820 dedup_ratio=0.342143 router_tokens_per_expert=[596, 77, 3641, 967, 3117, 0, 6786, 5733, 0, 154, 0, 409, 0, 849, 0, 449, 322, 364, 2941, 77, 973, 44, 431, 137, 3, 77, 526, 0, 0, 136, 0, 553, 6205, 298, 0, 77, 224, 1629, 824, 0, 405, 0, 0, 0, 0, 0, 0, 243, 2501, 0, 0, 72, 0, 1075, 150, 0, 2591, 0, 2, 1908, 1499, 87, 0, 0] payload_rows_per_dst=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] dup_rows_per_dst=[19814, 1870, 7462, 226, 1356, 486, 556, 1864] combine_rows_per_src=[15180, 9134, 15526, 4262, 4748, 810, 7040, 7970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202711040 dispatch_meta_bytes=253240 combine_payload_bytes=202711040 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=202964280 ingress_bytes=410329088 link_bytes=410329088 token_bytes=405675320 weight_movement_bytes=207618048 total_nvlink_bytes=613293368 weight_to_token_ratio=0.511784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544035324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258514944 dispatch_meta_bytes=324360 combine_payload_bytes=258514944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258839304 ingress_bytes=293117952 link_bytes=293117952 token_bytes=517354248 weight_movement_bytes=34603008 total_nvlink_bytes=551957256 weight_to_token_ratio=0.066885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102979255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=77152 dup_rows=11366 dup_rows_remote=9662 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 0, 265, 0, 0, 0, 0, 412, 128, 1151, 0, 541, 343, 0, 91, 52, 0, 0, 0, 0, 94, 0, 0, 0, 6295, 0, 315, 1, 2005, 0, 32, 353, 4761, 0, 0, 0, 2322, 0, 639, 2782, 766, 1536, 0, 550, 997, 2768, 0, 0, 134, 0, 465, 0, 0, 53, 0, 598, 0, 7275, 6414, 4742, 89, 0, 183] payload_rows_per_dst=[13040, 8884, 14776, 9312, 9786, 9714, 10700, 10726] dup_rows_per_dst=[318, 2598, 66, 1834, 1704, 2644, 1704, 498] combine_rows_per_src=[13040, 8884, 14776, 9312, 9786, 9714, 10700, 10726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258514944 dispatch_meta_bytes=324360 combine_payload_bytes=258514944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258839304 ingress_bytes=293117952 link_bytes=293117952 token_bytes=517354248 weight_movement_bytes=34603008 total_nvlink_bytes=551957256 weight_to_token_ratio=0.066885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102979255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87136 payload_rows_remote=73966 dup_rows=11168 dup_rows_remote=7738 dedup_ratio=0.113607 router_tokens_per_expert=[0, 0, 0, 248, 0, 0, 0, 0, 367, 110, 837, 0, 596, 288, 0, 77, 48, 0, 0, 0, 0, 66, 0, 0, 0, 6523, 0, 502, 0, 1760, 0, 191, 229, 4968, 0, 0, 0, 2474, 0, 637, 4464, 439, 910, 0, 568, 1415, 2601, 0, 0, 112, 0, 484, 0, 0, 10, 0, 646, 0, 7294, 6481, 3648, 4, 0, 155] payload_rows_per_dst=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] dup_rows_per_dst=[406, 850, 90, 3608, 28, 3430, 2502, 254] combine_rows_per_src=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281108480 dispatch_meta_bytes=328072 combine_payload_bytes=281108480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281436552 ingress_bytes=315711488 link_bytes=315711488 token_bytes=562545032 weight_movement_bytes=34603008 total_nvlink_bytes=597148040 weight_to_token_ratio=0.061512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187996912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=360400 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268951504 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537542608 weight_movement_bytes=34603008 total_nvlink_bytes=572145616 weight_to_token_ratio=0.064373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=295165952 dispatch_meta_bytes=322344 combine_payload_bytes=295165952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295488296 ingress_bytes=329768960 link_bytes=329768960 token_bytes=590654248 weight_movement_bytes=34603008 total_nvlink_bytes=625257256 weight_to_token_ratio=0.058584 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240894048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281108480 dispatch_meta_bytes=328072 combine_payload_bytes=281108480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281436552 ingress_bytes=315711488 link_bytes=315711488 token_bytes=562545032 weight_movement_bytes=34603008 total_nvlink_bytes=597148040 weight_to_token_ratio=0.061512 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187996912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=332570624 dispatch_meta_bytes=363256 combine_payload_bytes=332570624 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=332933880 ingress_bytes=436379648 link_bytes=436379648 token_bytes=665504504 weight_movement_bytes=103809024 total_nvlink_bytes=769313528 weight_to_token_ratio=0.155985 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642061484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64942 payload_rows_remote=49204 dup_rows=33362 dup_rows_remote=10194 dedup_ratio=0.339376 router_tokens_per_expert=[0, 0, 0, 316, 0, 0, 0, 0, 386, 122, 1290, 0, 592, 324, 0, 84, 46, 0, 0, 0, 0, 88, 0, 0, 0, 6386, 0, 302, 0, 2175, 0, 21, 393, 4449, 0, 0, 0, 2477, 0, 733, 2947, 699, 1504, 0, 624, 998, 2439, 0, 0, 100, 0, 509, 0, 0, 50, 0, 596, 0, 7284, 6451, 4518, 81, 0, 168] payload_rows_per_dst=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] dup_rows_per_dst=[116, 1498, 0, 4186, 366, 2538, 1490, 23168] combine_rows_per_src=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87136 payload_rows_remote=73966 dup_rows=11168 dup_rows_remote=7738 dedup_ratio=0.113607 router_tokens_per_expert=[0, 0, 0, 248, 0, 0, 0, 0, 367, 110, 837, 0, 596, 288, 0, 77, 48, 0, 0, 0, 0, 66, 0, 0, 0, 6523, 0, 502, 0, 1760, 0, 191, 229, 4968, 0, 0, 0, 2474, 0, 637, 4464, 439, 910, 0, 568, 1415, 2601, 0, 0, 112, 0, 484, 0, 0, 10, 0, 646, 0, 7294, 6481, 3648, 4, 0, 155] payload_rows_per_dst=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] dup_rows_per_dst=[406, 850, 90, 3608, 28, 3430, 2502, 254] combine_rows_per_src=[13052, 3700, 14726, 14344, 6652, 13170, 12840, 8652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69840 payload_rows_remote=53770 dup_rows=28464 dup_rows_remote=15634 dedup_ratio=0.289551 router_tokens_per_expert=[0, 7756, 2895, 0, 0, 59, 248, 3492, 0, 0, 0, 0, 0, 1146, 2905, 137, 0, 77, 37, 328, 0, 0, 0, 1082, 317, 246, 0, 127, 0, 59, 243, 0, 229, 7824, 77, 0, 4200, 0, 1968, 0, 400, 0, 0, 0, 2141, 245, 0, 0, 77, 0, 262, 366, 0, 59, 0, 0, 0, 425, 6744, 1219, 79, 899, 0, 784] payload_rows_per_dst=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] dup_rows_per_dst=[12830, 818, 1346, 330, 12512, 100, 154, 374] combine_rows_per_src=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83292 payload_rows_remote=71270 dup_rows=15012 dup_rows_remote=12822 dedup_ratio=0.152710 router_tokens_per_expert=[1, 0, 0, 237, 0, 2, 0, 1, 331, 96, 941, 1, 638, 352, 1, 107, 66, 1, 0, 0, 1, 49, 0, 0, 0, 6510, 1, 406, 27, 1866, 0, 207, 320, 4928, 0, 0, 0, 2340, 0, 656, 3882, 659, 1186, 1, 596, 1010, 2525, 0, 0, 152, 1, 479, 0, 0, 66, 1, 678, 1, 7042, 6378, 4124, 71, 0, 214] payload_rows_per_dst=[12852, 3704, 8682, 14282, 6516, 13220, 12022, 12014] dup_rows_per_dst=[386, 1230, 2, 3752, 116, 3540, 2190, 3796] combine_rows_per_src=[12852, 3704, 8682, 14282, 6516, 13220, 12022, 12014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=202711040 dispatch_meta_bytes=253240 combine_payload_bytes=202711040 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=202964280 ingress_bytes=410329088 link_bytes=410329088 token_bytes=405675320 weight_movement_bytes=207618048 total_nvlink_bytes=613293368 weight_to_token_ratio=0.511784 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544035324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64942 payload_rows_remote=49204 dup_rows=33362 dup_rows_remote=10194 dedup_ratio=0.339376 router_tokens_per_expert=[0, 0, 0, 316, 0, 0, 0, 0, 386, 122, 1290, 0, 592, 324, 0, 84, 46, 0, 0, 0, 0, 88, 0, 0, 0, 6386, 0, 302, 0, 2175, 0, 21, 393, 4449, 0, 0, 0, 2477, 0, 733, 2947, 699, 1504, 0, 624, 998, 2439, 0, 0, 100, 0, 509, 0, 0, 50, 0, 596, 0, 7284, 6451, 4518, 81, 0, 168] payload_rows_per_dst=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] dup_rows_per_dst=[116, 1498, 0, 4186, 366, 2538, 1490, 23168] combine_rows_per_src=[4746, 4098, 268, 13582, 6840, 9070, 10600, 15738] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316014592 dispatch_meta_bytes=347256 combine_payload_bytes=316014592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316361848 ingress_bytes=385220608 link_bytes=385220608 token_bytes=632376440 weight_movement_bytes=69206016 total_nvlink_bytes=701582456 weight_to_token_ratio=0.109438 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302964736 dispatch_meta_bytes=326816 combine_payload_bytes=302964736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303291552 ingress_bytes=406773760 link_bytes=406773760 token_bytes=606256288 weight_movement_bytes=103809024 total_nvlink_bytes=710065312 weight_to_token_ratio=0.171230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530656911e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=201539584 dispatch_meta_bytes=237592 combine_payload_bytes=201539584 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=201777176 ingress_bytes=339951616 link_bytes=339951616 token_bytes=403316760 weight_movement_bytes=138412032 total_nvlink_bytes=541728792 weight_to_token_ratio=0.343184 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279210563e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87554 payload_rows_remote=82600 dup_rows=10750 dup_rows_remote=10750 dedup_ratio=0.109355 router_tokens_per_expert=[0, 0, 0, 203, 0, 0, 0, 0, 373, 115, 617, 0, 594, 260, 0, 64, 33, 0, 0, 0, 0, 79, 0, 0, 0, 6894, 0, 368, 0, 2105, 0, 4, 205, 5047, 0, 0, 0, 2567, 0, 640, 4450, 327, 989, 0, 580, 1667, 2438, 0, 0, 52, 0, 437, 0, 0, 7, 0, 686, 0, 7408, 6495, 3318, 1, 0, 129] payload_rows_per_dst=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] dup_rows_per_dst=[378, 3168, 66, 0, 3206, 1452, 2252, 228] combine_rows_per_src=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83292 payload_rows_remote=71270 dup_rows=15012 dup_rows_remote=12822 dedup_ratio=0.152710 router_tokens_per_expert=[1, 0, 0, 237, 0, 2, 0, 1, 331, 96, 941, 1, 638, 352, 1, 107, 66, 1, 0, 0, 1, 49, 0, 0, 0, 6510, 1, 406, 27, 1866, 0, 207, 320, 4928, 0, 0, 0, 2340, 0, 656, 3882, 659, 1186, 1, 596, 1010, 2525, 0, 0, 152, 1, 479, 0, 0, 66, 1, 678, 1, 7042, 6378, 4124, 71, 0, 214] payload_rows_per_dst=[12852, 3704, 8682, 14282, 6516, 13220, 12022, 12014] dup_rows_per_dst=[386, 1230, 2, 3752, 116, 3540, 2190, 3796] combine_rows_per_src=[12852, 3704, 8682, 14282, 6516, 13220, 12022, 12014] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=73124 dup_rows=10264 dup_rows_remote=10258 dedup_ratio=0.104411 router_tokens_per_expert=[0, 0, 0, 185, 0, 0, 0, 0, 380, 101, 966, 0, 557, 301, 0, 73, 3, 0, 0, 0, 0, 86, 0, 0, 0, 6309, 0, 491, 0, 2035, 0, 0, 198, 5055, 0, 0, 0, 2592, 0, 571, 4231, 423, 1000, 0, 598, 1547, 2509, 0, 0, 82, 0, 447, 0, 0, 77, 0, 682, 0, 7372, 6452, 3681, 1, 0, 147] payload_rows_per_dst=[12910, 14250, 14916, 5052, 13460, 7850, 10884, 8718] dup_rows_per_dst=[364, 3124, 6, 0, 3372, 1870, 1224, 304] combine_rows_per_src=[12910, 14250, 14916, 5052, 13460, 7850, 10884, 8718] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87270 payload_rows_remote=72770 dup_rows=11034 dup_rows_remote=6974 dedup_ratio=0.112244 router_tokens_per_expert=[0, 0, 0, 293, 0, 0, 0, 0, 412, 115, 1235, 0, 503, 323, 0, 72, 53, 0, 0, 0, 0, 95, 0, 0, 0, 6620, 0, 340, 1, 1974, 0, 60, 304, 4811, 0, 0, 0, 2240, 0, 681, 3762, 577, 1201, 0, 616, 1134, 2477, 0, 0, 80, 0, 486, 0, 0, 7, 0, 590, 0, 7249, 6481, 4187, 29, 0, 144] payload_rows_per_dst=[13130, 14500, 14702, 8938, 13140, 9716, 8624, 4520] dup_rows_per_dst=[418, 4060, 92, 1084, 2932, 2294, 46, 108] combine_rows_per_src=[13130, 14500, 14702, 8938, 13140, 9716, 8624, 4520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86938 payload_rows_remote=77152 dup_rows=11366 dup_rows_remote=9662 dedup_ratio=0.115621 router_tokens_per_expert=[0, 0, 0, 265, 0, 0, 0, 0, 412, 128, 1151, 0, 541, 343, 0, 91, 52, 0, 0, 0, 0, 94, 0, 0, 0, 6295, 0, 315, 1, 2005, 0, 32, 353, 4761, 0, 0, 0, 2322, 0, 639, 2782, 766, 1536, 0, 550, 997, 2768, 0, 0, 134, 0, 465, 0, 0, 53, 0, 598, 0, 7275, 6414, 4742, 89, 0, 183] payload_rows_per_dst=[13040, 8884, 14776, 9312, 9786, 9714, 10700, 10726] dup_rows_per_dst=[318, 2598, 66, 1834, 1704, 2644, 1704, 498] combine_rows_per_src=[13040, 8884, 14776, 9312, 9786, 9714, 10700, 10726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220241920 dispatch_meta_bytes=277616 combine_payload_bytes=220241920 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=220519536 ingress_bytes=358653952 link_bytes=358653952 token_bytes=440761456 weight_movement_bytes=138412032 total_nvlink_bytes=579173488 weight_to_token_ratio=0.314029 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349585947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291921920 dispatch_meta_bytes=336368 combine_payload_bytes=291921920 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=292258288 ingress_bytes=361127936 link_bytes=361127936 token_bytes=584180208 weight_movement_bytes=69206016 total_nvlink_bytes=653386224 weight_to_token_ratio=0.118467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87554 payload_rows_remote=82600 dup_rows=10750 dup_rows_remote=10750 dedup_ratio=0.109355 router_tokens_per_expert=[0, 0, 0, 203, 0, 0, 0, 0, 373, 115, 617, 0, 594, 260, 0, 64, 33, 0, 0, 0, 0, 79, 0, 0, 0, 6894, 0, 368, 0, 2105, 0, 4, 205, 5047, 0, 0, 0, 2567, 0, 640, 4450, 327, 989, 0, 580, 1667, 2438, 0, 0, 52, 0, 437, 0, 0, 7, 0, 686, 0, 7408, 6495, 3318, 1, 0, 129] payload_rows_per_dst=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] dup_rows_per_dst=[378, 3168, 66, 0, 3206, 1452, 2252, 228] combine_rows_per_src=[13018, 14666, 14974, 4954, 13712, 5674, 12516, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=373400 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=338703000 ingress_bytes=407535616 link_bytes=407535616 token_bytes=677032600 weight_movement_bytes=69206016 total_nvlink_bytes=746238616 weight_to_token_ratio=0.102220 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533523714e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69840 payload_rows_remote=53770 dup_rows=28464 dup_rows_remote=15634 dedup_ratio=0.289551 router_tokens_per_expert=[0, 7756, 2895, 0, 0, 59, 248, 3492, 0, 0, 0, 0, 0, 1146, 2905, 137, 0, 77, 37, 328, 0, 0, 0, 1082, 317, 246, 0, 127, 0, 59, 243, 0, 229, 7824, 77, 0, 4200, 0, 1968, 0, 400, 0, 0, 0, 2141, 245, 0, 0, 77, 0, 262, 366, 0, 59, 0, 0, 0, 425, 6744, 1219, 79, 899, 0, 784] payload_rows_per_dst=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] dup_rows_per_dst=[12830, 818, 1346, 330, 12512, 100, 154, 374] combine_rows_per_src=[16070, 7558, 15190, 1654, 16084, 5472, 1374, 6438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=291921920 dispatch_meta_bytes=336368 combine_payload_bytes=291921920 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=292258288 ingress_bytes=361127936 link_bytes=361127936 token_bytes=584180208 weight_movement_bytes=69206016 total_nvlink_bytes=653386224 weight_to_token_ratio=0.118467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.358895350e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=81108 dup_rows=7098 dup_rows_remote=5624 dedup_ratio=0.072205 router_tokens_per_expert=[0, 66, 252, 95, 0, 5632, 0, 408, 0, 1, 5, 0, 91, 0, 0, 0, 3564, 0, 0, 0, 0, 1992, 0, 1881, 0, 0, 12, 6947, 0, 0, 0, 35, 4529, 0, 7, 319, 0, 0, 185, 746, 253, 0, 0, 1807, 0, 0, 0, 0, 738, 1, 4828, 91, 6476, 66, 604, 0, 0, 595, 1620, 690, 961, 3564, 0, 91] payload_rows_per_dst=[12842, 13134, 7374, 13894, 10098, 10378, 11586, 11900] dup_rows_per_dst=[64, 12, 372, 94, 1474, 870, 1070, 3142] combine_rows_per_src=[12842, 13134, 7374, 13894, 10098, 10378, 11586, 11900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90832 payload_rows_remote=80036 dup_rows=7472 dup_rows_remote=5952 dedup_ratio=0.076009 router_tokens_per_expert=[0, 50, 251, 72, 0, 4935, 0, 444, 0, 0, 8, 0, 78, 0, 0, 0, 3619, 0, 0, 0, 0, 2225, 0, 2352, 0, 0, 25, 7320, 0, 0, 0, 45, 4548, 3, 78, 256, 0, 0, 143, 441, 192, 0, 0, 2347, 0, 0, 0, 0, 1241, 0, 4485, 77, 6684, 49, 296, 0, 0, 416, 1262, 640, 946, 3504, 0, 120] payload_rows_per_dst=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] dup_rows_per_dst=[24, 14, 512, 140, 1270, 1520, 1250, 2742] combine_rows_per_src=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=299515904 dispatch_meta_bytes=333528 combine_payload_bytes=299515904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299849432 ingress_bytes=334118912 link_bytes=334118912 token_bytes=599365336 weight_movement_bytes=34603008 total_nvlink_bytes=633968344 weight_to_token_ratio=0.057733 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=48256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=373400 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=338703000 ingress_bytes=407535616 link_bytes=407535616 token_bytes=677032600 weight_movement_bytes=69206016 total_nvlink_bytes=746238616 weight_to_token_ratio=0.102220 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533523714e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=298065920 dispatch_meta_bytes=318976 combine_payload_bytes=298065920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298384896 ingress_bytes=332668928 link_bytes=332668928 token_bytes=596450816 weight_movement_bytes=34603008 total_nvlink_bytes=631053824 weight_to_token_ratio=0.058015 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251806394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316014592 dispatch_meta_bytes=347256 combine_payload_bytes=316014592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=316361848 ingress_bytes=385220608 link_bytes=385220608 token_bytes=632376440 weight_movement_bytes=69206016 total_nvlink_bytes=701582456 weight_to_token_ratio=0.109438 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449554135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90832 payload_rows_remote=80036 dup_rows=7472 dup_rows_remote=5952 dedup_ratio=0.076009 router_tokens_per_expert=[0, 50, 251, 72, 0, 4935, 0, 444, 0, 0, 8, 0, 78, 0, 0, 0, 3619, 0, 0, 0, 0, 2225, 0, 2352, 0, 0, 25, 7320, 0, 0, 0, 45, 4548, 3, 78, 256, 0, 0, 143, 441, 192, 0, 0, 2347, 0, 0, 0, 0, 1241, 0, 4485, 77, 6684, 49, 296, 0, 0, 416, 1262, 640, 946, 3504, 0, 120] payload_rows_per_dst=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] dup_rows_per_dst=[24, 14, 512, 140, 1270, 1520, 1250, 2742] combine_rows_per_src=[11480, 13526, 8642, 14640, 9668, 10796, 11046, 11034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84264 payload_rows_remote=72470 dup_rows=14040 dup_rows_remote=10032 dedup_ratio=0.142822 router_tokens_per_expert=[0, 57, 241, 89, 0, 5503, 0, 415, 0, 0, 15, 0, 85, 0, 0, 0, 3655, 0, 0, 0, 0, 1805, 0, 1827, 0, 0, 12, 7007, 0, 0, 0, 55, 4898, 0, 9, 273, 0, 0, 172, 823, 228, 0, 0, 1731, 0, 0, 0, 0, 820, 0, 4994, 84, 6454, 57, 637, 0, 0, 576, 1526, 660, 999, 3360, 0, 85] payload_rows_per_dst=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] dup_rows_per_dst=[6, 20, 140, 132, 1480, 2062, 6192, 4008] combine_rows_per_src=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84264 payload_rows_remote=72470 dup_rows=14040 dup_rows_remote=10032 dedup_ratio=0.142822 router_tokens_per_expert=[0, 57, 241, 89, 0, 5503, 0, 415, 0, 0, 15, 0, 85, 0, 0, 0, 3655, 0, 0, 0, 0, 1805, 0, 1827, 0, 0, 12, 7007, 0, 0, 0, 55, 4898, 0, 9, 273, 0, 0, 172, 823, 228, 0, 0, 1731, 0, 0, 0, 0, 820, 0, 4994, 84, 6454, 57, 637, 0, 0, 576, 1526, 660, 999, 3360, 0, 85] payload_rows_per_dst=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] dup_rows_per_dst=[6, 20, 140, 132, 1480, 2062, 6192, 4008] combine_rows_per_src=[12604, 5216, 5232, 14016, 10870, 11058, 13474, 11794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=220241920 dispatch_meta_bytes=277616 combine_payload_bytes=220241920 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=220519536 ingress_bytes=358653952 link_bytes=358653952 token_bytes=440761456 weight_movement_bytes=138412032 total_nvlink_bytes=579173488 weight_to_token_ratio=0.314029 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349585947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332218368 dispatch_meta_bytes=346928 combine_payload_bytes=332218368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332565296 ingress_bytes=366821376 link_bytes=366821376 token_bytes=664783664 weight_movement_bytes=34603008 total_nvlink_bytes=699386672 weight_to_token_ratio=0.052052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380319306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=327827456 dispatch_meta_bytes=343952 combine_payload_bytes=327827456 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=328171408 ingress_bytes=397033472 link_bytes=397033472 token_bytes=655998864 weight_movement_bytes=69206016 total_nvlink_bytes=725204880 weight_to_token_ratio=0.105497 gemm_rows_defined=50176 gemm_rows_charged=51200 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.020000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494004991e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86990 payload_rows_remote=74226 dup_rows=11314 dup_rows_remote=10908 dedup_ratio=0.115092 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 387, 129, 859, 0, 621, 316, 0, 77, 41, 0, 0, 0, 0, 82, 0, 0, 0, 6598, 0, 289, 0, 2001, 0, 243, 301, 4955, 0, 0, 0, 2349, 0, 632, 3785, 525, 1191, 0, 634, 1245, 2466, 0, 0, 86, 0, 463, 0, 0, 21, 0, 603, 0, 7346, 6376, 4155, 13, 0, 154] payload_rows_per_dst=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] dup_rows_per_dst=[406, 3456, 78, 1984, 3088, 2228, 74, 0] combine_rows_per_src=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90446 payload_rows_remote=78148 dup_rows=7858 dup_rows_remote=6372 dedup_ratio=0.079936 router_tokens_per_expert=[0, 37, 245, 49, 0, 5070, 0, 470, 0, 0, 3, 0, 107, 0, 0, 1, 3064, 0, 0, 0, 1, 1828, 0, 2240, 0, 0, 37, 7144, 2, 0, 0, 37, 4442, 10, 107, 305, 0, 1, 156, 620, 193, 1, 1, 2356, 3, 0, 2, 1, 959, 1, 5183, 107, 6345, 37, 604, 1, 0, 447, 1584, 714, 971, 3550, 1, 115] payload_rows_per_dst=[11742, 12906, 7784, 14292, 9934, 10060, 12298, 11430] dup_rows_per_dst=[0, 6, 354, 148, 1348, 1182, 1486, 3334] combine_rows_per_src=[11742, 12906, 7784, 14292, 9934, 10060, 12298, 11430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=330008 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=297167128 ingress_bytes=366043136 link_bytes=366043136 token_bytes=594004248 weight_movement_bytes=69206016 total_nvlink_bytes=663210264 weight_to_token_ratio=0.116508 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377390852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90446 payload_rows_remote=78148 dup_rows=7858 dup_rows_remote=6372 dedup_ratio=0.079936 router_tokens_per_expert=[0, 37, 245, 49, 0, 5070, 0, 470, 0, 0, 3, 0, 107, 0, 0, 1, 3064, 0, 0, 0, 1, 1828, 0, 2240, 0, 0, 37, 7144, 2, 0, 0, 37, 4442, 10, 107, 305, 0, 1, 156, 620, 193, 1, 1, 2356, 3, 0, 2, 1, 959, 1, 5183, 107, 6345, 37, 604, 1, 0, 447, 1584, 714, 971, 3550, 1, 115] payload_rows_per_dst=[11742, 12906, 7784, 14292, 9934, 10060, 12298, 11430] dup_rows_per_dst=[0, 6, 354, 148, 1348, 1182, 1486, 3334] combine_rows_per_src=[11742, 12906, 7784, 14292, 9934, 10060, 12298, 11430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88290 payload_rows_remote=77790 dup_rows=10014 dup_rows_remote=7120 dedup_ratio=0.101868 router_tokens_per_expert=[0, 51, 244, 87, 0, 4523, 0, 371, 0, 0, 9, 0, 74, 0, 0, 0, 3932, 0, 0, 0, 0, 2276, 0, 2046, 0, 0, 22, 7392, 0, 0, 0, 50, 4290, 0, 74, 241, 0, 0, 158, 531, 210, 0, 0, 2737, 0, 0, 0, 0, 1403, 0, 4303, 69, 6575, 51, 363, 0, 0, 374, 1046, 633, 1217, 3725, 0, 75] payload_rows_per_dst=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] dup_rows_per_dst=[4, 18, 2894, 98, 1496, 1970, 1388, 2146] combine_rows_per_src=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90940 payload_rows_remote=75972 dup_rows=7364 dup_rows_remote=7220 dedup_ratio=0.074910 router_tokens_per_expert=[0, 50, 244, 84, 0, 4891, 0, 399, 0, 0, 16, 0, 65, 0, 0, 0, 3948, 0, 0, 0, 0, 2320, 0, 1933, 0, 0, 33, 7484, 0, 0, 0, 39, 4550, 0, 64, 154, 0, 0, 143, 470, 193, 0, 0, 2747, 0, 0, 0, 0, 1276, 0, 4508, 62, 6690, 50, 291, 0, 0, 298, 975, 602, 883, 3617, 0, 73] payload_rows_per_dst=[11304, 13510, 8128, 14968, 9472, 11710, 11084, 10764] dup_rows_per_dst=[32, 32, 378, 144, 1290, 2066, 1290, 2132] combine_rows_per_src=[11304, 13510, 8128, 14968, 9472, 11710, 11084, 10764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85222 payload_rows_remote=71898 dup_rows=13082 dup_rows_remote=13050 dedup_ratio=0.133077 router_tokens_per_expert=[0, 59, 258, 96, 0, 5142, 0, 375, 0, 0, 16, 0, 72, 0, 0, 0, 3631, 0, 0, 0, 0, 2068, 0, 1998, 0, 0, 25, 7258, 0, 0, 0, 59, 4796, 2, 71, 247, 0, 0, 167, 658, 226, 0, 0, 1949, 0, 0, 0, 0, 1078, 0, 4666, 72, 6590, 59, 629, 0, 0, 453, 1363, 613, 1006, 3371, 0, 79] payload_rows_per_dst=[11858, 13324, 15566, 6886, 11082, 10470, 10482, 5554] dup_rows_per_dst=[2, 32, 7082, 24, 1958, 1142, 1368, 1474] combine_rows_per_src=[11858, 13324, 15566, 6886, 11082, 10470, 10482, 5554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88290 payload_rows_remote=77790 dup_rows=10014 dup_rows_remote=7120 dedup_ratio=0.101868 router_tokens_per_expert=[0, 51, 244, 87, 0, 4523, 0, 371, 0, 0, 9, 0, 74, 0, 0, 0, 3932, 0, 0, 0, 0, 2276, 0, 2046, 0, 0, 22, 7392, 0, 0, 0, 50, 4290, 0, 74, 241, 0, 0, 158, 531, 210, 0, 0, 2737, 0, 0, 0, 0, 1403, 0, 4303, 69, 6575, 51, 363, 0, 0, 374, 1046, 633, 1217, 3725, 0, 75] payload_rows_per_dst=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] dup_rows_per_dst=[4, 18, 2894, 98, 1496, 1970, 1388, 2146] combine_rows_per_src=[10548, 13298, 10500, 12182, 9092, 11788, 10990, 9892] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=81108 dup_rows=7098 dup_rows_remote=5624 dedup_ratio=0.072205 router_tokens_per_expert=[0, 66, 252, 95, 0, 5632, 0, 408, 0, 1, 5, 0, 91, 0, 0, 0, 3564, 0, 0, 0, 0, 1992, 0, 1881, 0, 0, 12, 6947, 0, 0, 0, 35, 4529, 0, 7, 319, 0, 0, 185, 746, 253, 0, 0, 1807, 0, 0, 0, 0, 738, 1, 4828, 91, 6476, 66, 604, 0, 0, 595, 1620, 690, 961, 3564, 0, 91] payload_rows_per_dst=[12842, 13134, 7374, 13894, 10098, 10378, 11586, 11900] dup_rows_per_dst=[64, 12, 372, 94, 1474, 870, 1070, 3142] combine_rows_per_src=[12842, 13134, 7374, 13894, 10098, 10378, 11586, 11900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304029696 dispatch_meta_bytes=340536 combine_payload_bytes=304029696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304370232 ingress_bytes=338632704 link_bytes=338632704 token_bytes=608399928 weight_movement_bytes=34603008 total_nvlink_bytes=643002936 weight_to_token_ratio=0.056875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274247603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=320094208 dispatch_meta_bytes=338080 combine_payload_bytes=320094208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320432288 ingress_bytes=458506240 link_bytes=458506240 token_bytes=640526496 weight_movement_bytes=138412032 total_nvlink_bytes=778938528 weight_to_token_ratio=0.216091 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725322068e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311181312 dispatch_meta_bytes=332768 combine_payload_bytes=311181312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311514080 ingress_bytes=380387328 link_bytes=380387328 token_bytes=622695392 weight_movement_bytes=69206016 total_nvlink_bytes=691901408 weight_to_token_ratio=0.111139 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431366892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87418 payload_rows_remote=76518 dup_rows=10886 dup_rows_remote=8706 dedup_ratio=0.110738 router_tokens_per_expert=[136, 0, 66, 1, 66, 0, 0, 0, 0, 3699, 0, 138, 89, 41, 419, 1674, 30, 7254, 0, 91, 0, 0, 54, 0, 7436, 0, 133, 253, 0, 1970, 91, 6490, 7405, 1632, 0, 1, 0, 0, 0, 0, 91, 3, 2784, 0, 0, 1700, 252, 0, 14, 0, 0, 2949, 0, 0, 0, 0, 0, 1927, 263, 0, 0, 0, 0, 0] payload_rows_per_dst=[15042, 9816, 14510, 4714, 10900, 8764, 8948, 14724] dup_rows_per_dst=[368, 2304, 348, 180, 2180, 896, 1974, 2636] combine_rows_per_src=[15042, 9816, 14510, 4714, 10900, 8764, 8948, 14724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86990 payload_rows_remote=74226 dup_rows=11314 dup_rows_remote=10908 dedup_ratio=0.115092 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 387, 129, 859, 0, 621, 316, 0, 77, 41, 0, 0, 0, 0, 82, 0, 0, 0, 6598, 0, 289, 0, 2001, 0, 243, 301, 4955, 0, 0, 0, 2349, 0, 632, 3785, 525, 1191, 0, 634, 1245, 2466, 0, 0, 86, 0, 463, 0, 0, 21, 0, 603, 0, 7346, 6376, 4155, 13, 0, 154] payload_rows_per_dst=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] dup_rows_per_dst=[406, 3456, 78, 1984, 3088, 2228, 74, 0] combine_rows_per_src=[12764, 14518, 14860, 11392, 13386, 9894, 8636, 1540] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=320094208 dispatch_meta_bytes=338080 combine_payload_bytes=320094208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=320432288 ingress_bytes=458506240 link_bytes=458506240 token_bytes=640526496 weight_movement_bytes=138412032 total_nvlink_bytes=778938528 weight_to_token_ratio=0.216091 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.725322068e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=318627840 dispatch_meta_bytes=339640 combine_payload_bytes=318627840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=318967480 ingress_bytes=422436864 link_bytes=422436864 token_bytes=637595320 weight_movement_bytes=103809024 total_nvlink_bytes=741404344 weight_to_token_ratio=0.162813 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589595910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85314 payload_rows_remote=76126 dup_rows=12990 dup_rows_remote=11806 dedup_ratio=0.132141 router_tokens_per_expert=[566, 0, 49, 0, 50, 0, 0, 0, 0, 3085, 0, 112, 6, 1, 360, 1872, 59, 7337, 0, 78, 0, 0, 59, 0, 7461, 0, 228, 251, 0, 2088, 77, 6566, 6855, 1593, 0, 0, 0, 0, 0, 0, 78, 0, 1987, 0, 0, 2870, 251, 0, 0, 0, 0, 3168, 0, 0, 0, 0, 0, 1924, 118, 0, 0, 0, 0, 3] payload_rows_per_dst=[15500, 8556, 14674, 5118, 3186, 9188, 14272, 14820] dup_rows_per_dst=[752, 2316, 392, 170, 0, 1184, 5774, 2402] combine_rows_per_src=[15500, 8556, 14674, 5118, 3186, 9188, 14272, 14820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311181312 dispatch_meta_bytes=332768 combine_payload_bytes=311181312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=311514080 ingress_bytes=380387328 link_bytes=380387328 token_bytes=622695392 weight_movement_bytes=69206016 total_nvlink_bytes=691901408 weight_to_token_ratio=0.111139 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431366892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=294494208 dispatch_meta_bytes=339792 combine_payload_bytes=294494208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294834000 ingress_bytes=329097216 link_bytes=329097216 token_bytes=589328208 weight_movement_bytes=34603008 total_nvlink_bytes=623931216 weight_to_token_ratio=0.058716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.238366329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=318627840 dispatch_meta_bytes=339640 combine_payload_bytes=318627840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=318967480 ingress_bytes=422436864 link_bytes=422436864 token_bytes=637595320 weight_movement_bytes=103809024 total_nvlink_bytes=741404344 weight_to_token_ratio=0.162813 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.589595910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=332218368 dispatch_meta_bytes=346928 combine_payload_bytes=332218368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332565296 ingress_bytes=366821376 link_bytes=366821376 token_bytes=664783664 weight_movement_bytes=34603008 total_nvlink_bytes=699386672 weight_to_token_ratio=0.052052 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.380319306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78106 payload_rows_remote=63072 dup_rows=20198 dup_rows_remote=17616 dedup_ratio=0.205465 router_tokens_per_expert=[158, 0, 57, 0, 57, 0, 0, 0, 0, 3604, 0, 100, 83, 33, 426, 1730, 42, 7290, 0, 85, 0, 0, 50, 0, 7482, 0, 115, 241, 0, 1848, 84, 6619, 7480, 1618, 0, 0, 0, 0, 0, 0, 85, 0, 2535, 0, 0, 1843, 241, 0, 133, 0, 0, 2924, 0, 0, 0, 0, 0, 2045, 142, 0, 0, 0, 0, 2] payload_rows_per_dst=[430, 9628, 270, 15132, 6490, 15362, 15760, 15034] dup_rows_per_dst=[114, 2324, 84, 712, 442, 8626, 5314, 2582] combine_rows_per_src=[430, 9628, 270, 15132, 6490, 15362, 15760, 15034] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304029696 dispatch_meta_bytes=340536 combine_payload_bytes=304029696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304370232 ingress_bytes=338632704 link_bytes=338632704 token_bytes=608399928 weight_movement_bytes=34603008 total_nvlink_bytes=643002936 weight_to_token_ratio=0.056875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274247603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=313417728 dispatch_meta_bytes=340896 combine_payload_bytes=313417728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313758624 ingress_bytes=382623744 link_bytes=382623744 token_bytes=627176352 weight_movement_bytes=69206016 total_nvlink_bytes=696382368 weight_to_token_ratio=0.110345 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439782345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=311812096 dispatch_meta_bytes=351728 combine_payload_bytes=311812096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312163824 ingress_bytes=346415104 link_bytes=346415104 token_bytes=623975920 weight_movement_bytes=34603008 total_nvlink_bytes=658578928 weight_to_token_ratio=0.055456 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303532148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80236 payload_rows_remote=67132 dup_rows=18068 dup_rows_remote=15378 dedup_ratio=0.183797 router_tokens_per_expert=[0, 59, 246, 82, 0, 5242, 0, 441, 0, 0, 24, 0, 77, 0, 0, 0, 3276, 0, 0, 0, 0, 2057, 0, 2294, 0, 0, 21, 7322, 0, 0, 0, 57, 4807, 0, 71, 245, 0, 0, 159, 668, 218, 0, 0, 2122, 0, 0, 0, 0, 930, 0, 4763, 77, 6527, 59, 584, 0, 0, 521, 1348, 627, 906, 3245, 0, 77] payload_rows_per_dst=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] dup_rows_per_dst=[2690, 48, 7694, 24, 4644, 1082, 288, 1598] combine_rows_per_src=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79906 payload_rows_remote=65432 dup_rows=18398 dup_rows_remote=12764 dedup_ratio=0.187154 router_tokens_per_expert=[158, 0, 37, 0, 37, 0, 0, 0, 2, 3322, 0, 156, 86, 59, 434, 1783, 29, 7243, 1, 107, 0, 0, 67, 1, 7416, 1, 162, 245, 1, 2075, 107, 6586, 6859, 1562, 2, 0, 2, 0, 0, 1, 107, 0, 2404, 2, 1, 2591, 245, 0, 86, 0, 0, 3109, 0, 0, 0, 0, 1, 1848, 209, 0, 0, 0, 0, 8] payload_rows_per_dst=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] dup_rows_per_dst=[100, 2532, 250, 2892, 286, 4332, 5634, 2372] combine_rows_per_src=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=258342912 dispatch_meta_bytes=322752 combine_payload_bytes=258342912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258665664 ingress_bytes=292945920 link_bytes=292945920 token_bytes=517008576 weight_movement_bytes=34603008 total_nvlink_bytes=551611584 weight_to_token_ratio=0.066929 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102331913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79906 payload_rows_remote=65432 dup_rows=18398 dup_rows_remote=12764 dedup_ratio=0.187154 router_tokens_per_expert=[158, 0, 37, 0, 37, 0, 0, 0, 2, 3322, 0, 156, 86, 59, 434, 1783, 29, 7243, 1, 107, 0, 0, 67, 1, 7416, 1, 162, 245, 1, 2075, 107, 6586, 6859, 1562, 2, 0, 2, 0, 0, 1, 107, 0, 2404, 2, 1, 2591, 245, 0, 86, 0, 0, 3109, 0, 0, 0, 0, 1, 1848, 209, 0, 0, 0, 0, 8] payload_rows_per_dst=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] dup_rows_per_dst=[100, 2532, 250, 2892, 286, 4332, 5634, 2372] combine_rows_per_src=[2262, 9152, 7444, 14062, 5346, 12234, 14474, 14932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86080 payload_rows_remote=71428 dup_rows=12224 dup_rows_remote=11726 dedup_ratio=0.124349 router_tokens_per_expert=[258, 0, 51, 0, 51, 0, 0, 0, 0, 3223, 0, 82, 0, 5, 416, 1881, 87, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 253, 244, 0, 2639, 73, 6461, 6920, 1519, 0, 0, 0, 0, 0, 0, 74, 0, 2218, 0, 0, 1949, 244, 0, 2, 0, 0, 3492, 0, 0, 0, 0, 0, 1646, 362, 0, 0, 0, 0, 0] payload_rows_per_dst=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] dup_rows_per_dst=[592, 2288, 498, 4544, 2770, 984, 2, 546] combine_rows_per_src=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88000 payload_rows_remote=82886 dup_rows=10304 dup_rows_remote=10050 dedup_ratio=0.104818 router_tokens_per_expert=[699, 0, 47, 0, 50, 0, 0, 0, 0, 3085, 0, 52, 0, 3, 391, 1988, 71, 7391, 0, 65, 0, 0, 71, 0, 7541, 0, 255, 244, 0, 1901, 64, 6571, 6886, 1730, 0, 0, 0, 0, 0, 0, 65, 0, 2143, 0, 0, 2450, 244, 0, 2, 0, 0, 3172, 0, 0, 0, 0, 0, 1902, 69, 0, 0, 0, 0, 0] payload_rows_per_dst=[15760, 8850, 14782, 5114, 14024, 8732, 6348, 14390] dup_rows_per_dst=[914, 2188, 414, 254, 3208, 1072, 0, 2254] combine_rows_per_src=[15760, 8850, 14782, 5114, 14024, 8732, 6348, 14390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87006 payload_rows_remote=77876 dup_rows=11298 dup_rows_remote=9040 dedup_ratio=0.114929 router_tokens_per_expert=[251, 0, 59, 0, 59, 0, 0, 0, 0, 3230, 0, 80, 52, 4, 410, 1918, 53, 7313, 0, 72, 0, 0, 53, 0, 7530, 0, 167, 258, 0, 2163, 72, 6687, 7004, 2013, 0, 0, 0, 0, 0, 0, 72, 0, 2182, 0, 0, 1947, 258, 0, 6, 0, 0, 3198, 0, 0, 0, 0, 0, 1928, 113, 0, 0, 0, 0, 0] payload_rows_per_dst=[15278, 9130, 14626, 14856, 14742, 7954, 6404, 4016] dup_rows_per_dst=[520, 2258, 356, 3838, 3292, 964, 4, 66] combine_rows_per_src=[15278, 9130, 14626, 14856, 14742, 7954, 6404, 4016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86080 payload_rows_remote=71428 dup_rows=12224 dup_rows_remote=11726 dedup_ratio=0.124349 router_tokens_per_expert=[258, 0, 51, 0, 51, 0, 0, 0, 0, 3223, 0, 82, 0, 5, 416, 1881, 87, 7326, 0, 74, 0, 0, 88, 0, 7514, 0, 253, 244, 0, 2639, 73, 6461, 6920, 1519, 0, 0, 0, 0, 0, 0, 74, 0, 2218, 0, 0, 1949, 244, 0, 2, 0, 0, 3492, 0, 0, 0, 0, 0, 1646, 362, 0, 0, 0, 0, 0] payload_rows_per_dst=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] dup_rows_per_dst=[592, 2288, 498, 4544, 2770, 984, 2, 546] combine_rows_per_src=[15156, 8926, 14652, 14796, 14108, 7986, 6986, 3470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=330040 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275302712 ingress_bytes=309575680 link_bytes=309575680 token_bytes=550275384 weight_movement_bytes=34603008 total_nvlink_bytes=584878392 weight_to_token_ratio=0.062883 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164908361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87418 payload_rows_remote=76518 dup_rows=10886 dup_rows_remote=8706 dedup_ratio=0.110738 router_tokens_per_expert=[136, 0, 66, 1, 66, 0, 0, 0, 0, 3699, 0, 138, 89, 41, 419, 1674, 30, 7254, 0, 91, 0, 0, 54, 0, 7436, 0, 133, 253, 0, 1970, 91, 6490, 7405, 1632, 0, 1, 0, 0, 0, 0, 91, 3, 2784, 0, 0, 1700, 252, 0, 14, 0, 0, 2949, 0, 0, 0, 0, 0, 1927, 263, 0, 0, 0, 0, 0] payload_rows_per_dst=[15042, 9816, 14510, 4714, 10900, 8764, 8948, 14724] dup_rows_per_dst=[368, 2304, 348, 180, 2180, 896, 1974, 2636] combine_rows_per_src=[15042, 9816, 14510, 4714, 10900, 8764, 8948, 14724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=312784 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268322256 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536331728 weight_movement_bytes=34603008 total_nvlink_bytes=570934736 weight_to_token_ratio=0.064518 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=312784 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268322256 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536331728 weight_movement_bytes=34603008 total_nvlink_bytes=570934736 weight_to_token_ratio=0.064518 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83668 payload_rows_remote=74212 dup_rows=14636 dup_rows_remote=13192 dedup_ratio=0.148885 router_tokens_per_expert=[0, 0, 134, 511, 0, 0, 2329, 0, 0, 4645, 1156, 0, 0, 0, 768, 0, 282, 351, 0, 478, 4, 53, 0, 0, 4367, 0, 0, 292, 0, 0, 0, 0, 54, 342, 0, 0, 3118, 1310, 0, 91, 91, 0, 5192, 0, 0, 0, 0, 0, 581, 0, 91, 0, 7542, 1, 83, 0, 6825, 0, 0, 0, 7252, 475, 734, 0] payload_rows_per_dst=[5808, 11760, 14910, 9318, 9456, 14232, 15620, 2564] dup_rows_per_dst=[140, 1378, 1930, 0, 1444, 8758, 976, 10] combine_rows_per_src=[5808, 11760, 14910, 9318, 9456, 14232, 15620, 2564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80236 payload_rows_remote=67132 dup_rows=18068 dup_rows_remote=15378 dedup_ratio=0.183797 router_tokens_per_expert=[0, 59, 246, 82, 0, 5242, 0, 441, 0, 0, 24, 0, 77, 0, 0, 0, 3276, 0, 0, 0, 0, 2057, 0, 2294, 0, 0, 21, 7322, 0, 0, 0, 57, 4807, 0, 71, 245, 0, 0, 159, 668, 218, 0, 0, 2122, 0, 0, 0, 0, 930, 0, 4763, 77, 6527, 59, 584, 0, 0, 521, 1348, 627, 906, 3245, 0, 77] payload_rows_per_dst=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] dup_rows_per_dst=[2690, 48, 7694, 24, 4644, 1082, 288, 1598] combine_rows_per_src=[13104, 13208, 15652, 6622, 13128, 10150, 3012, 5360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292569088 dispatch_meta_bytes=332616 combine_payload_bytes=292569088 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=292901704 ingress_bytes=361775104 link_bytes=361775104 token_bytes=585470792 weight_movement_bytes=69206016 total_nvlink_bytes=654676808 weight_to_token_ratio=0.118206 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86906 payload_rows_remote=76288 dup_rows=11398 dup_rows_remote=11244 dedup_ratio=0.115946 router_tokens_per_expert=[0, 0, 112, 458, 0, 6, 3041, 0, 0, 3409, 1387, 0, 0, 0, 728, 0, 192, 318, 0, 329, 13, 59, 0, 0, 3820, 0, 0, 326, 0, 0, 0, 0, 59, 329, 0, 0, 3472, 1791, 0, 77, 77, 0, 5309, 0, 0, 0, 0, 0, 701, 0, 77, 0, 7611, 0, 66, 0, 6888, 0, 0, 0, 7218, 570, 709, 0] payload_rows_per_dst=[7094, 9704, 9960, 8292, 10176, 10618, 15718, 15344] dup_rows_per_dst=[140, 1344, 1006, 0, 1280, 154, 1192, 6282] combine_rows_per_src=[7094, 9704, 9960, 8292, 10176, 10618, 15718, 15344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=339501056 dispatch_meta_bytes=371744 combine_payload_bytes=339501056 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=339872800 ingress_bytes=512516096 link_bytes=512516096 token_bytes=679373856 weight_movement_bytes=173015040 total_nvlink_bytes=852388896 weight_to_token_ratio=0.254668 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.928556808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=318980096 dispatch_meta_bytes=347664 combine_payload_bytes=318980096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=319327760 ingress_bytes=353583104 link_bytes=353583104 token_bytes=638307856 weight_movement_bytes=34603008 total_nvlink_bytes=672910864 weight_to_token_ratio=0.054211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.330504755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292569088 dispatch_meta_bytes=332616 combine_payload_bytes=292569088 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=292901704 ingress_bytes=361775104 link_bytes=361775104 token_bytes=585470792 weight_movement_bytes=69206016 total_nvlink_bytes=654676808 weight_to_token_ratio=0.118206 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361330591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=313417728 dispatch_meta_bytes=340896 combine_payload_bytes=313417728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313758624 ingress_bytes=382623744 link_bytes=382623744 token_bytes=627176352 weight_movement_bytes=69206016 total_nvlink_bytes=696382368 weight_to_token_ratio=0.110345 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.439782345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77454 payload_rows_remote=61246 dup_rows=20850 dup_rows_remote=3668 dedup_ratio=0.212097 router_tokens_per_expert=[0, 0, 100, 608, 0, 1, 2214, 0, 0, 4384, 1211, 0, 0, 0, 756, 0, 221, 330, 0, 453, 2, 50, 0, 0, 4355, 0, 0, 294, 0, 0, 0, 0, 50, 326, 0, 0, 3372, 1245, 0, 84, 84, 0, 5262, 0, 0, 0, 0, 0, 673, 0, 84, 0, 7587, 0, 79, 0, 6770, 0, 0, 0, 7276, 536, 745, 0] payload_rows_per_dst=[5716, 11482, 1758, 9298, 9326, 10524, 13142, 16208] dup_rows_per_dst=[130, 1220, 354, 0, 828, 168, 968, 17182] combine_rows_per_src=[5716, 11482, 1758, 9298, 9326, 10524, 13142, 16208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312475648 dispatch_meta_bytes=350128 combine_payload_bytes=312475648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312825776 ingress_bytes=347078656 link_bytes=347078656 token_bytes=625301424 weight_movement_bytes=34603008 total_nvlink_bytes=659904432 weight_to_token_ratio=0.055338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306029040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=303972352 dispatch_meta_bytes=349616 combine_payload_bytes=303972352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304321968 ingress_bytes=338575360 link_bytes=338575360 token_bytes=608294320 weight_movement_bytes=34603008 total_nvlink_bytes=642897328 weight_to_token_ratio=0.056885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274031822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274972672 dispatch_meta_bytes=330040 combine_payload_bytes=274972672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275302712 ingress_bytes=309575680 link_bytes=309575680 token_bytes=550275384 weight_movement_bytes=34603008 total_nvlink_bytes=584878392 weight_to_token_ratio=0.062883 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164908361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85862 payload_rows_remote=70812 dup_rows=12442 dup_rows_remote=11976 dedup_ratio=0.126567 router_tokens_per_expert=[182, 0, 59, 0, 59, 0, 0, 0, 0, 3183, 0, 86, 66, 5, 431, 1708, 66, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 145, 246, 0, 2451, 77, 6581, 6844, 1821, 0, 0, 0, 0, 0, 0, 77, 0, 2224, 0, 0, 2466, 246, 0, 0, 0, 0, 3205, 0, 0, 0, 0, 0, 1933, 97, 0, 0, 0, 0, 0] payload_rows_per_dst=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] dup_rows_per_dst=[466, 5410, 418, 1912, 3122, 1108, 0, 6] combine_rows_per_src=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79992 payload_rows_remote=64304 dup_rows=18312 dup_rows_remote=17182 dedup_ratio=0.186279 router_tokens_per_expert=[0, 0, 152, 562, 0, 1, 2607, 0, 0, 4151, 1253, 0, 1, 3, 768, 0, 233, 296, 2, 337, 1, 66, 0, 0, 4405, 0, 1, 319, 0, 0, 0, 2, 67, 352, 1, 2, 2994, 1348, 0, 107, 107, 0, 5239, 0, 0, 0, 1, 2, 676, 1, 107, 0, 7578, 1, 46, 0, 6809, 0, 0, 0, 7184, 554, 816, 0] payload_rows_per_dst=[6494, 11000, 1670, 9454, 8922, 10484, 15688, 16280] dup_rows_per_dst=[150, 1352, 200, 0, 820, 214, 1130, 14446] combine_rows_per_src=[6494, 11000, 1670, 9454, 8922, 10484, 15688, 16280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312475648 dispatch_meta_bytes=350128 combine_payload_bytes=312475648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312825776 ingress_bytes=347078656 link_bytes=347078656 token_bytes=625301424 weight_movement_bytes=34603008 total_nvlink_bytes=659904432 weight_to_token_ratio=0.055338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.306029040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=250863616 dispatch_meta_bytes=259656 combine_payload_bytes=250863616 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=251123272 ingress_bytes=423878656 link_bytes=423878656 token_bytes=501986888 weight_movement_bytes=173015040 total_nvlink_bytes=675001928 weight_to_token_ratio=0.344660 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595021257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79992 payload_rows_remote=64304 dup_rows=18312 dup_rows_remote=17182 dedup_ratio=0.186279 router_tokens_per_expert=[0, 0, 152, 562, 0, 1, 2607, 0, 0, 4151, 1253, 0, 1, 3, 768, 0, 233, 296, 2, 337, 1, 66, 0, 0, 4405, 0, 1, 319, 0, 0, 0, 2, 67, 352, 1, 2, 2994, 1348, 0, 107, 107, 0, 5239, 0, 0, 0, 1, 2, 676, 1, 107, 0, 7578, 1, 46, 0, 6809, 0, 0, 0, 7184, 554, 816, 0] payload_rows_per_dst=[6494, 11000, 1670, 9454, 8922, 10484, 15688, 16280] dup_rows_per_dst=[150, 1352, 200, 0, 820, 214, 1130, 14446] combine_rows_per_src=[6494, 11000, 1670, 9454, 8922, 10484, 15688, 16280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86048 payload_rows_remote=70990 dup_rows=12256 dup_rows_remote=10808 dedup_ratio=0.124674 router_tokens_per_expert=[0, 0, 82, 529, 0, 0, 2967, 0, 0, 3247, 1258, 0, 0, 0, 763, 0, 211, 330, 0, 311, 5, 88, 0, 0, 3548, 0, 0, 316, 0, 0, 0, 0, 88, 318, 0, 0, 3228, 2520, 0, 73, 73, 0, 5564, 0, 0, 0, 0, 0, 620, 0, 73, 0, 7576, 0, 86, 0, 6787, 0, 0, 0, 7308, 480, 703, 0] payload_rows_per_dst=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] dup_rows_per_dst=[5690, 1150, 1448, 1196, 1692, 146, 934, 0] combine_rows_per_src=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85126 payload_rows_remote=77000 dup_rows=13178 dup_rows_remote=13178 dedup_ratio=0.134054 router_tokens_per_expert=[0, 0, 52, 494, 0, 0, 2898, 0, 0, 3198, 1338, 0, 0, 0, 730, 0, 209, 323, 0, 267, 0, 71, 0, 0, 3765, 0, 0, 298, 0, 0, 0, 0, 71, 309, 0, 0, 3545, 2228, 0, 64, 64, 0, 5347, 0, 0, 0, 0, 0, 653, 0, 64, 0, 7619, 0, 79, 0, 6878, 0, 0, 0, 7353, 524, 711, 0] payload_rows_per_dst=[9216, 9318, 15092, 8126, 14524, 10694, 15686, 2470] dup_rows_per_dst=[1930, 1214, 1354, 0, 7408, 128, 1144, 0] combine_rows_per_src=[9216, 9318, 15092, 8126, 14524, 10694, 15686, 2470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80972 payload_rows_remote=69816 dup_rows=17332 dup_rows_remote=16090 dedup_ratio=0.176310 router_tokens_per_expert=[0, 0, 80, 502, 0, 0, 2665, 0, 0, 4203, 1238, 0, 0, 0, 758, 0, 270, 354, 0, 432, 2, 53, 0, 0, 4041, 0, 0, 309, 0, 0, 0, 0, 53, 330, 0, 0, 3386, 1471, 0, 72, 72, 0, 5130, 0, 0, 0, 0, 0, 627, 0, 72, 0, 7583, 0, 94, 0, 6783, 0, 0, 0, 7330, 512, 730, 0] payload_rows_per_dst=[15026, 11156, 15070, 15784, 9606, 10260, 1586, 2484] dup_rows_per_dst=[5034, 1242, 1812, 8082, 1018, 144, 0, 0] combine_rows_per_src=[15026, 11156, 15070, 15784, 9606, 10260, 1586, 2484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86048 payload_rows_remote=70990 dup_rows=12256 dup_rows_remote=10808 dedup_ratio=0.124674 router_tokens_per_expert=[0, 0, 82, 529, 0, 0, 2967, 0, 0, 3247, 1258, 0, 0, 0, 763, 0, 211, 330, 0, 311, 5, 88, 0, 0, 3548, 0, 0, 316, 0, 0, 0, 0, 88, 318, 0, 0, 3228, 2520, 0, 73, 73, 0, 5564, 0, 0, 0, 0, 0, 620, 0, 73, 0, 7576, 0, 86, 0, 6787, 0, 0, 0, 7308, 480, 703, 0] payload_rows_per_dst=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] dup_rows_per_dst=[5690, 1150, 1448, 1196, 1692, 146, 934, 0] combine_rows_per_src=[15040, 9386, 15058, 9388, 10762, 11128, 12920, 2366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=290045952 dispatch_meta_bytes=331152 combine_payload_bytes=290045952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290377104 ingress_bytes=324648960 link_bytes=324648960 token_bytes=580423056 weight_movement_bytes=34603008 total_nvlink_bytes=615026064 weight_to_token_ratio=0.059617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221627900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83668 payload_rows_remote=74212 dup_rows=14636 dup_rows_remote=13192 dedup_ratio=0.148885 router_tokens_per_expert=[0, 0, 134, 511, 0, 0, 2329, 0, 0, 4645, 1156, 0, 0, 0, 768, 0, 282, 351, 0, 478, 4, 53, 0, 0, 4367, 0, 0, 292, 0, 0, 0, 0, 54, 342, 0, 0, 3118, 1310, 0, 91, 91, 0, 5192, 0, 0, 0, 0, 0, 581, 0, 91, 0, 7542, 1, 83, 0, 6825, 0, 0, 0, 7252, 475, 734, 0] payload_rows_per_dst=[5808, 11760, 14910, 9318, 9456, 14232, 15620, 2564] dup_rows_per_dst=[140, 1378, 1930, 0, 1444, 8758, 976, 10] combine_rows_per_src=[5808, 11760, 14910, 9318, 9456, 14232, 15620, 2564] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=263389184 dispatch_meta_bytes=325944 combine_payload_bytes=263389184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263715128 ingress_bytes=367198208 link_bytes=367198208 token_bytes=527104312 weight_movement_bytes=103809024 total_nvlink_bytes=630913336 weight_to_token_ratio=0.196942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381737295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87972 payload_rows_remote=84400 dup_rows=10332 dup_rows_remote=10332 dedup_ratio=0.105103 router_tokens_per_expert=[3821, 0, 0, 78, 700, 5164, 0, 0, 2029, 65, 347, 0, 0, 0, 0, 0, 0, 36, 124, 0, 0, 77, 0, 0, 188, 0, 0, 0, 1, 78, 0, 0, 0, 0, 2556, 0, 0, 0, 1016, 301, 0, 6428, 5816, 8142, 0, 0, 2774, 0, 0, 0, 0, 78, 0, 0, 0, 6900, 981, 0, 0, 363, 636, 173, 251, 29] payload_rows_per_dst=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] dup_rows_per_dst=[776, 2550, 290, 376, 2018, 0, 0, 4322] combine_rows_per_src=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88272 payload_rows_remote=75946 dup_rows=10032 dup_rows_remote=7468 dedup_ratio=0.102051 router_tokens_per_expert=[4141, 0, 0, 92, 764, 4809, 0, 0, 1701, 94, 434, 0, 0, 0, 0, 0, 0, 70, 160, 0, 0, 91, 0, 0, 203, 0, 0, 0, 0, 91, 0, 0, 0, 0, 2716, 0, 0, 0, 1075, 327, 1, 6225, 5501, 8125, 0, 0, 3327, 1, 0, 0, 0, 91, 0, 0, 0, 6911, 863, 0, 0, 315, 569, 196, 251, 8] payload_rows_per_dst=[9252, 12096, 16384, 12632, 12326, 4, 14004, 11574] dup_rows_per_dst=[742, 1980, 508, 406, 2564, 0, 0, 3832] combine_rows_per_src=[9252, 12096, 16384, 12632, 12326, 4, 14004, 11574] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85862 payload_rows_remote=70812 dup_rows=12442 dup_rows_remote=11976 dedup_ratio=0.126567 router_tokens_per_expert=[182, 0, 59, 0, 59, 0, 0, 0, 0, 3183, 0, 86, 66, 5, 431, 1708, 66, 7293, 0, 77, 0, 0, 66, 0, 7458, 0, 145, 246, 0, 2451, 77, 6581, 6844, 1821, 0, 0, 0, 0, 0, 0, 77, 0, 2224, 0, 0, 2466, 246, 0, 0, 0, 0, 3205, 0, 0, 0, 0, 0, 1933, 97, 0, 0, 0, 0, 0] payload_rows_per_dst=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] dup_rows_per_dst=[466, 5410, 418, 1912, 3122, 1108, 0, 6] combine_rows_per_src=[15050, 12626, 14586, 10010, 14208, 8918, 6410, 4054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290775040 dispatch_meta_bytes=327192 combine_payload_bytes=290775040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291102232 ingress_bytes=325378048 link_bytes=325378048 token_bytes=581877272 weight_movement_bytes=34603008 total_nvlink_bytes=616480280 weight_to_token_ratio=0.059468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224371400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315392000 dispatch_meta_bytes=360712 combine_payload_bytes=315392000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315752712 ingress_bytes=349995008 link_bytes=349995008 token_bytes=631144712 weight_movement_bytes=34603008 total_nvlink_bytes=665747720 weight_to_token_ratio=0.054826 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.317003038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87972 payload_rows_remote=84400 dup_rows=10332 dup_rows_remote=10332 dedup_ratio=0.105103 router_tokens_per_expert=[3821, 0, 0, 78, 700, 5164, 0, 0, 2029, 65, 347, 0, 0, 0, 0, 0, 0, 36, 124, 0, 0, 77, 0, 0, 188, 0, 0, 0, 1, 78, 0, 0, 0, 0, 2556, 0, 0, 0, 1016, 301, 0, 6428, 5816, 8142, 0, 0, 2774, 0, 0, 0, 0, 78, 0, 0, 0, 6900, 981, 0, 0, 363, 636, 173, 251, 29] payload_rows_per_dst=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] dup_rows_per_dst=[776, 2550, 290, 376, 2018, 0, 0, 4322] combine_rows_per_src=[8422, 12660, 12896, 13014, 11276, 3572, 13956, 12176] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=343624 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286309960 ingress_bytes=320569344 link_bytes=320569344 token_bytes=572276296 weight_movement_bytes=34603008 total_nvlink_bytes=606879304 weight_to_token_ratio=0.060466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.206276634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290775040 dispatch_meta_bytes=327192 combine_payload_bytes=290775040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291102232 ingress_bytes=325378048 link_bytes=325378048 token_bytes=581877272 weight_movement_bytes=34603008 total_nvlink_bytes=616480280 weight_to_token_ratio=0.059468 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224371400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=303972352 dispatch_meta_bytes=349616 combine_payload_bytes=303972352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304321968 ingress_bytes=338575360 link_bytes=338575360 token_bytes=608294320 weight_movement_bytes=34603008 total_nvlink_bytes=642897328 weight_to_token_ratio=0.056885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274031822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=345702400 dispatch_meta_bytes=378928 combine_payload_bytes=345702400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=346081328 ingress_bytes=518717440 link_bytes=518717440 token_bytes=691783728 weight_movement_bytes=173015040 total_nvlink_bytes=864798768 weight_to_token_ratio=0.250100 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.951891967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=311074816 dispatch_meta_bytes=333656 combine_payload_bytes=311074816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311408472 ingress_bytes=345677824 link_bytes=345677824 token_bytes=622483288 weight_movement_bytes=34603008 total_nvlink_bytes=657086296 weight_to_token_ratio=0.055589 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300757822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=62694 dup_rows=24240 dup_rows_remote=20420 dedup_ratio=0.246582 router_tokens_per_expert=[4094, 0, 0, 85, 781, 4675, 0, 0, 1743, 88, 431, 0, 0, 0, 0, 0, 0, 59, 148, 0, 0, 83, 0, 0, 213, 0, 0, 0, 0, 85, 0, 0, 0, 0, 2799, 0, 0, 0, 1088, 299, 0, 6352, 5462, 8135, 0, 0, 3428, 0, 0, 0, 0, 85, 0, 0, 0, 6886, 816, 0, 0, 294, 624, 153, 241, 5] payload_rows_per_dst=[7578, 11896, 580, 5000, 7062, 16340, 14238, 11370] dup_rows_per_dst=[634, 1978, 0, 144, 1310, 14942, 1412, 3820] combine_rows_per_src=[7578, 11896, 580, 5000, 7062, 16340, 14238, 11370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=290045952 dispatch_meta_bytes=331152 combine_payload_bytes=290045952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=290377104 ingress_bytes=324648960 link_bytes=324648960 token_bytes=580423056 weight_movement_bytes=34603008 total_nvlink_bytes=615026064 weight_to_token_ratio=0.059617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221627900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65268 dup_rows=17956 dup_rows_remote=12340 dedup_ratio=0.182658 router_tokens_per_expert=[0, 0, 86, 527, 0, 3, 2888, 0, 0, 3596, 1288, 0, 0, 0, 779, 0, 250, 328, 0, 408, 7, 66, 0, 0, 3926, 0, 0, 308, 0, 0, 0, 0, 66, 323, 0, 0, 3675, 1380, 0, 77, 77, 0, 5249, 0, 0, 0, 0, 0, 641, 0, 77, 0, 7585, 0, 82, 0, 6844, 0, 0, 0, 7282, 567, 767, 0] payload_rows_per_dst=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] dup_rows_per_dst=[5616, 3552, 1722, 5772, 1140, 154, 0, 0] combine_rows_per_src=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=345702400 dispatch_meta_bytes=378928 combine_payload_bytes=345702400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=346081328 ingress_bytes=518717440 link_bytes=518717440 token_bytes=691783728 weight_movement_bytes=173015040 total_nvlink_bytes=864798768 weight_to_token_ratio=0.250100 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.951891967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85156 payload_rows_remote=71290 dup_rows=13148 dup_rows_remote=13140 dedup_ratio=0.133748 router_tokens_per_expert=[3798, 0, 3, 108, 791, 4677, 1, 0, 2058, 49, 409, 0, 0, 1, 0, 0, 0, 78, 92, 2, 0, 107, 1, 0, 212, 0, 0, 1, 1, 107, 0, 1, 2, 0, 2904, 0, 0, 0, 1139, 302, 0, 6319, 5501, 8148, 0, 1, 2879, 0, 0, 1, 0, 107, 0, 0, 4, 6825, 1086, 3, 0, 362, 604, 220, 245, 3] payload_rows_per_dst=[8648, 12096, 560, 12856, 9196, 16298, 13866, 11636] dup_rows_per_dst=[754, 2292, 0, 426, 2064, 3192, 8, 4412] combine_rows_per_src=[8648, 12096, 560, 12856, 9196, 16298, 13866, 11636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=256794624 dispatch_meta_bytes=332456 combine_payload_bytes=256794624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257127080 ingress_bytes=291397632 link_bytes=291397632 token_bytes=513921704 weight_movement_bytes=34603008 total_nvlink_bytes=548524712 weight_to_token_ratio=0.067331 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096505830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87910 payload_rows_remote=74774 dup_rows=10394 dup_rows_remote=10042 dedup_ratio=0.105733 router_tokens_per_expert=[4397, 0, 0, 65, 699, 5581, 0, 0, 2198, 79, 362, 0, 0, 0, 0, 0, 0, 8, 176, 0, 0, 64, 0, 0, 176, 0, 0, 0, 0, 65, 0, 0, 0, 0, 2025, 0, 0, 0, 884, 294, 0, 6503, 5582, 8142, 0, 0, 2406, 0, 0, 0, 0, 65, 0, 0, 0, 7057, 998, 0, 0, 307, 603, 125, 244, 47] payload_rows_per_dst=[9004, 13120, 16384, 13136, 10170, 0, 14244, 11852] dup_rows_per_dst=[1318, 3320, 396, 352, 1048, 0, 0, 3960] combine_rows_per_src=[9004, 13120, 16384, 13136, 10170, 0, 14244, 11852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89438 payload_rows_remote=73054 dup_rows=8866 dup_rows_remote=8490 dedup_ratio=0.090190 router_tokens_per_expert=[4543, 0, 0, 74, 732, 5365, 0, 0, 1934, 86, 358, 0, 0, 0, 0, 0, 0, 21, 145, 0, 0, 73, 0, 0, 208, 0, 0, 0, 0, 74, 0, 0, 0, 0, 2098, 0, 0, 0, 866, 295, 0, 6428, 5750, 8141, 0, 0, 2438, 0, 0, 0, 0, 74, 0, 0, 0, 7041, 1064, 0, 0, 296, 566, 170, 244, 68] payload_rows_per_dst=[9730, 12284, 16384, 13004, 10226, 6060, 14230, 7520] dup_rows_per_dst=[1714, 2456, 376, 416, 1168, 0, 0, 2736] combine_rows_per_src=[9730, 12284, 16384, 13004, 10226, 6060, 14230, 7520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86726 payload_rows_remote=81666 dup_rows=11578 dup_rows_remote=11574 dedup_ratio=0.117778 router_tokens_per_expert=[4305, 0, 0, 72, 717, 4963, 0, 0, 2087, 95, 350, 0, 0, 0, 0, 0, 0, 30, 164, 0, 0, 72, 0, 0, 180, 0, 0, 0, 0, 72, 0, 0, 0, 0, 2692, 0, 0, 0, 1137, 321, 0, 6386, 5374, 8133, 0, 0, 2856, 0, 0, 0, 0, 72, 0, 0, 0, 6959, 797, 0, 0, 302, 607, 133, 258, 18] payload_rows_per_dst=[14542, 5060, 16384, 12916, 11754, 11454, 12192, 2424] dup_rows_per_dst=[5572, 4, 414, 360, 2258, 1164, 0, 1806] combine_rows_per_src=[14542, 5060, 16384, 12916, 11754, 11454, 12192, 2424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=310432 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648160 ingress_bytes=301940736 link_bytes=301940736 token_bytes=534985888 weight_movement_bytes=34603008 total_nvlink_bytes=569588896 weight_to_token_ratio=0.064680 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292003840 dispatch_meta_bytes=337720 combine_payload_bytes=292003840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292341560 ingress_bytes=361209856 link_bytes=361209856 token_bytes=584345400 weight_movement_bytes=69206016 total_nvlink_bytes=653551416 weight_to_token_ratio=0.118433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359203608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88272 payload_rows_remote=75946 dup_rows=10032 dup_rows_remote=7468 dedup_ratio=0.102051 router_tokens_per_expert=[4141, 0, 0, 92, 764, 4809, 0, 0, 1701, 94, 434, 0, 0, 0, 0, 0, 0, 70, 160, 0, 0, 91, 0, 0, 203, 0, 0, 0, 0, 91, 0, 0, 0, 0, 2716, 0, 0, 0, 1075, 327, 1, 6225, 5501, 8125, 0, 0, 3327, 1, 0, 0, 0, 91, 0, 0, 0, 6911, 863, 0, 0, 315, 569, 196, 251, 8] payload_rows_per_dst=[9252, 12096, 16384, 12632, 12326, 4, 14004, 11574] dup_rows_per_dst=[742, 1980, 508, 406, 2564, 0, 0, 3832] combine_rows_per_src=[9252, 12096, 16384, 12632, 12326, 4, 14004, 11574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89478 payload_rows_remote=75492 dup_rows=8826 dup_rows_remote=5542 dedup_ratio=0.089783 router_tokens_per_expert=[3362, 466, 0, 0, 0, 0, 1525, 27, 0, 973, 0, 3098, 0, 0, 248, 0, 0, 155, 66, 0, 0, 452, 0, 0, 0, 0, 7014, 254, 425, 0, 100, 4574, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 1940, 116, 7000, 78, 7321, 0, 6930, 0, 0, 153, 0, 77, 0, 0, 66, 0, 0, 0, 95, 0, 0, 693] payload_rows_per_dst=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] dup_rows_per_dst=[1950, 728, 494, 244, 272, 3284, 456, 1398] combine_rows_per_src=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89154 payload_rows_remote=74066 dup_rows=9150 dup_rows_remote=5882 dedup_ratio=0.093079 router_tokens_per_expert=[4318, 520, 0, 0, 0, 0, 1166, 33, 0, 1282, 0, 3304, 0, 0, 459, 0, 0, 182, 90, 0, 0, 412, 0, 0, 0, 0, 6902, 256, 496, 0, 123, 3042, 0, 0, 0, 0, 0, 0, 0, 2229, 0, 0, 1534, 160, 6949, 92, 7330, 0, 7145, 0, 0, 174, 0, 91, 0, 0, 94, 0, 0, 0, 92, 0, 0, 677] payload_rows_per_dst=[10078, 8930, 15518, 7696, 15088, 3384, 14292, 14168] dup_rows_per_dst=[1996, 1160, 510, 138, 3268, 188, 528, 1362] combine_rows_per_src=[10078, 8930, 15518, 7696, 15088, 3384, 14292, 14168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292003840 dispatch_meta_bytes=337720 combine_payload_bytes=292003840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292341560 ingress_bytes=361209856 link_bytes=361209856 token_bytes=584345400 weight_movement_bytes=69206016 total_nvlink_bytes=653551416 weight_to_token_ratio=0.118433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359203608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65268 dup_rows=17956 dup_rows_remote=12340 dedup_ratio=0.182658 router_tokens_per_expert=[0, 0, 86, 527, 0, 3, 2888, 0, 0, 3596, 1288, 0, 0, 0, 779, 0, 250, 328, 0, 408, 7, 66, 0, 0, 3926, 0, 0, 308, 0, 0, 0, 0, 66, 323, 0, 0, 3675, 1380, 0, 77, 77, 0, 5249, 0, 0, 0, 0, 0, 641, 0, 77, 0, 7585, 0, 82, 0, 6844, 0, 0, 0, 7282, 567, 767, 0] payload_rows_per_dst=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] dup_rows_per_dst=[5616, 3552, 1722, 5772, 1140, 154, 0, 0] combine_rows_per_src=[15080, 12046, 14960, 13594, 9902, 10498, 1600, 2668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=306274304 dispatch_meta_bytes=339264 combine_payload_bytes=306274304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306613568 ingress_bytes=340877312 link_bytes=340877312 token_bytes=612887872 weight_movement_bytes=34603008 total_nvlink_bytes=647490880 weight_to_token_ratio=0.056459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282693882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=299229184 dispatch_meta_bytes=326176 combine_payload_bytes=299229184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299555360 ingress_bytes=333832192 link_bytes=333832192 token_bytes=598784544 weight_movement_bytes=34603008 total_nvlink_bytes=633387552 weight_to_token_ratio=0.057789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256183663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=334503936 dispatch_meta_bytes=372960 combine_payload_bytes=334503936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=334876896 ingress_bytes=369106944 link_bytes=369106944 token_bytes=669380832 weight_movement_bytes=34603008 total_nvlink_bytes=703983840 weight_to_token_ratio=0.051694 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=324136 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=309539368 ingress_bytes=482230272 link_bytes=482230272 token_bytes=618754600 weight_movement_bytes=173015040 total_nvlink_bytes=791769640 weight_to_token_ratio=0.279618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.814593691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=311074816 dispatch_meta_bytes=333656 combine_payload_bytes=311074816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311408472 ingress_bytes=345677824 link_bytes=345677824 token_bytes=622483288 weight_movement_bytes=34603008 total_nvlink_bytes=657086296 weight_to_token_ratio=0.055589 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.300757822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89478 payload_rows_remote=75492 dup_rows=8826 dup_rows_remote=5542 dedup_ratio=0.089783 router_tokens_per_expert=[3362, 466, 0, 0, 0, 0, 1525, 27, 0, 973, 0, 3098, 0, 0, 248, 0, 0, 155, 66, 0, 0, 452, 0, 0, 0, 0, 7014, 254, 425, 0, 100, 4574, 0, 0, 0, 0, 0, 0, 0, 1944, 0, 0, 1940, 116, 7000, 78, 7321, 0, 6930, 0, 0, 153, 0, 77, 0, 0, 66, 0, 0, 0, 95, 0, 0, 693] payload_rows_per_dst=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] dup_rows_per_dst=[1950, 728, 494, 244, 272, 3284, 456, 1398] combine_rows_per_src=[8810, 7910, 14930, 10462, 5178, 13986, 13864, 14338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=60350 dup_rows=23908 dup_rows_remote=22612 dedup_ratio=0.243205 router_tokens_per_expert=[4191, 406, 0, 0, 0, 0, 1111, 10, 0, 1258, 0, 3537, 0, 0, 411, 0, 0, 169, 88, 0, 0, 446, 0, 0, 0, 0, 6854, 244, 540, 0, 130, 3055, 0, 0, 0, 0, 0, 0, 0, 2500, 0, 0, 1494, 145, 7009, 88, 7305, 0, 7116, 0, 0, 144, 0, 84, 0, 0, 88, 0, 0, 0, 93, 0, 0, 636] payload_rows_per_dst=[9606, 9300, 1406, 4518, 5000, 15738, 14782, 14046] dup_rows_per_dst=[1830, 1112, 0, 60, 0, 12300, 7310, 1296] combine_rows_per_src=[9606, 9300, 1406, 4518, 5000, 15738, 14782, 14046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=310432 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648160 ingress_bytes=301940736 link_bytes=301940736 token_bytes=534985888 weight_movement_bytes=34603008 total_nvlink_bytes=569588896 weight_to_token_ratio=0.064680 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=22656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=303374336 dispatch_meta_bytes=319792 combine_payload_bytes=303374336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303694128 ingress_bytes=337977344 link_bytes=337977344 token_bytes=607068464 weight_movement_bytes=34603008 total_nvlink_bytes=641671472 weight_to_token_ratio=0.057000 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271781536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78654 payload_rows_remote=64280 dup_rows=19650 dup_rows_remote=14406 dedup_ratio=0.199890 router_tokens_per_expert=[3894, 0, 0, 77, 784, 5054, 0, 0, 1788, 82, 386, 0, 0, 0, 0, 0, 0, 23, 141, 0, 0, 77, 0, 0, 198, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2686, 0, 0, 0, 1028, 313, 0, 6358, 5708, 8133, 0, 0, 2905, 0, 0, 0, 0, 77, 0, 0, 0, 6931, 1038, 0, 0, 340, 649, 155, 246, 4] payload_rows_per_dst=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] dup_rows_per_dst=[5244, 6, 364, 396, 2282, 9380, 0, 1978] combine_rows_per_src=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76180 payload_rows_remote=62260 dup_rows=22124 dup_rows_remote=21526 dedup_ratio=0.225057 router_tokens_per_expert=[3571, 647, 0, 0, 0, 0, 1306, 40, 0, 1292, 1, 3120, 1, 0, 251, 0, 1, 214, 62, 0, 2, 434, 0, 1, 0, 1, 6796, 246, 541, 0, 104, 3732, 3, 0, 0, 0, 3, 0, 0, 2326, 0, 2, 1693, 86, 7063, 107, 7334, 0, 6960, 1, 0, 191, 0, 107, 0, 0, 49, 0, 0, 3, 64, 0, 0, 797] payload_rows_per_dst=[9008, 8440, 1426, 9018, 4658, 15680, 13920, 14030] dup_rows_per_dst=[2120, 890, 2, 230, 6, 16890, 598, 1388] combine_rows_per_src=[9008, 8440, 1426, 9018, 4658, 15680, 13920, 14030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=324136 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=309539368 ingress_bytes=482230272 link_bytes=482230272 token_bytes=618754600 weight_movement_bytes=173015040 total_nvlink_bytes=791769640 weight_to_token_ratio=0.279618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.814593691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=247193600 dispatch_meta_bytes=331848 combine_payload_bytes=247193600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=247525448 ingress_bytes=281796608 link_bytes=281796608 token_bytes=494719048 weight_movement_bytes=34603008 total_nvlink_bytes=529322056 weight_to_token_ratio=0.069945 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.060377949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76180 payload_rows_remote=62260 dup_rows=22124 dup_rows_remote=21526 dedup_ratio=0.225057 router_tokens_per_expert=[3571, 647, 0, 0, 0, 0, 1306, 40, 0, 1292, 1, 3120, 1, 0, 251, 0, 1, 214, 62, 0, 2, 434, 0, 1, 0, 1, 6796, 246, 541, 0, 104, 3732, 3, 0, 0, 0, 3, 0, 0, 2326, 0, 2, 1693, 86, 7063, 107, 7334, 0, 6960, 1, 0, 191, 0, 107, 0, 0, 49, 0, 0, 3, 64, 0, 0, 797] payload_rows_per_dst=[9008, 8440, 1426, 9018, 4658, 15680, 13920, 14030] dup_rows_per_dst=[2120, 890, 2, 230, 6, 16890, 598, 1388] combine_rows_per_src=[9008, 8440, 1426, 9018, 4658, 15680, 13920, 14030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90258 payload_rows_remote=79742 dup_rows=8046 dup_rows_remote=7882 dedup_ratio=0.081848 router_tokens_per_expert=[3280, 465, 0, 0, 0, 0, 1617, 28, 0, 778, 0, 3212, 0, 0, 247, 0, 0, 129, 33, 0, 0, 454, 0, 0, 0, 0, 7018, 244, 304, 0, 85, 4707, 0, 0, 0, 0, 0, 0, 0, 1899, 0, 0, 1883, 129, 7201, 65, 7372, 0, 7033, 0, 0, 152, 0, 64, 0, 0, 79, 0, 0, 0, 87, 0, 0, 587] payload_rows_per_dst=[8688, 7800, 15446, 10516, 15452, 3996, 14066, 14294] dup_rows_per_dst=[2092, 674, 530, 164, 2748, 158, 432, 1248] combine_rows_per_src=[8688, 7800, 15446, 10516, 15452, 3996, 14066, 14294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90040 payload_rows_remote=74602 dup_rows=8264 dup_rows_remote=7812 dedup_ratio=0.084066 router_tokens_per_expert=[3362, 663, 0, 0, 0, 0, 1522, 21, 0, 915, 0, 2642, 0, 0, 249, 0, 0, 147, 46, 0, 0, 408, 0, 0, 0, 0, 7025, 244, 378, 0, 106, 4689, 0, 0, 0, 0, 0, 0, 0, 2014, 0, 0, 2155, 137, 7056, 74, 7344, 0, 6913, 0, 0, 184, 0, 73, 0, 0, 86, 0, 0, 0, 84, 0, 0, 615] payload_rows_per_dst=[9008, 6978, 15438, 10654, 15228, 4556, 13832, 14346] dup_rows_per_dst=[2128, 634, 452, 180, 2912, 176, 508, 1274] combine_rows_per_src=[9008, 6978, 15438, 10654, 15228, 4556, 13832, 14346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78322 payload_rows_remote=64058 dup_rows=19982 dup_rows_remote=13606 dedup_ratio=0.203267 router_tokens_per_expert=[3651, 412, 0, 0, 0, 0, 1441, 1, 0, 1170, 0, 3225, 0, 0, 275, 0, 0, 144, 118, 0, 0, 479, 0, 0, 0, 0, 6918, 261, 546, 0, 130, 4099, 0, 0, 0, 0, 0, 0, 0, 2069, 0, 0, 1689, 154, 6978, 72, 7296, 0, 7048, 0, 0, 146, 0, 72, 0, 0, 95, 0, 0, 0, 83, 0, 0, 580] payload_rows_per_dst=[8948, 14264, 15488, 14544, 15162, 3640, 3038, 3238] dup_rows_per_dst=[2062, 6376, 586, 7490, 2932, 190, 194, 152] combine_rows_per_src=[8948, 14264, 15488, 14544, 15162, 3640, 3038, 3238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263290880 dispatch_meta_bytes=314744 combine_payload_bytes=263290880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=263605624 ingress_bytes=332496896 link_bytes=332496896 token_bytes=526896504 weight_movement_bytes=69206016 total_nvlink_bytes=596102520 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251159051e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91066 payload_rows_remote=74682 dup_rows=7238 dup_rows_remote=6434 dedup_ratio=0.073629 router_tokens_per_expert=[0, 0, 467, 0, 0, 0, 966, 0, 0, 0, 127, 65, 7313, 0, 0, 6800, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 8115, 0, 0, 7742, 0, 0, 0, 77, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 78, 0, 3129, 0, 0, 507, 1610, 0, 0] payload_rows_per_dst=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] dup_rows_per_dst=[180, 0, 5298, 0, 0, 804, 738, 218] combine_rows_per_src=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255016960 dispatch_meta_bytes=335144 combine_payload_bytes=255016960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255352104 ingress_bytes=324222976 link_bytes=324222976 token_bytes=510369064 weight_movement_bytes=69206016 total_nvlink_bytes=579575080 weight_to_token_ratio=0.135600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220024957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89154 payload_rows_remote=74066 dup_rows=9150 dup_rows_remote=5882 dedup_ratio=0.093079 router_tokens_per_expert=[4318, 520, 0, 0, 0, 0, 1166, 33, 0, 1282, 0, 3304, 0, 0, 459, 0, 0, 182, 90, 0, 0, 412, 0, 0, 0, 0, 6902, 256, 496, 0, 123, 3042, 0, 0, 0, 0, 0, 0, 0, 2229, 0, 0, 1534, 160, 6949, 92, 7330, 0, 7145, 0, 0, 174, 0, 91, 0, 0, 94, 0, 0, 0, 92, 0, 0, 677] payload_rows_per_dst=[10078, 8930, 15518, 7696, 15088, 3384, 14292, 14168] dup_rows_per_dst=[1996, 1160, 510, 138, 3268, 188, 528, 1362] combine_rows_per_src=[10078, 8930, 15518, 7696, 15088, 3384, 14292, 14168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255016960 dispatch_meta_bytes=335144 combine_payload_bytes=255016960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255352104 ingress_bytes=324222976 link_bytes=324222976 token_bytes=510369064 weight_movement_bytes=69206016 total_nvlink_bytes=579575080 weight_to_token_ratio=0.135600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220024957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94010 payload_rows_remote=78500 dup_rows=4294 dup_rows_remote=4294 dedup_ratio=0.043681 router_tokens_per_expert=[0, 0, 415, 0, 0, 0, 827, 0, 0, 0, 157, 94, 7357, 0, 0, 6726, 47, 0, 0, 0, 0, 4477, 0, 0, 0, 0, 0, 8101, 0, 0, 7638, 0, 0, 0, 91, 0, 0, 7664, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 91, 0, 3101, 0, 0, 464, 963, 0, 0] payload_rows_per_dst=[15958, 12662, 9576, 0, 15510, 16384, 14984, 8936] dup_rows_per_dst=[1802, 0, 764, 0, 0, 954, 654, 120] combine_rows_per_src=[15958, 12662, 9576, 0, 15510, 16384, 14984, 8936] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=326623232 dispatch_meta_bytes=350496 combine_payload_bytes=326623232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=326973728 ingress_bytes=395829248 link_bytes=395829248 token_bytes=653596960 weight_movement_bytes=69206016 total_nvlink_bytes=722802976 weight_to_token_ratio=0.105885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.489473593e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78654 payload_rows_remote=64280 dup_rows=19650 dup_rows_remote=14406 dedup_ratio=0.199890 router_tokens_per_expert=[3894, 0, 0, 77, 784, 5054, 0, 0, 1788, 82, 386, 0, 0, 0, 0, 0, 0, 23, 141, 0, 0, 77, 0, 0, 198, 0, 0, 0, 0, 77, 0, 0, 0, 0, 2686, 0, 0, 0, 1028, 313, 0, 6358, 5708, 8133, 0, 0, 2905, 0, 0, 0, 0, 77, 0, 0, 0, 6931, 1038, 0, 0, 340, 649, 155, 246, 4] payload_rows_per_dst=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] dup_rows_per_dst=[5244, 6, 364, 396, 2282, 9380, 0, 1978] combine_rows_per_src=[14374, 4506, 16384, 12870, 11582, 15898, 154, 2886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=305569792 dispatch_meta_bytes=329656 combine_payload_bytes=305569792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305899448 ingress_bytes=340172800 link_bytes=340172800 token_bytes=611469240 weight_movement_bytes=34603008 total_nvlink_bytes=646072248 weight_to_token_ratio=0.056590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280042860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=262381568 dispatch_meta_bytes=310656 combine_payload_bytes=262381568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262692224 ingress_bytes=296984576 link_bytes=296984576 token_bytes=525073792 weight_movement_bytes=34603008 total_nvlink_bytes=559676800 weight_to_token_ratio=0.065901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.117529050e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=305897472 dispatch_meta_bytes=324464 combine_payload_bytes=305897472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306221936 ingress_bytes=340500480 link_bytes=340500480 token_bytes=612119408 weight_movement_bytes=34603008 total_nvlink_bytes=646722416 weight_to_token_ratio=0.056530 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281275894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91066 payload_rows_remote=74682 dup_rows=7238 dup_rows_remote=6434 dedup_ratio=0.073629 router_tokens_per_expert=[0, 0, 467, 0, 0, 0, 966, 0, 0, 0, 127, 65, 7313, 0, 0, 6800, 0, 0, 0, 0, 0, 3356, 0, 0, 0, 0, 0, 8115, 0, 0, 7742, 0, 0, 0, 77, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 507, 0, 78, 0, 3129, 0, 0, 507, 1610, 0, 0] payload_rows_per_dst=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] dup_rows_per_dst=[180, 0, 5298, 0, 0, 804, 738, 218] combine_rows_per_src=[2978, 384, 15014, 15192, 15782, 16384, 15058, 10274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83652 payload_rows_remote=67268 dup_rows=14652 dup_rows_remote=6532 dedup_ratio=0.149048 router_tokens_per_expert=[0, 0, 426, 0, 0, 0, 817, 0, 0, 0, 141, 88, 7318, 0, 0, 6785, 6, 0, 0, 0, 0, 4412, 0, 0, 0, 0, 0, 8108, 0, 0, 7687, 0, 0, 0, 84, 0, 0, 7731, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 397, 0, 85, 0, 3029, 0, 0, 501, 991, 0, 0] payload_rows_per_dst=[2386, 458, 13104, 15374, 4692, 16384, 14870, 16384] dup_rows_per_dst=[100, 0, 4778, 0, 0, 924, 730, 8120] combine_rows_per_src=[2386, 458, 13104, 15374, 4692, 16384, 14870, 16384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=303374336 dispatch_meta_bytes=319792 combine_payload_bytes=303374336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303694128 ingress_bytes=337977344 link_bytes=337977344 token_bytes=607068464 weight_movement_bytes=34603008 total_nvlink_bytes=641671472 weight_to_token_ratio=0.057000 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=31872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271781536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=321536000 dispatch_meta_bytes=331176 combine_payload_bytes=321536000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=321867176 ingress_bytes=425345024 link_bytes=425345024 token_bytes=643403176 weight_movement_bytes=103809024 total_nvlink_bytes=747212200 weight_to_token_ratio=0.161344 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.600539082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263290880 dispatch_meta_bytes=314744 combine_payload_bytes=263290880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=263605624 ingress_bytes=332496896 link_bytes=332496896 token_bytes=526896504 weight_movement_bytes=69206016 total_nvlink_bytes=596102520 weight_to_token_ratio=0.131347 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251159051e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=61244 dup_rows=23912 dup_rows_remote=17080 dedup_ratio=0.243245 router_tokens_per_expert=[3817, 422, 0, 0, 0, 0, 1632, 8, 0, 1176, 0, 3105, 0, 0, 268, 0, 0, 154, 80, 0, 0, 465, 0, 0, 0, 0, 6856, 268, 547, 0, 128, 3396, 0, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 1851, 141, 7004, 77, 7340, 0, 7005, 0, 0, 165, 0, 77, 0, 0, 82, 0, 0, 0, 111, 0, 0, 670] payload_rows_per_dst=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] dup_rows_per_dst=[6832, 7674, 524, 7268, 1296, 164, 154, 0] combine_rows_per_src=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=305897472 dispatch_meta_bytes=324464 combine_payload_bytes=305897472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306221936 ingress_bytes=340500480 link_bytes=340500480 token_bytes=612119408 weight_movement_bytes=34603008 total_nvlink_bytes=646722416 weight_to_token_ratio=0.056530 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281275894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=275529728 dispatch_meta_bytes=295200 combine_payload_bytes=275529728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275824928 ingress_bytes=310132736 link_bytes=310132736 token_bytes=551354656 weight_movement_bytes=34603008 total_nvlink_bytes=585957664 weight_to_token_ratio=0.062760 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.167004518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87664 payload_rows_remote=72742 dup_rows=10640 dup_rows_remote=9842 dedup_ratio=0.108236 router_tokens_per_expert=[0, 1, 393, 0, 0, 0, 978, 0, 0, 0, 144, 49, 7267, 0, 3, 6686, 6, 0, 0, 1, 1, 4301, 0, 1, 0, 0, 0, 8079, 0, 3, 7716, 0, 0, 0, 108, 0, 0, 7689, 3, 0, 1, 0, 516, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 486, 0, 107, 0, 3037, 0, 0, 527, 1043, 1, 0] payload_rows_per_dst=[2612, 392, 15152, 15438, 11946, 16384, 14922, 10818] dup_rows_per_dst=[132, 0, 6840, 0, 0, 818, 798, 2052] combine_rows_per_src=[2612, 392, 15152, 15438, 11946, 16384, 14922, 10818] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94558 payload_rows_remote=94558 dup_rows=3746 dup_rows_remote=3746 dedup_ratio=0.038106 router_tokens_per_expert=[0, 0, 514, 0, 0, 0, 862, 0, 0, 0, 114, 79, 7362, 0, 0, 6993, 5, 0, 0, 0, 0, 2996, 0, 0, 0, 0, 0, 8128, 0, 0, 7754, 0, 0, 0, 64, 0, 0, 7890, 0, 0, 0, 0, 420, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 487, 0, 60, 0, 2947, 0, 0, 448, 2029, 0, 0] payload_rows_per_dst=[16118, 14372, 6002, 0, 15908, 16384, 15134, 10640] dup_rows_per_dst=[2142, 0, 0, 0, 0, 712, 684, 208] combine_rows_per_src=[16118, 14372, 6002, 0, 15908, 16384, 15134, 10640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87664 payload_rows_remote=72742 dup_rows=10640 dup_rows_remote=9842 dedup_ratio=0.108236 router_tokens_per_expert=[0, 1, 393, 0, 0, 0, 978, 0, 0, 0, 144, 49, 7267, 0, 3, 6686, 6, 0, 0, 1, 1, 4301, 0, 1, 0, 0, 0, 8079, 0, 3, 7716, 0, 0, 0, 108, 0, 0, 7689, 3, 0, 1, 0, 516, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 486, 0, 107, 0, 3037, 0, 0, 527, 1043, 1, 0] payload_rows_per_dst=[2612, 392, 15152, 15438, 11946, 16384, 14922, 10818] dup_rows_per_dst=[132, 0, 6840, 0, 0, 818, 798, 2052] combine_rows_per_src=[2612, 392, 15152, 15438, 11946, 16384, 14922, 10818] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94850 payload_rows_remote=88886 dup_rows=3454 dup_rows_remote=3454 dedup_ratio=0.035136 router_tokens_per_expert=[0, 0, 492, 0, 0, 0, 844, 0, 0, 0, 124, 86, 7379, 0, 0, 6945, 11, 0, 0, 0, 0, 2971, 0, 0, 0, 0, 0, 8118, 0, 0, 7725, 0, 0, 0, 73, 0, 0, 7827, 0, 0, 0, 0, 498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 481, 0, 64, 0, 3130, 0, 0, 467, 1917, 0, 0] payload_rows_per_dst=[16080, 14310, 5964, 6950, 15800, 9740, 15154, 10852] dup_rows_per_dst=[2042, 0, 0, 0, 0, 542, 694, 176] combine_rows_per_src=[16080, 14310, 5964, 6950, 15800, 9740, 15154, 10852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250855424 dispatch_meta_bytes=313296 combine_payload_bytes=250855424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251168720 ingress_bytes=285458432 link_bytes=285458432 token_bytes=502024144 weight_movement_bytes=34603008 total_nvlink_bytes=536627152 weight_to_token_ratio=0.068927 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074157098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84880 payload_rows_remote=69080 dup_rows=13424 dup_rows_remote=2200 dedup_ratio=0.136556 router_tokens_per_expert=[0, 0, 374, 0, 0, 0, 850, 0, 0, 0, 131, 95, 7388, 0, 0, 6760, 4, 0, 0, 0, 0, 4141, 0, 0, 0, 0, 0, 8120, 0, 0, 7676, 0, 0, 0, 72, 0, 0, 7814, 0, 0, 0, 0, 479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 460, 0, 71, 0, 2813, 0, 0, 484, 1420, 0, 0] payload_rows_per_dst=[16012, 15800, 8290, 16240, 15772, 958, 2586, 9222] dup_rows_per_dst=[1788, 11224, 0, 0, 0, 0, 200, 212] combine_rows_per_src=[16012, 15800, 8290, 16240, 15772, 958, 2586, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=75590 dup_rows=6712 dup_rows_remote=6710 dedup_ratio=0.068278 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 50, 767, 0, 1832, 0, 155, 7851, 7942, 0, 0, 0, 0, 0, 857, 0, 7697, 0, 50, 0, 7255, 0, 3, 0, 0, 4567, 7080, 78, 0, 0, 0, 0, 0, 1, 0, 59, 0, 1087, 0, 435, 0, 0, 0, 0, 0, 0, 0, 233, 189, 0, 334, 367] payload_rows_per_dst=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] dup_rows_per_dst=[0, 746, 2580, 0, 162, 2, 2310, 912] combine_rows_per_src=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94010 payload_rows_remote=78500 dup_rows=4294 dup_rows_remote=4294 dedup_ratio=0.043681 router_tokens_per_expert=[0, 0, 415, 0, 0, 0, 827, 0, 0, 0, 157, 94, 7357, 0, 0, 6726, 47, 0, 0, 0, 0, 4477, 0, 0, 0, 0, 0, 8101, 0, 0, 7638, 0, 0, 0, 91, 0, 0, 7664, 0, 0, 0, 0, 568, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 371, 0, 91, 0, 3101, 0, 0, 464, 963, 0, 0] payload_rows_per_dst=[15958, 12662, 9576, 0, 15510, 16384, 14984, 8936] dup_rows_per_dst=[1802, 0, 764, 0, 0, 954, 654, 120] combine_rows_per_src=[15958, 12662, 9576, 0, 15510, 16384, 14984, 8936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=297951232 dispatch_meta_bytes=330336 combine_payload_bytes=297951232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298281568 ingress_bytes=332554240 link_bytes=332554240 token_bytes=596232800 weight_movement_bytes=34603008 total_nvlink_bytes=630835808 weight_to_token_ratio=0.058036 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251374832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93392 payload_rows_remote=81200 dup_rows=4912 dup_rows_remote=4282 dedup_ratio=0.049967 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 8, 0, 0, 0, 107, 0, 0, 66, 635, 0, 1495, 0, 182, 7712, 7906, 0, 0, 0, 0, 0, 593, 0, 7718, 0, 66, 0, 6783, 0, 0, 0, 224, 6096, 7092, 91, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1018, 0, 545, 0, 0, 0, 0, 0, 0, 0, 36, 105, 0, 315, 301] payload_rows_per_dst=[13590, 15764, 3354, 1318, 12192, 15920, 16070, 15184] dup_rows_per_dst=[0, 1276, 0, 0, 630, 0, 2492, 514] combine_rows_per_src=[13590, 15764, 3354, 1318, 12192, 15920, 16070, 15184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=387309568 dispatch_meta_bytes=393216 combine_payload_bytes=387309568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=387702784 ingress_bytes=491118592 link_bytes=491118592 token_bytes=775012352 weight_movement_bytes=103809024 total_nvlink_bytes=878821376 weight_to_token_ratio=0.133945 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.848039723e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=297951232 dispatch_meta_bytes=330336 combine_payload_bytes=297951232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298281568 ingress_bytes=332554240 link_bytes=332554240 token_bytes=596232800 weight_movement_bytes=34603008 total_nvlink_bytes=630835808 weight_to_token_ratio=0.058036 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.037500 gemm_active_groups=4 gemm_peak_group_rows=44928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251374832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=364077056 dispatch_meta_bytes=369360 combine_payload_bytes=364077056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=364446416 ingress_bytes=398680064 link_bytes=398680064 token_bytes=728523472 weight_movement_bytes=34603008 total_nvlink_bytes=763126480 weight_to_token_ratio=0.047497 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.500200984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=61244 dup_rows=23912 dup_rows_remote=17080 dedup_ratio=0.243245 router_tokens_per_expert=[3817, 422, 0, 0, 0, 0, 1632, 8, 0, 1176, 0, 3105, 0, 0, 268, 0, 0, 154, 80, 0, 0, 465, 0, 0, 0, 0, 6856, 268, 547, 0, 128, 3396, 0, 0, 0, 0, 0, 0, 0, 2307, 0, 0, 1851, 141, 7004, 77, 7340, 0, 7005, 0, 0, 165, 0, 77, 0, 0, 82, 0, 0, 0, 111, 0, 0, 670] payload_rows_per_dst=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] dup_rows_per_dst=[6832, 7674, 524, 7268, 1296, 164, 154, 0] combine_rows_per_src=[13148, 15434, 15554, 15122, 9104, 3974, 330, 1726] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=282951680 dispatch_meta_bytes=285120 combine_payload_bytes=282951680 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=283236800 ingress_bytes=386760704 link_bytes=386760704 token_bytes=566188480 weight_movement_bytes=103809024 total_nvlink_bytes=669997504 weight_to_token_ratio=0.183347 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.455349392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=75590 dup_rows=6712 dup_rows_remote=6710 dedup_ratio=0.068278 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 50, 767, 0, 1832, 0, 155, 7851, 7942, 0, 0, 0, 0, 0, 857, 0, 7697, 0, 50, 0, 7255, 0, 3, 0, 0, 4567, 7080, 78, 0, 0, 0, 0, 0, 1, 0, 59, 0, 1087, 0, 435, 0, 0, 0, 0, 0, 0, 0, 233, 189, 0, 334, 367] payload_rows_per_dst=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] dup_rows_per_dst=[0, 746, 2580, 0, 162, 2, 2310, 912] combine_rows_per_src=[14518, 5490, 13012, 1814, 9134, 16002, 16128, 15494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81148 payload_rows_remote=66082 dup_rows=17156 dup_rows_remote=16566 dedup_ratio=0.174520 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 111, 0, 0, 57, 732, 0, 1420, 0, 169, 7765, 7951, 0, 0, 0, 0, 0, 642, 0, 7719, 0, 57, 0, 6921, 0, 1, 0, 138, 6005, 7062, 85, 0, 0, 0, 0, 0, 0, 0, 49, 0, 983, 0, 517, 0, 0, 0, 0, 0, 0, 0, 52, 84, 0, 346, 284] payload_rows_per_dst=[4, 1766, 15694, 1398, 15120, 16000, 16100, 15066] dup_rows_per_dst=[0, 34, 2734, 0, 11460, 0, 2338, 590] combine_rows_per_src=[4, 1766, 15694, 1398, 15120, 16000, 16100, 15066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=329200 combine_payload_bytes=309616640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309945840 ingress_bytes=344219648 link_bytes=344219648 token_bytes=619562480 weight_movement_bytes=34603008 total_nvlink_bytes=654165488 weight_to_token_ratio=0.055851 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295270823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=321536000 dispatch_meta_bytes=331176 combine_payload_bytes=321536000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=321867176 ingress_bytes=425345024 link_bytes=425345024 token_bytes=643403176 weight_movement_bytes=103809024 total_nvlink_bytes=747212200 weight_to_token_ratio=0.161344 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.600539082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=332595200 dispatch_meta_bytes=341928 combine_payload_bytes=332595200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=332937128 ingress_bytes=471007232 link_bytes=471007232 token_bytes=665532328 weight_movement_bytes=138412032 total_nvlink_bytes=803944360 weight_to_token_ratio=0.207972 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.772362295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=250855424 dispatch_meta_bytes=313296 combine_payload_bytes=250855424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=251168720 ingress_bytes=285458432 link_bytes=285458432 token_bytes=502024144 weight_movement_bytes=34603008 total_nvlink_bytes=536627152 weight_to_token_ratio=0.068927 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.074157098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70446 payload_rows_remote=54368 dup_rows=27858 dup_rows_remote=25962 dedup_ratio=0.283386 router_tokens_per_expert=[0, 0, 439, 0, 0, 0, 830, 0, 0, 0, 136, 82, 7351, 0, 0, 6782, 2, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 8114, 0, 0, 7718, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 77, 0, 3116, 0, 0, 468, 1452, 0, 0] payload_rows_per_dst=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] dup_rows_per_dst=[1896, 12736, 0, 12880, 0, 0, 150, 196] combine_rows_per_src=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=270671872 dispatch_meta_bytes=330592 combine_payload_bytes=270671872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271002464 ingress_bytes=305274880 link_bytes=305274880 token_bytes=541674336 weight_movement_bytes=34603008 total_nvlink_bytes=576277344 weight_to_token_ratio=0.063882 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148724797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=73398 dup_rows=8814 dup_rows_remote=6310 dedup_ratio=0.089661 router_tokens_per_expert=[73, 0, 1, 0, 3, 0, 6, 0, 0, 0, 190, 0, 0, 37, 667, 0, 1666, 0, 214, 7607, 7787, 0, 3, 1, 0, 1, 683, 0, 7681, 0, 38, 0, 6876, 1, 0, 0, 235, 5560, 7060, 107, 0, 0, 0, 3, 2, 1, 0, 66, 0, 1051, 1, 563, 0, 2, 0, 0, 1, 0, 2, 104, 153, 3, 340, 363] payload_rows_per_dst=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] dup_rows_per_dst=[2, 20, 3086, 2, 2520, 2, 2504, 678] combine_rows_per_src=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=222691328 dispatch_meta_bytes=321320 combine_payload_bytes=222691328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223012648 ingress_bytes=257294336 link_bytes=257294336 token_bytes=445703976 weight_movement_bytes=34603008 total_nvlink_bytes=480306984 weight_to_token_ratio=0.077637 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.681778725e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=329200 combine_payload_bytes=309616640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309945840 ingress_bytes=344219648 link_bytes=344219648 token_bytes=619562480 weight_movement_bytes=34603008 total_nvlink_bytes=654165488 weight_to_token_ratio=0.055851 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.295270823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92108 payload_rows_remote=84856 dup_rows=6196 dup_rows_remote=4550 dedup_ratio=0.063029 router_tokens_per_expert=[12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 390, 0, 0, 50, 629, 0, 1306, 0, 129, 8025, 8003, 0, 0, 0, 0, 0, 1028, 0, 7742, 0, 50, 0, 7598, 0, 3, 0, 0, 4527, 7047, 65, 0, 0, 0, 0, 0, 0, 0, 62, 0, 1120, 0, 308, 0, 0, 0, 0, 0, 0, 0, 242, 141, 0, 346, 329] payload_rows_per_dst=[15220, 16252, 2870, 7252, 5064, 16130, 14034, 15286] dup_rows_per_dst=[0, 1936, 0, 1646, 72, 0, 1618, 924] combine_rows_per_src=[15220, 16252, 2870, 7252, 5064, 16130, 14034, 15286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89490 payload_rows_remote=73398 dup_rows=8814 dup_rows_remote=6310 dedup_ratio=0.089661 router_tokens_per_expert=[73, 0, 1, 0, 3, 0, 6, 0, 0, 0, 190, 0, 0, 37, 667, 0, 1666, 0, 214, 7607, 7787, 0, 3, 1, 0, 1, 683, 0, 7681, 0, 38, 0, 6876, 1, 0, 0, 235, 5560, 7060, 107, 0, 0, 0, 3, 2, 1, 0, 66, 0, 1051, 1, 563, 0, 2, 0, 0, 1, 0, 2, 104, 153, 3, 340, 363] payload_rows_per_dst=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] dup_rows_per_dst=[2, 20, 3086, 2, 2520, 2, 2504, 678] combine_rows_per_src=[11378, 1768, 15896, 1442, 11824, 15716, 16092, 15374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=270671872 dispatch_meta_bytes=330592 combine_payload_bytes=270671872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271002464 ingress_bytes=305274880 link_bytes=305274880 token_bytes=541674336 weight_movement_bytes=34603008 total_nvlink_bytes=576277344 weight_to_token_ratio=0.063882 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148724797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84930 payload_rows_remote=81982 dup_rows=13374 dup_rows_remote=13374 dedup_ratio=0.136047 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 51, 716, 0, 1327, 0, 147, 7985, 7947, 0, 0, 0, 0, 0, 974, 0, 7769, 0, 51, 0, 7566, 0, 2, 0, 0, 4635, 7032, 74, 0, 0, 0, 0, 0, 0, 0, 87, 0, 1033, 0, 401, 0, 0, 0, 0, 0, 0, 0, 217, 156, 0, 324, 292] payload_rows_per_dst=[15140, 16236, 2948, 16384, 1348, 16068, 2698, 14108] dup_rows_per_dst=[0, 1992, 0, 10474, 4, 0, 170, 734] combine_rows_per_src=[15140, 16236, 2948, 16384, 1348, 16068, 2698, 14108] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81850 payload_rows_remote=65734 dup_rows=16454 dup_rows_remote=15024 dedup_ratio=0.167379 router_tokens_per_expert=[7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 59, 799, 0, 1280, 0, 144, 7818, 7987, 0, 0, 0, 0, 0, 632, 0, 7758, 0, 59, 0, 7219, 0, 0, 0, 61, 5842, 7065, 72, 0, 0, 0, 0, 0, 0, 0, 53, 0, 933, 0, 443, 0, 0, 0, 0, 0, 0, 0, 100, 83, 0, 347, 294] payload_rows_per_dst=[14452, 16116, 14422, 16384, 14274, 2336, 2218, 1648] dup_rows_per_dst=[0, 1430, 2170, 12198, 122, 0, 534, 0] combine_rows_per_src=[14452, 16116, 14422, 16384, 14274, 2336, 2218, 1648] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90776 payload_rows_remote=75058 dup_rows=7528 dup_rows_remote=3650 dedup_ratio=0.076579 router_tokens_per_expert=[0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 639, 0, 0, 0, 265, 2330, 7310, 701, 0, 0, 7718, 0, 0, 7327, 112, 50, 0, 0, 0, 0, 199, 0, 0, 0, 0, 6927, 0, 2394, 0, 0, 0, 0, 77, 0, 0, 0, 0, 236, 6698, 0, 253, 152, 4502, 77, 0, 0, 0, 553, 4, 281] payload_rows_per_dst=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] dup_rows_per_dst=[382, 870, 834, 0, 154, 3878, 902, 508] combine_rows_per_src=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=300638208 dispatch_meta_bytes=318832 combine_payload_bytes=300638208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300957040 ingress_bytes=335241216 link_bytes=335241216 token_bytes=601595248 weight_movement_bytes=34603008 total_nvlink_bytes=636198256 weight_to_token_ratio=0.057519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93392 payload_rows_remote=81200 dup_rows=4912 dup_rows_remote=4282 dedup_ratio=0.049967 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 8, 0, 0, 0, 107, 0, 0, 66, 635, 0, 1495, 0, 182, 7712, 7906, 0, 0, 0, 0, 0, 593, 0, 7718, 0, 66, 0, 6783, 0, 0, 0, 224, 6096, 7092, 91, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1018, 0, 545, 0, 0, 0, 0, 0, 0, 0, 36, 105, 0, 315, 301] payload_rows_per_dst=[13590, 15764, 3354, 1318, 12192, 15920, 16070, 15184] dup_rows_per_dst=[0, 1276, 0, 0, 630, 0, 2492, 514] combine_rows_per_src=[13590, 15764, 3354, 1318, 12192, 15920, 16070, 15184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=347570176 dispatch_meta_bytes=357624 combine_payload_bytes=347570176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=347927800 ingress_bytes=416776192 link_bytes=416776192 token_bytes=695497976 weight_movement_bytes=69206016 total_nvlink_bytes=764703992 weight_to_token_ratio=0.099506 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.568295257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80220 payload_rows_remote=64254 dup_rows=18084 dup_rows_remote=10896 dedup_ratio=0.183960 router_tokens_per_expert=[0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 85, 0, 84, 503, 0, 0, 0, 360, 2863, 7376, 776, 0, 0, 7887, 0, 0, 7397, 109, 57, 0, 0, 0, 0, 184, 0, 0, 0, 0, 6904, 0, 2105, 0, 0, 0, 0, 84, 0, 0, 0, 0, 306, 6651, 0, 205, 148, 3836, 84, 0, 0, 0, 774, 1, 231] payload_rows_per_dst=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] dup_rows_per_dst=[0, 168, 6904, 218, 168, 3438, 0, 7188] combine_rows_per_src=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92260 payload_rows_remote=78118 dup_rows=6044 dup_rows_remote=5864 dedup_ratio=0.061483 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 1, 91, 0, 90, 465, 0, 0, 0, 402, 3065, 7438, 765, 0, 1, 7925, 0, 0, 7369, 158, 66, 0, 0, 0, 0, 225, 0, 0, 0, 0, 6936, 0, 1978, 0, 0, 0, 0, 91, 0, 0, 0, 0, 316, 6614, 0, 189, 176, 3694, 91, 0, 0, 0, 711, 0, 219] payload_rows_per_dst=[15850, 15540, 7952, 400, 14142, 15684, 13692, 9000] dup_rows_per_dst=[152, 630, 514, 48, 180, 3192, 898, 430] combine_rows_per_src=[15850, 15540, 7952, 400, 14142, 15684, 13692, 9000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=300638208 dispatch_meta_bytes=318832 combine_payload_bytes=300638208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=300957040 ingress_bytes=335241216 link_bytes=335241216 token_bytes=601595248 weight_movement_bytes=34603008 total_nvlink_bytes=636198256 weight_to_token_ratio=0.057519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=335798272 dispatch_meta_bytes=381424 combine_payload_bytes=335798272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=336179696 ingress_bytes=439607296 link_bytes=439607296 token_bytes=671977968 weight_movement_bytes=103809024 total_nvlink_bytes=775786992 weight_to_token_ratio=0.154483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654206864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=323032 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269569496 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538815960 weight_movement_bytes=34603008 total_nvlink_bytes=573418968 weight_to_token_ratio=0.064220 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=70446 payload_rows_remote=54368 dup_rows=27858 dup_rows_remote=25962 dedup_ratio=0.283386 router_tokens_per_expert=[0, 0, 439, 0, 0, 0, 830, 0, 0, 0, 136, 82, 7351, 0, 0, 6782, 2, 0, 0, 0, 0, 3759, 0, 0, 0, 0, 0, 8114, 0, 0, 7718, 0, 0, 0, 77, 0, 0, 7811, 0, 0, 0, 0, 466, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 77, 0, 3116, 0, 0, 468, 1452, 0, 0] payload_rows_per_dst=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] dup_rows_per_dst=[1896, 12736, 0, 12880, 0, 0, 150, 196] combine_rows_per_src=[16078, 15966, 7522, 16230, 2894, 932, 948, 9876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90776 payload_rows_remote=75058 dup_rows=7528 dup_rows_remote=3650 dedup_ratio=0.076579 router_tokens_per_expert=[0, 0, 0, 191, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 639, 0, 0, 0, 265, 2330, 7310, 701, 0, 0, 7718, 0, 0, 7327, 112, 50, 0, 0, 0, 0, 199, 0, 0, 0, 0, 6927, 0, 2394, 0, 0, 0, 0, 77, 0, 0, 0, 0, 236, 6698, 0, 253, 152, 4502, 77, 0, 0, 0, 553, 4, 281] payload_rows_per_dst=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] dup_rows_per_dst=[382, 870, 834, 0, 154, 3878, 902, 508] combine_rows_per_src=[15436, 14248, 6850, 324, 14098, 15718, 13776, 10326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=319971328 dispatch_meta_bytes=335928 combine_payload_bytes=319971328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320307256 ingress_bytes=423780352 link_bytes=423780352 token_bytes=640278584 weight_movement_bytes=103809024 total_nvlink_bytes=744087608 weight_to_token_ratio=0.162131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594651347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80220 payload_rows_remote=64254 dup_rows=18084 dup_rows_remote=10896 dedup_ratio=0.183960 router_tokens_per_expert=[0, 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, 85, 0, 84, 503, 0, 0, 0, 360, 2863, 7376, 776, 0, 0, 7887, 0, 0, 7397, 109, 57, 0, 0, 0, 0, 184, 0, 0, 0, 0, 6904, 0, 2105, 0, 0, 0, 0, 84, 0, 0, 0, 0, 306, 6651, 0, 205, 148, 3836, 84, 0, 0, 0, 774, 1, 231] payload_rows_per_dst=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] dup_rows_per_dst=[0, 168, 6904, 218, 168, 3438, 0, 7188] combine_rows_per_src=[284, 1176, 15846, 15888, 14008, 15734, 1318, 15966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=222691328 dispatch_meta_bytes=321320 combine_payload_bytes=222691328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=223012648 ingress_bytes=257294336 link_bytes=257294336 token_bytes=445703976 weight_movement_bytes=34603008 total_nvlink_bytes=480306984 weight_to_token_ratio=0.077637 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.681778725e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=307437568 dispatch_meta_bytes=314832 combine_payload_bytes=307437568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307752400 ingress_bytes=342040576 link_bytes=342040576 token_bytes=615189968 weight_movement_bytes=34603008 total_nvlink_bytes=649792976 weight_to_token_ratio=0.056248 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=30208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.287071151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80982 payload_rows_remote=66982 dup_rows=17322 dup_rows_remote=17322 dedup_ratio=0.176208 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 59, 735, 0, 1653, 0, 154, 7822, 7962, 0, 0, 0, 0, 0, 751, 0, 7692, 0, 59, 0, 6996, 0, 1, 0, 122, 5494, 6989, 77, 0, 0, 0, 0, 0, 0, 0, 64, 0, 973, 0, 526, 0, 0, 0, 0, 0, 0, 0, 82, 96, 0, 354, 364] payload_rows_per_dst=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] dup_rows_per_dst=[0, 1492, 3286, 11608, 246, 0, 686, 4] combine_rows_per_src=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=332595200 dispatch_meta_bytes=341928 combine_payload_bytes=332595200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=332937128 ingress_bytes=471007232 link_bytes=471007232 token_bytes=665532328 weight_movement_bytes=138412032 total_nvlink_bytes=803944360 weight_to_token_ratio=0.207972 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=24832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.772362295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=300600 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263484984 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526669368 weight_movement_bytes=34603008 total_nvlink_bytes=561272376 weight_to_token_ratio=0.065702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84846 payload_rows_remote=73218 dup_rows=13458 dup_rows_remote=12648 dedup_ratio=0.136902 router_tokens_per_expert=[0, 0, 1, 192, 1, 3, 0, 0, 0, 0, 0, 107, 1, 107, 356, 0, 0, 0, 403, 2590, 7284, 826, 1, 0, 7928, 2, 0, 7282, 187, 37, 0, 0, 0, 3, 260, 3, 1, 0, 0, 6726, 2, 2284, 0, 3, 0, 0, 108, 0, 0, 0, 0, 275, 6639, 1, 228, 156, 4203, 107, 0, 0, 2, 567, 3, 273] payload_rows_per_dst=[1984, 928, 15756, 14342, 13764, 15644, 11628, 10800] dup_rows_per_dst=[48, 214, 6452, 328, 222, 3714, 810, 1670] combine_rows_per_src=[1984, 928, 15756, 14342, 13764, 15644, 11628, 10800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274358272 dispatch_meta_bytes=337216 combine_payload_bytes=274358272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274695488 ingress_bytes=308961280 link_bytes=308961280 token_bytes=549053760 weight_movement_bytes=34603008 total_nvlink_bytes=583656768 weight_to_token_ratio=0.063023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162596423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95024 payload_rows_remote=81728 dup_rows=3280 dup_rows_remote=3278 dedup_ratio=0.033366 router_tokens_per_expert=[0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 65, 0, 65, 521, 0, 0, 0, 155, 2243, 7367, 667, 0, 0, 7876, 0, 0, 7346, 53, 50, 0, 0, 0, 0, 126, 0, 0, 0, 0, 7088, 0, 2730, 0, 0, 0, 0, 64, 0, 0, 0, 0, 242, 6734, 0, 233, 91, 4508, 64, 0, 0, 0, 443, 0, 254] payload_rows_per_dst=[15752, 15350, 5690, 13296, 14300, 6774, 13742, 10120] dup_rows_per_dst=[334, 686, 440, 2, 128, 414, 858, 418] combine_rows_per_src=[15752, 15350, 5690, 13296, 14300, 6774, 13742, 10120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299900928 dispatch_meta_bytes=343464 combine_payload_bytes=299900928 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=300244392 ingress_bytes=369106944 link_bytes=369106944 token_bytes=600145320 weight_movement_bytes=69206016 total_nvlink_bytes=669351336 weight_to_token_ratio=0.115315 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84846 payload_rows_remote=73218 dup_rows=13458 dup_rows_remote=12648 dedup_ratio=0.136902 router_tokens_per_expert=[0, 0, 1, 192, 1, 3, 0, 0, 0, 0, 0, 107, 1, 107, 356, 0, 0, 0, 403, 2590, 7284, 826, 1, 0, 7928, 2, 0, 7282, 187, 37, 0, 0, 0, 3, 260, 3, 1, 0, 0, 6726, 2, 2284, 0, 3, 0, 0, 108, 0, 0, 0, 0, 275, 6639, 1, 228, 156, 4203, 107, 0, 0, 2, 567, 3, 273] payload_rows_per_dst=[1984, 928, 15756, 14342, 13764, 15644, 11628, 10800] dup_rows_per_dst=[48, 214, 6452, 328, 222, 3714, 810, 1670] combine_rows_per_src=[1984, 928, 15756, 14342, 13764, 15644, 11628, 10800] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95366 payload_rows_remote=89170 dup_rows=2938 dup_rows_remote=2524 dedup_ratio=0.029887 router_tokens_per_expert=[0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 74, 0, 74, 545, 0, 0, 0, 173, 2497, 7373, 635, 0, 0, 7857, 0, 0, 7375, 82, 51, 0, 0, 0, 0, 155, 0, 0, 0, 0, 7067, 0, 2332, 0, 0, 0, 0, 73, 0, 0, 0, 0, 209, 6623, 0, 257, 130, 4548, 73, 0, 0, 0, 508, 0, 280] payload_rows_per_dst=[15714, 15382, 6196, 15016, 14298, 4810, 13588, 10362] dup_rows_per_dst=[322, 750, 414, 0, 146, 0, 850, 456] combine_rows_per_src=[15714, 15382, 6196, 15016, 14298, 4810, 13588, 10362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90948 payload_rows_remote=75470 dup_rows=7356 dup_rows_remote=6644 dedup_ratio=0.074829 router_tokens_per_expert=[0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 72, 0, 72, 535, 0, 0, 0, 242, 2586, 7416, 707, 0, 0, 7879, 0, 0, 7452, 83, 59, 0, 0, 0, 0, 155, 0, 0, 0, 0, 6992, 0, 2169, 0, 0, 0, 0, 72, 0, 0, 0, 0, 226, 6748, 0, 208, 136, 4283, 72, 0, 0, 0, 613, 0, 240] payload_rows_per_dst=[15758, 15478, 6678, 15258, 9422, 4482, 13842, 10030] dup_rows_per_dst=[270, 712, 392, 4712, 90, 0, 794, 386] combine_rows_per_src=[15758, 15478, 6678, 15258, 9422, 4482, 13842, 10030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92980 payload_rows_remote=76974 dup_rows=5324 dup_rows_remote=5116 dedup_ratio=0.054159 router_tokens_per_expert=[0, 693, 394, 1072, 8006, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 6683, 0, 0, 0, 0, 210, 7018, 77, 0, 0, 0, 0, 0, 2698, 0, 0, 3564, 2, 116, 0, 0, 178, 0, 0, 0, 104, 0, 0, 1251, 5, 0, 0, 0, 65, 7825, 106, 6668, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] dup_rows_per_dst=[2146, 0, 54, 0, 1330, 208, 1528, 58] combine_rows_per_src=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92260 payload_rows_remote=78118 dup_rows=6044 dup_rows_remote=5864 dedup_ratio=0.061483 router_tokens_per_expert=[0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 1, 91, 0, 90, 465, 0, 0, 0, 402, 3065, 7438, 765, 0, 1, 7925, 0, 0, 7369, 158, 66, 0, 0, 0, 0, 225, 0, 0, 0, 0, 6936, 0, 1978, 0, 0, 0, 0, 91, 0, 0, 0, 0, 316, 6614, 0, 189, 176, 3694, 91, 0, 0, 0, 711, 0, 219] payload_rows_per_dst=[15850, 15540, 7952, 400, 14142, 15684, 13692, 9000] dup_rows_per_dst=[152, 630, 514, 48, 180, 3192, 898, 430] combine_rows_per_src=[15850, 15540, 7952, 400, 14142, 15684, 13692, 9000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68446 payload_rows_remote=55080 dup_rows=29858 dup_rows_remote=28716 dedup_ratio=0.303731 router_tokens_per_expert=[0, 679, 412, 1115, 8001, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 6264, 0, 0, 0, 0, 437, 6976, 84, 0, 0, 0, 0, 0, 1885, 0, 0, 5218, 0, 145, 0, 0, 189, 0, 0, 0, 6, 0, 0, 1159, 0, 0, 0, 0, 88, 7835, 133, 6267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] dup_rows_per_dst=[1342, 0, 0, 0, 11630, 12, 15732, 1142] combine_rows_per_src=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93040 payload_rows_remote=80392 dup_rows=5264 dup_rows_remote=3240 dedup_ratio=0.053548 router_tokens_per_expert=[0, 676, 441, 1084, 7981, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 6349, 0, 0, 0, 0, 568, 6982, 91, 0, 0, 0, 0, 0, 1896, 0, 0, 5281, 0, 159, 0, 0, 228, 0, 0, 0, 2, 0, 0, 1104, 0, 0, 0, 0, 94, 7783, 142, 6033, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] dup_rows_per_dst=[2090, 0, 6, 488, 2024, 4, 548, 104] combine_rows_per_src=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=334757888 dispatch_meta_bytes=340024 combine_payload_bytes=334757888 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=335097912 ingress_bytes=438566912 link_bytes=438566912 token_bytes=669855800 weight_movement_bytes=103809024 total_nvlink_bytes=773664824 weight_to_token_ratio=0.154972 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650291982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80982 payload_rows_remote=66982 dup_rows=17322 dup_rows_remote=17322 dedup_ratio=0.176208 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 59, 735, 0, 1653, 0, 154, 7822, 7962, 0, 0, 0, 0, 0, 751, 0, 7692, 0, 59, 0, 6996, 0, 1, 0, 122, 5494, 6989, 77, 0, 0, 0, 0, 0, 0, 0, 64, 0, 973, 0, 526, 0, 0, 0, 0, 0, 0, 0, 82, 96, 0, 354, 364] payload_rows_per_dst=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] dup_rows_per_dst=[0, 1492, 3286, 11608, 246, 0, 686, 4] combine_rows_per_src=[14000, 15986, 16252, 16384, 14132, 128, 2312, 1788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299900928 dispatch_meta_bytes=343464 combine_payload_bytes=299900928 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=300244392 ingress_bytes=369106944 link_bytes=369106944 token_bytes=600145320 weight_movement_bytes=69206016 total_nvlink_bytes=669351336 weight_to_token_ratio=0.115315 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=365240320 dispatch_meta_bytes=366776 combine_payload_bytes=365240320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=365607096 ingress_bytes=434446336 link_bytes=434446336 token_bytes=730847416 weight_movement_bytes=69206016 total_nvlink_bytes=800053432 weight_to_token_ratio=0.094693 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.634786587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88724 payload_rows_remote=73080 dup_rows=9580 dup_rows_remote=9206 dedup_ratio=0.097453 router_tokens_per_expert=[0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 573, 0, 0, 0, 234, 2508, 7359, 834, 0, 0, 7822, 0, 0, 7358, 108, 59, 0, 0, 0, 0, 163, 0, 0, 0, 0, 6828, 0, 2334, 0, 0, 0, 0, 77, 0, 0, 0, 0, 298, 6633, 0, 185, 116, 4442, 77, 0, 0, 0, 578, 2, 223] payload_rows_per_dst=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] dup_rows_per_dst=[374, 816, 3906, 3418, 0, 0, 634, 432] combine_rows_per_src=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92980 payload_rows_remote=76974 dup_rows=5324 dup_rows_remote=5116 dedup_ratio=0.054159 router_tokens_per_expert=[0, 693, 394, 1072, 8006, 2336, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 6683, 0, 0, 0, 0, 210, 7018, 77, 0, 0, 0, 0, 0, 2698, 0, 0, 3564, 2, 116, 0, 0, 178, 0, 0, 0, 104, 0, 0, 1251, 5, 0, 0, 0, 65, 7825, 106, 6668, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] dup_rows_per_dst=[2146, 0, 54, 0, 1330, 208, 1528, 58] combine_rows_per_src=[6844, 16012, 13474, 574, 11430, 16006, 15150, 13490] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309125120 dispatch_meta_bytes=328456 combine_payload_bytes=309125120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309453576 ingress_bytes=343728128 link_bytes=343728128 token_bytes=618578696 weight_movement_bytes=34603008 total_nvlink_bytes=653181704 weight_to_token_ratio=0.055940 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293421273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68446 payload_rows_remote=55080 dup_rows=29858 dup_rows_remote=28716 dedup_ratio=0.303731 router_tokens_per_expert=[0, 679, 412, 1115, 8001, 2185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 6264, 0, 0, 0, 0, 437, 6976, 84, 0, 0, 0, 0, 0, 1885, 0, 0, 5218, 0, 145, 0, 0, 189, 0, 0, 0, 6, 0, 0, 1159, 0, 0, 0, 0, 88, 7835, 133, 6267, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] dup_rows_per_dst=[1342, 0, 0, 0, 11630, 12, 15732, 1142] combine_rows_per_src=[5732, 16002, 148, 1042, 15394, 378, 16384, 13366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=329285632 dispatch_meta_bytes=334528 combine_payload_bytes=329285632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329620160 ingress_bytes=398491648 link_bytes=398491648 token_bytes=658905792 weight_movement_bytes=69206016 total_nvlink_bytes=728111808 weight_to_token_ratio=0.105032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499491990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=274358272 dispatch_meta_bytes=337216 combine_payload_bytes=274358272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274695488 ingress_bytes=308961280 link_bytes=308961280 token_bytes=549053760 weight_movement_bytes=34603008 total_nvlink_bytes=583656768 weight_to_token_ratio=0.063023 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=49024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.162596423e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=315285504 dispatch_meta_bytes=328360 combine_payload_bytes=315285504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315613864 ingress_bytes=349888512 link_bytes=349888512 token_bytes=630899368 weight_movement_bytes=34603008 total_nvlink_bytes=665502376 weight_to_token_ratio=0.054847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316602302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=319971328 dispatch_meta_bytes=335928 combine_payload_bytes=319971328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320307256 ingress_bytes=423780352 link_bytes=423780352 token_bytes=640278584 weight_movement_bytes=103809024 total_nvlink_bytes=744087608 weight_to_token_ratio=0.162131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594651347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=225607680 dispatch_meta_bytes=335184 combine_payload_bytes=225607680 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=225942864 ingress_bytes=294813696 link_bytes=294813696 token_bytes=451550544 weight_movement_bytes=69206016 total_nvlink_bytes=520756560 weight_to_token_ratio=0.153263 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=47360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.109360204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75000 payload_rows_remote=58638 dup_rows=23304 dup_rows_remote=7892 dedup_ratio=0.237061 router_tokens_per_expert=[0, 710, 401, 1195, 7977, 2290, 0, 0, 0, 0, 1, 0, 1, 0, 1, 2, 3, 1, 48, 0, 2, 6504, 0, 0, 0, 0, 422, 6952, 104, 0, 2, 0, 0, 0, 1898, 0, 0, 4693, 1, 86, 0, 0, 201, 0, 1, 1, 62, 0, 0, 1297, 0, 0, 1, 3, 49, 7768, 174, 6295, 0, 3, 3, 0, 0, 0] payload_rows_per_dst=[6882, 15958, 8154, 1056, 13008, 770, 16362, 12810] dup_rows_per_dst=[2310, 6, 6, 0, 5304, 126, 15412, 140] combine_rows_per_src=[6882, 15958, 8154, 1056, 13008, 770, 16362, 12810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299335680 dispatch_meta_bytes=329144 combine_payload_bytes=299335680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299664824 ingress_bytes=333938688 link_bytes=333938688 token_bytes=599000504 weight_movement_bytes=34603008 total_nvlink_bytes=633603512 weight_to_token_ratio=0.057768 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92648 payload_rows_remote=78392 dup_rows=5656 dup_rows_remote=2958 dedup_ratio=0.057536 router_tokens_per_expert=[0, 626, 386, 1086, 8007, 2167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 137, 0, 0, 6893, 0, 0, 0, 0, 180, 7017, 64, 0, 0, 0, 0, 0, 3031, 0, 0, 3300, 0, 129, 0, 0, 131, 0, 0, 0, 52, 0, 0, 1101, 8, 0, 0, 0, 79, 7920, 67, 6771, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6574, 16014, 13924, 14256, 9748, 16102, 2360, 13670] dup_rows_per_dst=[1956, 0, 136, 2698, 740, 104, 16, 6] combine_rows_per_src=[6574, 16014, 13924, 14256, 9748, 16102, 2360, 13670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=240181248 dispatch_meta_bytes=266120 combine_payload_bytes=240181248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=240447368 ingress_bytes=309387264 link_bytes=309387264 token_bytes=480628616 weight_movement_bytes=69206016 total_nvlink_bytes=549834632 weight_to_token_ratio=0.143991 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88078 payload_rows_remote=73918 dup_rows=10226 dup_rows_remote=10116 dedup_ratio=0.104024 router_tokens_per_expert=[0, 657, 403, 1091, 7980, 2057, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 139, 0, 0, 6996, 0, 0, 0, 0, 285, 6987, 73, 0, 0, 0, 0, 0, 2648, 0, 0, 3915, 4, 137, 0, 0, 168, 0, 0, 0, 82, 0, 0, 1020, 0, 0, 0, 0, 86, 7855, 106, 6463, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6398, 15960, 14160, 14652, 5578, 16046, 2212, 13072] dup_rows_per_dst=[2018, 0, 110, 7868, 0, 164, 0, 66] combine_rows_per_src=[6398, 15960, 14160, 14652, 5578, 16046, 2212, 13072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75000 payload_rows_remote=58638 dup_rows=23304 dup_rows_remote=7892 dedup_ratio=0.237061 router_tokens_per_expert=[0, 710, 401, 1195, 7977, 2290, 0, 0, 0, 0, 1, 0, 1, 0, 1, 2, 3, 1, 48, 0, 2, 6504, 0, 0, 0, 0, 422, 6952, 104, 0, 2, 0, 0, 0, 1898, 0, 0, 4693, 1, 86, 0, 0, 201, 0, 1, 1, 62, 0, 0, 1297, 0, 0, 1, 3, 49, 7768, 174, 6295, 0, 3, 3, 0, 0, 0] payload_rows_per_dst=[6882, 15958, 8154, 1056, 13008, 770, 16362, 12810] dup_rows_per_dst=[2310, 6, 6, 0, 5304, 126, 15412, 140] combine_rows_per_src=[6882, 15958, 8154, 1056, 13008, 770, 16362, 12810] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88100 payload_rows_remote=76802 dup_rows=10204 dup_rows_remote=10124 dedup_ratio=0.103800 router_tokens_per_expert=[3014, 0, 0, 0, 0, 0, 0, 1463, 438, 0, 507, 0, 0, 0, 0, 981, 0, 0, 0, 4971, 193, 356, 0, 6391, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 0, 7817, 0, 0, 0, 7333, 0, 0, 62, 0, 0, 0, 0, 2741, 0, 7728, 0, 0, 0, 0, 0, 3517, 143, 445, 0, 78] payload_rows_per_dst=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] dup_rows_per_dst=[560, 878, 0, 0, 1540, 80, 5482, 1664] combine_rows_per_src=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81498 payload_rows_remote=79108 dup_rows=16806 dup_rows_remote=16806 dedup_ratio=0.170959 router_tokens_per_expert=[0, 657, 430, 987, 8008, 2081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 6494, 0, 0, 0, 0, 222, 6999, 72, 0, 0, 0, 0, 0, 2154, 0, 0, 4749, 0, 154, 0, 0, 175, 0, 0, 0, 48, 0, 0, 1090, 0, 0, 0, 0, 95, 7870, 112, 6647, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14898, 2390, 13178, 14518, 4616, 16090, 2370, 13438] dup_rows_per_dst=[7038, 0, 26, 9566, 0, 96, 0, 80] combine_rows_per_src=[14898, 2390, 13178, 14518, 4616, 16090, 2370, 13438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93040 payload_rows_remote=80392 dup_rows=5264 dup_rows_remote=3240 dedup_ratio=0.053548 router_tokens_per_expert=[0, 676, 441, 1084, 7981, 2160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 0, 6349, 0, 0, 0, 0, 568, 6982, 91, 0, 0, 0, 0, 0, 1896, 0, 0, 5281, 0, 159, 0, 0, 228, 0, 0, 0, 2, 0, 0, 1104, 0, 0, 0, 0, 94, 7783, 142, 6033, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] dup_rows_per_dst=[2090, 0, 6, 488, 2024, 4, 548, 104] combine_rows_per_src=[6632, 15962, 12886, 8982, 12648, 16022, 7660, 12248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84530 payload_rows_remote=68150 dup_rows=13774 dup_rows_remote=7792 dedup_ratio=0.140116 router_tokens_per_expert=[1471, 0, 0, 0, 0, 0, 0, 1628, 505, 0, 405, 0, 0, 0, 0, 1132, 0, 0, 0, 5820, 287, 381, 0, 6383, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 7345, 0, 0, 52, 0, 0, 0, 0, 2954, 0, 7791, 0, 0, 0, 0, 0, 3163, 172, 416, 0, 85] payload_rows_per_dst=[5946, 3072, 12832, 12766, 5330, 12618, 15586, 16380] dup_rows_per_dst=[252, 1012, 80, 0, 386, 94, 5968, 5982] combine_rows_per_src=[5946, 3072, 12832, 12766, 5330, 12618, 15586, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321093632 dispatch_meta_bytes=325400 combine_payload_bytes=321093632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=321419032 ingress_bytes=390299648 link_bytes=390299648 token_bytes=642512664 weight_movement_bytes=69206016 total_nvlink_bytes=711718680 weight_to_token_ratio=0.107712 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.468666154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86976 payload_rows_remote=71070 dup_rows=11328 dup_rows_remote=4312 dedup_ratio=0.115234 router_tokens_per_expert=[1595, 0, 0, 0, 0, 0, 0, 1627, 483, 0, 404, 0, 0, 0, 0, 1144, 0, 0, 0, 5961, 248, 428, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 7417, 0, 0, 63, 0, 0, 0, 0, 2923, 0, 7727, 0, 0, 0, 0, 0, 3119, 185, 408, 0, 91] payload_rows_per_dst=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] dup_rows_per_dst=[162, 1488, 86, 0, 7016, 108, 2286, 182] combine_rows_per_src=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88724 payload_rows_remote=73080 dup_rows=9580 dup_rows_remote=9206 dedup_ratio=0.097453 router_tokens_per_expert=[0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 77, 0, 77, 573, 0, 0, 0, 234, 2508, 7359, 834, 0, 0, 7822, 0, 0, 7358, 108, 59, 0, 0, 0, 0, 163, 0, 0, 0, 0, 6828, 0, 2334, 0, 0, 0, 0, 77, 0, 0, 0, 0, 298, 6633, 0, 185, 116, 4442, 77, 0, 0, 0, 578, 2, 223] payload_rows_per_dst=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] dup_rows_per_dst=[374, 816, 3906, 3418, 0, 0, 634, 432] combine_rows_per_src=[15644, 15356, 13480, 15054, 3184, 4822, 10972, 10212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88100 payload_rows_remote=76802 dup_rows=10204 dup_rows_remote=10124 dedup_ratio=0.103800 router_tokens_per_expert=[3014, 0, 0, 0, 0, 0, 0, 1463, 438, 0, 507, 0, 0, 0, 0, 981, 0, 0, 0, 4971, 193, 356, 0, 6391, 0, 0, 0, 0, 0, 0, 0, 0, 974, 0, 0, 0, 0, 0, 0, 7817, 0, 0, 0, 7333, 0, 0, 62, 0, 0, 0, 0, 2741, 0, 7728, 0, 0, 0, 0, 0, 3517, 143, 445, 0, 78] payload_rows_per_dst=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] dup_rows_per_dst=[560, 878, 0, 0, 1540, 80, 5482, 1664] combine_rows_per_src=[8394, 2974, 11040, 12782, 16042, 11298, 15456, 10114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=302768128 dispatch_meta_bytes=336136 combine_payload_bytes=302768128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303104264 ingress_bytes=371974144 link_bytes=371974144 token_bytes=605872392 weight_movement_bytes=69206016 total_nvlink_bytes=675078408 weight_to_token_ratio=0.114225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399708757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73386 payload_rows_remote=57002 dup_rows=24918 dup_rows_remote=15858 dedup_ratio=0.253479 router_tokens_per_expert=[0, 621, 407, 1195, 7990, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6559, 0, 0, 0, 0, 200, 6933, 77, 0, 0, 0, 0, 0, 2286, 0, 0, 4092, 0, 141, 0, 0, 168, 0, 0, 0, 60, 0, 0, 1202, 1, 0, 0, 0, 82, 7860, 127, 6553, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] dup_rows_per_dst=[9060, 0, 7476, 8234, 0, 120, 2, 26] combine_rows_per_src=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314580992 dispatch_meta_bytes=347704 combine_payload_bytes=314580992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314928696 ingress_bytes=383787008 link_bytes=383787008 token_bytes=629509688 weight_movement_bytes=69206016 total_nvlink_bytes=698715704 weight_to_token_ratio=0.109936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444159614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=240181248 dispatch_meta_bytes=266120 combine_payload_bytes=240181248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=240447368 ingress_bytes=309387264 link_bytes=309387264 token_bytes=480628616 weight_movement_bytes=69206016 total_nvlink_bytes=549834632 weight_to_token_ratio=0.143991 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164199367e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=324026368 dispatch_meta_bytes=383656 combine_payload_bytes=324026368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=324410024 ingress_bytes=358629376 link_bytes=358629376 token_bytes=648436392 weight_movement_bytes=34603008 total_nvlink_bytes=683039400 weight_to_token_ratio=0.053364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.349493470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=329285632 dispatch_meta_bytes=334528 combine_payload_bytes=329285632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=329620160 ingress_bytes=398491648 link_bytes=398491648 token_bytes=658905792 weight_movement_bytes=69206016 total_nvlink_bytes=728111808 weight_to_token_ratio=0.105032 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.499491990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84530 payload_rows_remote=68150 dup_rows=13774 dup_rows_remote=7792 dedup_ratio=0.140116 router_tokens_per_expert=[1471, 0, 0, 0, 0, 0, 0, 1628, 505, 0, 405, 0, 0, 0, 0, 1132, 0, 0, 0, 5820, 287, 381, 0, 6383, 0, 0, 0, 0, 0, 0, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 7785, 0, 0, 0, 7345, 0, 0, 52, 0, 0, 0, 0, 2954, 0, 7791, 0, 0, 0, 0, 0, 3163, 172, 416, 0, 85] payload_rows_per_dst=[5946, 3072, 12832, 12766, 5330, 12618, 15586, 16380] dup_rows_per_dst=[252, 1012, 80, 0, 386, 94, 5968, 5982] combine_rows_per_src=[5946, 3072, 12832, 12766, 5330, 12618, 15586, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=299335680 dispatch_meta_bytes=329144 combine_payload_bytes=299335680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299664824 ingress_bytes=333938688 link_bytes=333938688 token_bytes=599000504 weight_movement_bytes=34603008 total_nvlink_bytes=633603512 weight_to_token_ratio=0.057768 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=291102720 dispatch_meta_bytes=301528 combine_payload_bytes=291102720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291404248 ingress_bytes=394911744 link_bytes=394911744 token_bytes=582506968 weight_movement_bytes=103809024 total_nvlink_bytes=686315992 weight_to_token_ratio=0.178211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486021100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=314580992 dispatch_meta_bytes=347704 combine_payload_bytes=314580992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314928696 ingress_bytes=383787008 link_bytes=383787008 token_bytes=629509688 weight_movement_bytes=69206016 total_nvlink_bytes=698715704 weight_to_token_ratio=0.109936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.027500 gemm_active_groups=7 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444159614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=279142400 dispatch_meta_bytes=303768 combine_payload_bytes=279142400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279446168 ingress_bytes=313745408 link_bytes=313745408 token_bytes=558588568 weight_movement_bytes=34603008 total_nvlink_bytes=593191576 weight_to_token_ratio=0.061947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180598712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84020 payload_rows_remote=68646 dup_rows=14284 dup_rows_remote=9942 dedup_ratio=0.145304 router_tokens_per_expert=[2540, 2, 0, 0, 0, 0, 0, 1463, 512, 3, 504, 0, 1, 0, 1, 1131, 0, 2, 0, 5421, 294, 403, 0, 6424, 0, 0, 0, 0, 0, 0, 0, 1, 1049, 2, 0, 0, 0, 0, 0, 7768, 0, 3, 0, 7308, 2, 3, 73, 0, 0, 0, 0, 2170, 1, 7686, 0, 1, 0, 0, 0, 3652, 156, 469, 0, 107] payload_rows_per_dst=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] dup_rows_per_dst=[240, 1030, 12, 2, 1622, 10, 4342, 7026] combine_rows_per_src=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233480192 dispatch_meta_bytes=291440 combine_payload_bytes=233480192 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=233771632 ingress_bytes=302686208 link_bytes=302686208 token_bytes=467251824 weight_movement_bytes=69206016 total_nvlink_bytes=536457840 weight_to_token_ratio=0.148113 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138983832e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92748 payload_rows_remote=79304 dup_rows=5556 dup_rows_remote=5556 dedup_ratio=0.056519 router_tokens_per_expert=[3043, 0, 0, 0, 0, 0, 0, 1377, 456, 0, 556, 0, 0, 0, 0, 942, 0, 0, 0, 4579, 250, 278, 0, 6722, 0, 0, 0, 0, 0, 0, 0, 0, 759, 0, 0, 0, 0, 0, 0, 7789, 0, 0, 0, 7328, 0, 0, 71, 0, 0, 0, 0, 2949, 0, 7720, 0, 0, 0, 0, 0, 3725, 143, 400, 0, 65] payload_rows_per_dst=[8314, 15690, 10214, 13444, 15996, 14656, 5898, 8536] dup_rows_per_dst=[526, 3658, 0, 0, 1100, 142, 0, 130] combine_rows_per_src=[8314, 15690, 10214, 13444, 15996, 14656, 5898, 8536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281174016 dispatch_meta_bytes=314352 combine_payload_bytes=281174016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281488368 ingress_bytes=384983040 link_bytes=384983040 token_bytes=562662384 weight_movement_bytes=103809024 total_nvlink_bytes=666471408 weight_to_token_ratio=0.184496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90924 payload_rows_remote=80758 dup_rows=7380 dup_rows_remote=7380 dedup_ratio=0.075073 router_tokens_per_expert=[2385, 0, 0, 0, 0, 0, 0, 1296, 490, 0, 436, 0, 0, 0, 0, 1090, 0, 0, 0, 4515, 254, 314, 0, 6911, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 7298, 0, 0, 88, 0, 0, 0, 0, 3354, 0, 7708, 0, 0, 0, 0, 0, 3820, 159, 370, 0, 74] payload_rows_per_dst=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] dup_rows_per_dst=[2410, 3654, 0, 0, 962, 176, 0, 178] combine_rows_per_src=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92684 payload_rows_remote=86656 dup_rows=5620 dup_rows_remote=5506 dedup_ratio=0.057170 router_tokens_per_expert=[0, 0, 7653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 340, 0, 0, 0, 0, 0, 0, 6513, 7252, 0, 77, 0, 0, 0, 59, 0, 112, 206, 0, 0, 0, 7584, 8065, 0, 0, 370, 0, 0, 251, 0, 78, 0, 0, 0, 1594, 0, 0, 0, 0, 0] payload_rows_per_dst=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] dup_rows_per_dst=[0, 60, 572, 358, 272, 114, 1080, 3164] combine_rows_per_src=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90924 payload_rows_remote=80758 dup_rows=7380 dup_rows_remote=7380 dedup_ratio=0.075073 router_tokens_per_expert=[2385, 0, 0, 0, 0, 0, 0, 1296, 490, 0, 436, 0, 0, 0, 0, 1090, 0, 0, 0, 4515, 254, 314, 0, 6911, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 7737, 0, 0, 0, 7298, 0, 0, 88, 0, 0, 0, 0, 3354, 0, 7708, 0, 0, 0, 0, 0, 3820, 159, 370, 0, 74] payload_rows_per_dst=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] dup_rows_per_dst=[2410, 3654, 0, 0, 962, 176, 0, 178] combine_rows_per_src=[9698, 15794, 10166, 13822, 11472, 14596, 6708, 8668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=70502 dup_rows=11796 dup_rows_remote=7920 dedup_ratio=0.119995 router_tokens_per_expert=[2503, 0, 0, 0, 0, 0, 0, 1601, 481, 0, 535, 0, 0, 0, 0, 1075, 0, 0, 0, 5017, 277, 345, 0, 6586, 0, 0, 0, 0, 0, 0, 0, 0, 1171, 0, 0, 0, 0, 0, 0, 7686, 0, 0, 0, 7320, 0, 0, 53, 0, 0, 0, 0, 2878, 0, 7850, 0, 0, 0, 0, 0, 3160, 167, 375, 0, 72] payload_rows_per_dst=[15926, 16006, 11262, 13172, 2342, 14640, 5756, 7404] dup_rows_per_dst=[7654, 3876, 16, 0, 0, 106, 0, 144] combine_rows_per_src=[15926, 16006, 11262, 13172, 2342, 14640, 5756, 7404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86976 payload_rows_remote=71070 dup_rows=11328 dup_rows_remote=4312 dedup_ratio=0.115234 router_tokens_per_expert=[1595, 0, 0, 0, 0, 0, 0, 1627, 483, 0, 404, 0, 0, 0, 0, 1144, 0, 0, 0, 5961, 248, 428, 0, 6520, 0, 0, 0, 0, 0, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 7562, 0, 0, 0, 7417, 0, 0, 63, 0, 0, 0, 0, 2923, 0, 7727, 0, 0, 0, 0, 0, 3119, 185, 408, 0, 91] payload_rows_per_dst=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] dup_rows_per_dst=[162, 1488, 86, 0, 7016, 108, 2286, 182] combine_rows_per_src=[6282, 6258, 13188, 13040, 15906, 14852, 10026, 7424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84020 payload_rows_remote=68646 dup_rows=14284 dup_rows_remote=9942 dedup_ratio=0.145304 router_tokens_per_expert=[2540, 2, 0, 0, 0, 0, 0, 1463, 512, 3, 504, 0, 1, 0, 1, 1131, 0, 2, 0, 5421, 294, 403, 0, 6424, 0, 0, 0, 0, 0, 0, 0, 1, 1049, 2, 0, 0, 0, 0, 0, 7768, 0, 3, 0, 7308, 2, 3, 73, 0, 0, 0, 0, 2170, 1, 7686, 0, 1, 0, 0, 0, 3652, 156, 469, 0, 107] payload_rows_per_dst=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] dup_rows_per_dst=[240, 1030, 12, 2, 1622, 10, 4342, 7026] combine_rows_per_src=[7770, 3274, 12228, 12848, 16016, 152, 15374, 16358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324829184 dispatch_meta_bytes=339440 combine_payload_bytes=324829184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325168624 ingress_bytes=359432192 link_bytes=359432192 token_bytes=649997808 weight_movement_bytes=34603008 total_nvlink_bytes=684600816 weight_to_token_ratio=0.053236 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=49152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.352514402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78336 payload_rows_remote=62122 dup_rows=19968 dup_rows_remote=17344 dedup_ratio=0.203125 router_tokens_per_expert=[0, 0, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 247, 0, 0, 0, 0, 0, 0, 6553, 7436, 0, 84, 0, 0, 0, 50, 0, 134, 225, 0, 0, 0, 7568, 8051, 0, 0, 517, 0, 0, 241, 0, 85, 0, 0, 0, 1424, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 15994, 776, 13224, 268, 16384, 15476, 16214] dup_rows_per_dst=[0, 200, 114, 376, 0, 15308, 1346, 2624] combine_rows_per_src=[0, 15994, 776, 13224, 268, 16384, 15476, 16214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=78112 dup_rows=5550 dup_rows_remote=5252 dedup_ratio=0.056458 router_tokens_per_expert=[0, 0, 7585, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8064, 0, 482, 0, 0, 317, 66, 0, 0, 0, 66, 0, 302, 0, 0, 0, 0, 0, 0, 6625, 7321, 0, 93, 0, 0, 0, 56, 0, 130, 250, 0, 0, 0, 7537, 8039, 0, 0, 512, 0, 0, 252, 0, 91, 0, 0, 0, 1363, 0, 0, 0, 0, 0] payload_rows_per_dst=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] dup_rows_per_dst=[0, 2, 774, 456, 298, 8, 1358, 2654] combine_rows_per_src=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73386 payload_rows_remote=57002 dup_rows=24918 dup_rows_remote=15858 dedup_ratio=0.253479 router_tokens_per_expert=[0, 621, 407, 1195, 7990, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 0, 6559, 0, 0, 0, 0, 200, 6933, 77, 0, 0, 0, 0, 0, 2286, 0, 0, 4092, 0, 141, 0, 0, 168, 0, 0, 0, 60, 0, 0, 1202, 1, 0, 0, 0, 82, 7860, 127, 6553, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] dup_rows_per_dst=[9060, 0, 7476, 8234, 0, 120, 2, 26] combine_rows_per_src=[16384, 0, 14566, 14370, 4854, 16056, 2568, 4588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92684 payload_rows_remote=86656 dup_rows=5620 dup_rows_remote=5506 dedup_ratio=0.057170 router_tokens_per_expert=[0, 0, 7653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8102, 0, 429, 0, 0, 367, 50, 0, 0, 0, 50, 0, 340, 0, 0, 0, 0, 0, 0, 6513, 7252, 0, 77, 0, 0, 0, 59, 0, 112, 206, 0, 0, 0, 7584, 8065, 0, 0, 370, 0, 0, 251, 0, 78, 0, 0, 0, 1594, 0, 0, 0, 0, 0] payload_rows_per_dst=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] dup_rows_per_dst=[0, 60, 572, 358, 272, 114, 1080, 3164] combine_rows_per_src=[11286, 4818, 10986, 13348, 14504, 6028, 15486, 16228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=330784768 dispatch_meta_bytes=352552 combine_payload_bytes=330784768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=331137320 ingress_bytes=399990784 link_bytes=399990784 token_bytes=661922088 weight_movement_bytes=69206016 total_nvlink_bytes=731128104 weight_to_token_ratio=0.104553 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505133118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82150 payload_rows_remote=66224 dup_rows=16154 dup_rows_remote=7910 dedup_ratio=0.164327 router_tokens_per_expert=[2632, 0, 0, 0, 0, 0, 0, 1723, 508, 0, 434, 0, 0, 0, 0, 1067, 0, 0, 0, 4950, 241, 355, 0, 6258, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 7320, 0, 0, 68, 0, 0, 0, 0, 2711, 0, 7811, 0, 0, 0, 0, 0, 3380, 159, 441, 0, 77] payload_rows_per_dst=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] dup_rows_per_dst=[8244, 3694, 3930, 0, 0, 132, 0, 154] combine_rows_per_src=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=354942976 dispatch_meta_bytes=368648 combine_payload_bytes=354942976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=355311624 ingress_bytes=458752000 link_bytes=458752000 token_bytes=710254600 weight_movement_bytes=103809024 total_nvlink_bytes=814063624 weight_to_token_ratio=0.146157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.726246843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=288776192 dispatch_meta_bytes=313688 combine_payload_bytes=288776192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289089880 ingress_bytes=323379200 link_bytes=323379200 token_bytes=577866072 weight_movement_bytes=34603008 total_nvlink_bytes=612469080 weight_to_token_ratio=0.059881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216849895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=291102720 dispatch_meta_bytes=301528 combine_payload_bytes=291102720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=291404248 ingress_bytes=394911744 link_bytes=394911744 token_bytes=582506968 weight_movement_bytes=103809024 total_nvlink_bytes=686315992 weight_to_token_ratio=0.178211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.486021100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=281174016 dispatch_meta_bytes=314352 combine_payload_bytes=281174016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281488368 ingress_bytes=384983040 link_bytes=384983040 token_bytes=562662384 weight_movement_bytes=103809024 total_nvlink_bytes=666471408 weight_to_token_ratio=0.184496 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=26496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448660186e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78336 payload_rows_remote=62122 dup_rows=19968 dup_rows_remote=17344 dedup_ratio=0.203125 router_tokens_per_expert=[0, 0, 7668, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7995, 0, 429, 0, 0, 331, 57, 0, 0, 0, 57, 0, 247, 0, 0, 0, 0, 0, 0, 6553, 7436, 0, 84, 0, 0, 0, 50, 0, 134, 225, 0, 0, 0, 7568, 8051, 0, 0, 517, 0, 0, 241, 0, 85, 0, 0, 0, 1424, 0, 0, 0, 0, 0] payload_rows_per_dst=[0, 15994, 776, 13224, 268, 16384, 15476, 16214] dup_rows_per_dst=[0, 200, 114, 376, 0, 15308, 1346, 2624] combine_rows_per_src=[0, 15994, 776, 13224, 268, 16384, 15476, 16214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=319946752 dispatch_meta_bytes=333456 combine_payload_bytes=319946752 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=320280208 ingress_bytes=389152768 link_bytes=389152768 token_bytes=640226960 weight_movement_bytes=69206016 total_nvlink_bytes=709432976 weight_to_token_ratio=0.108096 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.464350537e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=233480192 dispatch_meta_bytes=291440 combine_payload_bytes=233480192 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=233771632 ingress_bytes=302686208 link_bytes=302686208 token_bytes=467251824 weight_movement_bytes=69206016 total_nvlink_bytes=536457840 weight_to_token_ratio=0.148113 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.035000 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138983832e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=354942976 dispatch_meta_bytes=368648 combine_payload_bytes=354942976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=355311624 ingress_bytes=458752000 link_bytes=458752000 token_bytes=710254600 weight_movement_bytes=103809024 total_nvlink_bytes=814063624 weight_to_token_ratio=0.146157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.726246843e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=254451712 dispatch_meta_bytes=317864 combine_payload_bytes=254451712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254769576 ingress_bytes=289054720 link_bytes=289054720 token_bytes=509221288 weight_movement_bytes=34603008 total_nvlink_bytes=543824296 weight_to_token_ratio=0.067953 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=36352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.087689641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271253504 dispatch_meta_bytes=296536 combine_payload_bytes=271253504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271550040 ingress_bytes=305856512 link_bytes=305856512 token_bytes=542803544 weight_movement_bytes=34603008 total_nvlink_bytes=577406552 weight_to_token_ratio=0.063749 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150913431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86552 payload_rows_remote=71170 dup_rows=11752 dup_rows_remote=10842 dedup_ratio=0.119548 router_tokens_per_expert=[0, 0, 7639, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8037, 0, 446, 0, 2, 362, 37, 0, 0, 0, 37, 0, 322, 1, 1, 0, 0, 0, 0, 6421, 7322, 0, 107, 0, 3, 0, 70, 0, 144, 251, 0, 0, 3, 7477, 8045, 0, 3, 314, 0, 0, 245, 0, 107, 0, 0, 3, 1747, 2, 0, 0, 0, 0] payload_rows_per_dst=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] dup_rows_per_dst=[0, 306, 74, 404, 226, 6420, 910, 3412] combine_rows_per_src=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94078 payload_rows_remote=80810 dup_rows=4226 dup_rows_remote=3962 dedup_ratio=0.042989 router_tokens_per_expert=[0, 0, 7597, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8099, 0, 375, 0, 0, 335, 50, 0, 0, 0, 50, 0, 310, 0, 0, 0, 0, 0, 0, 6456, 7297, 0, 64, 0, 0, 0, 71, 0, 103, 186, 0, 0, 0, 7603, 8079, 0, 0, 258, 0, 0, 244, 0, 65, 0, 0, 0, 1910, 0, 0, 0, 0, 0] payload_rows_per_dst=[15194, 8560, 16256, 13268, 14594, 578, 15486, 10142] dup_rows_per_dst=[0, 320, 772, 264, 270, 0, 854, 1746] combine_rows_per_src=[15194, 8560, 16256, 13268, 14594, 578, 15486, 10142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95236 payload_rows_remote=78998 dup_rows=3068 dup_rows_remote=2370 dedup_ratio=0.031209 router_tokens_per_expert=[0, 0, 7601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 412, 0, 0, 297, 51, 0, 0, 0, 51, 0, 339, 0, 0, 0, 0, 0, 0, 6445, 7337, 0, 71, 0, 0, 0, 92, 0, 114, 202, 0, 0, 0, 7583, 8069, 0, 0, 330, 0, 0, 244, 0, 74, 0, 0, 0, 1727, 0, 0, 0, 0, 0] payload_rows_per_dst=[15202, 16236, 16238, 13232, 14674, 4494, 11706, 3454] dup_rows_per_dst=[0, 814, 698, 336, 326, 66, 828, 0] combine_rows_per_src=[15202, 16236, 16238, 13232, 14674, 4494, 11706, 3454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=328048 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291840368 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583352688 weight_movement_bytes=34603008 total_nvlink_bytes=617955696 weight_to_token_ratio=0.059317 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91930 payload_rows_remote=75546 dup_rows=6374 dup_rows_remote=5650 dedup_ratio=0.064840 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 987, 0, 77, 7875, 5006, 0, 0, 77, 0, 0, 0, 0, 0, 49, 0, 903, 0, 0, 0, 0, 0, 0, 803, 7745, 0, 0, 0, 0, 0, 0, 0, 7673, 0, 8115, 78, 0, 0, 85, 0, 0, 276, 0, 259, 0, 0, 0, 0, 592, 0, 0, 0, 0, 329, 7911, 0, 0, 0, 312] payload_rows_per_dst=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] dup_rows_per_dst=[182, 154, 1894, 1318, 0, 724, 976, 1126] combine_rows_per_src=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=78112 dup_rows=5550 dup_rows_remote=5252 dedup_ratio=0.056458 router_tokens_per_expert=[0, 0, 7585, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8064, 0, 482, 0, 0, 317, 66, 0, 0, 0, 66, 0, 302, 0, 0, 0, 0, 0, 0, 6625, 7321, 0, 93, 0, 0, 0, 56, 0, 130, 250, 0, 0, 0, 7537, 8039, 0, 0, 512, 0, 0, 252, 0, 91, 0, 0, 0, 1363, 0, 0, 0, 0, 0] payload_rows_per_dst=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] dup_rows_per_dst=[0, 2, 774, 456, 298, 8, 1358, 2654] combine_rows_per_src=[15170, 964, 16202, 13398, 14642, 752, 15426, 16200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95236 payload_rows_remote=78998 dup_rows=3068 dup_rows_remote=2370 dedup_ratio=0.031209 router_tokens_per_expert=[0, 0, 7601, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 412, 0, 0, 297, 51, 0, 0, 0, 51, 0, 339, 0, 0, 0, 0, 0, 0, 6445, 7337, 0, 71, 0, 0, 0, 92, 0, 114, 202, 0, 0, 0, 7583, 8069, 0, 0, 330, 0, 0, 244, 0, 74, 0, 0, 0, 1727, 0, 0, 0, 0, 0] payload_rows_per_dst=[15202, 16236, 16238, 13232, 14674, 4494, 11706, 3454] dup_rows_per_dst=[0, 814, 698, 336, 326, 66, 828, 0] combine_rows_per_src=[15202, 16236, 16238, 13232, 14674, 4494, 11706, 3454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=95756 payload_rows_remote=79516 dup_rows=2548 dup_rows_remote=1684 dedup_ratio=0.025920 router_tokens_per_expert=[0, 0, 7577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 433, 0, 0, 303, 59, 0, 0, 0, 59, 0, 344, 0, 0, 0, 0, 0, 0, 6507, 7317, 0, 72, 0, 0, 0, 56, 0, 118, 203, 0, 0, 0, 7614, 8061, 0, 0, 464, 0, 0, 258, 0, 72, 0, 0, 0, 1516, 0, 0, 0, 0, 0] payload_rows_per_dst=[15154, 16240, 16240, 13352, 14634, 15660, 1444, 3032] dup_rows_per_dst=[0, 864, 724, 350, 256, 210, 144, 0] combine_rows_per_src=[15154, 16240, 16240, 13352, 14634, 15660, 1444, 3032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90626 payload_rows_remote=74650 dup_rows=7678 dup_rows_remote=5358 dedup_ratio=0.078105 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1208, 0, 83, 7863, 4610, 0, 0, 84, 0, 0, 0, 0, 0, 56, 0, 837, 0, 0, 0, 0, 0, 0, 850, 7912, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 8108, 85, 0, 0, 95, 0, 0, 94, 0, 246, 0, 0, 0, 0, 770, 0, 0, 0, 0, 326, 7903, 0, 0, 0, 340] payload_rows_per_dst=[2416, 168, 15750, 15824, 8062, 16384, 16046, 15976] dup_rows_per_dst=[0, 166, 1762, 1700, 0, 380, 1350, 2320] combine_rows_per_src=[2416, 168, 15750, 15824, 8062, 16384, 16046, 15976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86552 payload_rows_remote=71170 dup_rows=11752 dup_rows_remote=10842 dedup_ratio=0.119548 router_tokens_per_expert=[0, 0, 7639, 0, 0, 0, 0, 0, 3, 1, 0, 0, 8037, 0, 446, 0, 2, 362, 37, 0, 0, 0, 37, 0, 322, 1, 1, 0, 0, 0, 0, 6421, 7322, 0, 107, 0, 3, 0, 70, 0, 144, 251, 0, 0, 3, 7477, 8045, 0, 3, 314, 0, 0, 245, 0, 107, 0, 0, 3, 1747, 2, 0, 0, 0, 0] payload_rows_per_dst=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] dup_rows_per_dst=[0, 306, 74, 404, 226, 6420, 910, 3412] combine_rows_per_src=[0, 15872, 802, 13086, 8860, 16384, 15382, 16166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=330997760 dispatch_meta_bytes=339088 combine_payload_bytes=330997760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=331336848 ingress_bytes=400203776 link_bytes=400203776 token_bytes=662334608 weight_movement_bytes=69206016 total_nvlink_bytes=731540624 weight_to_token_ratio=0.104488 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.505934590e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90894 payload_rows_remote=90894 dup_rows=7410 dup_rows_remote=7410 dedup_ratio=0.075378 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1325, 0, 90, 7846, 4540, 0, 0, 91, 0, 0, 0, 0, 0, 65, 0, 604, 0, 0, 0, 0, 0, 0, 873, 7852, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 8101, 91, 0, 0, 94, 0, 0, 99, 0, 252, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 343, 7943, 0, 0, 0, 305] payload_rows_per_dst=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] dup_rows_per_dst=[2634, 180, 1338, 0, 0, 386, 1758, 1114] combine_rows_per_src=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91930 payload_rows_remote=75546 dup_rows=6374 dup_rows_remote=5650 dedup_ratio=0.064840 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 987, 0, 77, 7875, 5006, 0, 0, 77, 0, 0, 0, 0, 0, 49, 0, 903, 0, 0, 0, 0, 0, 0, 803, 7745, 0, 0, 0, 0, 0, 0, 0, 7673, 0, 8115, 78, 0, 0, 85, 0, 0, 276, 0, 259, 0, 0, 0, 0, 592, 0, 0, 0, 0, 329, 7911, 0, 0, 0, 312] payload_rows_per_dst=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] dup_rows_per_dst=[182, 154, 1894, 1318, 0, 724, 976, 1126] combine_rows_per_src=[4016, 6070, 15760, 13554, 4096, 16384, 16072, 15978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82150 payload_rows_remote=66224 dup_rows=16154 dup_rows_remote=7910 dedup_ratio=0.164327 router_tokens_per_expert=[2632, 0, 0, 0, 0, 0, 0, 1723, 508, 0, 434, 0, 0, 0, 0, 1067, 0, 0, 0, 4950, 241, 355, 0, 6258, 0, 0, 0, 0, 0, 0, 0, 0, 1287, 0, 0, 0, 0, 0, 0, 7730, 0, 0, 0, 7320, 0, 0, 68, 0, 0, 0, 0, 2711, 0, 7811, 0, 0, 0, 0, 0, 3380, 159, 441, 0, 77] payload_rows_per_dst=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] dup_rows_per_dst=[8244, 3694, 3930, 0, 0, 132, 0, 154] combine_rows_per_src=[15926, 15946, 13250, 6428, 2574, 14644, 5422, 7960] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=323575808 dispatch_meta_bytes=325472 combine_payload_bytes=323575808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323901280 ingress_bytes=358178816 link_bytes=358178816 token_bytes=647477088 weight_movement_bytes=34603008 total_nvlink_bytes=682080096 weight_to_token_ratio=0.053443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347798049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85228 payload_rows_remote=69648 dup_rows=13076 dup_rows_remote=2358 dedup_ratio=0.133016 router_tokens_per_expert=[0, 0, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 414, 0, 0, 364, 59, 0, 0, 0, 59, 0, 347, 0, 0, 0, 0, 0, 0, 6377, 7295, 0, 77, 0, 0, 0, 71, 0, 116, 216, 0, 0, 0, 7617, 8056, 0, 0, 367, 0, 0, 245, 0, 77, 0, 0, 0, 1634, 0, 0, 0, 0, 0] payload_rows_per_dst=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] dup_rows_per_dst=[10718, 824, 846, 44, 282, 202, 160, 0] combine_rows_per_src=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325697536 dispatch_meta_bytes=324800 combine_payload_bytes=325697536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326022336 ingress_bytes=394903552 link_bytes=394903552 token_bytes=651719872 weight_movement_bytes=69206016 total_nvlink_bytes=720925888 weight_to_token_ratio=0.106190 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485990274e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=328048 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291840368 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583352688 weight_movement_bytes=34603008 total_nvlink_bytes=617955696 weight_to_token_ratio=0.059317 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90626 payload_rows_remote=74650 dup_rows=7678 dup_rows_remote=5358 dedup_ratio=0.078105 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1208, 0, 83, 7863, 4610, 0, 0, 84, 0, 0, 0, 0, 0, 56, 0, 837, 0, 0, 0, 0, 0, 0, 850, 7912, 0, 0, 0, 0, 0, 0, 0, 7682, 0, 8108, 85, 0, 0, 95, 0, 0, 94, 0, 246, 0, 0, 0, 0, 770, 0, 0, 0, 0, 326, 7903, 0, 0, 0, 340] payload_rows_per_dst=[2416, 168, 15750, 15824, 8062, 16384, 16046, 15976] dup_rows_per_dst=[0, 166, 1762, 1700, 0, 380, 1350, 2320] combine_rows_per_src=[2416, 168, 15750, 15824, 8062, 16384, 16046, 15976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372301824 dispatch_meta_bytes=393216 combine_payload_bytes=372301824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372695040 ingress_bytes=476110848 link_bytes=476110848 token_bytes=744996864 weight_movement_bytes=103809024 total_nvlink_bytes=848805888 weight_to_token_ratio=0.139342 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791566791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309436416 dispatch_meta_bytes=324784 combine_payload_bytes=309436416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309761200 ingress_bytes=344039424 link_bytes=344039424 token_bytes=619197616 weight_movement_bytes=34603008 total_nvlink_bytes=653800624 weight_to_token_ratio=0.055884 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=46336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=271253504 dispatch_meta_bytes=296536 combine_payload_bytes=271253504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271550040 ingress_bytes=305856512 link_bytes=305856512 token_bytes=542803544 weight_movement_bytes=34603008 total_nvlink_bytes=577406552 weight_to_token_ratio=0.063749 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150913431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=319946752 dispatch_meta_bytes=333456 combine_payload_bytes=319946752 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=320280208 ingress_bytes=389152768 link_bytes=389152768 token_bytes=640226960 weight_movement_bytes=69206016 total_nvlink_bytes=709432976 weight_to_token_ratio=0.108096 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.464350537e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=325697536 dispatch_meta_bytes=324800 combine_payload_bytes=325697536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=326022336 ingress_bytes=394903552 link_bytes=394903552 token_bytes=651719872 weight_movement_bytes=69206016 total_nvlink_bytes=720925888 weight_to_token_ratio=0.106190 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.032500 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485990274e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=90634 dup_rows=6120 dup_rows_remote=6120 dedup_ratio=0.062256 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 963, 0, 64, 7869, 5007, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 995, 0, 0, 0, 0, 0, 0, 775, 7915, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 8128, 65, 0, 0, 75, 0, 0, 253, 0, 302, 0, 0, 0, 0, 245, 0, 0, 0, 0, 309, 7938, 0, 0, 0, 395] payload_rows_per_dst=[16206, 10142, 15824, 1550, 0, 16384, 16072, 16006] dup_rows_per_dst=[1550, 128, 2004, 0, 0, 658, 502, 1278] combine_rows_per_src=[16206, 10142, 15824, 1550, 0, 16384, 16072, 16006] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=305766400 dispatch_meta_bytes=320032 combine_payload_bytes=305766400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306086432 ingress_bytes=374972416 link_bytes=374972416 token_bytes=611852832 weight_movement_bytes=69206016 total_nvlink_bytes=681058848 weight_to_token_ratio=0.113109 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.035000 gemm_active_groups=5 gemm_peak_group_rows=43264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.410991014e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91906 payload_rows_remote=75964 dup_rows=6398 dup_rows_remote=4954 dedup_ratio=0.065084 router_tokens_per_expert=[0, 1, 2, 3, 3, 1, 1228, 0, 107, 7853, 4816, 0, 0, 107, 0, 1, 0, 0, 0, 34, 3, 580, 3, 0, 0, 0, 0, 0, 851, 7740, 1, 0, 0, 3, 0, 0, 1, 7573, 0, 8082, 107, 0, 0, 116, 2, 0, 191, 0, 246, 0, 0, 0, 0, 874, 0, 0, 0, 0, 352, 7887, 0, 0, 0, 384] payload_rows_per_dst=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] dup_rows_per_dst=[12, 214, 1148, 1702, 2, 618, 1444, 1258] combine_rows_per_src=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285278208 dispatch_meta_bytes=288024 combine_payload_bytes=285278208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285566232 ingress_bytes=354484224 link_bytes=354484224 token_bytes=570844440 weight_movement_bytes=69206016 total_nvlink_bytes=640050456 weight_to_token_ratio=0.121234 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333895597e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14038833200 prefetch_bytes=761266176 reduce_bytes=899678208 weight_movement_bytes=1660944384 egress_bytes=7023343664 ingress_bytes=8676433920 link_bytes=8676433920 total_nvlink_bytes=15699777584 weight_to_token_ratio=0.118311 prefetch_share_of_ingress=0.087740 dedup_ratio=0.132581 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=26 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.419509667e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13836634616 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6922226168 ingress_bytes=8471543808 link_bytes=8471543808 total_nvlink_bytes=15393769976 weight_to_token_ratio=0.112537 prefetch_share_of_ingress=0.098031 dedup_ratio=0.136417 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=24003 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.385988580e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92184 payload_rows_remote=90634 dup_rows=6120 dup_rows_remote=6120 dedup_ratio=0.062256 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 963, 0, 64, 7869, 5007, 0, 0, 64, 0, 0, 0, 0, 0, 50, 0, 995, 0, 0, 0, 0, 0, 0, 775, 7915, 0, 0, 0, 0, 0, 0, 0, 7740, 0, 8128, 65, 0, 0, 75, 0, 0, 253, 0, 302, 0, 0, 0, 0, 245, 0, 0, 0, 0, 309, 7938, 0, 0, 0, 395] payload_rows_per_dst=[16206, 10142, 15824, 1550, 0, 16384, 16072, 16006] dup_rows_per_dst=[1550, 128, 2004, 0, 0, 658, 502, 1278] combine_rows_per_src=[16206, 10142, 15824, 1550, 0, 16384, 16072, 16006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12050401408 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=6028830848 ingress_bytes=7336484864 link_bytes=7336484864 total_nvlink_bytes=13365315712 weight_to_token_ratio=0.109118 prefetch_share_of_ingress=0.099048 dedup_ratio=0.210752 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200287039e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82962 payload_rows_remote=75512 dup_rows=15342 dup_rows_remote=14586 dedup_ratio=0.156067 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 997, 0, 73, 7862, 4929, 0, 0, 73, 0, 0, 0, 0, 0, 49, 0, 849, 0, 0, 0, 0, 0, 0, 829, 7789, 0, 0, 0, 0, 0, 0, 0, 7719, 0, 8119, 74, 0, 0, 76, 0, 0, 255, 0, 277, 0, 0, 0, 0, 537, 0, 0, 0, 0, 318, 7936, 0, 0, 0, 391] payload_rows_per_dst=[15942, 13506, 7450, 9464, 8648, 11740, 8242, 7970] dup_rows_per_dst=[1630, 5958, 756, 936, 4556, 518, 410, 578] combine_rows_per_src=[15942, 13506, 7450, 9464, 8648, 11740, 8242, 7970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=311148544 dispatch_meta_bytes=323672 combine_payload_bytes=311148544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311472216 ingress_bytes=345751552 link_bytes=345751552 token_bytes=622620760 weight_movement_bytes=34603008 total_nvlink_bytes=657223768 weight_to_token_ratio=0.055576 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301035255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90894 payload_rows_remote=90894 dup_rows=7410 dup_rows_remote=7410 dedup_ratio=0.075378 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1325, 0, 90, 7846, 4540, 0, 0, 91, 0, 0, 0, 0, 0, 65, 0, 604, 0, 0, 0, 0, 0, 0, 873, 7852, 0, 0, 0, 0, 0, 0, 0, 7624, 0, 8101, 91, 0, 0, 94, 0, 0, 99, 0, 252, 0, 0, 0, 0, 1014, 0, 0, 0, 0, 343, 7943, 0, 0, 0, 305] payload_rows_per_dst=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] dup_rows_per_dst=[2634, 180, 1338, 0, 0, 386, 1758, 1114] combine_rows_per_src=[15720, 9262, 15692, 1746, 0, 16384, 16022, 16068] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69022 payload_rows_remote=53164 dup_rows=29282 dup_rows_remote=19280 dedup_ratio=0.297872 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1115, 0, 72, 7839, 4947, 0, 0, 72, 0, 0, 0, 0, 0, 57, 0, 751, 0, 0, 0, 0, 0, 0, 844, 7865, 0, 0, 0, 0, 0, 0, 0, 7710, 0, 8120, 72, 0, 0, 80, 0, 0, 172, 0, 259, 0, 0, 0, 0, 618, 0, 0, 0, 0, 330, 7911, 0, 0, 0, 318] payload_rows_per_dst=[15932, 15858, 1616, 15822, 16240, 504, 1754, 1296] dup_rows_per_dst=[2028, 10002, 0, 1688, 15420, 144, 0, 0] combine_rows_per_src=[15932, 15858, 1616, 15822, 16240, 504, 1754, 1296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91906 payload_rows_remote=75964 dup_rows=6398 dup_rows_remote=4954 dedup_ratio=0.065084 router_tokens_per_expert=[0, 1, 2, 3, 3, 1, 1228, 0, 107, 7853, 4816, 0, 0, 107, 0, 1, 0, 0, 0, 34, 3, 580, 3, 0, 0, 0, 0, 0, 851, 7740, 1, 0, 0, 3, 0, 0, 1, 7573, 0, 8082, 107, 0, 0, 116, 2, 0, 191, 0, 246, 0, 0, 0, 0, 874, 0, 0, 0, 0, 352, 7887, 0, 0, 0, 384] payload_rows_per_dst=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] dup_rows_per_dst=[12, 214, 1148, 1702, 2, 618, 1444, 1258] combine_rows_per_src=[2464, 216, 15798, 15482, 9638, 16378, 15942, 15988] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=371236864 dispatch_meta_bytes=387016 combine_payload_bytes=371236864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=371623880 ingress_bytes=440442880 link_bytes=440442880 token_bytes=742860744 weight_movement_bytes=69206016 total_nvlink_bytes=812066760 weight_to_token_ratio=0.093161 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657351099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12050401408 prefetch_bytes=726663168 reduce_bytes=588251136 weight_movement_bytes=1314914304 egress_bytes=6028830848 ingress_bytes=7336484864 link_bytes=7336484864 total_nvlink_bytes=13365315712 weight_to_token_ratio=0.109118 prefetch_share_of_ingress=0.099048 dedup_ratio=0.210752 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=17 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030435 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200287039e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=371236864 dispatch_meta_bytes=387016 combine_payload_bytes=371236864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=371623880 ingress_bytes=440442880 link_bytes=440442880 token_bytes=742860744 weight_movement_bytes=69206016 total_nvlink_bytes=812066760 weight_to_token_ratio=0.093161 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.032500 gemm_active_groups=5 gemm_peak_group_rows=22400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657351099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309297152 dispatch_meta_bytes=360392 combine_payload_bytes=309297152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309657544 ingress_bytes=343900160 link_bytes=343900160 token_bytes=618954696 weight_movement_bytes=34603008 total_nvlink_bytes=653557704 weight_to_token_ratio=0.055906 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294068616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68916 payload_rows_remote=53068 dup_rows=29388 dup_rows_remote=27664 dedup_ratio=0.298950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 981, 0, 77, 7864, 5122, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 697, 0, 0, 0, 0, 0, 0, 847, 7805, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 8115, 77, 0, 0, 87, 0, 0, 195, 0, 246, 0, 0, 0, 0, 611, 0, 0, 0, 0, 323, 7912, 0, 0, 0, 351] payload_rows_per_dst=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] dup_rows_per_dst=[1724, 10404, 0, 1694, 15412, 154, 0, 0] combine_rows_per_src=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12852893120 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6430234048 ingress_bytes=7979794432 link_bytes=7979794432 total_nvlink_bytes=14410028480 weight_to_token_ratio=0.121151 prefetch_share_of_ingress=0.086727 dedup_ratio=0.173503 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.305535827e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372301824 dispatch_meta_bytes=393216 combine_payload_bytes=372301824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372695040 ingress_bytes=476110848 link_bytes=476110848 token_bytes=744996864 weight_movement_bytes=103809024 total_nvlink_bytes=848805888 weight_to_token_ratio=0.139342 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791566791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85228 payload_rows_remote=69648 dup_rows=13076 dup_rows_remote=2358 dedup_ratio=0.133016 router_tokens_per_expert=[0, 0, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 414, 0, 0, 364, 59, 0, 0, 0, 59, 0, 347, 0, 0, 0, 0, 0, 0, 6377, 7295, 0, 77, 0, 0, 0, 71, 0, 116, 216, 0, 0, 0, 7617, 8056, 0, 0, 367, 0, 0, 245, 0, 77, 0, 0, 0, 1634, 0, 0, 0, 0, 0] payload_rows_per_dst=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] dup_rows_per_dst=[10718, 824, 846, 44, 282, 202, 160, 0] combine_rows_per_src=[15580, 16230, 16230, 2808, 14198, 15696, 1218, 3268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217759744 dispatch_meta_bytes=289776 combine_payload_bytes=217759744 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=218049520 ingress_bytes=321568768 link_bytes=321568768 token_bytes=435809264 weight_movement_bytes=103809024 total_nvlink_bytes=539618288 weight_to_token_ratio=0.238198 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.030000 gemm_active_groups=6 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.210037386e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=311148544 dispatch_meta_bytes=323672 combine_payload_bytes=311148544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311472216 ingress_bytes=345751552 link_bytes=345751552 token_bytes=622620760 weight_movement_bytes=34603008 total_nvlink_bytes=657223768 weight_to_token_ratio=0.055576 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.035000 gemm_active_groups=4 gemm_peak_group_rows=41856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301035255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14360435568 prefetch_bytes=795869184 reduce_bytes=761266176 weight_movement_bytes=1557135360 egress_bytes=7184243568 ingress_bytes=8733327360 link_bytes=8733327360 total_nvlink_bytes=15917570928 weight_to_token_ratio=0.108432 prefetch_share_of_ingress=0.091130 dedup_ratio=0.124944 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=22 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029891 gemm_inner_pad_upper_bound=22479 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.428817729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13792672568 prefetch_bytes=622854144 reduce_bytes=657457152 weight_movement_bytes=1280311296 egress_bytes=6900251448 ingress_bytes=8172732416 link_bytes=8172732416 total_nvlink_bytes=15072983864 weight_to_token_ratio=0.092825 prefetch_share_of_ingress=0.076211 dedup_ratio=0.137555 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=19 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030109 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.337101484e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=322928 combine_payload_bytes=217366528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=217689456 ingress_bytes=251969536 link_bytes=251969536 token_bytes=435055984 weight_movement_bytes=34603008 total_nvlink_bytes=469658992 weight_to_token_ratio=0.079537 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.481410788e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14038833200 prefetch_bytes=761266176 reduce_bytes=899678208 weight_movement_bytes=1660944384 egress_bytes=7023343664 ingress_bytes=8676433920 link_bytes=8676433920 total_nvlink_bytes=15699777584 weight_to_token_ratio=0.118311 prefetch_share_of_ingress=0.087740 dedup_ratio=0.132581 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=26 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22733 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.419509667e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=285278208 dispatch_meta_bytes=288024 combine_payload_bytes=285278208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285566232 ingress_bytes=354484224 link_bytes=354484224 token_bytes=570844440 weight_movement_bytes=69206016 total_nvlink_bytes=640050456 weight_to_token_ratio=0.121234 gemm_rows_defined=49280 gemm_rows_charged=51200 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.037500 gemm_active_groups=3 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.333895597e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13167645768 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6587683912 ingress_bytes=7825670144 link_bytes=7825670144 total_nvlink_bytes=14413354056 weight_to_token_ratio=0.094604 prefetch_share_of_ingress=0.092856 dedup_ratio=0.174217 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030217 gemm_inner_pad_upper_bound=22860 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.280320293e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12852893120 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6430234048 ingress_bytes=7979794432 link_bytes=7979794432 total_nvlink_bytes=14410028480 weight_to_token_ratio=0.121151 prefetch_share_of_ingress=0.086727 dedup_ratio=0.173503 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029783 gemm_inner_pad_upper_bound=22225 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.305535827e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68916 payload_rows_remote=53068 dup_rows=29388 dup_rows_remote=27664 dedup_ratio=0.298950 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 981, 0, 77, 7864, 5122, 0, 0, 77, 0, 0, 0, 0, 0, 59, 0, 697, 0, 0, 0, 0, 0, 0, 847, 7805, 0, 0, 0, 0, 0, 0, 0, 7706, 0, 8115, 77, 0, 0, 87, 0, 0, 195, 0, 246, 0, 0, 0, 0, 611, 0, 0, 0, 0, 323, 7912, 0, 0, 0, 351] payload_rows_per_dst=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] dup_rows_per_dst=[1724, 10404, 0, 1694, 15412, 154, 0, 0] combine_rows_per_src=[15848, 15876, 1512, 15824, 16230, 564, 1714, 1348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=217366528 dispatch_meta_bytes=322928 combine_payload_bytes=217366528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=217689456 ingress_bytes=251969536 link_bytes=251969536 token_bytes=435055984 weight_movement_bytes=34603008 total_nvlink_bytes=469658992 weight_to_token_ratio=0.079537 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.025000 gemm_active_groups=8 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.481410788e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12035531992 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6021424344 ingress_bytes=7467433984 link_bytes=7467433984 total_nvlink_bytes=13488858328 weight_to_token_ratio=0.120753 prefetch_share_of_ingress=0.088043 dedup_ratio=0.211343 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.030870 gemm_inner_pad_upper_bound=21209 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.221710996e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:55:27] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 798.8 | throughput per GPU (TFLOP/s/GPU): 159.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.620937E+00 | loss scale: 1.0 | grad norm: 2.990 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75798 payload_rows_remote=66022 dup_rows=22506 dup_rows_remote=19916 dedup_ratio=0.228943 router_tokens_per_expert=[1105, 324, 806, 1212, 672, 691, 794, 1081, 1279, 327, 892, 342, 380, 1280, 987, 1118, 703, 1008, 864, 648, 680, 738, 378, 1473, 556, 691, 768, 355, 822, 90, 782, 724, 1013, 845, 1148, 841, 343, 206, 477, 909, 584, 592, 560, 857, 1096, 582, 734, 694, 794, 554, 875, 435, 1053, 1223, 998, 824, 1230, 281, 1226, 590, 219, 866, 1013, 920] payload_rows_per_dst=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] dup_rows_per_dst=[4466, 2590, 2586, 2894, 2704, 3476, 1962, 1828] combine_rows_per_src=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] + [2026-08-26 18:07:35] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 802.9 | throughput per GPU (TFLOP/s/GPU): 158.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.620937E+00 | loss scale: 1.0 | grad norm: 2.990 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75464 payload_rows_remote=66130 dup_rows=22840 dup_rows_remote=19858 dedup_ratio=0.232340 router_tokens_per_expert=[1191, 317, 771, 1225, 620, 867, 748, 935, 1412, 300, 978, 378, 330, 1472, 847, 1099, 666, 996, 824, 613, 636, 727, 392, 1364, 466, 723, 810, 358, 771, 83, 844, 556, 1065, 831, 1305, 888, 502, 231, 528, 808, 587, 617, 596, 744, 1136, 600, 591, 624, 659, 583, 845, 526, 974, 1354, 865, 710, 1235, 306, 1488, 626, 208, 782, 1107, 912] payload_rows_per_dst=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] dup_rows_per_dst=[3334, 3174, 2556, 2836, 2982, 2446, 2992, 2520] combine_rows_per_src=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74706 payload_rows_remote=64140 dup_rows=23598 dup_rows_remote=20964 dedup_ratio=0.240051 router_tokens_per_expert=[1154, 393, 843, 1263, 588, 906, 730, 1035, 1376, 355, 835, 395, 379, 1203, 783, 1069, 674, 1015, 876, 533, 769, 754, 422, 1393, 457, 669, 798, 324, 782, 100, 863, 752, 1084, 803, 1235, 876, 422, 209, 493, 771, 670, 645, 526, 821, 1106, 596, 708, 592, 767, 575, 815, 522, 999, 1243, 903, 773, 1221, 300, 1344, 673, 205, 777, 1071, 919] payload_rows_per_dst=[6974, 9068, 8552, 10596, 10244, 8790, 9916, 10566] dup_rows_per_dst=[2016, 3722, 1534, 3728, 4148, 2538, 3278, 2634] combine_rows_per_src=[6974, 9068, 8552, 10596, 10244, 8790, 9916, 10566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75302 payload_rows_remote=66778 dup_rows=23002 dup_rows_remote=20538 dedup_ratio=0.233988 router_tokens_per_expert=[1080, 319, 849, 1190, 688, 835, 856, 1024, 1386, 361, 947, 343, 339, 1361, 1059, 945, 765, 1040, 800, 676, 718, 731, 395, 1415, 481, 608, 779, 351, 831, 109, 922, 626, 1060, 775, 1236, 862, 366, 225, 470, 803, 629, 635, 520, 775, 1076, 637, 577, 645, 870, 616, 817, 518, 934, 1279, 866, 774, 1105, 320, 1452, 566, 233, 881, 978, 823] payload_rows_per_dst=[8366, 9582, 10268, 9030, 8826, 8524, 10188, 10518] dup_rows_per_dst=[2936, 3900, 2812, 2764, 2768, 2464, 3160, 2198] combine_rows_per_src=[8366, 9582, 10268, 9030, 8826, 8524, 10188, 10518] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=64546 dup_rows=23626 dup_rows_remote=20440 dedup_ratio=0.240336 router_tokens_per_expert=[1129, 366, 807, 1161, 644, 960, 870, 850, 1315, 409, 952, 369, 294, 1480, 855, 1106, 820, 1060, 800, 593, 668, 690, 397, 1338, 332, 653, 865, 397, 798, 94, 833, 647, 1068, 802, 1240, 886, 344, 254, 505, 631, 610, 642, 597, 721, 1228, 612, 583, 566, 883, 538, 826, 575, 940, 1323, 841, 733, 1185, 267, 1435, 635, 217, 925, 1161, 827] payload_rows_per_dst=[7990, 9530, 9390, 8994, 9218, 8774, 10132, 10650] dup_rows_per_dst=[2736, 4030, 2280, 3092, 3304, 2344, 3186, 2654] combine_rows_per_src=[7990, 9530, 9390, 8994, 9218, 8774, 10132, 10650] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74706 payload_rows_remote=64140 dup_rows=23598 dup_rows_remote=20964 dedup_ratio=0.240051 router_tokens_per_expert=[1154, 393, 843, 1263, 588, 906, 730, 1035, 1376, 355, 835, 395, 379, 1203, 783, 1069, 674, 1015, 876, 533, 769, 754, 422, 1393, 457, 669, 798, 324, 782, 100, 863, 752, 1084, 803, 1235, 876, 422, 209, 493, 771, 670, 645, 526, 821, 1106, 596, 708, 592, 767, 575, 815, 522, 999, 1243, 903, 773, 1221, 300, 1344, 673, 205, 777, 1071, 919] payload_rows_per_dst=[6974, 9068, 8552, 10596, 10244, 8790, 9916, 10566] dup_rows_per_dst=[2016, 3722, 1534, 3728, 4148, 2538, 3278, 2634] combine_rows_per_src=[6974, 9068, 8552, 10596, 10244, 8790, 9916, 10566] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75798 payload_rows_remote=66022 dup_rows=22506 dup_rows_remote=19916 dedup_ratio=0.228943 router_tokens_per_expert=[1105, 324, 806, 1212, 672, 691, 794, 1081, 1279, 327, 892, 342, 380, 1280, 987, 1118, 703, 1008, 864, 648, 680, 738, 378, 1473, 556, 691, 768, 355, 822, 90, 782, 724, 1013, 845, 1148, 841, 343, 206, 477, 909, 584, 592, 560, 857, 1096, 582, 734, 694, 794, 554, 875, 435, 1053, 1223, 998, 824, 1230, 281, 1226, 590, 219, 866, 1013, 920] payload_rows_per_dst=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] dup_rows_per_dst=[4466, 2590, 2586, 2894, 2704, 3476, 1962, 1828] combine_rows_per_src=[9038, 9776, 10398, 9106, 8860, 10368, 9104, 9148] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74992 payload_rows_remote=65998 dup_rows=23312 dup_rows_remote=20566 dedup_ratio=0.237142 router_tokens_per_expert=[1075, 355, 856, 1157, 705, 882, 861, 961, 1362, 365, 924, 336, 320, 1349, 947, 1113, 724, 1015, 823, 622, 757, 725, 357, 1352, 496, 644, 795, 420, 822, 114, 779, 643, 1055, 790, 1181, 901, 407, 214, 458, 901, 600, 683, 529, 726, 1059, 604, 624, 583, 862, 534, 843, 491, 1020, 1290, 926, 662, 1240, 292, 1261, 709, 259, 861, 1035, 896] payload_rows_per_dst=[9396, 8462, 10012, 8994, 8894, 9242, 9580, 10412] dup_rows_per_dst=[4718, 2246, 2738, 2746, 2920, 2692, 2558, 2694] combine_rows_per_src=[9396, 8462, 10012, 8994, 8894, 9242, 9580, 10412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75210 payload_rows_remote=65162 dup_rows=23094 dup_rows_remote=20354 dedup_ratio=0.234924 router_tokens_per_expert=[1093, 317, 749, 1191, 740, 831, 805, 1007, 1291, 316, 951, 389, 345, 1269, 920, 1038, 732, 1018, 867, 626, 705, 714, 418, 1314, 517, 675, 798, 400, 816, 135, 997, 596, 1111, 755, 1257, 934, 438, 197, 511, 874, 612, 618, 572, 769, 1145, 620, 673, 670, 704, 551, 794, 489, 1037, 1271, 851, 758, 1202, 312, 1340, 622, 232, 780, 1069, 804] payload_rows_per_dst=[9178, 8252, 10048, 9148, 9160, 10326, 8652, 10446] dup_rows_per_dst=[4488, 2204, 2740, 3102, 2994, 3574, 1716, 2276] combine_rows_per_src=[9178, 8252, 10048, 9148, 9160, 10326, 8652, 10446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75464 payload_rows_remote=66130 dup_rows=22840 dup_rows_remote=19858 dedup_ratio=0.232340 router_tokens_per_expert=[1191, 317, 771, 1225, 620, 867, 748, 935, 1412, 300, 978, 378, 330, 1472, 847, 1099, 666, 996, 824, 613, 636, 727, 392, 1364, 466, 723, 810, 358, 771, 83, 844, 556, 1065, 831, 1305, 888, 502, 231, 528, 808, 587, 617, 596, 744, 1136, 600, 591, 624, 659, 583, 845, 526, 974, 1354, 865, 710, 1235, 306, 1488, 626, 208, 782, 1107, 912] payload_rows_per_dst=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] dup_rows_per_dst=[3334, 3174, 2556, 2836, 2982, 2446, 2992, 2520] combine_rows_per_src=[8756, 9266, 9880, 8836, 9334, 8544, 10040, 10808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=343752 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270769864 ingress_bytes=339632128 link_bytes=339632128 token_bytes=541195976 weight_movement_bytes=69206016 total_nvlink_bytes=610401992 weight_to_token_ratio=0.127876 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278008355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270868480 dispatch_meta_bytes=343952 combine_payload_bytes=270868480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271212432 ingress_bytes=305471488 link_bytes=305471488 token_bytes=542080912 weight_movement_bytes=34603008 total_nvlink_bytes=576683920 weight_to_token_ratio=0.063834 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149464617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=273522688 dispatch_meta_bytes=349264 combine_payload_bytes=273522688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273871952 ingress_bytes=308125696 link_bytes=308125696 token_bytes=547394640 weight_movement_bytes=34603008 total_nvlink_bytes=581997648 weight_to_token_ratio=0.063214 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159452188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=339944 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264720360 ingress_bytes=333586432 link_bytes=333586432 token_bytes=529100776 weight_movement_bytes=69206016 total_nvlink_bytes=598306792 weight_to_token_ratio=0.130799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255258888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262717440 dispatch_meta_bytes=340416 combine_payload_bytes=262717440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263057856 ingress_bytes=366526464 link_bytes=366526464 token_bytes=525775296 weight_movement_bytes=103809024 total_nvlink_bytes=629584320 weight_to_token_ratio=0.197440 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379209576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346256 combine_payload_bytes=270327808 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270674064 ingress_bytes=339533824 link_bytes=339533824 token_bytes=541001872 weight_movement_bytes=69206016 total_nvlink_bytes=610207888 weight_to_token_ratio=0.127922 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.277638445e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=339944 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264720360 ingress_bytes=333586432 link_bytes=333586432 token_bytes=529100776 weight_movement_bytes=69206016 total_nvlink_bytes=598306792 weight_to_token_ratio=0.130799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255258888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270426112 dispatch_meta_bytes=343752 combine_payload_bytes=270426112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270769864 ingress_bytes=339632128 link_bytes=339632128 token_bytes=541195976 weight_movement_bytes=69206016 total_nvlink_bytes=610401992 weight_to_token_ratio=0.127876 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278008355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=342064 combine_payload_bytes=266903552 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267245616 ingress_bytes=336109568 link_bytes=336109568 token_bytes=534149168 weight_movement_bytes=69206016 total_nvlink_bytes=603355184 weight_to_token_ratio=0.129563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264753245e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74996 payload_rows_remote=65646 dup_rows=23308 dup_rows_remote=18746 dedup_ratio=0.237101 router_tokens_per_expert=[1171, 375, 819, 1272, 638, 738, 795, 1055, 1365, 338, 919, 343, 434, 1330, 845, 914, 739, 1013, 770, 588, 632, 734, 388, 1391, 468, 674, 832, 351, 757, 103, 922, 840, 1083, 860, 1248, 846, 423, 216, 493, 805, 703, 703, 618, 821, 975, 631, 786, 629, 697, 547, 851, 517, 944, 1235, 835, 834, 1203, 252, 1128, 675, 185, 764, 1039, 1053] payload_rows_per_dst=[9350, 9796, 10032, 9334, 9034, 10474, 9436, 7540] dup_rows_per_dst=[4562, 2706, 2478, 3104, 2914, 3728, 2404, 1412] combine_rows_per_src=[9350, 9796, 10032, 9334, 9034, 10474, 9436, 7540] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270868480 dispatch_meta_bytes=343952 combine_payload_bytes=270868480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271212432 ingress_bytes=305471488 link_bytes=305471488 token_bytes=542080912 weight_movement_bytes=34603008 total_nvlink_bytes=576683920 weight_to_token_ratio=0.063834 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=9856 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149464617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76160 payload_rows_remote=66356 dup_rows=22144 dup_rows_remote=20008 dedup_ratio=0.225260 router_tokens_per_expert=[790, 947, 900, 462, 902, 315, 260, 1070, 57, 983, 1443, 374, 592, 976, 951, 1498, 971, 1148, 810, 268, 1479, 167, 1867, 1216, 665, 717, 749, 896, 184, 294, 218, 96, 342, 240, 419, 201, 948, 1601, 1174, 1045, 2136, 1503, 1018, 1294, 406, 1503, 28, 280, 943, 1018, 965, 14, 143, 1534, 172, 1255, 844, 362, 915, 574, 890, 402, 331, 387] payload_rows_per_dst=[8702, 10120, 9560, 9612, 9804, 8436, 9852, 10074] dup_rows_per_dst=[2590, 3628, 2558, 2298, 2136, 3628, 2236, 3070] combine_rows_per_src=[8702, 10120, 9560, 9612, 9804, 8436, 9852, 10074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76454 payload_rows_remote=67742 dup_rows=21850 dup_rows_remote=18014 dedup_ratio=0.222270 router_tokens_per_expert=[649, 1029, 953, 431, 762, 249, 147, 1052, 71, 962, 1333, 300, 560, 898, 914, 1270, 1039, 1137, 831, 480, 1466, 189, 1791, 1205, 648, 695, 634, 838, 309, 470, 202, 78, 485, 420, 458, 199, 834, 1793, 1237, 1147, 2144, 1617, 1175, 1443, 457, 1294, 29, 259, 784, 1043, 881, 14, 208, 1602, 198, 1309, 743, 360, 860, 526, 1044, 314, 296, 387] payload_rows_per_dst=[8458, 9626, 9964, 9550, 10524, 8712, 9878, 9742] dup_rows_per_dst=[2086, 2990, 2730, 2486, 2622, 3836, 2200, 2900] combine_rows_per_src=[8458, 9626, 9964, 9550, 10524, 8712, 9878, 9742] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76836 payload_rows_remote=66676 dup_rows=21468 dup_rows_remote=19382 dedup_ratio=0.218384 router_tokens_per_expert=[706, 964, 927, 317, 880, 329, 326, 977, 60, 861, 1451, 371, 625, 922, 911, 1351, 829, 1242, 854, 379, 1593, 201, 2090, 1204, 693, 616, 685, 829, 189, 312, 168, 87, 492, 352, 429, 182, 893, 1699, 1248, 1127, 1932, 1599, 1139, 1284, 393, 1506, 38, 246, 839, 1097, 823, 18, 171, 1664, 246, 1265, 891, 424, 728, 512, 963, 367, 255, 381] payload_rows_per_dst=[8742, 9822, 9962, 9078, 9062, 9836, 10160, 10174] dup_rows_per_dst=[2110, 3282, 2642, 1944, 1688, 4668, 2086, 3048] combine_rows_per_src=[8742, 9822, 9962, 9078, 9062, 9836, 10160, 10174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76170 payload_rows_remote=68448 dup_rows=22134 dup_rows_remote=19804 dedup_ratio=0.225159 router_tokens_per_expert=[756, 1025, 968, 409, 798, 271, 206, 1079, 69, 946, 1120, 291, 569, 903, 1128, 1301, 1156, 1150, 713, 342, 1585, 129, 1821, 1151, 575, 653, 644, 761, 250, 391, 152, 58, 445, 281, 511, 252, 961, 1964, 1157, 1096, 2213, 1592, 1360, 1512, 469, 1213, 16, 218, 643, 1267, 685, 17, 176, 1789, 191, 1230, 673, 353, 891, 530, 1083, 315, 292, 387] payload_rows_per_dst=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] dup_rows_per_dst=[3116, 2330, 2758, 3964, 2712, 2176, 2162, 2916] combine_rows_per_src=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75940 payload_rows_remote=66446 dup_rows=22364 dup_rows_remote=19726 dedup_ratio=0.227498 router_tokens_per_expert=[778, 919, 975, 369, 786, 344, 406, 1025, 85, 1104, 1400, 349, 631, 1000, 913, 1389, 980, 1250, 742, 293, 1457, 158, 1773, 1076, 618, 612, 663, 822, 140, 274, 252, 108, 484, 352, 490, 280, 968, 1791, 1169, 1209, 2138, 1586, 1255, 1345, 435, 1215, 33, 269, 710, 1193, 680, 28, 216, 1733, 199, 1390, 718, 313, 757, 536, 1086, 296, 273, 314] payload_rows_per_dst=[8770, 10094, 8052, 9224, 9182, 10490, 10634, 9494] dup_rows_per_dst=[2434, 3648, 1368, 2030, 1814, 5368, 3064, 2638] combine_rows_per_src=[8770, 10094, 8052, 9224, 9182, 10490, 10634, 9494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76438 payload_rows_remote=66992 dup_rows=21866 dup_rows_remote=19558 dedup_ratio=0.222432 router_tokens_per_expert=[689, 1071, 957, 409, 884, 303, 256, 1140, 79, 974, 1435, 317, 546, 1008, 899, 1330, 1011, 1185, 742, 317, 1555, 240, 1627, 1234, 648, 669, 696, 857, 271, 381, 251, 113, 373, 322, 436, 182, 903, 1812, 1213, 1076, 1991, 1611, 1101, 1353, 501, 1393, 36, 227, 728, 1156, 842, 19, 175, 1595, 229, 1227, 846, 344, 838, 559, 895, 395, 311, 369] payload_rows_per_dst=[9586, 9016, 9680, 9446, 10432, 8792, 9890, 9596] dup_rows_per_dst=[2796, 3196, 2888, 2308, 2202, 3652, 2052, 2772] combine_rows_per_src=[9586, 9016, 9680, 9446, 10432, 8792, 9890, 9596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76170 payload_rows_remote=68448 dup_rows=22134 dup_rows_remote=19804 dedup_ratio=0.225159 router_tokens_per_expert=[756, 1025, 968, 409, 798, 271, 206, 1079, 69, 946, 1120, 291, 569, 903, 1128, 1301, 1156, 1150, 713, 342, 1585, 129, 1821, 1151, 575, 653, 644, 761, 250, 391, 152, 58, 445, 281, 511, 252, 961, 1964, 1157, 1096, 2213, 1592, 1360, 1512, 469, 1213, 16, 218, 643, 1267, 685, 17, 176, 1789, 191, 1230, 673, 353, 891, 530, 1083, 315, 292, 387] payload_rows_per_dst=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] dup_rows_per_dst=[3116, 2330, 2758, 3964, 2712, 2176, 2162, 2916] combine_rows_per_src=[10510, 7722, 9840, 10438, 10622, 7576, 9834, 9628] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268886016 dispatch_meta_bytes=337568 combine_payload_bytes=268886016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269223584 ingress_bytes=372695040 link_bytes=372695040 token_bytes=538109600 weight_movement_bytes=103809024 total_nvlink_bytes=641918624 weight_to_token_ratio=0.192914 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.402421431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76452 payload_rows_remote=66994 dup_rows=21852 dup_rows_remote=19304 dedup_ratio=0.222290 router_tokens_per_expert=[729, 985, 925, 437, 923, 256, 310, 1179, 98, 928, 1412, 355, 577, 943, 1021, 1381, 966, 1161, 679, 358, 1396, 153, 1672, 1290, 631, 730, 642, 813, 295, 357, 192, 125, 440, 286, 500, 181, 874, 1731, 1225, 1011, 2075, 1414, 1177, 1326, 464, 1349, 37, 240, 891, 1120, 947, 16, 157, 1624, 196, 1163, 785, 380, 1000, 591, 890, 408, 299, 436] payload_rows_per_dst=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] dup_rows_per_dst=[3314, 2650, 2548, 2222, 2324, 3604, 2150, 3040] combine_rows_per_src=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76160 payload_rows_remote=66356 dup_rows=22144 dup_rows_remote=20008 dedup_ratio=0.225260 router_tokens_per_expert=[790, 947, 900, 462, 902, 315, 260, 1070, 57, 983, 1443, 374, 592, 976, 951, 1498, 971, 1148, 810, 268, 1479, 167, 1867, 1216, 665, 717, 749, 896, 184, 294, 218, 96, 342, 240, 419, 201, 948, 1601, 1174, 1045, 2136, 1503, 1018, 1294, 406, 1503, 28, 280, 943, 1018, 965, 14, 143, 1534, 172, 1255, 844, 362, 915, 574, 890, 402, 331, 387] payload_rows_per_dst=[8702, 10120, 9560, 9612, 9804, 8436, 9852, 10074] dup_rows_per_dst=[2590, 3628, 2558, 2298, 2136, 3628, 2236, 3070] combine_rows_per_src=[8702, 10120, 9560, 9612, 9804, 8436, 9852, 10074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277471232 dispatch_meta_bytes=343024 combine_payload_bytes=277471232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277814256 ingress_bytes=381280256 link_bytes=381280256 token_bytes=555285488 weight_movement_bytes=103809024 total_nvlink_bytes=659094512 weight_to_token_ratio=0.186947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434726908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271794176 dispatch_meta_bytes=345456 combine_payload_bytes=271794176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272139632 ingress_bytes=341000192 link_bytes=341000192 token_bytes=543933808 weight_movement_bytes=69206016 total_nvlink_bytes=613139824 weight_to_token_ratio=0.127232 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283156270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76452 payload_rows_remote=66994 dup_rows=21852 dup_rows_remote=19304 dedup_ratio=0.222290 router_tokens_per_expert=[729, 985, 925, 437, 923, 256, 310, 1179, 98, 928, 1412, 355, 577, 943, 1021, 1381, 966, 1161, 679, 358, 1396, 153, 1672, 1290, 631, 730, 642, 813, 295, 357, 192, 125, 440, 286, 500, 181, 874, 1731, 1225, 1011, 2075, 1414, 1177, 1326, 464, 1349, 37, 240, 891, 1120, 947, 16, 157, 1624, 196, 1163, 785, 380, 1000, 591, 890, 408, 299, 436] payload_rows_per_dst=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] dup_rows_per_dst=[3314, 2650, 2548, 2222, 2324, 3604, 2150, 3040] combine_rows_per_src=[10936, 8018, 9458, 9498, 10172, 8410, 10078, 9882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273104896 dispatch_meta_bytes=344232 combine_payload_bytes=273104896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273449128 ingress_bytes=307707904 link_bytes=307707904 token_bytes=546554024 weight_movement_bytes=34603008 total_nvlink_bytes=581157032 weight_to_token_ratio=0.063311 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.157880070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=353008 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280716016 ingress_bytes=349569024 link_bytes=349569024 token_bytes=561079024 weight_movement_bytes=69206016 total_nvlink_bytes=630285040 weight_to_token_ratio=0.123345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315400095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277471232 dispatch_meta_bytes=343024 combine_payload_bytes=277471232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277814256 ingress_bytes=381280256 link_bytes=381280256 token_bytes=555285488 weight_movement_bytes=103809024 total_nvlink_bytes=659094512 weight_to_token_ratio=0.186947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434726908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272162816 dispatch_meta_bytes=344688 combine_payload_bytes=272162816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272507504 ingress_bytes=306765824 link_bytes=306765824 token_bytes=544670320 weight_movement_bytes=34603008 total_nvlink_bytes=579273328 weight_to_token_ratio=0.063530 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154335099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274399232 dispatch_meta_bytes=346200 combine_payload_bytes=274399232 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=274745432 ingress_bytes=343605248 link_bytes=343605248 token_bytes=549144664 weight_movement_bytes=69206016 total_nvlink_bytes=618350680 weight_to_token_ratio=0.126025 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292958886e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=280363008 dispatch_meta_bytes=353008 combine_payload_bytes=280363008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280716016 ingress_bytes=349569024 link_bytes=349569024 token_bytes=561079024 weight_movement_bytes=69206016 total_nvlink_bytes=630285040 weight_to_token_ratio=0.123345 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315400095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274407424 dispatch_meta_bytes=345192 combine_payload_bytes=274407424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=274752616 ingress_bytes=412819456 link_bytes=412819456 token_bytes=549160040 weight_movement_bytes=138412032 total_nvlink_bytes=687572072 weight_to_token_ratio=0.252043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.553406378e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271794176 dispatch_meta_bytes=345456 combine_payload_bytes=271794176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272139632 ingress_bytes=341000192 link_bytes=341000192 token_bytes=543933808 weight_movement_bytes=69206016 total_nvlink_bytes=613139824 weight_to_token_ratio=0.127232 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283156270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=64516 dup_rows=23628 dup_rows_remote=19026 dedup_ratio=0.240356 router_tokens_per_expert=[57, 917, 523, 344, 38, 1072, 908, 220, 570, 363, 780, 1471, 1528, 1027, 209, 1714, 916, 1587, 316, 721, 97, 181, 1184, 908, 519, 914, 892, 770, 249, 622, 892, 204, 473, 1057, 773, 557, 2132, 1873, 1895, 516, 643, 109, 278, 358, 1518, 1761, 159, 1641, 1050, 147, 795, 1163, 33, 191, 220, 710, 1072, 516, 75, 574, 69, 1476, 305, 2300] payload_rows_per_dst=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] dup_rows_per_dst=[2720, 1792, 2178, 2850, 4602, 3948, 2846, 2692] combine_rows_per_src=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76324 payload_rows_remote=65330 dup_rows=21980 dup_rows_remote=18820 dedup_ratio=0.223592 router_tokens_per_expert=[650, 1003, 1031, 462, 922, 279, 308, 1053, 97, 1177, 1207, 329, 670, 1029, 995, 1369, 1027, 1252, 726, 191, 1425, 156, 1852, 1096, 639, 522, 592, 743, 194, 298, 219, 83, 463, 271, 539, 283, 852, 1901, 1044, 1151, 2223, 1590, 1200, 1410, 427, 1223, 26, 274, 767, 1183, 704, 38, 194, 1691, 175, 1262, 730, 375, 865, 551, 1144, 320, 332, 348] payload_rows_per_dst=[10994, 8592, 11116, 10476, 10380, 7122, 9968, 7676] dup_rows_per_dst=[3160, 3570, 3180, 3730, 2628, 1998, 2060, 1654] combine_rows_per_src=[10994, 8592, 11116, 10476, 10380, 7122, 9968, 7676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76434 payload_rows_remote=67262 dup_rows=21870 dup_rows_remote=18076 dedup_ratio=0.222473 router_tokens_per_expert=[172, 862, 479, 374, 78, 902, 1024, 224, 746, 270, 899, 1281, 1202, 1341, 773, 2047, 880, 1306, 301, 689, 97, 406, 1240, 1114, 300, 876, 932, 632, 370, 614, 803, 295, 405, 919, 686, 618, 1761, 1266, 1988, 466, 702, 196, 373, 296, 1676, 1410, 217, 1613, 1112, 683, 908, 912, 63, 253, 411, 763, 880, 266, 76, 697, 278, 1351, 487, 1891] payload_rows_per_dst=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] dup_rows_per_dst=[2934, 1800, 2258, 2584, 3210, 3794, 3544, 1746] combine_rows_per_src=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76542 payload_rows_remote=66362 dup_rows=21762 dup_rows_remote=18688 dedup_ratio=0.221375 router_tokens_per_expert=[150, 938, 424, 426, 58, 897, 932, 273, 531, 311, 865, 1282, 1184, 1335, 649, 1895, 880, 1291, 376, 737, 85, 314, 1192, 1050, 300, 951, 976, 782, 397, 662, 926, 242, 403, 1028, 614, 506, 1648, 1503, 1853, 528, 702, 153, 397, 298, 1705, 1687, 161, 1617, 1263, 645, 599, 934, 62, 266, 314, 649, 982, 304, 116, 861, 135, 1523, 489, 1896] payload_rows_per_dst=[9216, 8348, 9650, 10090, 9170, 9352, 10180, 10536] dup_rows_per_dst=[2686, 1402, 2200, 2946, 3290, 4088, 3074, 2076] combine_rows_per_src=[9216, 8348, 9650, 10090, 9170, 9352, 10180, 10536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76434 payload_rows_remote=67262 dup_rows=21870 dup_rows_remote=18076 dedup_ratio=0.222473 router_tokens_per_expert=[172, 862, 479, 374, 78, 902, 1024, 224, 746, 270, 899, 1281, 1202, 1341, 773, 2047, 880, 1306, 301, 689, 97, 406, 1240, 1114, 300, 876, 932, 632, 370, 614, 803, 295, 405, 919, 686, 618, 1761, 1266, 1988, 466, 702, 196, 373, 296, 1676, 1410, 217, 1613, 1112, 683, 908, 912, 63, 253, 411, 763, 880, 266, 76, 697, 278, 1351, 487, 1891] payload_rows_per_dst=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] dup_rows_per_dst=[2934, 1800, 2258, 2584, 3210, 3794, 3544, 1746] combine_rows_per_src=[9272, 8662, 9808, 9622, 9032, 9172, 10760, 10106] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76308 payload_rows_remote=65594 dup_rows=21996 dup_rows_remote=19810 dedup_ratio=0.223755 router_tokens_per_expert=[297, 871, 431, 390, 121, 1151, 1155, 274, 651, 275, 814, 1104, 1129, 1498, 902, 2197, 869, 1149, 313, 696, 72, 452, 1153, 1013, 231, 886, 769, 525, 294, 704, 885, 475, 465, 907, 694, 646, 1589, 1344, 1986, 524, 500, 188, 296, 257, 1512, 1526, 185, 1790, 1083, 777, 972, 735, 98, 248, 270, 783, 955, 289, 123, 879, 151, 1312, 600, 1722] payload_rows_per_dst=[8098, 8858, 9476, 9514, 9192, 9740, 10716, 10714] dup_rows_per_dst=[2258, 1680, 1958, 2232, 3554, 4518, 3610, 2186] combine_rows_per_src=[8098, 8858, 9476, 9514, 9192, 9740, 10716, 10714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75286 payload_rows_remote=63130 dup_rows=23018 dup_rows_remote=17344 dedup_ratio=0.234151 router_tokens_per_expert=[282, 786, 543, 327, 139, 1035, 1037, 261, 685, 281, 652, 1215, 1141, 1352, 977, 2266, 813, 1137, 243, 665, 89, 564, 1153, 844, 207, 868, 748, 497, 264, 628, 836, 354, 407, 1087, 787, 639, 1561, 1454, 2241, 548, 426, 209, 344, 174, 1731, 1514, 199, 1671, 1182, 634, 867, 801, 150, 446, 333, 899, 946, 345, 79, 726, 243, 1257, 575, 1788] payload_rows_per_dst=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] dup_rows_per_dst=[3548, 5674, 3036, 2024, 2088, 2626, 2420, 1602] combine_rows_per_src=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75832 payload_rows_remote=65842 dup_rows=22472 dup_rows_remote=19792 dedup_ratio=0.228597 router_tokens_per_expert=[150, 1002, 484, 384, 67, 1155, 974, 246, 697, 283, 909, 1404, 1395, 1198, 551, 1954, 939, 1376, 304, 667, 88, 289, 1184, 1085, 381, 864, 928, 710, 313, 629, 840, 266, 473, 1030, 752, 535, 1704, 1624, 1928, 480, 656, 152, 317, 273, 1457, 1574, 171, 1523, 1047, 437, 1028, 952, 61, 278, 315, 662, 942, 349, 80, 654, 174, 1453, 443, 1912] payload_rows_per_dst=[9554, 8290, 9702, 9990, 9540, 8592, 10188, 9976] dup_rows_per_dst=[3226, 1776, 2162, 2680, 3656, 3654, 3280, 2038] combine_rows_per_src=[9554, 8290, 9702, 9990, 9540, 8592, 10188, 9976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75286 payload_rows_remote=63130 dup_rows=23018 dup_rows_remote=17344 dedup_ratio=0.234151 router_tokens_per_expert=[282, 786, 543, 327, 139, 1035, 1037, 261, 685, 281, 652, 1215, 1141, 1352, 977, 2266, 813, 1137, 243, 665, 89, 564, 1153, 844, 207, 868, 748, 497, 264, 628, 836, 354, 407, 1087, 787, 639, 1561, 1454, 2241, 548, 426, 209, 344, 174, 1731, 1514, 199, 1671, 1182, 634, 867, 801, 150, 446, 333, 899, 946, 345, 79, 726, 243, 1257, 575, 1788] payload_rows_per_dst=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] dup_rows_per_dst=[3548, 5674, 3036, 2024, 2088, 2626, 2420, 1602] combine_rows_per_src=[9754, 12156, 10512, 9210, 7756, 7378, 8204, 10316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=334168 combine_payload_bytes=264257536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264591704 ingress_bytes=402669568 link_bytes=402669568 token_bytes=528849240 weight_movement_bytes=138412032 total_nvlink_bytes=667261272 weight_to_token_ratio=0.261723 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515213167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75092 payload_rows_remote=65776 dup_rows=23212 dup_rows_remote=21302 dedup_ratio=0.236125 router_tokens_per_expert=[117, 986, 599, 366, 85, 1088, 854, 192, 734, 327, 902, 1302, 1357, 1026, 392, 1585, 942, 1308, 294, 745, 81, 270, 1014, 959, 527, 995, 1024, 726, 335, 550, 844, 275, 435, 1077, 847, 580, 1861, 1796, 1957, 459, 648, 178, 369, 238, 1569, 1884, 172, 1567, 1011, 432, 892, 929, 108, 210, 276, 654, 1061, 514, 101, 570, 148, 1390, 397, 2021] payload_rows_per_dst=[9404, 9592, 9316, 10002, 9278, 9140, 8186, 10174] dup_rows_per_dst=[3084, 2978, 1910, 3154, 3484, 4110, 2262, 2230] combine_rows_per_src=[9404, 9592, 9316, 10002, 9278, 9140, 8186, 10174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275505152 dispatch_meta_bytes=341352 combine_payload_bytes=275505152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275846504 ingress_bytes=344711168 link_bytes=344711168 token_bytes=551351656 weight_movement_bytes=69206016 total_nvlink_bytes=620557672 weight_to_token_ratio=0.125521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297120374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=336600 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267928280 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535519960 weight_movement_bytes=34603008 total_nvlink_bytes=570122968 weight_to_token_ratio=0.064616 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=64516 dup_rows=23628 dup_rows_remote=19026 dedup_ratio=0.240356 router_tokens_per_expert=[57, 917, 523, 344, 38, 1072, 908, 220, 570, 363, 780, 1471, 1528, 1027, 209, 1714, 916, 1587, 316, 721, 97, 181, 1184, 908, 519, 914, 892, 770, 249, 622, 892, 204, 473, 1057, 773, 557, 2132, 1873, 1895, 516, 643, 109, 278, 358, 1518, 1761, 159, 1641, 1050, 147, 795, 1163, 33, 191, 220, 710, 1072, 516, 75, 574, 69, 1476, 305, 2300] payload_rows_per_dst=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] dup_rows_per_dst=[2720, 1792, 2178, 2850, 4602, 3948, 2846, 2692] combine_rows_per_src=[9228, 7162, 9642, 10216, 10160, 8986, 9200, 10082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=340200 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272158952 ingress_bytes=306421760 link_bytes=306421760 token_bytes=543977704 weight_movement_bytes=34603008 total_nvlink_bytes=578580712 weight_to_token_ratio=0.063611 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153040414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=268673024 dispatch_meta_bytes=341616 combine_payload_bytes=268673024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269014640 ingress_bytes=303276032 link_bytes=303276032 token_bytes=537687664 weight_movement_bytes=34603008 total_nvlink_bytes=572290672 weight_to_token_ratio=0.064355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.141203293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271818752 dispatch_meta_bytes=340200 combine_payload_bytes=271818752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272158952 ingress_bytes=306421760 link_bytes=306421760 token_bytes=543977704 weight_movement_bytes=34603008 total_nvlink_bytes=578580712 weight_to_token_ratio=0.063611 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153040414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=275505152 dispatch_meta_bytes=341352 combine_payload_bytes=275505152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275846504 ingress_bytes=344711168 link_bytes=344711168 token_bytes=551351656 weight_movement_bytes=69206016 total_nvlink_bytes=620557672 weight_to_token_ratio=0.125521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297120374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269688832 dispatch_meta_bytes=342536 combine_payload_bytes=269688832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270031368 ingress_bytes=304291840 link_bytes=304291840 token_bytes=539720200 weight_movement_bytes=34603008 total_nvlink_bytes=574323208 weight_to_token_ratio=0.064113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145025696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=258580480 dispatch_meta_bytes=321896 combine_payload_bytes=258580480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=258902376 ingress_bytes=362389504 link_bytes=362389504 token_bytes=517482856 weight_movement_bytes=103809024 total_nvlink_bytes=621291880 weight_to_token_ratio=0.200604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363642529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269688832 dispatch_meta_bytes=342536 combine_payload_bytes=269688832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270031368 ingress_bytes=304291840 link_bytes=304291840 token_bytes=539720200 weight_movement_bytes=34603008 total_nvlink_bytes=574323208 weight_to_token_ratio=0.064113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.145025696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=348312 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269766808 ingress_bytes=304021504 link_bytes=304021504 token_bytes=539185304 weight_movement_bytes=34603008 total_nvlink_bytes=573788312 weight_to_token_ratio=0.064176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144008444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=334168 combine_payload_bytes=264257536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=264591704 ingress_bytes=402669568 link_bytes=402669568 token_bytes=528849240 weight_movement_bytes=138412032 total_nvlink_bytes=667261272 weight_to_token_ratio=0.261723 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.515213167e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75952 payload_rows_remote=68552 dup_rows=22352 dup_rows_remote=20298 dedup_ratio=0.227376 router_tokens_per_expert=[373, 851, 1220, 480, 404, 436, 996, 657, 1625, 2505, 236, 350, 693, 713, 1079, 432, 1618, 63, 2204, 519, 605, 373, 9, 527, 774, 729, 621, 521, 656, 945, 1462, 1137, 770, 691, 262, 281, 866, 593, 1872, 722, 420, 1963, 137, 511, 378, 638, 680, 3103, 257, 402, 344, 987, 608, 997, 451, 632, 241, 527, 1400, 598, 723, 702, 276, 307] payload_rows_per_dst=[8804, 7040, 8738, 10598, 10664, 7400, 11830, 10878] dup_rows_per_dst=[2030, 844, 3098, 3092, 3822, 2054, 3732, 3680] combine_rows_per_src=[8804, 7040, 8738, 10598, 10664, 7400, 11830, 10878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=65168 dup_rows=23252 dup_rows_remote=19814 dedup_ratio=0.236532 router_tokens_per_expert=[188, 900, 575, 338, 100, 1139, 892, 248, 674, 272, 760, 1261, 1230, 1420, 781, 2057, 895, 1240, 254, 737, 70, 387, 1086, 950, 261, 871, 769, 515, 260, 633, 768, 428, 380, 1094, 759, 697, 1702, 1571, 2281, 645, 469, 185, 376, 211, 1666, 1533, 163, 1629, 959, 692, 878, 951, 92, 266, 238, 841, 914, 365, 91, 733, 131, 1254, 473, 1954] payload_rows_per_dst=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] dup_rows_per_dst=[3438, 5852, 3426, 2034, 2226, 2632, 1932, 1712] combine_rows_per_src=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77208 payload_rows_remote=68678 dup_rows=21096 dup_rows_remote=19336 dedup_ratio=0.214600 router_tokens_per_expert=[214, 481, 1547, 268, 528, 561, 1129, 759, 2158, 3027, 304, 300, 1038, 612, 1422, 452, 2044, 40, 2556, 428, 812, 446, 5, 554, 974, 968, 654, 435, 517, 1070, 1433, 725, 592, 549, 165, 173, 1050, 411, 1491, 714, 370, 2487, 119, 491, 243, 475, 405, 3059, 25, 234, 230, 1162, 478, 869, 111, 165, 192, 298, 1722, 636, 648, 737, 153, 237] payload_rows_per_dst=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] dup_rows_per_dst=[2038, 2418, 3894, 3112, 1760, 1846, 2220, 3808] combine_rows_per_src=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=64184 dup_rows=22524 dup_rows_remote=19348 dedup_ratio=0.229126 router_tokens_per_expert=[361, 878, 1338, 495, 478, 377, 959, 541, 1214, 2471, 188, 427, 779, 662, 1102, 452, 2008, 76, 2225, 454, 625, 435, 19, 628, 1107, 720, 613, 542, 557, 1073, 1732, 982, 571, 653, 190, 256, 961, 546, 2118, 781, 476, 1722, 176, 505, 336, 661, 590, 3190, 187, 404, 274, 1017, 829, 891, 203, 391, 255, 456, 1335, 574, 785, 730, 254, 317] payload_rows_per_dst=[8850, 6490, 9336, 11064, 10652, 6984, 11596, 10808] dup_rows_per_dst=[2004, 730, 3604, 3588, 3928, 1948, 3176, 3546] combine_rows_per_src=[8850, 6490, 9336, 11064, 10652, 6984, 11596, 10808] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74852 payload_rows_remote=64206 dup_rows=23452 dup_rows_remote=19924 dedup_ratio=0.238566 router_tokens_per_expert=[443, 982, 1048, 571, 387, 334, 1007, 662, 1307, 2285, 225, 315, 662, 566, 1012, 391, 1721, 77, 2019, 542, 655, 344, 48, 623, 598, 598, 682, 862, 758, 928, 1552, 1009, 705, 624, 213, 283, 718, 719, 1836, 830, 522, 1834, 170, 564, 443, 799, 747, 3089, 343, 466, 416, 913, 870, 1067, 334, 632, 175, 481, 1452, 605, 870, 644, 362, 213] payload_rows_per_dst=[8688, 5742, 8882, 9934, 10414, 8386, 12160, 10646] dup_rows_per_dst=[2180, 600, 3176, 3286, 4056, 2526, 4100, 3528] combine_rows_per_src=[8688, 5742, 8882, 9934, 10414, 8386, 12160, 10646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74836 payload_rows_remote=63764 dup_rows=23468 dup_rows_remote=20736 dedup_ratio=0.238729 router_tokens_per_expert=[548, 1207, 1124, 483, 352, 325, 915, 545, 1258, 2420, 195, 351, 662, 649, 945, 422, 1604, 46, 1927, 471, 625, 287, 19, 561, 688, 476, 644, 704, 593, 695, 1411, 1092, 566, 776, 281, 274, 735, 664, 1959, 803, 724, 1910, 154, 561, 354, 965, 934, 3011, 585, 433, 517, 802, 642, 1129, 577, 829, 198, 568, 1386, 642, 767, 595, 333, 234] payload_rows_per_dst=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] dup_rows_per_dst=[3532, 2732, 2808, 1616, 2678, 5900, 2652, 1550] combine_rows_per_src=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280788992 dispatch_meta_bytes=355400 combine_payload_bytes=280788992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281144392 ingress_bytes=384598016 link_bytes=384598016 token_bytes=561933384 weight_movement_bytes=103809024 total_nvlink_bytes=665742408 weight_to_token_ratio=0.184735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.447211372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75052 payload_rows_remote=65168 dup_rows=23252 dup_rows_remote=19814 dedup_ratio=0.236532 router_tokens_per_expert=[188, 900, 575, 338, 100, 1139, 892, 248, 674, 272, 760, 1261, 1230, 1420, 781, 2057, 895, 1240, 254, 737, 70, 387, 1086, 950, 261, 871, 769, 515, 260, 633, 768, 428, 380, 1094, 759, 697, 1702, 1571, 2281, 645, 469, 185, 376, 211, 1666, 1533, 163, 1629, 959, 692, 878, 951, 92, 266, 238, 841, 914, 365, 91, 733, 131, 1254, 473, 1954] payload_rows_per_dst=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] dup_rows_per_dst=[3438, 5852, 3426, 2034, 2226, 2632, 1932, 1712] combine_rows_per_src=[9884, 12282, 11070, 9156, 8066, 6574, 7902, 10118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76296 payload_rows_remote=66208 dup_rows=22008 dup_rows_remote=19348 dedup_ratio=0.223877 router_tokens_per_expert=[353, 656, 1453, 267, 525, 445, 1049, 848, 1577, 2844, 283, 378, 845, 682, 1214, 369, 1756, 120, 2197, 553, 850, 343, 10, 654, 927, 817, 651, 444, 570, 1048, 1408, 876, 686, 669, 173, 265, 922, 541, 1506, 725, 510, 2234, 161, 460, 344, 626, 391, 3201, 112, 338, 245, 1130, 524, 1028, 249, 342, 234, 372, 1623, 596, 729, 769, 211, 224] payload_rows_per_dst=[9430, 10840, 9280, 10088, 8962, 8750, 9634, 9312] dup_rows_per_dst=[2496, 3420, 3686, 2660, 2012, 2974, 2432, 2328] combine_rows_per_src=[9430, 10840, 9280, 10088, 8962, 8750, 9634, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75952 payload_rows_remote=68552 dup_rows=22352 dup_rows_remote=20298 dedup_ratio=0.227376 router_tokens_per_expert=[373, 851, 1220, 480, 404, 436, 996, 657, 1625, 2505, 236, 350, 693, 713, 1079, 432, 1618, 63, 2204, 519, 605, 373, 9, 527, 774, 729, 621, 521, 656, 945, 1462, 1137, 770, 691, 262, 281, 866, 593, 1872, 722, 420, 1963, 137, 511, 378, 638, 680, 3103, 257, 402, 344, 987, 608, 997, 451, 632, 241, 527, 1400, 598, 723, 702, 276, 307] payload_rows_per_dst=[8804, 7040, 8738, 10598, 10664, 7400, 11830, 10878] dup_rows_per_dst=[2030, 844, 3098, 3092, 3822, 2054, 3732, 3680] combine_rows_per_src=[8804, 7040, 8738, 10598, 10664, 7400, 11830, 10878] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74836 payload_rows_remote=63764 dup_rows=23468 dup_rows_remote=20736 dedup_ratio=0.238729 router_tokens_per_expert=[548, 1207, 1124, 483, 352, 325, 915, 545, 1258, 2420, 195, 351, 662, 649, 945, 422, 1604, 46, 1927, 471, 625, 287, 19, 561, 688, 476, 644, 704, 593, 695, 1411, 1092, 566, 776, 281, 274, 735, 664, 1959, 803, 724, 1910, 154, 561, 354, 965, 934, 3011, 585, 433, 517, 802, 642, 1129, 577, 829, 198, 568, 1386, 642, 767, 595, 333, 234] payload_rows_per_dst=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] dup_rows_per_dst=[3532, 2732, 2808, 1616, 2678, 5900, 2652, 1550] combine_rows_per_src=[10288, 11072, 8272, 8168, 9438, 11326, 8376, 7896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76296 payload_rows_remote=66208 dup_rows=22008 dup_rows_remote=19348 dedup_ratio=0.223877 router_tokens_per_expert=[353, 656, 1453, 267, 525, 445, 1049, 848, 1577, 2844, 283, 378, 845, 682, 1214, 369, 1756, 120, 2197, 553, 850, 343, 10, 654, 927, 817, 651, 444, 570, 1048, 1408, 876, 686, 669, 173, 265, 922, 541, 1506, 725, 510, 2234, 161, 460, 344, 626, 391, 3201, 112, 338, 245, 1130, 524, 1028, 249, 342, 234, 372, 1623, 596, 729, 769, 211, 224] payload_rows_per_dst=[9430, 10840, 9280, 10088, 8962, 8750, 9634, 9312] dup_rows_per_dst=[2496, 3420, 3686, 2660, 2012, 2974, 2432, 2328] combine_rows_per_src=[9430, 10840, 9280, 10088, 8962, 8750, 9634, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281305088 dispatch_meta_bytes=352056 combine_payload_bytes=281305088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281657144 ingress_bytes=315908096 link_bytes=315908096 token_bytes=562962232 weight_movement_bytes=34603008 total_nvlink_bytes=597565240 weight_to_token_ratio=0.061466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188736733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65776 dup_rows=23088 dup_rows_remote=19640 dedup_ratio=0.234863 router_tokens_per_expert=[328, 581, 1699, 231, 501, 693, 1166, 882, 2005, 2902, 367, 334, 1035, 558, 1472, 298, 1685, 59, 2336, 503, 850, 376, 13, 622, 799, 1066, 607, 416, 407, 1036, 1402, 735, 615, 631, 175, 189, 915, 483, 1228, 677, 462, 2697, 158, 467, 251, 542, 384, 3087, 54, 271, 271, 1120, 445, 922, 194, 253, 167, 335, 1807, 580, 645, 775, 143, 245] payload_rows_per_dst=[11128, 12762, 9440, 8560, 8266, 11224, 5786, 8050] dup_rows_per_dst=[3838, 5180, 3448, 1572, 1560, 4872, 1274, 1344] combine_rows_per_src=[11128, 12762, 9440, 8560, 8266, 11224, 5786, 8050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77208 payload_rows_remote=68678 dup_rows=21096 dup_rows_remote=19336 dedup_ratio=0.214600 router_tokens_per_expert=[214, 481, 1547, 268, 528, 561, 1129, 759, 2158, 3027, 304, 300, 1038, 612, 1422, 452, 2044, 40, 2556, 428, 812, 446, 5, 554, 974, 968, 654, 435, 517, 1070, 1433, 725, 592, 549, 165, 173, 1050, 411, 1491, 714, 370, 2487, 119, 491, 243, 475, 405, 3059, 25, 234, 230, 1162, 478, 869, 111, 165, 192, 298, 1722, 636, 648, 737, 153, 237] payload_rows_per_dst=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] dup_rows_per_dst=[2038, 2418, 3894, 3112, 1760, 1846, 2220, 3808] combine_rows_per_src=[8936, 10154, 9876, 10440, 8530, 7334, 10446, 11492] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=339928 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267268056 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534196184 weight_movement_bytes=34603008 total_nvlink_bytes=568799192 weight_to_token_ratio=0.064776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262897664 dispatch_meta_bytes=334128 combine_payload_bytes=262897664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263231792 ingress_bytes=297500672 link_bytes=297500672 token_bytes=526129456 weight_movement_bytes=34603008 total_nvlink_bytes=560732464 weight_to_token_ratio=0.065769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=14592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119471078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271187968 dispatch_meta_bytes=342224 combine_payload_bytes=271187968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271530192 ingress_bytes=340393984 link_bytes=340393984 token_bytes=542718160 weight_movement_bytes=69206016 total_nvlink_bytes=611924176 weight_to_token_ratio=0.127517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280875158e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262987776 dispatch_meta_bytes=336520 combine_payload_bytes=262987776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263324296 ingress_bytes=297590784 link_bytes=297590784 token_bytes=526312072 weight_movement_bytes=34603008 total_nvlink_bytes=560915080 weight_to_token_ratio=0.065746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119810162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=339928 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267268056 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534196184 weight_movement_bytes=34603008 total_nvlink_bytes=568799192 weight_to_token_ratio=0.064776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=280788992 dispatch_meta_bytes=355400 combine_payload_bytes=280788992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=281144392 ingress_bytes=384598016 link_bytes=384598016 token_bytes=561933384 weight_movement_bytes=103809024 total_nvlink_bytes=665742408 weight_to_token_ratio=0.184735 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.447211372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=261177344 dispatch_meta_bytes=338000 combine_payload_bytes=261177344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=261515344 ingress_bytes=364986368 link_bytes=364986368 token_bytes=522692688 weight_movement_bytes=103809024 total_nvlink_bytes=626501712 weight_to_token_ratio=0.198604 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.373414319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271187968 dispatch_meta_bytes=342224 combine_payload_bytes=271187968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=271530192 ingress_bytes=340393984 link_bytes=340393984 token_bytes=542718160 weight_movement_bytes=69206016 total_nvlink_bytes=611924176 weight_to_token_ratio=0.127517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280875158e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269418496 dispatch_meta_bytes=341664 combine_payload_bytes=269418496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=269760160 ingress_bytes=338624512 link_bytes=338624512 token_bytes=539178656 weight_movement_bytes=69206016 total_nvlink_bytes=608384672 weight_to_token_ratio=0.128355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274216777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=281305088 dispatch_meta_bytes=352056 combine_payload_bytes=281305088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281657144 ingress_bytes=315908096 link_bytes=315908096 token_bytes=562962232 weight_movement_bytes=34603008 total_nvlink_bytes=597565240 weight_to_token_ratio=0.061466 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188736733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73892 payload_rows_remote=63440 dup_rows=24412 dup_rows_remote=21234 dedup_ratio=0.248332 router_tokens_per_expert=[8, 1820, 1489, 520, 1142, 1198, 920, 855, 347, 443, 1676, 114, 321, 299, 377, 1597, 1581, 419, 256, 179, 445, 270, 210, 227, 549, 373, 2211, 591, 1543, 1003, 968, 96, 406, 32, 125, 395, 1639, 693, 301, 1343, 160, 285, 742, 1056, 837, 23, 2159, 10, 436, 394, 278, 1230, 561, 1028, 734, 1327, 831, 591, 236, 1333, 948, 1976, 1846, 1150] payload_rows_per_dst=[8050, 8386, 9260, 8894, 9798, 10452, 9196, 9856] dup_rows_per_dst=[4058, 1962, 1606, 2844, 3710, 3178, 2780, 4274] combine_rows_per_src=[8050, 8386, 9260, 8894, 9798, 10452, 9196, 9856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=65296 dup_rows=22524 dup_rows_remote=18946 dedup_ratio=0.229126 router_tokens_per_expert=[557, 1045, 1173, 491, 362, 337, 965, 595, 1158, 2491, 218, 413, 830, 559, 970, 460, 1686, 51, 2249, 558, 671, 396, 30, 645, 1001, 756, 636, 595, 539, 820, 1506, 916, 554, 712, 181, 280, 893, 540, 1900, 721, 639, 1977, 168, 503, 344, 695, 682, 3520, 291, 367, 299, 924, 710, 1145, 279, 430, 177, 373, 1517, 613, 848, 693, 270, 228] payload_rows_per_dst=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] dup_rows_per_dst=[3578, 5768, 1932, 2014, 2236, 3842, 1668, 1486] combine_rows_per_src=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=67436 dup_rows=23670 dup_rows_remote=21428 dedup_ratio=0.240784 router_tokens_per_expert=[0, 1368, 1449, 1053, 887, 1440, 675, 1329, 303, 803, 2483, 70, 176, 319, 250, 1039, 2092, 394, 17, 61, 135, 738, 290, 161, 598, 184, 2565, 335, 2834, 713, 698, 0, 273, 0, 7, 659, 1134, 127, 104, 1048, 71, 285, 28, 980, 489, 0, 3212, 0, 16, 887, 290, 817, 454, 1630, 1041, 1122, 583, 960, 50, 1545, 1216, 1637, 1664, 1364] payload_rows_per_dst=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] dup_rows_per_dst=[4472, 1934, 1714, 3654, 2242, 1968, 3230, 4456] combine_rows_per_src=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75110 payload_rows_remote=66030 dup_rows=23194 dup_rows_remote=20306 dedup_ratio=0.235942 router_tokens_per_expert=[17, 1976, 1459, 395, 1006, 1625, 891, 851, 561, 360, 1597, 154, 336, 340, 410, 1419, 1940, 443, 163, 206, 368, 240, 256, 204, 407, 369, 2059, 606, 1925, 1172, 1109, 101, 323, 43, 97, 497, 1328, 646, 315, 1105, 146, 408, 876, 943, 619, 7, 2104, 7, 388, 535, 217, 1156, 540, 1080, 788, 1280, 894, 737, 169, 1361, 969, 1819, 1691, 1099] payload_rows_per_dst=[7914, 8272, 9372, 10710, 9476, 10552, 9080, 9734] dup_rows_per_dst=[1656, 2082, 1650, 3854, 3184, 3518, 2888, 4362] combine_rows_per_src=[7914, 8272, 9372, 10710, 9476, 10552, 9080, 9734] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65468 dup_rows=23628 dup_rows_remote=19526 dedup_ratio=0.240356 router_tokens_per_expert=[17, 1788, 1433, 525, 1151, 1558, 1147, 741, 400, 356, 1483, 178, 570, 301, 375, 1583, 1340, 474, 143, 195, 584, 321, 181, 208, 392, 399, 1710, 607, 1593, 1209, 1260, 158, 486, 143, 313, 298, 1131, 918, 393, 968, 216, 318, 1245, 910, 867, 61, 2000, 28, 460, 351, 269, 1667, 573, 887, 725, 1256, 937, 568, 259, 1151, 1122, 1470, 1696, 1086] payload_rows_per_dst=[8154, 8114, 8592, 10554, 9766, 11002, 9286, 9208] dup_rows_per_dst=[1928, 2378, 1692, 3782, 3306, 3474, 2966, 4102] combine_rows_per_src=[8154, 8114, 8592, 10554, 9766, 11002, 9286, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259850240 dispatch_meta_bytes=338696 combine_payload_bytes=259850240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260188936 ingress_bytes=294453248 link_bytes=294453248 token_bytes=520039176 weight_movement_bytes=34603008 total_nvlink_bytes=554642184 weight_to_token_ratio=0.066539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108003867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73952 payload_rows_remote=64696 dup_rows=24352 dup_rows_remote=21968 dedup_ratio=0.247721 router_tokens_per_expert=[5, 1621, 1379, 673, 1299, 1289, 645, 1049, 324, 599, 2212, 84, 221, 309, 313, 1206, 1480, 371, 95, 88, 278, 522, 329, 255, 585, 325, 2296, 453, 2477, 1017, 870, 13, 295, 6, 28, 590, 1311, 296, 168, 1315, 100, 268, 245, 980, 697, 0, 2854, 4, 127, 636, 289, 955, 418, 1433, 857, 1306, 738, 766, 125, 1508, 1055, 1769, 2094, 1237] payload_rows_per_dst=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] dup_rows_per_dst=[4168, 4326, 2412, 2384, 2812, 1178, 3122, 3950] combine_rows_per_src=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72962 payload_rows_remote=64404 dup_rows=25342 dup_rows_remote=22342 dedup_ratio=0.257792 router_tokens_per_expert=[58, 2128, 1426, 385, 994, 1082, 1264, 671, 453, 305, 1392, 147, 449, 288, 475, 1198, 1633, 469, 259, 220, 491, 265, 208, 173, 375, 424, 1072, 553, 1204, 1160, 1198, 516, 635, 422, 358, 255, 1358, 1034, 421, 1138, 210, 317, 1612, 989, 901, 84, 2033, 104, 684, 257, 195, 1953, 599, 919, 672, 1000, 837, 465, 306, 996, 891, 1796, 1353, 1423] payload_rows_per_dst=[8190, 8558, 10588, 8496, 10836, 10062, 9106, 7126] dup_rows_per_dst=[4148, 3000, 3146, 2364, 4662, 2438, 3452, 2132] combine_rows_per_src=[8190, 8558, 10588, 8496, 10836, 10062, 9106, 7126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73892 payload_rows_remote=63440 dup_rows=24412 dup_rows_remote=21234 dedup_ratio=0.248332 router_tokens_per_expert=[8, 1820, 1489, 520, 1142, 1198, 920, 855, 347, 443, 1676, 114, 321, 299, 377, 1597, 1581, 419, 256, 179, 445, 270, 210, 227, 549, 373, 2211, 591, 1543, 1003, 968, 96, 406, 32, 125, 395, 1639, 693, 301, 1343, 160, 285, 742, 1056, 837, 23, 2159, 10, 436, 394, 278, 1230, 561, 1028, 734, 1327, 831, 591, 236, 1333, 948, 1976, 1846, 1150] payload_rows_per_dst=[8050, 8386, 9260, 8894, 9798, 10452, 9196, 9856] dup_rows_per_dst=[4058, 1962, 1606, 2844, 3710, 3178, 2780, 4274] combine_rows_per_src=[8050, 8386, 9260, 8894, 9798, 10452, 9196, 9856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75780 payload_rows_remote=65296 dup_rows=22524 dup_rows_remote=18946 dedup_ratio=0.229126 router_tokens_per_expert=[557, 1045, 1173, 491, 362, 337, 965, 595, 1158, 2491, 218, 413, 830, 559, 970, 460, 1686, 51, 2249, 558, 671, 396, 30, 645, 1001, 756, 636, 595, 539, 820, 1506, 916, 554, 712, 181, 280, 893, 540, 1900, 721, 639, 1977, 168, 503, 344, 695, 682, 3520, 291, 367, 299, 924, 710, 1145, 279, 430, 177, 373, 1517, 613, 848, 693, 270, 228] payload_rows_per_dst=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] dup_rows_per_dst=[3578, 5768, 1932, 2014, 2236, 3842, 1668, 1486] combine_rows_per_src=[10484, 12512, 8834, 8512, 9326, 10938, 7222, 7952] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73952 payload_rows_remote=64696 dup_rows=24352 dup_rows_remote=21968 dedup_ratio=0.247721 router_tokens_per_expert=[5, 1621, 1379, 673, 1299, 1289, 645, 1049, 324, 599, 2212, 84, 221, 309, 313, 1206, 1480, 371, 95, 88, 278, 522, 329, 255, 585, 325, 2296, 453, 2477, 1017, 870, 13, 295, 6, 28, 590, 1311, 296, 168, 1315, 100, 268, 245, 980, 697, 0, 2854, 4, 127, 636, 289, 955, 418, 1433, 857, 1306, 738, 766, 125, 1508, 1055, 1769, 2094, 1237] payload_rows_per_dst=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] dup_rows_per_dst=[4168, 4326, 2412, 2384, 2812, 1178, 3122, 3950] combine_rows_per_src=[8510, 10642, 9806, 9256, 8448, 9118, 8920, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=336968 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267789384 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535241800 weight_movement_bytes=34603008 total_nvlink_bytes=569844808 weight_to_token_ratio=0.064649 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=64430 dup_rows=23080 dup_rows_remote=19396 dedup_ratio=0.234782 router_tokens_per_expert=[0, 952, 1465, 1114, 990, 1125, 629, 1300, 255, 733, 2617, 68, 168, 291, 325, 938, 1709, 394, 48, 47, 166, 751, 366, 236, 658, 253, 2297, 246, 2818, 789, 662, 3, 312, 1, 23, 691, 1197, 208, 128, 1232, 83, 279, 42, 1015, 563, 0, 3090, 0, 56, 968, 224, 1045, 393, 1816, 959, 1186, 721, 780, 93, 1597, 1080, 1749, 1773, 1435] payload_rows_per_dst=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] dup_rows_per_dst=[4152, 4494, 3684, 1804, 1982, 886, 3312, 2766] combine_rows_per_src=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270458880 dispatch_meta_bytes=345344 combine_payload_bytes=270458880 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270804224 ingress_bytes=305061888 link_bytes=305061888 token_bytes=541263104 weight_movement_bytes=34603008 total_nvlink_bytes=575866112 weight_to_token_ratio=0.063930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147923325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74634 payload_rows_remote=67436 dup_rows=23670 dup_rows_remote=21428 dedup_ratio=0.240784 router_tokens_per_expert=[0, 1368, 1449, 1053, 887, 1440, 675, 1329, 303, 803, 2483, 70, 176, 319, 250, 1039, 2092, 394, 17, 61, 135, 738, 290, 161, 598, 184, 2565, 335, 2834, 713, 698, 0, 273, 0, 7, 659, 1134, 127, 104, 1048, 71, 285, 28, 980, 489, 0, 3212, 0, 16, 887, 290, 817, 454, 1630, 1041, 1122, 583, 960, 50, 1545, 1216, 1637, 1664, 1364] payload_rows_per_dst=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] dup_rows_per_dst=[4472, 1934, 1714, 3654, 2242, 1968, 3230, 4456] combine_rows_per_src=[9194, 8952, 9390, 9842, 7198, 10520, 9284, 10254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=339976 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268496904 ingress_bytes=406568960 link_bytes=406568960 token_bytes=536653832 weight_movement_bytes=138412032 total_nvlink_bytes=675065864 weight_to_token_ratio=0.257917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529886265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72962 payload_rows_remote=64404 dup_rows=25342 dup_rows_remote=22342 dedup_ratio=0.257792 router_tokens_per_expert=[58, 2128, 1426, 385, 994, 1082, 1264, 671, 453, 305, 1392, 147, 449, 288, 475, 1198, 1633, 469, 259, 220, 491, 265, 208, 173, 375, 424, 1072, 553, 1204, 1160, 1198, 516, 635, 422, 358, 255, 1358, 1034, 421, 1138, 210, 317, 1612, 989, 901, 84, 2033, 104, 684, 257, 195, 1953, 599, 919, 672, 1000, 837, 465, 306, 996, 891, 1796, 1353, 1423] payload_rows_per_dst=[8190, 8558, 10588, 8496, 10836, 10062, 9106, 7126] dup_rows_per_dst=[4148, 3000, 3146, 2364, 4662, 2438, 3452, 2132] combine_rows_per_src=[8190, 8558, 10588, 8496, 10836, 10062, 9106, 7126] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75224 payload_rows_remote=64430 dup_rows=23080 dup_rows_remote=19396 dedup_ratio=0.234782 router_tokens_per_expert=[0, 952, 1465, 1114, 990, 1125, 629, 1300, 255, 733, 2617, 68, 168, 291, 325, 938, 1709, 394, 48, 47, 166, 751, 366, 236, 658, 253, 2297, 246, 2818, 789, 662, 3, 312, 1, 23, 691, 1197, 208, 128, 1232, 83, 279, 42, 1015, 563, 0, 3090, 0, 56, 968, 224, 1045, 393, 1816, 959, 1186, 721, 780, 93, 1597, 1080, 1749, 1773, 1435] payload_rows_per_dst=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] dup_rows_per_dst=[4152, 4494, 3684, 1804, 1982, 886, 3312, 2766] combine_rows_per_src=[9094, 10890, 10794, 9054, 7506, 9258, 9982, 8646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276217856 dispatch_meta_bytes=355456 combine_payload_bytes=276217856 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276573312 ingress_bytes=345423872 link_bytes=345423872 token_bytes=552791168 weight_movement_bytes=69206016 total_nvlink_bytes=621997184 weight_to_token_ratio=0.125194 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299802221e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=339976 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268496904 ingress_bytes=406568960 link_bytes=406568960 token_bytes=536653832 weight_movement_bytes=138412032 total_nvlink_bytes=675065864 weight_to_token_ratio=0.257917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529886265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=346656 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265341472 ingress_bytes=368803840 link_bytes=368803840 token_bytes=530336288 weight_movement_bytes=103809024 total_nvlink_bytes=634145312 weight_to_token_ratio=0.195742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387779159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259850240 dispatch_meta_bytes=338696 combine_payload_bytes=259850240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260188936 ingress_bytes=294453248 link_bytes=294453248 token_bytes=520039176 weight_movement_bytes=34603008 total_nvlink_bytes=554642184 weight_to_token_ratio=0.066539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108003867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=336968 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267789384 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535241800 weight_movement_bytes=34603008 total_nvlink_bytes=569844808 weight_to_token_ratio=0.064649 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263798784 dispatch_meta_bytes=346984 combine_payload_bytes=263798784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264145768 ingress_bytes=298401792 link_bytes=298401792 token_bytes=527944552 weight_movement_bytes=34603008 total_nvlink_bytes=562547560 weight_to_token_ratio=0.065543 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122861920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=346656 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265341472 ingress_bytes=368803840 link_bytes=368803840 token_bytes=530336288 weight_movement_bytes=103809024 total_nvlink_bytes=634145312 weight_to_token_ratio=0.195742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387779159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263905280 dispatch_meta_bytes=335304 combine_payload_bytes=263905280 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=264240584 ingress_bytes=333111296 link_bytes=333111296 token_bytes=528145864 weight_movement_bytes=69206016 total_nvlink_bytes=597351880 weight_to_token_ratio=0.131036 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=14080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253470989e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=276217856 dispatch_meta_bytes=355456 combine_payload_bytes=276217856 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276573312 ingress_bytes=345423872 link_bytes=345423872 token_bytes=552791168 weight_movement_bytes=69206016 total_nvlink_bytes=621997184 weight_to_token_ratio=0.125194 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299802221e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77354 payload_rows_remote=66308 dup_rows=20950 dup_rows_remote=15774 dedup_ratio=0.213114 router_tokens_per_expert=[882, 1566, 1136, 272, 367, 206, 3544, 511, 16, 1, 1038, 834, 1284, 0, 9, 1238, 715, 1007, 3383, 82, 564, 389, 312, 1, 186, 2353, 920, 128, 359, 313, 2805, 487, 1582, 587, 162, 1211, 574, 341, 1246, 958, 219, 0, 167, 1825, 159, 460, 1225, 512, 889, 970, 1063, 678, 167, 681, 0, 1295, 796, 446, 445, 739, 256, 578, 1212, 801] payload_rows_per_dst=[8216, 7542, 10794, 11186, 8750, 11046, 9348, 10472] dup_rows_per_dst=[1664, 1298, 2112, 3916, 1408, 5176, 2138, 3238] combine_rows_per_src=[8216, 7542, 10794, 11186, 8750, 11046, 9348, 10472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73468 payload_rows_remote=63678 dup_rows=24836 dup_rows_remote=20010 dedup_ratio=0.252645 router_tokens_per_expert=[11, 1922, 1576, 693, 975, 1172, 920, 795, 555, 494, 1656, 92, 353, 277, 359, 1567, 1873, 462, 138, 165, 340, 415, 166, 170, 694, 390, 1804, 591, 1464, 972, 1144, 151, 382, 44, 192, 333, 1716, 768, 315, 1017, 153, 351, 779, 900, 848, 21, 2282, 16, 406, 510, 270, 1176, 539, 1014, 762, 1173, 974, 748, 194, 1166, 983, 1688, 1882, 1194] payload_rows_per_dst=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] dup_rows_per_dst=[4826, 4014, 3692, 2352, 3564, 1760, 2858, 1770] combine_rows_per_src=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271597568 dispatch_meta_bytes=328328 combine_payload_bytes=271597568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271925896 ingress_bytes=306200576 link_bytes=306200576 token_bytes=543523464 weight_movement_bytes=34603008 total_nvlink_bytes=578126472 weight_to_token_ratio=0.063664 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152208116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77912 payload_rows_remote=68230 dup_rows=20392 dup_rows_remote=17986 dedup_ratio=0.207438 router_tokens_per_expert=[916, 1911, 1089, 348, 410, 299, 2695, 395, 28, 8, 946, 835, 1749, 0, 7, 1298, 759, 733, 3147, 83, 634, 426, 384, 12, 100, 1876, 945, 171, 344, 250, 2517, 413, 1718, 669, 198, 1417, 503, 331, 1760, 758, 283, 0, 61, 1796, 266, 401, 1431, 480, 1044, 1253, 1091, 446, 201, 626, 0, 1383, 869, 486, 440, 696, 336, 565, 1246, 670] payload_rows_per_dst=[9000, 8170, 10296, 10298, 9466, 10266, 9682, 10734] dup_rows_per_dst=[1736, 1572, 2060, 2934, 1806, 4560, 2406, 3318] combine_rows_per_src=[9000, 8170, 10296, 10298, 9466, 10266, 9682, 10734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75542 payload_rows_remote=64554 dup_rows=22762 dup_rows_remote=19270 dedup_ratio=0.231547 router_tokens_per_expert=[867, 1221, 1178, 286, 527, 157, 2796, 349, 75, 61, 1228, 907, 1267, 0, 41, 1220, 695, 982, 3257, 120, 602, 278, 391, 185, 223, 1902, 837, 130, 387, 308, 2390, 481, 1417, 721, 247, 1407, 840, 410, 1412, 792, 310, 8, 178, 2001, 176, 506, 1191, 479, 957, 1087, 963, 592, 234, 751, 0, 1302, 737, 493, 452, 727, 297, 627, 1473, 1017] payload_rows_per_dst=[7572, 7996, 6828, 11484, 10432, 10616, 9626, 10988] dup_rows_per_dst=[1598, 1602, 576, 5098, 3134, 4674, 2588, 3492] combine_rows_per_src=[7572, 7996, 6828, 11484, 10432, 10616, 9626, 10988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76848 payload_rows_remote=69816 dup_rows=21456 dup_rows_remote=20888 dedup_ratio=0.218262 router_tokens_per_expert=[586, 1697, 674, 282, 87, 65, 2837, 504, 0, 0, 832, 681, 2350, 0, 0, 822, 480, 1004, 4349, 20, 471, 1303, 405, 0, 75, 3164, 1018, 93, 151, 35, 3663, 236, 1705, 448, 94, 810, 457, 307, 1146, 538, 49, 0, 11, 1229, 62, 678, 1975, 699, 1020, 1118, 1438, 1142, 116, 555, 0, 1178, 1139, 398, 112, 498, 229, 415, 838, 864] payload_rows_per_dst=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] dup_rows_per_dst=[1934, 1184, 3492, 4956, 568, 3444, 3154, 2724] combine_rows_per_src=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75542 payload_rows_remote=64554 dup_rows=22762 dup_rows_remote=19270 dedup_ratio=0.231547 router_tokens_per_expert=[867, 1221, 1178, 286, 527, 157, 2796, 349, 75, 61, 1228, 907, 1267, 0, 41, 1220, 695, 982, 3257, 120, 602, 278, 391, 185, 223, 1902, 837, 130, 387, 308, 2390, 481, 1417, 721, 247, 1407, 840, 410, 1412, 792, 310, 8, 178, 2001, 176, 506, 1191, 479, 957, 1087, 963, 592, 234, 751, 0, 1302, 737, 493, 452, 727, 297, 627, 1473, 1017] payload_rows_per_dst=[7572, 7996, 6828, 11484, 10432, 10616, 9626, 10988] dup_rows_per_dst=[1598, 1602, 576, 5098, 3134, 4674, 2588, 3492] combine_rows_per_src=[7572, 7996, 6828, 11484, 10432, 10616, 9626, 10988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279470080 dispatch_meta_bytes=344864 combine_payload_bytes=279470080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279814944 ingress_bytes=314073088 link_bytes=314073088 token_bytes=559285024 weight_movement_bytes=34603008 total_nvlink_bytes=593888032 weight_to_token_ratio=0.061870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181831745e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77386 payload_rows_remote=68390 dup_rows=20918 dup_rows_remote=19444 dedup_ratio=0.212789 router_tokens_per_expert=[774, 1449, 749, 245, 171, 137, 3233, 527, 2, 4, 854, 708, 1716, 0, 12, 784, 536, 1051, 4099, 50, 529, 638, 352, 2, 119, 2728, 951, 149, 218, 146, 3234, 409, 1731, 523, 152, 962, 563, 318, 1119, 771, 118, 5, 97, 1608, 110, 673, 1599, 502, 1062, 1107, 1166, 1084, 168, 615, 6, 1292, 1046, 390, 340, 639, 297, 570, 1067, 876] payload_rows_per_dst=[10810, 10396, 11776, 8996, 9006, 7454, 9832, 9116] dup_rows_per_dst=[3760, 3202, 2738, 1474, 2256, 1970, 3168, 2350] combine_rows_per_src=[10810, 10396, 11776, 8996, 9006, 7454, 9832, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=64498 dup_rows=22530 dup_rows_remote=17930 dedup_ratio=0.229187 router_tokens_per_expert=[1029, 931, 1476, 270, 554, 33, 2072, 364, 153, 159, 1276, 911, 989, 2, 87, 1722, 827, 783, 2101, 69, 613, 175, 304, 455, 460, 1344, 679, 327, 511, 438, 2639, 468, 1436, 788, 516, 2255, 744, 615, 1399, 811, 359, 75, 394, 1951, 347, 491, 925, 587, 885, 949, 820, 443, 256, 675, 39, 1495, 528, 512, 504, 1070, 206, 507, 1678, 671] payload_rows_per_dst=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] dup_rows_per_dst=[3412, 4600, 1532, 1058, 5492, 2298, 2076, 2062] combine_rows_per_src=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260825088 dispatch_meta_bytes=334752 combine_payload_bytes=260825088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261159840 ingress_bytes=399237120 link_bytes=399237120 token_bytes=521984928 weight_movement_bytes=138412032 total_nvlink_bytes=660396960 weight_to_token_ratio=0.265165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77526 payload_rows_remote=65678 dup_rows=20778 dup_rows_remote=17942 dedup_ratio=0.211365 router_tokens_per_expert=[549, 1616, 583, 222, 131, 8, 3012, 539, 1, 0, 716, 604, 1986, 0, 1, 739, 515, 945, 4047, 43, 432, 1047, 313, 0, 130, 3073, 1035, 76, 154, 103, 3778, 259, 2192, 434, 128, 780, 588, 252, 963, 828, 54, 0, 31, 1388, 68, 854, 1755, 613, 976, 1026, 1319, 1278, 141, 621, 0, 1201, 1131, 359, 313, 527, 268, 439, 885, 1083] payload_rows_per_dst=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] dup_rows_per_dst=[3196, 4354, 2836, 972, 2748, 1822, 3106, 1744] combine_rows_per_src=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279470080 dispatch_meta_bytes=344864 combine_payload_bytes=279470080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279814944 ingress_bytes=314073088 link_bytes=314073088 token_bytes=559285024 weight_movement_bytes=34603008 total_nvlink_bytes=593888032 weight_to_token_ratio=0.061870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.181831745e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76848 payload_rows_remote=69816 dup_rows=21456 dup_rows_remote=20888 dedup_ratio=0.218262 router_tokens_per_expert=[586, 1697, 674, 282, 87, 65, 2837, 504, 0, 0, 832, 681, 2350, 0, 0, 822, 480, 1004, 4349, 20, 471, 1303, 405, 0, 75, 3164, 1018, 93, 151, 35, 3663, 236, 1705, 448, 94, 810, 457, 307, 1146, 538, 49, 0, 11, 1229, 62, 678, 1975, 699, 1020, 1118, 1438, 1142, 116, 555, 0, 1178, 1139, 398, 112, 498, 229, 415, 838, 864] payload_rows_per_dst=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] dup_rows_per_dst=[1934, 1184, 3492, 4956, 568, 3444, 3154, 2724] combine_rows_per_src=[8176, 8186, 12572, 11914, 7032, 9316, 9980, 9672] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77354 payload_rows_remote=66308 dup_rows=20950 dup_rows_remote=15774 dedup_ratio=0.213114 router_tokens_per_expert=[882, 1566, 1136, 272, 367, 206, 3544, 511, 16, 1, 1038, 834, 1284, 0, 9, 1238, 715, 1007, 3383, 82, 564, 389, 312, 1, 186, 2353, 920, 128, 359, 313, 2805, 487, 1582, 587, 162, 1211, 574, 341, 1246, 958, 219, 0, 167, 1825, 159, 460, 1225, 512, 889, 970, 1063, 678, 167, 681, 0, 1295, 796, 446, 445, 739, 256, 578, 1212, 801] payload_rows_per_dst=[8216, 7542, 10794, 11186, 8750, 11046, 9348, 10472] dup_rows_per_dst=[1664, 1298, 2112, 3916, 1408, 5176, 2138, 3238] combine_rows_per_src=[8216, 7542, 10794, 11186, 8750, 11046, 9348, 10472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73468 payload_rows_remote=63678 dup_rows=24836 dup_rows_remote=20010 dedup_ratio=0.252645 router_tokens_per_expert=[11, 1922, 1576, 693, 975, 1172, 920, 795, 555, 494, 1656, 92, 353, 277, 359, 1567, 1873, 462, 138, 165, 340, 415, 166, 170, 694, 390, 1804, 591, 1464, 972, 1144, 151, 382, 44, 192, 333, 1716, 768, 315, 1017, 153, 351, 779, 900, 848, 21, 2282, 16, 406, 510, 270, 1176, 539, 1014, 762, 1173, 974, 748, 194, 1166, 983, 1688, 1882, 1194] payload_rows_per_dst=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] dup_rows_per_dst=[4826, 4014, 3692, 2352, 3564, 1760, 2858, 1770] combine_rows_per_src=[9790, 10300, 10906, 8460, 9814, 8940, 8842, 6416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=64498 dup_rows=22530 dup_rows_remote=17930 dedup_ratio=0.229187 router_tokens_per_expert=[1029, 931, 1476, 270, 554, 33, 2072, 364, 153, 159, 1276, 911, 989, 2, 87, 1722, 827, 783, 2101, 69, 613, 175, 304, 455, 460, 1344, 679, 327, 511, 438, 2639, 468, 1436, 788, 516, 2255, 744, 615, 1399, 811, 359, 75, 394, 1951, 347, 491, 925, 587, 885, 949, 820, 443, 256, 675, 39, 1495, 528, 512, 504, 1070, 206, 507, 1678, 671] payload_rows_per_dst=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] dup_rows_per_dst=[3412, 4600, 1532, 1058, 5492, 2298, 2076, 2062] combine_rows_per_src=[10046, 11276, 9122, 7396, 11636, 7960, 9048, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=362816 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286329152 ingress_bytes=355172352 link_bytes=355172352 token_bytes=572295488 weight_movement_bytes=69206016 total_nvlink_bytes=641501504 weight_to_token_ratio=0.120927 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.336484967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=335296 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264748480 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529161664 weight_movement_bytes=34603008 total_nvlink_bytes=563764672 weight_to_token_ratio=0.065392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77526 payload_rows_remote=65678 dup_rows=20778 dup_rows_remote=17942 dedup_ratio=0.211365 router_tokens_per_expert=[549, 1616, 583, 222, 131, 8, 3012, 539, 1, 0, 716, 604, 1986, 0, 1, 739, 515, 945, 4047, 43, 432, 1047, 313, 0, 130, 3073, 1035, 76, 154, 103, 3778, 259, 2192, 434, 128, 780, 588, 252, 963, 828, 54, 0, 31, 1388, 68, 854, 1755, 613, 976, 1026, 1319, 1278, 141, 621, 0, 1201, 1131, 359, 313, 527, 268, 439, 885, 1083] payload_rows_per_dst=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] dup_rows_per_dst=[3196, 4354, 2836, 972, 2748, 1822, 3106, 1744] combine_rows_per_src=[10124, 11296, 11848, 8688, 9582, 7704, 10018, 8266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=280125440 dispatch_meta_bytes=351336 combine_payload_bytes=280125440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=280476776 ingress_bytes=349331456 link_bytes=349331456 token_bytes=560602216 weight_movement_bytes=69206016 total_nvlink_bytes=629808232 weight_to_token_ratio=0.123449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.314506145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81560 payload_rows_remote=73576 dup_rows=16744 dup_rows_remote=14890 dedup_ratio=0.170329 router_tokens_per_expert=[696, 135, 0, 3379, 100, 17, 233, 67, 0, 327, 1674, 857, 1050, 522, 459, 1586, 0, 1, 667, 4, 173, 571, 474, 2538, 112, 56, 653, 2, 370, 591, 0, 5, 67, 1729, 249, 17, 0, 0, 0, 2173, 1490, 183, 220, 1944, 4977, 317, 1230, 3254, 390, 295, 0, 942, 340, 1119, 5960, 0, 4104, 2, 0, 5, 339, 0, 421, 66] payload_rows_per_dst=[8260, 10264, 7694, 12274, 7730, 7984, 14760, 12594] dup_rows_per_dst=[994, 2686, 1162, 1258, 1670, 1854, 3332, 3788] combine_rows_per_src=[8260, 10264, 7694, 12274, 7730, 7984, 14760, 12594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=271597568 dispatch_meta_bytes=328328 combine_payload_bytes=271597568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271925896 ingress_bytes=306200576 link_bytes=306200576 token_bytes=543523464 weight_movement_bytes=34603008 total_nvlink_bytes=578126472 weight_to_token_ratio=0.063664 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.152208116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=260825088 dispatch_meta_bytes=334752 combine_payload_bytes=260825088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=261159840 ingress_bytes=399237120 link_bytes=399237120 token_bytes=521984928 weight_movement_bytes=138412032 total_nvlink_bytes=660396960 weight_to_token_ratio=0.265165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.502297141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=329712 combine_payload_bytes=264183808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264513520 ingress_bytes=298786816 link_bytes=298786816 token_bytes=528697328 weight_movement_bytes=34603008 total_nvlink_bytes=563300336 weight_to_token_ratio=0.065450 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124310734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269017088 dispatch_meta_bytes=334480 combine_payload_bytes=269017088 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=269351568 ingress_bytes=407429120 link_bytes=407429120 token_bytes=538368656 weight_movement_bytes=138412032 total_nvlink_bytes=676780688 weight_to_token_ratio=0.257095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.533122978e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=285966336 dispatch_meta_bytes=362816 combine_payload_bytes=285966336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=286329152 ingress_bytes=355172352 link_bytes=355172352 token_bytes=572295488 weight_movement_bytes=69206016 total_nvlink_bytes=641501504 weight_to_token_ratio=0.120927 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.336484967e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=301367296 dispatch_meta_bytes=353864 combine_payload_bytes=301367296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=301721160 ingress_bytes=474382336 link_bytes=474382336 token_bytes=603088456 weight_movement_bytes=173015040 total_nvlink_bytes=776103496 weight_to_token_ratio=0.286882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.785062539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76332 payload_rows_remote=64624 dup_rows=21972 dup_rows_remote=16620 dedup_ratio=0.223511 router_tokens_per_expert=[747, 1642, 1115, 307, 364, 45, 2777, 322, 28, 12, 1015, 842, 1487, 0, 23, 1189, 748, 880, 3063, 81, 647, 292, 348, 13, 111, 1902, 872, 124, 396, 238, 3219, 324, 1486, 569, 212, 1460, 621, 304, 1760, 1082, 185, 7, 187, 2098, 164, 406, 1505, 447, 980, 1209, 953, 782, 183, 700, 6, 1254, 750, 483, 450, 892, 290, 549, 1165, 840] payload_rows_per_dst=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] dup_rows_per_dst=[5352, 4248, 1352, 754, 4002, 1994, 2384, 1886] combine_rows_per_src=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80950 payload_rows_remote=65924 dup_rows=17354 dup_rows_remote=13802 dedup_ratio=0.176534 router_tokens_per_expert=[921, 181, 1, 3299, 76, 21, 249, 85, 0, 443, 1236, 803, 1096, 442, 505, 1609, 0, 7, 724, 16, 106, 641, 483, 2693, 147, 86, 543, 66, 394, 570, 0, 9, 91, 1600, 434, 30, 16, 0, 0, 2147, 1392, 267, 230, 2183, 4763, 246, 1287, 2677, 363, 536, 0, 672, 345, 1291, 6082, 0, 4224, 8, 0, 25, 357, 0, 350, 84] payload_rows_per_dst=[8450, 9670, 8142, 11918, 7078, 8718, 15026, 11948] dup_rows_per_dst=[1216, 2598, 1198, 1238, 1558, 2492, 3552, 3502] combine_rows_per_src=[8450, 9670, 8142, 11918, 7078, 8718, 15026, 11948] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67034 dup_rows=19806 dup_rows_remote=16772 dedup_ratio=0.201477 router_tokens_per_expert=[797, 207, 0, 3547, 78, 33, 243, 85, 0, 427, 1602, 914, 1104, 529, 460, 1850, 1, 2, 684, 5, 445, 689, 480, 2455, 113, 83, 539, 12, 406, 661, 0, 13, 89, 1695, 363, 58, 1, 1, 5, 2490, 1415, 240, 241, 1782, 4746, 278, 1411, 2495, 408, 389, 1, 1051, 285, 1003, 5486, 1, 3910, 3, 3, 30, 353, 0, 284, 171] payload_rows_per_dst=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] dup_rows_per_dst=[1332, 3148, 1444, 314, 1764, 5602, 3168, 3034] combine_rows_per_src=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=69620 dup_rows=16202 dup_rows_remote=15530 dedup_ratio=0.164815 router_tokens_per_expert=[703, 135, 1, 4301, 81, 1, 210, 89, 1, 315, 948, 561, 935, 682, 435, 1422, 0, 0, 602, 1, 6, 572, 395, 2246, 137, 65, 404, 0, 147, 485, 0, 4, 88, 1875, 171, 3, 7, 6, 8, 2905, 1330, 86, 189, 1780, 5335, 184, 1640, 3456, 374, 108, 1, 1354, 222, 1108, 6164, 4, 4304, 1, 3, 17, 139, 1, 343, 62] payload_rows_per_dst=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] dup_rows_per_dst=[1138, 2184, 5952, 672, 2882, 3006, 210, 158] combine_rows_per_src=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264699904 dispatch_meta_bytes=324976 combine_payload_bytes=264699904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265024880 ingress_bytes=333905920 link_bytes=333905920 token_bytes=529724784 weight_movement_bytes=69206016 total_nvlink_bytes=598930800 weight_to_token_ratio=0.130645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256461095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77458 payload_rows_remote=67018 dup_rows=20846 dup_rows_remote=18136 dedup_ratio=0.212056 router_tokens_per_expert=[939, 357, 29, 1569, 120, 333, 358, 136, 31, 786, 1376, 847, 1179, 266, 279, 1737, 6, 121, 722, 61, 1504, 737, 692, 1822, 164, 494, 713, 442, 929, 765, 15, 114, 105, 1523, 539, 193, 72, 100, 27, 2855, 1320, 262, 563, 1587, 3112, 1036, 646, 1593, 438, 2063, 95, 379, 564, 869, 4700, 75, 2932, 453, 27, 165, 637, 36, 327, 216] payload_rows_per_dst=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] dup_rows_per_dst=[3358, 2710, 3950, 2762, 3722, 1964, 1250, 1130] combine_rows_per_src=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83548 payload_rows_remote=69246 dup_rows=14756 dup_rows_remote=9548 dedup_ratio=0.150106 router_tokens_per_expert=[547, 88, 0, 4639, 86, 0, 166, 74, 0, 278, 661, 420, 930, 735, 396, 1039, 0, 0, 531, 0, 9, 486, 434, 1921, 103, 72, 307, 0, 140, 378, 0, 0, 74, 1847, 208, 2, 0, 0, 0, 3230, 1240, 109, 177, 1937, 5453, 224, 1385, 3836, 377, 98, 0, 1410, 100, 1645, 6374, 0, 4582, 0, 0, 0, 0, 0, 352, 52] payload_rows_per_dst=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] dup_rows_per_dst=[818, 1628, 5208, 458, 3058, 2910, 548, 128] combine_rows_per_src=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=69784 dup_rows=16874 dup_rows_remote=13396 dedup_ratio=0.171651 router_tokens_per_expert=[677, 114, 0, 4593, 80, 0, 165, 97, 0, 258, 449, 426, 1004, 523, 444, 1152, 0, 0, 596, 2, 0, 519, 460, 1579, 107, 59, 260, 0, 82, 488, 0, 0, 97, 1947, 205, 4, 0, 0, 0, 3416, 1169, 82, 209, 1893, 5516, 158, 1553, 4086, 420, 89, 0, 1302, 110, 1706, 6371, 0, 4304, 0, 0, 0, 0, 0, 313, 68] payload_rows_per_dst=[10352, 6888, 6406, 12562, 11646, 10010, 12764, 10802] dup_rows_per_dst=[1100, 1624, 1256, 462, 3478, 3816, 3322, 1816] combine_rows_per_src=[10352, 6888, 6406, 12562, 11646, 10010, 12764, 10802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67034 dup_rows=19806 dup_rows_remote=16772 dedup_ratio=0.201477 router_tokens_per_expert=[797, 207, 0, 3547, 78, 33, 243, 85, 0, 427, 1602, 914, 1104, 529, 460, 1850, 1, 2, 684, 5, 445, 689, 480, 2455, 113, 83, 539, 12, 406, 661, 0, 13, 89, 1695, 363, 58, 1, 1, 5, 2490, 1415, 240, 241, 1782, 4746, 278, 1411, 2495, 408, 389, 1, 1051, 285, 1003, 5486, 1, 3910, 3, 3, 30, 353, 0, 284, 171] payload_rows_per_dst=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] dup_rows_per_dst=[1332, 3148, 1444, 314, 1764, 5602, 3168, 3034] combine_rows_per_src=[8648, 10624, 8078, 4996, 7640, 12968, 14080, 11464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=270024704 dispatch_meta_bytes=318904 combine_payload_bytes=270024704 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270343608 ingress_bytes=373833728 link_bytes=373833728 token_bytes=540368312 weight_movement_bytes=103809024 total_nvlink_bytes=644177336 weight_to_token_ratio=0.192108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406706222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82102 payload_rows_remote=69620 dup_rows=16202 dup_rows_remote=15530 dedup_ratio=0.164815 router_tokens_per_expert=[703, 135, 1, 4301, 81, 1, 210, 89, 1, 315, 948, 561, 935, 682, 435, 1422, 0, 0, 602, 1, 6, 572, 395, 2246, 137, 65, 404, 0, 147, 485, 0, 4, 88, 1875, 171, 3, 7, 6, 8, 2905, 1330, 86, 189, 1780, 5335, 184, 1640, 3456, 374, 108, 1, 1354, 222, 1108, 6164, 4, 4304, 1, 3, 17, 139, 1, 343, 62] payload_rows_per_dst=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] dup_rows_per_dst=[1138, 2184, 5952, 672, 2882, 3006, 210, 158] combine_rows_per_src=[9904, 8414, 14020, 12482, 10804, 10764, 6132, 9582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81560 payload_rows_remote=73576 dup_rows=16744 dup_rows_remote=14890 dedup_ratio=0.170329 router_tokens_per_expert=[696, 135, 0, 3379, 100, 17, 233, 67, 0, 327, 1674, 857, 1050, 522, 459, 1586, 0, 1, 667, 4, 173, 571, 474, 2538, 112, 56, 653, 2, 370, 591, 0, 5, 67, 1729, 249, 17, 0, 0, 0, 2173, 1490, 183, 220, 1944, 4977, 317, 1230, 3254, 390, 295, 0, 942, 340, 1119, 5960, 0, 4104, 2, 0, 5, 339, 0, 421, 66] payload_rows_per_dst=[8260, 10264, 7694, 12274, 7730, 7984, 14760, 12594] dup_rows_per_dst=[994, 2686, 1162, 1258, 1670, 1854, 3332, 3788] combine_rows_per_src=[8260, 10264, 7694, 12274, 7730, 7984, 14760, 12594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76332 payload_rows_remote=64624 dup_rows=21972 dup_rows_remote=16620 dedup_ratio=0.223511 router_tokens_per_expert=[747, 1642, 1115, 307, 364, 45, 2777, 322, 28, 12, 1015, 842, 1487, 0, 23, 1189, 748, 880, 3063, 81, 647, 292, 348, 13, 111, 1902, 872, 124, 396, 238, 3219, 324, 1486, 569, 212, 1460, 621, 304, 1760, 1082, 185, 7, 187, 2098, 164, 406, 1505, 447, 980, 1209, 953, 782, 183, 700, 6, 1254, 750, 483, 450, 892, 290, 549, 1165, 840] payload_rows_per_dst=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] dup_rows_per_dst=[5352, 4248, 1352, 754, 4002, 1994, 2384, 1886] combine_rows_per_src=[11708, 11382, 8370, 7180, 10986, 8004, 9750, 8952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285835264 dispatch_meta_bytes=332720 combine_payload_bytes=285835264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286167984 ingress_bytes=320438272 link_bytes=320438272 token_bytes=572003248 weight_movement_bytes=34603008 total_nvlink_bytes=606606256 weight_to_token_ratio=0.060494 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205783420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274571264 dispatch_meta_bytes=335224 combine_payload_bytes=274571264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274906488 ingress_bytes=309174272 link_bytes=309174272 token_bytes=549477752 weight_movement_bytes=34603008 total_nvlink_bytes=584080760 weight_to_token_ratio=0.062974 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163397895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77458 payload_rows_remote=67018 dup_rows=20846 dup_rows_remote=18136 dedup_ratio=0.212056 router_tokens_per_expert=[939, 357, 29, 1569, 120, 333, 358, 136, 31, 786, 1376, 847, 1179, 266, 279, 1737, 6, 121, 722, 61, 1504, 737, 692, 1822, 164, 494, 713, 442, 929, 765, 15, 114, 105, 1523, 539, 193, 72, 100, 27, 2855, 1320, 262, 563, 1587, 3112, 1036, 646, 1593, 438, 2063, 95, 379, 564, 869, 4700, 75, 2932, 453, 27, 165, 637, 36, 327, 216] payload_rows_per_dst=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] dup_rows_per_dst=[3358, 2710, 3950, 2762, 3722, 1964, 1250, 1130] combine_rows_per_src=[10540, 10440, 10564, 10734, 10280, 8728, 7716, 8456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83548 payload_rows_remote=69246 dup_rows=14756 dup_rows_remote=9548 dedup_ratio=0.150106 router_tokens_per_expert=[547, 88, 0, 4639, 86, 0, 166, 74, 0, 278, 661, 420, 930, 735, 396, 1039, 0, 0, 531, 0, 9, 486, 434, 1921, 103, 72, 307, 0, 140, 378, 0, 0, 74, 1847, 208, 2, 0, 0, 0, 3230, 1240, 109, 177, 1937, 5453, 224, 1385, 3836, 377, 98, 0, 1410, 100, 1645, 6374, 0, 4582, 0, 0, 0, 0, 0, 352, 52] payload_rows_per_dst=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] dup_rows_per_dst=[818, 1628, 5208, 458, 3058, 2910, 548, 128] combine_rows_per_src=[10382, 7290, 14302, 12448, 11538, 11032, 6712, 9844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285163520 dispatch_meta_bytes=340600 combine_payload_bytes=285163520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285504120 ingress_bytes=319766528 link_bytes=319766528 token_bytes=570667640 weight_movement_bytes=34603008 total_nvlink_bytes=605270648 weight_to_token_ratio=0.060636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203255702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79182 payload_rows_remote=69036 dup_rows=19122 dup_rows_remote=15440 dedup_ratio=0.194519 router_tokens_per_expert=[1394, 4752, 72, 189, 948, 146, 56, 40, 0, 216, 140, 307, 160, 314, 564, 777, 407, 669, 0, 1, 634, 100, 2663, 610, 2570, 3876, 639, 393, 54, 0, 7, 573, 0, 224, 2051, 0, 100, 1252, 942, 193, 741, 225, 0, 54, 2711, 6139, 490, 0, 5716, 0, 1487, 1031, 476, 132, 348, 60, 5, 364, 0, 0, 0, 628, 107, 405] payload_rows_per_dst=[4866, 10868, 8494, 9442, 8078, 10146, 14262, 13026] dup_rows_per_dst=[824, 1840, 1674, 3148, 1446, 3682, 4238, 2270] combine_rows_per_src=[4866, 10868, 8494, 9442, 8078, 10146, 14262, 13026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=301367296 dispatch_meta_bytes=353864 combine_payload_bytes=301367296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=301721160 ingress_bytes=474382336 link_bytes=474382336 token_bytes=603088456 weight_movement_bytes=173015040 total_nvlink_bytes=776103496 weight_to_token_ratio=0.286882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.785062539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285163520 dispatch_meta_bytes=340600 combine_payload_bytes=285163520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285504120 ingress_bytes=319766528 link_bytes=319766528 token_bytes=570667640 weight_movement_bytes=34603008 total_nvlink_bytes=605270648 weight_to_token_ratio=0.060636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203255702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264699904 dispatch_meta_bytes=324976 combine_payload_bytes=264699904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265024880 ingress_bytes=333905920 link_bytes=333905920 token_bytes=529724784 weight_movement_bytes=69206016 total_nvlink_bytes=598930800 weight_to_token_ratio=0.130645 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256461095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=274505728 dispatch_meta_bytes=340616 combine_payload_bytes=274505728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274846344 ingress_bytes=309108736 link_bytes=309108736 token_bytes=549352072 weight_movement_bytes=34603008 total_nvlink_bytes=583955080 weight_to_token_ratio=0.062989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163151288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283631616 dispatch_meta_bytes=315176 combine_payload_bytes=283631616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283946792 ingress_bytes=318234624 link_bytes=318234624 token_bytes=567578408 weight_movement_bytes=34603008 total_nvlink_bytes=602181416 weight_to_token_ratio=0.060966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197491270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=285835264 dispatch_meta_bytes=332720 combine_payload_bytes=285835264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286167984 ingress_bytes=320438272 link_bytes=320438272 token_bytes=572003248 weight_movement_bytes=34603008 total_nvlink_bytes=606606256 weight_to_token_ratio=0.060494 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205783420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=282771456 dispatch_meta_bytes=337904 combine_payload_bytes=282771456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283109360 ingress_bytes=351977472 link_bytes=351977472 token_bytes=565880816 weight_movement_bytes=69206016 total_nvlink_bytes=635086832 weight_to_token_ratio=0.122298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324462891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78634 payload_rows_remote=64004 dup_rows=19670 dup_rows_remote=12488 dedup_ratio=0.200094 router_tokens_per_expert=[875, 175, 0, 3261, 76, 9, 261, 72, 3, 340, 1396, 712, 1040, 578, 432, 1723, 1, 0, 674, 21, 114, 531, 452, 2671, 113, 94, 703, 4, 370, 541, 0, 2, 68, 1921, 478, 13, 0, 5, 1, 2435, 1278, 243, 199, 1807, 5036, 272, 1236, 2844, 385, 285, 1, 813, 323, 1055, 6177, 4, 4287, 2, 0, 22, 386, 0, 285, 47] payload_rows_per_dst=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] dup_rows_per_dst=[7182, 5548, 1046, 1392, 3254, 840, 178, 230] combine_rows_per_src=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79278 payload_rows_remote=65348 dup_rows=19026 dup_rows_remote=15450 dedup_ratio=0.193542 router_tokens_per_expert=[1703, 4512, 73, 223, 1036, 155, 70, 82, 11, 425, 177, 566, 82, 335, 540, 804, 409, 707, 0, 8, 666, 93, 2208, 762, 2498, 3942, 820, 298, 177, 38, 45, 497, 0, 216, 1942, 0, 75, 1646, 1006, 310, 702, 168, 0, 68, 2325, 5788, 560, 0, 5765, 0, 1237, 747, 437, 126, 368, 73, 33, 374, 0, 0, 0, 680, 184, 360] payload_rows_per_dst=[5660, 11306, 8076, 7246, 8604, 11868, 13930, 12588] dup_rows_per_dst=[1024, 2458, 1630, 1500, 1786, 4802, 3576, 2250] combine_rows_per_src=[5660, 11306, 8076, 7246, 8604, 11868, 13930, 12588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79576 payload_rows_remote=67238 dup_rows=18728 dup_rows_remote=16938 dedup_ratio=0.190511 router_tokens_per_expert=[1452, 3458, 64, 391, 1113, 198, 71, 145, 3, 348, 164, 255, 52, 326, 549, 668, 396, 772, 4, 17, 755, 102, 2821, 678, 2493, 4062, 765, 359, 195, 271, 70, 644, 1, 264, 2663, 0, 76, 1313, 962, 305, 632, 292, 2, 71, 2570, 5710, 624, 0, 5586, 2, 1399, 641, 461, 123, 344, 96, 5, 386, 4, 0, 0, 530, 114, 315] payload_rows_per_dst=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] dup_rows_per_dst=[1016, 1264, 1904, 3200, 1994, 3824, 3736, 1790] combine_rows_per_src=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283631616 dispatch_meta_bytes=315176 combine_payload_bytes=283631616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283946792 ingress_bytes=318234624 link_bytes=318234624 token_bytes=567578408 weight_movement_bytes=34603008 total_nvlink_bytes=602181416 weight_to_token_ratio=0.060966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197491270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78014 payload_rows_remote=71184 dup_rows=20290 dup_rows_remote=19558 dedup_ratio=0.206401 router_tokens_per_expert=[2392, 6909, 0, 134, 596, 0, 68, 0, 0, 132, 68, 301, 12, 341, 475, 725, 365, 1427, 0, 0, 364, 107, 1061, 671, 4488, 4131, 1173, 382, 0, 0, 72, 325, 0, 262, 812, 0, 80, 1882, 719, 26, 701, 22, 0, 59, 1394, 6437, 9, 0, 5724, 0, 475, 1023, 487, 75, 187, 35, 1, 177, 0, 0, 0, 1470, 141, 235] payload_rows_per_dst=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] dup_rows_per_dst=[2018, 1474, 1572, 8648, 732, 244, 2546, 3056] combine_rows_per_src=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79576 payload_rows_remote=67238 dup_rows=18728 dup_rows_remote=16938 dedup_ratio=0.190511 router_tokens_per_expert=[1452, 3458, 64, 391, 1113, 198, 71, 145, 3, 348, 164, 255, 52, 326, 549, 668, 396, 772, 4, 17, 755, 102, 2821, 678, 2493, 4062, 765, 359, 195, 271, 70, 644, 1, 264, 2663, 0, 76, 1313, 962, 305, 632, 292, 2, 71, 2570, 5710, 624, 0, 5586, 2, 1399, 641, 461, 123, 344, 96, 5, 386, 4, 0, 0, 530, 114, 315] payload_rows_per_dst=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] dup_rows_per_dst=[1016, 1264, 1904, 3200, 1994, 3824, 3736, 1790] combine_rows_per_src=[5852, 7758, 9186, 10226, 9174, 11474, 13568, 12338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267665408 dispatch_meta_bytes=323192 combine_payload_bytes=267665408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267988600 ingress_bytes=371474432 link_bytes=371474432 token_bytes=535654008 weight_movement_bytes=103809024 total_nvlink_bytes=639463032 weight_to_token_ratio=0.193799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397828381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79404 payload_rows_remote=70586 dup_rows=18900 dup_rows_remote=17364 dedup_ratio=0.192261 router_tokens_per_expert=[2004, 6354, 18, 142, 597, 23, 62, 13, 0, 170, 70, 320, 95, 330, 443, 763, 402, 1004, 12, 5, 476, 107, 1913, 599, 3301, 4309, 943, 421, 4, 1, 72, 435, 1, 272, 1183, 1, 76, 1739, 797, 132, 665, 49, 1, 63, 1963, 6449, 58, 12, 5760, 14, 754, 824, 461, 83, 295, 58, 2, 338, 11, 0, 3, 1255, 137, 288] payload_rows_per_dst=[13886, 11342, 7420, 8818, 12876, 5322, 5952, 13788] dup_rows_per_dst=[4540, 1658, 1616, 1536, 5262, 300, 810, 3178] combine_rows_per_src=[13886, 11342, 7420, 8818, 12876, 5322, 5952, 13788] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=262160384 dispatch_meta_bytes=305968 combine_payload_bytes=262160384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262466352 ingress_bytes=296763392 link_bytes=296763392 token_bytes=524626736 weight_movement_bytes=34603008 total_nvlink_bytes=559229744 weight_to_token_ratio=0.065957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116696753e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267665408 dispatch_meta_bytes=323192 combine_payload_bytes=267665408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267988600 ingress_bytes=371474432 link_bytes=371474432 token_bytes=535654008 weight_movement_bytes=103809024 total_nvlink_bytes=639463032 weight_to_token_ratio=0.193799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397828381e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74890 payload_rows_remote=65074 dup_rows=23414 dup_rows_remote=21374 dedup_ratio=0.238180 router_tokens_per_expert=[1191, 1442, 199, 531, 2469, 520, 290, 362, 349, 503, 527, 719, 161, 427, 607, 781, 414, 1309, 179, 226, 713, 94, 2714, 1525, 2353, 1854, 765, 128, 333, 346, 247, 807, 141, 350, 2187, 24, 97, 1144, 1395, 165, 847, 451, 25, 336, 1866, 3513, 455, 47, 4180, 206, 2654, 596, 537, 207, 329, 308, 164, 495, 79, 13, 5, 162, 497, 592] payload_rows_per_dst=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] dup_rows_per_dst=[3660, 2040, 6836, 1804, 4050, 4040, 854, 130] combine_rows_per_src=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79182 payload_rows_remote=69036 dup_rows=19122 dup_rows_remote=15440 dedup_ratio=0.194519 router_tokens_per_expert=[1394, 4752, 72, 189, 948, 146, 56, 40, 0, 216, 140, 307, 160, 314, 564, 777, 407, 669, 0, 1, 634, 100, 2663, 610, 2570, 3876, 639, 393, 54, 0, 7, 573, 0, 224, 2051, 0, 100, 1252, 942, 193, 741, 225, 0, 54, 2711, 6139, 490, 0, 5716, 0, 1487, 1031, 476, 132, 348, 60, 5, 364, 0, 0, 0, 628, 107, 405] payload_rows_per_dst=[4866, 10868, 8494, 9442, 8078, 10146, 14262, 13026] dup_rows_per_dst=[824, 1840, 1674, 3148, 1446, 3682, 4238, 2270] combine_rows_per_src=[4866, 10868, 8494, 9442, 8078, 10146, 14262, 13026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=291569664 dispatch_meta_bytes=362968 combine_payload_bytes=291569664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291932632 ingress_bytes=326172672 link_bytes=326172672 token_bytes=583502296 weight_movement_bytes=34603008 total_nvlink_bytes=618105304 weight_to_token_ratio=0.059302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227361506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78634 payload_rows_remote=64004 dup_rows=19670 dup_rows_remote=12488 dedup_ratio=0.200094 router_tokens_per_expert=[875, 175, 0, 3261, 76, 9, 261, 72, 3, 340, 1396, 712, 1040, 578, 432, 1723, 1, 0, 674, 21, 114, 531, 452, 2671, 113, 94, 703, 4, 370, 541, 0, 2, 68, 1921, 478, 13, 0, 5, 1, 2435, 1278, 243, 199, 1807, 5036, 272, 1236, 2844, 385, 285, 1, 813, 323, 1055, 6177, 4, 4287, 2, 0, 22, 386, 0, 285, 47] payload_rows_per_dst=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] dup_rows_per_dst=[7182, 5548, 1046, 1392, 3254, 840, 178, 230] combine_rows_per_src=[14630, 12588, 7882, 12334, 10202, 5616, 5554, 9828] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78014 payload_rows_remote=71184 dup_rows=20290 dup_rows_remote=19558 dedup_ratio=0.206401 router_tokens_per_expert=[2392, 6909, 0, 134, 596, 0, 68, 0, 0, 132, 68, 301, 12, 341, 475, 725, 365, 1427, 0, 0, 364, 107, 1061, 671, 4488, 4131, 1173, 382, 0, 0, 72, 325, 0, 262, 812, 0, 80, 1882, 719, 26, 701, 22, 0, 59, 1394, 6437, 9, 0, 5724, 0, 475, 1023, 487, 75, 187, 35, 1, 177, 0, 0, 0, 1470, 141, 235] payload_rows_per_dst=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] dup_rows_per_dst=[2018, 1474, 1572, 8648, 732, 244, 2546, 3056] combine_rows_per_src=[8256, 10896, 6418, 14156, 6830, 4126, 13466, 13866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80266 payload_rows_remote=73400 dup_rows=18038 dup_rows_remote=16600 dedup_ratio=0.183492 router_tokens_per_expert=[2139, 6977, 0, 112, 601, 8, 53, 0, 0, 119, 54, 292, 96, 334, 451, 790, 391, 1118, 0, 0, 338, 77, 1606, 622, 4045, 4200, 1125, 372, 0, 0, 78, 312, 0, 278, 728, 0, 90, 1681, 722, 56, 704, 39, 0, 72, 1369, 6530, 14, 0, 5565, 0, 794, 1044, 456, 94, 247, 62, 0, 253, 0, 0, 0, 1566, 161, 317] payload_rows_per_dst=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] dup_rows_per_dst=[4884, 1572, 1438, 1682, 4752, 1092, 402, 2216] combine_rows_per_src=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275406848 dispatch_meta_bytes=336704 combine_payload_bytes=275406848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275743552 ingress_bytes=310009856 link_bytes=310009856 token_bytes=551150400 weight_movement_bytes=34603008 total_nvlink_bytes=585753408 weight_to_token_ratio=0.062783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166542130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80668 payload_rows_remote=66844 dup_rows=17636 dup_rows_remote=17094 dedup_ratio=0.179403 router_tokens_per_expert=[587, 473, 4273, 2230, 1873, 0, 6299, 6028, 0, 199, 0, 506, 0, 1285, 0, 181, 305, 361, 2480, 136, 955, 49, 377, 121, 12, 99, 999, 0, 0, 155, 0, 460, 5849, 663, 0, 100, 236, 956, 754, 0, 413, 0, 22, 0, 0, 0, 79, 370, 2984, 0, 6, 179, 2, 1031, 58, 0, 2288, 0, 59, 1214, 1286, 160, 0, 0] payload_rows_per_dst=[5312, 10264, 11364, 13960, 3432, 13824, 14124, 8388] dup_rows_per_dst=[554, 2624, 2664, 1546, 1986, 542, 6094, 1626] combine_rows_per_src=[5312, 10264, 11364, 13960, 3432, 13824, 14124, 8388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74890 payload_rows_remote=65074 dup_rows=23414 dup_rows_remote=21374 dedup_ratio=0.238180 router_tokens_per_expert=[1191, 1442, 199, 531, 2469, 520, 290, 362, 349, 503, 527, 719, 161, 427, 607, 781, 414, 1309, 179, 226, 713, 94, 2714, 1525, 2353, 1854, 765, 128, 333, 346, 247, 807, 141, 350, 2187, 24, 97, 1144, 1395, 165, 847, 451, 25, 336, 1866, 3513, 455, 47, 4180, 206, 2654, 596, 537, 207, 329, 308, 164, 495, 79, 13, 5, 162, 497, 592] payload_rows_per_dst=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] dup_rows_per_dst=[3660, 2040, 6836, 1804, 4050, 4040, 854, 130] combine_rows_per_src=[10348, 9816, 12752, 8154, 10076, 11040, 8820, 3884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80266 payload_rows_remote=73400 dup_rows=18038 dup_rows_remote=16600 dedup_ratio=0.183492 router_tokens_per_expert=[2139, 6977, 0, 112, 601, 8, 53, 0, 0, 119, 54, 292, 96, 334, 451, 790, 391, 1118, 0, 0, 338, 77, 1606, 622, 4045, 4200, 1125, 372, 0, 0, 78, 312, 0, 278, 728, 0, 90, 1681, 722, 56, 704, 39, 0, 72, 1369, 6530, 14, 0, 5565, 0, 794, 1044, 456, 94, 247, 62, 0, 253, 0, 0, 0, 1566, 161, 317] payload_rows_per_dst=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] dup_rows_per_dst=[4884, 1572, 1438, 1682, 4752, 1092, 402, 2216] combine_rows_per_src=[14896, 11100, 6866, 10182, 13488, 7676, 4992, 11066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=289120256 dispatch_meta_bytes=351800 combine_payload_bytes=289120256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=289472056 ingress_bytes=358326272 link_bytes=358326272 token_bytes=578592312 weight_movement_bytes=69206016 total_nvlink_bytes=647798328 weight_to_token_ratio=0.119611 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.348352914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=282771456 dispatch_meta_bytes=337904 combine_payload_bytes=282771456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=283109360 ingress_bytes=351977472 link_bytes=351977472 token_bytes=565880816 weight_movement_bytes=69206016 total_nvlink_bytes=635086832 weight_to_token_ratio=0.122298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.324462891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=262160384 dispatch_meta_bytes=305968 combine_payload_bytes=262160384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262466352 ingress_bytes=296763392 link_bytes=296763392 token_bytes=524626736 weight_movement_bytes=34603008 total_nvlink_bytes=559229744 weight_to_token_ratio=0.065957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.116696753e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79358 payload_rows_remote=68062 dup_rows=18946 dup_rows_remote=14914 dedup_ratio=0.192729 router_tokens_per_expert=[584, 434, 3744, 2989, 1499, 0, 6672, 5992, 0, 149, 0, 412, 0, 1299, 0, 235, 330, 368, 2178, 93, 905, 50, 363, 160, 45, 74, 977, 0, 4, 145, 0, 208, 5819, 609, 0, 75, 224, 1066, 755, 0, 409, 0, 98, 0, 0, 0, 230, 394, 2715, 0, 27, 193, 15, 1495, 68, 0, 2137, 4, 17, 1279, 1531, 83, 0, 0] payload_rows_per_dst=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] dup_rows_per_dst=[504, 2174, 3068, 1568, 1786, 998, 4032, 4816] combine_rows_per_src=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345792 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888896 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533432000 weight_movement_bytes=34603008 total_nvlink_bytes=568035008 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=291569664 dispatch_meta_bytes=362968 combine_payload_bytes=291569664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291932632 ingress_bytes=326172672 link_bytes=326172672 token_bytes=583502296 weight_movement_bytes=34603008 total_nvlink_bytes=618105304 weight_to_token_ratio=0.059302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227361506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=300646400 dispatch_meta_bytes=360000 combine_payload_bytes=300646400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301006400 ingress_bytes=335249408 link_bytes=335249408 token_bytes=601652800 weight_movement_bytes=34603008 total_nvlink_bytes=636255808 weight_to_token_ratio=0.057513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273793024 dispatch_meta_bytes=335752 combine_payload_bytes=273793024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274128776 ingress_bytes=308396032 link_bytes=308396032 token_bytes=547921800 weight_movement_bytes=34603008 total_nvlink_bytes=582524808 weight_to_token_ratio=0.063153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160469440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=62582 dup_rows=23648 dup_rows_remote=19148 dedup_ratio=0.240560 router_tokens_per_expert=[1587, 4635, 78, 208, 983, 163, 48, 87, 3, 248, 148, 396, 65, 322, 554, 872, 394, 877, 6, 8, 745, 87, 2821, 814, 2463, 3848, 839, 288, 156, 27, 26, 482, 5, 232, 1672, 0, 72, 1422, 971, 204, 690, 223, 0, 75, 2165, 5835, 341, 5, 5957, 8, 1812, 743, 456, 93, 370, 94, 13, 393, 5, 2, 4, 554, 115, 343] payload_rows_per_dst=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] dup_rows_per_dst=[4500, 2042, 8560, 1478, 1466, 5214, 364, 24] combine_rows_per_src=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79358 payload_rows_remote=68062 dup_rows=18946 dup_rows_remote=14914 dedup_ratio=0.192729 router_tokens_per_expert=[584, 434, 3744, 2989, 1499, 0, 6672, 5992, 0, 149, 0, 412, 0, 1299, 0, 235, 330, 368, 2178, 93, 905, 50, 363, 160, 45, 74, 977, 0, 4, 145, 0, 208, 5819, 609, 0, 75, 224, 1066, 755, 0, 409, 0, 98, 0, 0, 0, 230, 394, 2715, 0, 27, 193, 15, 1495, 68, 0, 2137, 4, 17, 1279, 1531, 83, 0, 0] payload_rows_per_dst=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] dup_rows_per_dst=[504, 2174, 3068, 1568, 1786, 998, 4032, 4816] combine_rows_per_src=[4530, 9504, 11804, 13322, 3672, 14608, 11296, 10622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81038 payload_rows_remote=74550 dup_rows=17266 dup_rows_remote=14750 dedup_ratio=0.175639 router_tokens_per_expert=[548, 169, 3742, 1952, 1934, 0, 6268, 5113, 0, 160, 0, 408, 0, 1107, 0, 276, 317, 380, 2734, 83, 982, 43, 421, 152, 13, 80, 649, 0, 0, 148, 0, 597, 6029, 703, 0, 80, 266, 2256, 752, 0, 423, 0, 0, 0, 0, 0, 0, 265, 3097, 0, 0, 87, 0, 1059, 123, 0, 1609, 0, 14, 1987, 2045, 81, 0, 0] payload_rows_per_dst=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] dup_rows_per_dst=[414, 1930, 2790, 774, 2516, 492, 5902, 2448] combine_rows_per_src=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278781952 dispatch_meta_bytes=331904 combine_payload_bytes=278781952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279113856 ingress_bytes=313384960 link_bytes=313384960 token_bytes=557895808 weight_movement_bytes=34603008 total_nvlink_bytes=592498816 weight_to_token_ratio=0.062024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179242375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79100 payload_rows_remote=65396 dup_rows=19204 dup_rows_remote=11954 dedup_ratio=0.195353 router_tokens_per_expert=[605, 533, 3468, 2779, 1957, 2, 6353, 6300, 3, 149, 1, 368, 4, 1408, 0, 181, 319, 355, 2422, 106, 903, 79, 390, 171, 42, 73, 1205, 1, 4, 138, 8, 260, 5842, 684, 0, 76, 337, 948, 670, 1, 398, 0, 95, 0, 0, 0, 273, 329, 2948, 6, 3, 247, 1, 976, 92, 4, 2260, 0, 61, 1055, 1202, 55, 0, 2] payload_rows_per_dst=[1976, 10158, 11922, 14110, 5190, 14214, 7826, 13704] dup_rows_per_dst=[490, 2880, 3126, 1952, 2096, 682, 728, 7250] combine_rows_per_src=[1976, 10158, 11922, 14110, 5190, 14214, 7826, 13704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84100 payload_rows_remote=70692 dup_rows=14204 dup_rows_remote=13128 dedup_ratio=0.144491 router_tokens_per_expert=[544, 356, 4364, 1983, 2239, 1, 6284, 5707, 10, 155, 4, 431, 16, 1107, 0, 239, 305, 318, 2701, 86, 925, 53, 380, 137, 13, 77, 753, 14, 0, 137, 16, 525, 5927, 652, 0, 78, 255, 1496, 723, 4, 404, 4, 12, 5, 0, 10, 14, 284, 2700, 4, 2, 117, 9, 1092, 122, 8, 1742, 0, 15, 1719, 1748, 116, 10, 0] payload_rows_per_dst=[7158, 10508, 9682, 13408, 13696, 13512, 7438, 8698] dup_rows_per_dst=[1286, 2144, 1930, 1076, 4574, 522, 670, 2002] combine_rows_per_src=[7158, 10508, 9682, 13408, 13696, 13512, 7438, 8698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=256335872 dispatch_meta_bytes=326920 combine_payload_bytes=256335872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256662792 ingress_bytes=360144896 link_bytes=360144896 token_bytes=512998664 weight_movement_bytes=103809024 total_nvlink_bytes=616807688 weight_to_token_ratio=0.202357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355196250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278781952 dispatch_meta_bytes=331904 combine_payload_bytes=278781952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279113856 ingress_bytes=313384960 link_bytes=313384960 token_bytes=557895808 weight_movement_bytes=34603008 total_nvlink_bytes=592498816 weight_to_token_ratio=0.062024 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179242375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81038 payload_rows_remote=74550 dup_rows=17266 dup_rows_remote=14750 dedup_ratio=0.175639 router_tokens_per_expert=[548, 169, 3742, 1952, 1934, 0, 6268, 5113, 0, 160, 0, 408, 0, 1107, 0, 276, 317, 380, 2734, 83, 982, 43, 421, 152, 13, 80, 649, 0, 0, 148, 0, 597, 6029, 703, 0, 80, 266, 2256, 752, 0, 423, 0, 0, 0, 0, 0, 0, 265, 3097, 0, 0, 87, 0, 1059, 123, 0, 1609, 0, 14, 1987, 2045, 81, 0, 0] payload_rows_per_dst=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] dup_rows_per_dst=[414, 1930, 2790, 774, 2516, 492, 5902, 2448] combine_rows_per_src=[4888, 9456, 11338, 12426, 6488, 13420, 13998, 9024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80668 payload_rows_remote=66844 dup_rows=17636 dup_rows_remote=17094 dedup_ratio=0.179403 router_tokens_per_expert=[587, 473, 4273, 2230, 1873, 0, 6299, 6028, 0, 199, 0, 506, 0, 1285, 0, 181, 305, 361, 2480, 136, 955, 49, 377, 121, 12, 99, 999, 0, 0, 155, 0, 460, 5849, 663, 0, 100, 236, 956, 754, 0, 413, 0, 22, 0, 0, 0, 79, 370, 2984, 0, 6, 179, 2, 1031, 58, 0, 2288, 0, 59, 1214, 1286, 160, 0, 0] payload_rows_per_dst=[5312, 10264, 11364, 13960, 3432, 13824, 14124, 8388] dup_rows_per_dst=[554, 2624, 2664, 1546, 1986, 542, 6094, 1626] combine_rows_per_src=[5312, 10264, 11364, 13960, 3432, 13824, 14124, 8388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=357200 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305714000 ingress_bytes=409165824 link_bytes=409165824 token_bytes=611070800 weight_movement_bytes=103809024 total_nvlink_bytes=714879824 weight_to_token_ratio=0.169881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539658055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=309400 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268171416 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536033432 weight_movement_bytes=34603008 total_nvlink_bytes=570636440 weight_to_token_ratio=0.064554 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=62582 dup_rows=23648 dup_rows_remote=19148 dedup_ratio=0.240560 router_tokens_per_expert=[1587, 4635, 78, 208, 983, 163, 48, 87, 3, 248, 148, 396, 65, 322, 554, 872, 394, 877, 6, 8, 745, 87, 2821, 814, 2463, 3848, 839, 288, 156, 27, 26, 482, 5, 232, 1672, 0, 72, 1422, 971, 204, 690, 223, 0, 75, 2165, 5835, 341, 5, 5957, 8, 1812, 743, 456, 93, 370, 94, 13, 393, 5, 2, 4, 554, 115, 343] payload_rows_per_dst=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] dup_rows_per_dst=[4500, 2042, 8560, 1478, 1466, 5214, 364, 24] combine_rows_per_src=[12074, 10870, 13862, 7084, 7690, 13454, 6788, 2834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72724 payload_rows_remote=63128 dup_rows=25580 dup_rows_remote=22516 dedup_ratio=0.260213 router_tokens_per_expert=[679, 1416, 2855, 1911, 857, 142, 5235, 4341, 142, 188, 95, 488, 195, 1699, 27, 641, 440, 451, 1776, 146, 860, 49, 622, 612, 59, 251, 1637, 59, 253, 151, 246, 1030, 4780, 479, 41, 172, 359, 1565, 699, 147, 457, 260, 842, 340, 191, 150, 475, 483, 1861, 26, 140, 396, 180, 937, 171, 167, 1486, 474, 176, 370, 1201, 132, 429, 13] payload_rows_per_dst=[13452, 9596, 8546, 6220, 12378, 8192, 6792, 7548] dup_rows_per_dst=[12156, 3064, 1366, 1152, 4106, 1758, 964, 1014] combine_rows_per_src=[13452, 9596, 8546, 6220, 12378, 8192, 6792, 7548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80892 payload_rows_remote=68134 dup_rows=17412 dup_rows_remote=12484 dedup_ratio=0.177124 router_tokens_per_expert=[0, 7277, 2150, 0, 64, 55, 418, 5058, 0, 0, 0, 0, 0, 686, 2538, 169, 49, 101, 160, 329, 67, 0, 173, 1054, 907, 247, 0, 281, 234, 55, 319, 0, 58, 7338, 105, 0, 1337, 0, 1897, 0, 691, 0, 0, 0, 2847, 247, 0, 0, 100, 0, 183, 470, 43, 479, 95, 0, 0, 376, 5639, 369, 256, 2366, 0, 1865] payload_rows_per_dst=[5296, 6740, 3534, 12976, 15264, 12758, 15470, 8854] dup_rows_per_dst=[78, 46, 332, 2388, 6206, 4928, 1824, 1610] combine_rows_per_src=[5296, 6740, 3534, 12976, 15264, 12758, 15470, 8854] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83362 payload_rows_remote=74550 dup_rows=14942 dup_rows_remote=13774 dedup_ratio=0.151998 router_tokens_per_expert=[581, 255, 4052, 1697, 2398, 0, 6175, 5125, 0, 176, 0, 498, 0, 1115, 0, 191, 341, 335, 2752, 91, 912, 45, 377, 137, 11, 89, 642, 0, 0, 143, 0, 555, 6099, 646, 0, 90, 292, 1704, 711, 0, 428, 0, 0, 0, 0, 0, 0, 276, 2806, 0, 1, 123, 0, 1251, 245, 0, 2008, 1, 12, 1993, 1635, 138, 0, 0] payload_rows_per_dst=[8530, 10000, 8812, 11936, 13968, 13228, 7860, 9028] dup_rows_per_dst=[1686, 2064, 1168, 840, 5116, 530, 992, 2546] combine_rows_per_src=[8530, 10000, 8812, 11936, 13968, 13228, 7860, 9028] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=289554432 dispatch_meta_bytes=335280 combine_payload_bytes=289554432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289889712 ingress_bytes=324157440 link_bytes=324157440 token_bytes=579444144 weight_movement_bytes=34603008 total_nvlink_bytes=614047152 weight_to_token_ratio=0.059718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.219778350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=309400 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268171416 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536033432 weight_movement_bytes=34603008 total_nvlink_bytes=570636440 weight_to_token_ratio=0.064554 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273793024 dispatch_meta_bytes=335752 combine_payload_bytes=273793024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274128776 ingress_bytes=308396032 link_bytes=308396032 token_bytes=547921800 weight_movement_bytes=34603008 total_nvlink_bytes=582524808 weight_to_token_ratio=0.063153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.160469440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=357200 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=305714000 ingress_bytes=409165824 link_bytes=409165824 token_bytes=611070800 weight_movement_bytes=103809024 total_nvlink_bytes=714879824 weight_to_token_ratio=0.169881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539658055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=256335872 dispatch_meta_bytes=326920 combine_payload_bytes=256335872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=256662792 ingress_bytes=360144896 link_bytes=360144896 token_bytes=512998664 weight_movement_bytes=103809024 total_nvlink_bytes=616807688 weight_to_token_ratio=0.202357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.355196250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76416 payload_rows_remote=60936 dup_rows=21888 dup_rows_remote=20256 dedup_ratio=0.222656 router_tokens_per_expert=[0, 7301, 2397, 0, 19, 70, 446, 4465, 0, 0, 0, 0, 0, 626, 2509, 169, 160, 86, 82, 341, 84, 0, 241, 1093, 885, 252, 0, 404, 251, 70, 204, 3, 53, 7268, 84, 0, 1419, 0, 1817, 0, 561, 0, 0, 0, 2642, 251, 0, 0, 75, 0, 81, 514, 26, 485, 74, 0, 0, 363, 6056, 309, 222, 2501, 0, 2193] payload_rows_per_dst=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] dup_rows_per_dst=[102, 4, 636, 938, 6204, 3706, 1632, 8666] combine_rows_per_src=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=258572288 dispatch_meta_bytes=342576 combine_payload_bytes=258572288 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=258914864 ingress_bytes=327778304 link_bytes=327778304 token_bytes=517487152 weight_movement_bytes=69206016 total_nvlink_bytes=586693168 weight_to_token_ratio=0.133735 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=305356800 dispatch_meta_bytes=353296 combine_payload_bytes=305356800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305710096 ingress_bytes=339959808 link_bytes=339959808 token_bytes=611066896 weight_movement_bytes=34603008 total_nvlink_bytes=645669904 weight_to_token_ratio=0.056627 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279241388e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=279076864 dispatch_meta_bytes=322472 combine_payload_bytes=279076864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279399336 ingress_bytes=313679872 link_bytes=313679872 token_bytes=558476200 weight_movement_bytes=34603008 total_nvlink_bytes=593079208 weight_to_token_ratio=0.061960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180352105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66686 payload_rows_remote=52268 dup_rows=31618 dup_rows_remote=10668 dedup_ratio=0.321635 router_tokens_per_expert=[626, 517, 3853, 2492, 1794, 7, 6334, 5914, 5, 144, 9, 392, 7, 1253, 1, 305, 342, 367, 2072, 113, 867, 51, 390, 160, 33, 73, 1223, 0, 10, 119, 9, 540, 5858, 588, 2, 74, 274, 1130, 721, 3, 394, 2, 57, 3, 0, 5, 143, 287, 2799, 2, 12, 204, 5, 1681, 89, 2, 2722, 18, 70, 1026, 893, 63, 1, 2] payload_rows_per_dst=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] dup_rows_per_dst=[20950, 2328, 1192, 576, 3804, 516, 1016, 1236] combine_rows_per_src=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76416 payload_rows_remote=60936 dup_rows=21888 dup_rows_remote=20256 dedup_ratio=0.222656 router_tokens_per_expert=[0, 7301, 2397, 0, 19, 70, 446, 4465, 0, 0, 0, 0, 0, 626, 2509, 169, 160, 86, 82, 341, 84, 0, 241, 1093, 885, 252, 0, 404, 251, 70, 204, 3, 53, 7268, 84, 0, 1419, 0, 1817, 0, 561, 0, 0, 0, 2642, 251, 0, 0, 75, 0, 81, 514, 26, 485, 74, 0, 0, 363, 6056, 309, 222, 2501, 0, 2193] payload_rows_per_dst=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] dup_rows_per_dst=[102, 4, 636, 938, 6204, 3706, 1632, 8666] combine_rows_per_src=[5762, 6604, 3538, 3748, 15078, 12132, 15480, 14074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85126 payload_rows_remote=71826 dup_rows=13178 dup_rows_remote=10884 dedup_ratio=0.134054 router_tokens_per_expert=[2, 7485, 1983, 2, 1, 62, 387, 4630, 0, 16, 4, 0, 1, 803, 2607, 146, 1, 84, 116, 327, 88, 9, 130, 938, 820, 240, 2, 240, 238, 61, 277, 0, 116, 7366, 79, 9, 2570, 1, 1735, 1, 733, 0, 20, 0, 2803, 246, 0, 18, 77, 17, 118, 380, 5, 400, 61, 17, 3, 374, 5919, 898, 246, 2055, 0, 1185] payload_rows_per_dst=[10156, 6958, 15594, 13300, 7894, 10438, 15728, 5058] dup_rows_per_dst=[2470, 196, 2524, 2294, 1128, 2820, 1392, 354] combine_rows_per_src=[10156, 6958, 15594, 13300, 7894, 10438, 15728, 5058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69700 payload_rows_remote=54084 dup_rows=28604 dup_rows_remote=10576 dedup_ratio=0.290975 router_tokens_per_expert=[2, 7162, 1779, 0, 55, 66, 388, 5147, 0, 8, 0, 1, 5, 552, 2574, 169, 250, 82, 53, 375, 86, 1, 253, 1067, 737, 240, 2, 396, 240, 64, 206, 1, 53, 7262, 98, 2, 1223, 0, 1626, 0, 649, 3, 7, 1, 2710, 240, 4, 1, 75, 5, 106, 523, 77, 532, 71, 5, 3, 388, 5947, 342, 200, 2467, 0, 2571] payload_rows_per_dst=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] dup_rows_per_dst=[0, 40, 786, 764, 2418, 4636, 1932, 18028] combine_rows_per_src=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249593856 dispatch_meta_bytes=324768 combine_payload_bytes=249593856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249918624 ingress_bytes=284196864 link_bytes=284196864 token_bytes=499512480 weight_movement_bytes=34603008 total_nvlink_bytes=534115488 weight_to_token_ratio=0.069274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069409920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85686 payload_rows_remote=75670 dup_rows=12618 dup_rows_remote=10124 dedup_ratio=0.128357 router_tokens_per_expert=[0, 7580, 2274, 0, 0, 68, 503, 4110, 0, 0, 0, 0, 0, 892, 3081, 153, 0, 82, 81, 344, 76, 0, 38, 966, 668, 247, 0, 199, 151, 68, 259, 0, 120, 7564, 80, 0, 3291, 0, 1293, 0, 798, 0, 0, 0, 2307, 247, 0, 0, 80, 0, 145, 395, 0, 363, 51, 0, 0, 418, 5981, 1146, 263, 1781, 0, 989] payload_rows_per_dst=[8274, 7956, 13248, 13384, 10016, 11588, 15920, 5300] dup_rows_per_dst=[926, 296, 2112, 1762, 2494, 3336, 1308, 384] combine_rows_per_src=[8274, 7956, 13248, 13384, 10016, 11588, 15920, 5300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=251744 combine_payload_bytes=214089728 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=214341472 ingress_bytes=421707776 link_bytes=421707776 token_bytes=428431200 weight_movement_bytes=207618048 total_nvlink_bytes=636049248 weight_to_token_ratio=0.484601 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.586852411e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=249593856 dispatch_meta_bytes=324768 combine_payload_bytes=249593856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=249918624 ingress_bytes=284196864 link_bytes=284196864 token_bytes=499512480 weight_movement_bytes=34603008 total_nvlink_bytes=534115488 weight_to_token_ratio=0.069274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.069409920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69700 payload_rows_remote=54084 dup_rows=28604 dup_rows_remote=10576 dedup_ratio=0.290975 router_tokens_per_expert=[2, 7162, 1779, 0, 55, 66, 388, 5147, 0, 8, 0, 1, 5, 552, 2574, 169, 250, 82, 53, 375, 86, 1, 253, 1067, 737, 240, 2, 396, 240, 64, 206, 1, 53, 7262, 98, 2, 1223, 0, 1626, 0, 649, 3, 7, 1, 2710, 240, 4, 1, 75, 5, 106, 523, 77, 532, 71, 5, 3, 388, 5947, 342, 200, 2467, 0, 2571] payload_rows_per_dst=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] dup_rows_per_dst=[0, 40, 786, 764, 2418, 4636, 1932, 18028] combine_rows_per_src=[4580, 6578, 3548, 3008, 8302, 12888, 15180, 15616] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75792 payload_rows_remote=61462 dup_rows=22512 dup_rows_remote=15950 dedup_ratio=0.229004 router_tokens_per_expert=[37, 5884, 3964, 192, 191, 167, 334, 2321, 31, 552, 131, 79, 58, 620, 2904, 187, 379, 143, 391, 284, 128, 197, 321, 1013, 425, 332, 416, 679, 256, 52, 475, 7, 327, 5018, 208, 83, 602, 379, 1988, 8, 238, 55, 497, 173, 1171, 328, 46, 216, 43, 221, 161, 686, 471, 675, 295, 361, 371, 429, 4057, 255, 370, 2819, 507, 2944] payload_rows_per_dst=[12502, 14330, 12730, 11084, 6374, 4762, 5150, 8860] dup_rows_per_dst=[7548, 6562, 3018, 2314, 816, 686, 676, 892] combine_rows_per_src=[12502, 14330, 12730, 11084, 6374, 4762, 5150, 8860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85126 payload_rows_remote=71826 dup_rows=13178 dup_rows_remote=10884 dedup_ratio=0.134054 router_tokens_per_expert=[2, 7485, 1983, 2, 1, 62, 387, 4630, 0, 16, 4, 0, 1, 803, 2607, 146, 1, 84, 116, 327, 88, 9, 130, 938, 820, 240, 2, 240, 238, 61, 277, 0, 116, 7366, 79, 9, 2570, 1, 1735, 1, 733, 0, 20, 0, 2803, 246, 0, 18, 77, 17, 118, 380, 5, 400, 61, 17, 3, 374, 5919, 898, 246, 2055, 0, 1185] payload_rows_per_dst=[10156, 6958, 15594, 13300, 7894, 10438, 15728, 5058] dup_rows_per_dst=[2470, 196, 2524, 2294, 1128, 2820, 1392, 354] combine_rows_per_src=[10156, 6958, 15594, 13300, 7894, 10438, 15728, 5058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80892 payload_rows_remote=68134 dup_rows=17412 dup_rows_remote=12484 dedup_ratio=0.177124 router_tokens_per_expert=[0, 7277, 2150, 0, 64, 55, 418, 5058, 0, 0, 0, 0, 0, 686, 2538, 169, 49, 101, 160, 329, 67, 0, 173, 1054, 907, 247, 0, 281, 234, 55, 319, 0, 58, 7338, 105, 0, 1337, 0, 1897, 0, 691, 0, 0, 0, 2847, 247, 0, 0, 100, 0, 183, 470, 43, 479, 95, 0, 0, 376, 5639, 369, 256, 2366, 0, 1865] payload_rows_per_dst=[5296, 6740, 3534, 12976, 15264, 12758, 15470, 8854] dup_rows_per_dst=[78, 46, 332, 2388, 6206, 4928, 1824, 1610] combine_rows_per_src=[5296, 6740, 3534, 12976, 15264, 12758, 15470, 8854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=221528064 dispatch_meta_bytes=258640 combine_payload_bytes=221528064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221786704 ingress_bytes=325337088 link_bytes=325337088 token_bytes=443314768 weight_movement_bytes=103809024 total_nvlink_bytes=547123792 weight_to_token_ratio=0.234165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224217270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343176 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310287496 ingress_bytes=379150336 link_bytes=379150336 token_bytes=620231816 weight_movement_bytes=69206016 total_nvlink_bytes=689437832 weight_to_token_ratio=0.111581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426712190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66686 payload_rows_remote=52268 dup_rows=31618 dup_rows_remote=10668 dedup_ratio=0.321635 router_tokens_per_expert=[626, 517, 3853, 2492, 1794, 7, 6334, 5914, 5, 144, 9, 392, 7, 1253, 1, 305, 342, 367, 2072, 113, 867, 51, 390, 160, 33, 73, 1223, 0, 10, 119, 9, 540, 5858, 588, 2, 74, 274, 1130, 721, 3, 394, 2, 57, 3, 0, 5, 143, 287, 2799, 2, 12, 204, 5, 1681, 89, 2, 2722, 18, 70, 1026, 893, 63, 1, 2] payload_rows_per_dst=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] dup_rows_per_dst=[20950, 2328, 1192, 576, 3804, 516, 1016, 1236] combine_rows_per_src=[14418, 9610, 7532, 3438, 13496, 1266, 8572, 8354] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80526 payload_rows_remote=64614 dup_rows=17778 dup_rows_remote=15348 dedup_ratio=0.180847 router_tokens_per_expert=[0, 7561, 1868, 0, 3, 53, 442, 4403, 0, 0, 0, 0, 0, 888, 3100, 153, 0, 94, 119, 339, 54, 0, 24, 941, 717, 265, 0, 262, 192, 53, 300, 0, 83, 7600, 90, 0, 3121, 0, 1559, 0, 731, 0, 0, 0, 2576, 265, 0, 0, 90, 0, 153, 378, 0, 312, 58, 0, 0, 374, 5941, 970, 214, 1890, 0, 916] payload_rows_per_dst=[12406, 15060, 15912, 13376, 8600, 6962, 3516, 4694] dup_rows_per_dst=[4912, 6432, 2430, 2084, 1106, 182, 378, 254] combine_rows_per_src=[12406, 15060, 15912, 13376, 8600, 6962, 3516, 4694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87458 payload_rows_remote=78174 dup_rows=10846 dup_rows_remote=9060 dedup_ratio=0.110331 router_tokens_per_expert=[0, 0, 0, 331, 0, 0, 0, 0, 369, 98, 1511, 0, 598, 346, 0, 100, 54, 0, 1, 0, 0, 67, 0, 0, 0, 5586, 0, 284, 228, 1447, 0, 323, 1448, 3209, 0, 0, 0, 1515, 0, 886, 2011, 1500, 3552, 0, 555, 567, 902, 0, 0, 160, 0, 2614, 52, 0, 54, 0, 538, 0, 5889, 6055, 5246, 857, 0, 199] payload_rows_per_dst=[12366, 5056, 12352, 13364, 11638, 9284, 10986, 12412] dup_rows_per_dst=[74, 988, 2, 2372, 2478, 1786, 1878, 1268] combine_rows_per_src=[12366, 5056, 12352, 13364, 11638, 9284, 10986, 12412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75792 payload_rows_remote=61462 dup_rows=22512 dup_rows_remote=15950 dedup_ratio=0.229004 router_tokens_per_expert=[37, 5884, 3964, 192, 191, 167, 334, 2321, 31, 552, 131, 79, 58, 620, 2904, 187, 379, 143, 391, 284, 128, 197, 321, 1013, 425, 332, 416, 679, 256, 52, 475, 7, 327, 5018, 208, 83, 602, 379, 1988, 8, 238, 55, 497, 173, 1171, 328, 46, 216, 43, 221, 161, 686, 471, 675, 295, 361, 371, 429, 4057, 255, 370, 2819, 507, 2944] payload_rows_per_dst=[12502, 14330, 12730, 11084, 6374, 4762, 5150, 8860] dup_rows_per_dst=[7548, 6562, 3018, 2314, 816, 686, 676, 892] combine_rows_per_src=[12502, 14330, 12730, 11084, 6374, 4762, 5150, 8860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=294199296 dispatch_meta_bytes=330840 combine_payload_bytes=294199296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294530136 ingress_bytes=328802304 link_bytes=328802304 token_bytes=588729432 weight_movement_bytes=34603008 total_nvlink_bytes=623332440 weight_to_token_ratio=0.058776 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237256599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=221528064 dispatch_meta_bytes=258640 combine_payload_bytes=221528064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=221786704 ingress_bytes=325337088 link_bytes=325337088 token_bytes=443314768 weight_movement_bytes=103809024 total_nvlink_bytes=547123792 weight_to_token_ratio=0.234165 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224217270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343176 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310287496 ingress_bytes=379150336 link_bytes=379150336 token_bytes=620231816 weight_movement_bytes=69206016 total_nvlink_bytes=689437832 weight_to_token_ratio=0.111581 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.426712190e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251748352 dispatch_meta_bytes=309648 combine_payload_bytes=251748352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252058000 ingress_bytes=286351360 link_bytes=286351360 token_bytes=503806352 weight_movement_bytes=34603008 total_nvlink_bytes=538409360 weight_to_token_ratio=0.068683 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077517115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=320200704 dispatch_meta_bytes=348936 combine_payload_bytes=320200704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320549640 ingress_bytes=424009728 link_bytes=424009728 token_bytes=640750344 weight_movement_bytes=103809024 total_nvlink_bytes=744559368 weight_to_token_ratio=0.162012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595514471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=319848 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264978792 ingress_bytes=299261952 link_bytes=299261952 token_bytes=529637736 weight_movement_bytes=34603008 total_nvlink_bytes=564240744 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=279076864 dispatch_meta_bytes=322472 combine_payload_bytes=279076864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279399336 ingress_bytes=313679872 link_bytes=313679872 token_bytes=558476200 weight_movement_bytes=34603008 total_nvlink_bytes=593079208 weight_to_token_ratio=0.061960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180352105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86610 payload_rows_remote=74668 dup_rows=11694 dup_rows_remote=9072 dedup_ratio=0.118958 router_tokens_per_expert=[0, 0, 0, 348, 0, 0, 0, 0, 405, 82, 1447, 0, 757, 325, 0, 108, 68, 0, 6, 0, 0, 84, 0, 0, 0, 4883, 0, 510, 259, 1036, 0, 341, 1012, 3474, 0, 0, 0, 1916, 0, 986, 2030, 2071, 3969, 0, 557, 539, 876, 0, 0, 84, 0, 3140, 21, 0, 68, 0, 840, 0, 5185, 5936, 4710, 930, 0, 149] payload_rows_per_dst=[11004, 5230, 12176, 12726, 11850, 10140, 11942, 11542] dup_rows_per_dst=[62, 1018, 12, 1332, 2926, 2006, 2622, 1716] combine_rows_per_src=[11004, 5230, 12176, 12726, 11850, 10140, 11942, 11542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76966 payload_rows_remote=63176 dup_rows=21338 dup_rows_remote=11836 dedup_ratio=0.217061 router_tokens_per_expert=[1, 7301, 2351, 1, 66, 50, 410, 4856, 0, 7, 2, 2, 1, 698, 2925, 135, 56, 93, 63, 343, 64, 6, 198, 1060, 475, 262, 2, 412, 256, 47, 186, 0, 49, 7298, 84, 8, 1454, 6, 1601, 0, 683, 3, 11, 5, 2267, 258, 5, 9, 77, 7, 158, 496, 19, 522, 72, 8, 0, 385, 5802, 220, 232, 3091, 1, 1992] payload_rows_per_dst=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] dup_rows_per_dst=[9502, 5572, 2824, 1994, 660, 138, 300, 348] combine_rows_per_src=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=319848 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264978792 ingress_bytes=299261952 link_bytes=299261952 token_bytes=529637736 weight_movement_bytes=34603008 total_nvlink_bytes=564240744 weight_to_token_ratio=0.065333 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126098633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=214089728 dispatch_meta_bytes=251744 combine_payload_bytes=214089728 prefetch_bytes=0 reduce_bytes=207618048 egress_bytes=214341472 ingress_bytes=421707776 link_bytes=421707776 token_bytes=428431200 weight_movement_bytes=207618048 total_nvlink_bytes=636049248 weight_to_token_ratio=0.484601 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.586852411e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=251748352 dispatch_meta_bytes=309648 combine_payload_bytes=251748352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=252058000 ingress_bytes=286351360 link_bytes=286351360 token_bytes=503806352 weight_movement_bytes=34603008 total_nvlink_bytes=538409360 weight_to_token_ratio=0.068683 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.077517115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88136 payload_rows_remote=77098 dup_rows=10168 dup_rows_remote=9448 dedup_ratio=0.103434 router_tokens_per_expert=[0, 0, 0, 343, 3, 0, 1, 21, 388, 166, 1437, 12, 642, 315, 7, 76, 62, 0, 0, 0, 11, 89, 0, 7, 1, 5650, 5, 334, 252, 1399, 16, 280, 715, 3592, 0, 2, 0, 1550, 5, 887, 2895, 1147, 3323, 3, 558, 1103, 1142, 2, 0, 116, 21, 1774, 6, 4, 63, 0, 603, 17, 6278, 6212, 4908, 558, 0, 151] payload_rows_per_dst=[13224, 13126, 12758, 11038, 5822, 10974, 9576, 11618] dup_rows_per_dst=[68, 4260, 4, 720, 496, 2726, 1038, 856] combine_rows_per_src=[13224, 13126, 12758, 11038, 5822, 10974, 9576, 11618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258768896 dispatch_meta_bytes=300048 combine_payload_bytes=258768896 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259068944 ingress_bytes=397180928 link_bytes=397180928 token_bytes=517837840 weight_movement_bytes=138412032 total_nvlink_bytes=656249872 weight_to_token_ratio=0.267288 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494559856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=305840128 dispatch_meta_bytes=334960 combine_payload_bytes=305840128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306175088 ingress_bytes=340443136 link_bytes=340443136 token_bytes=612015216 weight_movement_bytes=34603008 total_nvlink_bytes=646618224 weight_to_token_ratio=0.056539 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281060113e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84626 payload_rows_remote=71440 dup_rows=13678 dup_rows_remote=10300 dedup_ratio=0.139140 router_tokens_per_expert=[4, 1, 0, 334, 0, 1, 0, 5, 394, 80, 1122, 5, 787, 313, 3, 90, 71, 0, 9, 0, 7, 85, 0, 3, 1, 4792, 0, 303, 237, 966, 3, 262, 1819, 2524, 0, 2, 0, 1622, 0, 906, 1571, 1823, 4516, 0, 568, 517, 623, 0, 0, 81, 8, 3612, 199, 3, 71, 1, 789, 9, 5505, 5879, 5402, 1077, 0, 147] payload_rows_per_dst=[9912, 4536, 12088, 11880, 11054, 8686, 13284, 13186] dup_rows_per_dst=[72, 1052, 20, 1248, 2692, 1518, 3698, 3378] combine_rows_per_src=[9912, 4536, 12088, 11880, 11054, 8686, 13284, 13186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87900 payload_rows_remote=77158 dup_rows=10404 dup_rows_remote=8736 dedup_ratio=0.105835 router_tokens_per_expert=[0, 0, 0, 334, 0, 0, 0, 0, 412, 153, 1673, 0, 646, 326, 0, 80, 59, 0, 1, 0, 0, 97, 0, 0, 0, 5821, 0, 362, 272, 1783, 0, 49, 477, 4395, 0, 0, 0, 1486, 0, 912, 3311, 815, 2448, 0, 629, 1128, 1985, 0, 0, 81, 0, 1213, 0, 0, 59, 0, 728, 0, 6469, 6227, 4301, 260, 0, 160] payload_rows_per_dst=[13588, 10900, 12766, 9756, 10742, 12602, 7184, 10362] dup_rows_per_dst=[18, 3294, 2, 1334, 1668, 3134, 418, 536] combine_rows_per_src=[13588, 10900, 12766, 9756, 10742, 12602, 7184, 10362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89684 payload_rows_remote=79460 dup_rows=8620 dup_rows_remote=7982 dedup_ratio=0.087687 router_tokens_per_expert=[0, 55, 248, 67, 24, 4858, 0, 468, 0, 0, 38, 0, 100, 0, 0, 0, 1551, 0, 0, 0, 0, 1563, 0, 1480, 0, 50, 7, 1684, 0, 0, 0, 55, 5077, 38, 100, 451, 0, 0, 166, 1093, 242, 0, 0, 1156, 67, 0, 0, 0, 357, 0, 4654, 102, 6164, 55, 4679, 0, 17, 2861, 3966, 685, 2203, 2513, 0, 258] payload_rows_per_dst=[11402, 12530, 8238, 11916, 11800, 10224, 10370, 13204] dup_rows_per_dst=[38, 74, 950, 984, 2050, 638, 16, 3870] combine_rows_per_src=[11402, 12530, 8238, 11916, 11800, 10224, 10370, 13204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=70528 dup_rows=16018 dup_rows_remote=12196 dedup_ratio=0.162944 router_tokens_per_expert=[223, 155, 0, 494, 311, 80, 27, 410, 412, 120, 686, 272, 1464, 490, 210, 150, 155, 135, 36, 2, 375, 96, 88, 471, 66, 3986, 26, 426, 449, 1551, 477, 555, 596, 1321, 1, 201, 10, 1800, 84, 1257, 2347, 3968, 3547, 97, 715, 392, 1007, 94, 7, 185, 355, 2480, 344, 446, 272, 85, 934, 355, 3994, 3231, 3101, 1343, 2, 183] payload_rows_per_dst=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] dup_rows_per_dst=[830, 3822, 718, 1678, 2606, 1420, 3276, 1668] combine_rows_per_src=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=305840128 dispatch_meta_bytes=334960 combine_payload_bytes=305840128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306175088 ingress_bytes=340443136 link_bytes=340443136 token_bytes=612015216 weight_movement_bytes=34603008 total_nvlink_bytes=646618224 weight_to_token_ratio=0.056539 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281060113e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88136 payload_rows_remote=77098 dup_rows=10168 dup_rows_remote=9448 dedup_ratio=0.103434 router_tokens_per_expert=[0, 0, 0, 343, 3, 0, 1, 21, 388, 166, 1437, 12, 642, 315, 7, 76, 62, 0, 0, 0, 11, 89, 0, 7, 1, 5650, 5, 334, 252, 1399, 16, 280, 715, 3592, 0, 2, 0, 1550, 5, 887, 2895, 1147, 3323, 3, 558, 1103, 1142, 2, 0, 116, 21, 1774, 6, 4, 63, 0, 603, 17, 6278, 6212, 4908, 558, 0, 151] payload_rows_per_dst=[13224, 13126, 12758, 11038, 5822, 10974, 9576, 11618] dup_rows_per_dst=[68, 4260, 4, 720, 496, 2726, 1038, 856] combine_rows_per_src=[13224, 13126, 12758, 11038, 5822, 10974, 9576, 11618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87458 payload_rows_remote=78174 dup_rows=10846 dup_rows_remote=9060 dedup_ratio=0.110331 router_tokens_per_expert=[0, 0, 0, 331, 0, 0, 0, 0, 369, 98, 1511, 0, 598, 346, 0, 100, 54, 0, 1, 0, 0, 67, 0, 0, 0, 5586, 0, 284, 228, 1447, 0, 323, 1448, 3209, 0, 0, 0, 1515, 0, 886, 2011, 1500, 3552, 0, 555, 567, 902, 0, 0, 160, 0, 2614, 52, 0, 54, 0, 538, 0, 5889, 6055, 5246, 857, 0, 199] payload_rows_per_dst=[12366, 5056, 12352, 13364, 11638, 9284, 10986, 12412] dup_rows_per_dst=[74, 988, 2, 2372, 2478, 1786, 1878, 1268] combine_rows_per_src=[12366, 5056, 12352, 13364, 11638, 9284, 10986, 12412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=343576 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316382744 ingress_bytes=385245184 link_bytes=385245184 token_bytes=632421912 weight_movement_bytes=69206016 total_nvlink_bytes=701627928 weight_to_token_ratio=0.109430 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449646613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292618240 dispatch_meta_bytes=326960 combine_payload_bytes=292618240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=292945200 ingress_bytes=431030272 link_bytes=431030272 token_bytes=585563440 weight_movement_bytes=138412032 total_nvlink_bytes=723975472 weight_to_token_ratio=0.236374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.621932213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88036 payload_rows_remote=75060 dup_rows=10268 dup_rows_remote=10268 dedup_ratio=0.104451 router_tokens_per_expert=[2, 0, 0, 293, 0, 0, 0, 0, 392, 156, 1845, 0, 581, 354, 0, 89, 70, 0, 0, 0, 0, 74, 0, 0, 0, 5561, 0, 298, 205, 1795, 0, 2, 532, 4102, 0, 0, 0, 1602, 0, 894, 3311, 771, 2416, 0, 555, 1136, 1526, 0, 0, 138, 0, 1268, 0, 0, 72, 0, 687, 0, 6682, 6344, 4795, 427, 0, 177] payload_rows_per_dst=[13912, 13116, 12976, 11498, 10514, 7548, 7256, 11216] dup_rows_per_dst=[42, 4840, 0, 1306, 1902, 690, 532, 956] combine_rows_per_src=[13912, 13116, 12976, 11498, 10514, 7548, 7256, 11216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76966 payload_rows_remote=63176 dup_rows=21338 dup_rows_remote=11836 dedup_ratio=0.217061 router_tokens_per_expert=[1, 7301, 2351, 1, 66, 50, 410, 4856, 0, 7, 2, 2, 1, 698, 2925, 135, 56, 93, 63, 343, 64, 6, 198, 1060, 475, 262, 2, 412, 256, 47, 186, 0, 49, 7298, 84, 8, 1454, 6, 1601, 0, 683, 3, 11, 5, 2267, 258, 5, 9, 77, 7, 158, 496, 19, 522, 72, 8, 0, 385, 5802, 220, 232, 3091, 1, 1992] payload_rows_per_dst=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] dup_rows_per_dst=[9502, 5572, 2824, 1994, 660, 138, 300, 348] combine_rows_per_src=[13790, 14930, 15538, 12890, 5744, 6344, 2418, 5312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82286 payload_rows_remote=70528 dup_rows=16018 dup_rows_remote=12196 dedup_ratio=0.162944 router_tokens_per_expert=[223, 155, 0, 494, 311, 80, 27, 410, 412, 120, 686, 272, 1464, 490, 210, 150, 155, 135, 36, 2, 375, 96, 88, 471, 66, 3986, 26, 426, 449, 1551, 477, 555, 596, 1321, 1, 201, 10, 1800, 84, 1257, 2347, 3968, 3547, 97, 715, 392, 1007, 94, 7, 185, 355, 2480, 344, 446, 272, 85, 934, 355, 3994, 3231, 3101, 1343, 2, 183] payload_rows_per_dst=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] dup_rows_per_dst=[830, 3822, 718, 1678, 2606, 1420, 3276, 1668] combine_rows_per_src=[10558, 11758, 8460, 8064, 9986, 11126, 12166, 10168] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=315793408 dispatch_meta_bytes=346184 combine_payload_bytes=315793408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316139592 ingress_bytes=384999424 link_bytes=384999424 token_bytes=631933000 weight_movement_bytes=69206016 total_nvlink_bytes=701139016 weight_to_token_ratio=0.109515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.448721838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=292618240 dispatch_meta_bytes=326960 combine_payload_bytes=292618240 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=292945200 ingress_bytes=431030272 link_bytes=431030272 token_bytes=585563440 weight_movement_bytes=138412032 total_nvlink_bytes=723975472 weight_to_token_ratio=0.236374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.621932213e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=343576 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=316382744 ingress_bytes=385245184 link_bytes=385245184 token_bytes=632421912 weight_movement_bytes=69206016 total_nvlink_bytes=701627928 weight_to_token_ratio=0.109430 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.449646613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325468160 dispatch_meta_bytes=349768 combine_payload_bytes=325468160 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=325817928 ingress_bytes=394674176 link_bytes=394674176 token_bytes=651286088 weight_movement_bytes=69206016 total_nvlink_bytes=720492104 weight_to_token_ratio=0.106261 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=288882688 dispatch_meta_bytes=330896 combine_payload_bytes=288882688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289213584 ingress_bytes=323485696 link_bytes=323485696 token_bytes=578096272 weight_movement_bytes=34603008 total_nvlink_bytes=612699280 weight_to_token_ratio=0.059857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217250631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307445760 dispatch_meta_bytes=341312 combine_payload_bytes=307445760 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=307787072 ingress_bytes=376651776 link_bytes=376651776 token_bytes=615232832 weight_movement_bytes=69206016 total_nvlink_bytes=684438848 weight_to_token_ratio=0.112488 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417310310e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84774 payload_rows_remote=72286 dup_rows=13530 dup_rows_remote=13444 dedup_ratio=0.137634 router_tokens_per_expert=[0, 1, 2, 286, 1, 0, 0, 9, 367, 51, 1160, 12, 695, 330, 6, 85, 75, 2, 3, 0, 6, 64, 0, 3, 1, 4948, 3, 243, 224, 1182, 2, 291, 1002, 3048, 0, 4, 1, 2074, 3, 914, 1967, 1652, 4096, 0, 558, 438, 1042, 1, 0, 62, 11, 2726, 120, 7, 76, 4, 896, 10, 5988, 6166, 5063, 1025, 0, 146] payload_rows_per_dst=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] dup_rows_per_dst=[86, 3202, 1078, 358, 2578, 3138, 1904, 1186] combine_rows_per_src=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=320200704 dispatch_meta_bytes=348936 combine_payload_bytes=320200704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=320549640 ingress_bytes=424009728 link_bytes=424009728 token_bytes=640750344 weight_movement_bytes=103809024 total_nvlink_bytes=744559368 weight_to_token_ratio=0.162012 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.595514471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84990 payload_rows_remote=71678 dup_rows=13314 dup_rows_remote=7910 dedup_ratio=0.135437 router_tokens_per_expert=[0, 70, 270, 85, 20, 4898, 0, 439, 0, 0, 18, 0, 75, 0, 0, 0, 1888, 0, 0, 0, 0, 1626, 0, 1854, 0, 16, 7, 1442, 0, 0, 0, 70, 5426, 0, 68, 339, 0, 0, 158, 1191, 265, 0, 0, 1326, 84, 0, 0, 8, 398, 0, 5271, 80, 5955, 70, 4205, 0, 15, 2520, 3229, 623, 2359, 2636, 0, 148] payload_rows_per_dst=[11494, 12064, 9394, 4292, 12232, 9116, 13312, 13086] dup_rows_per_dst=[70, 32, 1342, 110, 2132, 708, 5404, 3516] combine_rows_per_src=[11494, 12064, 9394, 4292, 12232, 9116, 13312, 13086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307445760 dispatch_meta_bytes=341312 combine_payload_bytes=307445760 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=307787072 ingress_bytes=376651776 link_bytes=376651776 token_bytes=615232832 weight_movement_bytes=69206016 total_nvlink_bytes=684438848 weight_to_token_ratio=0.112488 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.417310310e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258768896 dispatch_meta_bytes=300048 combine_payload_bytes=258768896 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=259068944 ingress_bytes=397180928 link_bytes=397180928 token_bytes=517837840 weight_movement_bytes=138412032 total_nvlink_bytes=656249872 weight_to_token_ratio=0.267288 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.494559856e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=288882688 dispatch_meta_bytes=330896 combine_payload_bytes=288882688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289213584 ingress_bytes=323485696 link_bytes=323485696 token_bytes=578096272 weight_movement_bytes=34603008 total_nvlink_bytes=612699280 weight_to_token_ratio=0.059857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217250631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=75204 dup_rows=10020 dup_rows_remote=7210 dedup_ratio=0.101929 router_tokens_per_expert=[2, 61, 243, 113, 16, 4284, 4, 395, 0, 2, 82, 9, 78, 12, 0, 0, 2343, 1, 0, 0, 10, 1863, 5, 1435, 0, 0, 47, 3404, 0, 1, 0, 68, 5232, 36, 82, 353, 1, 18, 164, 805, 225, 14, 24, 1267, 88, 2, 22, 0, 555, 0, 4425, 76, 6407, 61, 3975, 7, 10, 2206, 3575, 619, 1706, 2519, 7, 193] payload_rows_per_dst=[10198, 13014, 9878, 13080, 11800, 9668, 9020, 11626] dup_rows_per_dst=[38, 166, 1436, 2810, 1582, 766, 328, 2894] combine_rows_per_src=[10198, 13014, 9878, 13080, 11800, 9668, 9020, 11626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296083456 dispatch_meta_bytes=342920 combine_payload_bytes=296083456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296426376 ingress_bytes=330686464 link_bytes=330686464 token_bytes=592509832 weight_movement_bytes=34603008 total_nvlink_bytes=627112840 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244346542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86772 payload_rows_remote=75670 dup_rows=11532 dup_rows_remote=10116 dedup_ratio=0.117310 router_tokens_per_expert=[0, 68, 247, 98, 0, 4499, 0, 460, 0, 0, 48, 0, 80, 0, 0, 0, 2673, 0, 0, 0, 0, 2075, 0, 1704, 0, 0, 31, 5004, 0, 0, 0, 68, 4981, 1, 80, 259, 0, 0, 177, 761, 245, 0, 0, 1537, 72, 0, 0, 0, 595, 0, 4770, 80, 6330, 68, 2980, 0, 0, 1506, 3000, 653, 1324, 2517, 0, 161] payload_rows_per_dst=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] dup_rows_per_dst=[2, 92, 1686, 5264, 1416, 806, 368, 1898] combine_rows_per_src=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=293593088 dispatch_meta_bytes=318352 combine_payload_bytes=293593088 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=293911440 ingress_bytes=432005120 link_bytes=432005120 token_bytes=587504528 weight_movement_bytes=138412032 total_nvlink_bytes=725916560 weight_to_token_ratio=0.235593 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.625600487e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80144 payload_rows_remote=66920 dup_rows=18160 dup_rows_remote=13378 dedup_ratio=0.184733 router_tokens_per_expert=[1, 64, 302, 91, 58, 5499, 2, 425, 0, 0, 47, 2, 74, 7, 0, 3, 1942, 0, 0, 0, 0, 1655, 1, 1348, 0, 52, 25, 1398, 0, 0, 2, 64, 5235, 56, 73, 448, 0, 7, 158, 1379, 297, 6, 10, 1022, 85, 1, 9, 6, 402, 11, 4870, 87, 5858, 64, 4327, 4, 24, 3165, 3448, 628, 2198, 2083, 1, 128] payload_rows_per_dst=[12656, 11890, 8866, 3064, 6352, 9110, 14982, 13224] dup_rows_per_dst=[228, 92, 1026, 18, 970, 658, 10386, 4782] combine_rows_per_src=[12656, 11890, 8866, 3064, 6352, 9110, 14982, 13224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88410 payload_rows_remote=79450 dup_rows=9894 dup_rows_remote=8854 dedup_ratio=0.100647 router_tokens_per_expert=[0, 0, 55, 0, 55, 0, 0, 0, 0, 2972, 0, 259, 100, 290, 30, 1523, 1, 7111, 0, 142, 0, 0, 54, 0, 7386, 0, 205, 247, 0, 1140, 99, 5597, 7529, 3751, 0, 202, 0, 0, 0, 0, 139, 48, 2646, 0, 0, 1920, 247, 0, 205, 0, 0, 1875, 0, 0, 0, 0, 0, 3117, 160, 0, 0, 0, 0, 47] payload_rows_per_dst=[14882, 8780, 14460, 3184, 7906, 8960, 15952, 14286] dup_rows_per_dst=[110, 1568, 156, 198, 0, 1040, 3266, 3556] combine_rows_per_src=[14882, 8780, 14460, 3184, 7906, 8960, 15952, 14286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86772 payload_rows_remote=75670 dup_rows=11532 dup_rows_remote=10116 dedup_ratio=0.117310 router_tokens_per_expert=[0, 68, 247, 98, 0, 4499, 0, 460, 0, 0, 48, 0, 80, 0, 0, 0, 2673, 0, 0, 0, 0, 2075, 0, 1704, 0, 0, 31, 5004, 0, 0, 0, 68, 4981, 1, 80, 259, 0, 0, 177, 761, 245, 0, 0, 1537, 72, 0, 0, 0, 595, 0, 4770, 80, 6330, 68, 2980, 0, 0, 1506, 3000, 653, 1324, 2517, 0, 161] payload_rows_per_dst=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] dup_rows_per_dst=[2, 92, 1686, 5264, 1416, 806, 368, 1898] combine_rows_per_src=[10742, 12824, 11218, 14482, 11102, 8902, 7078, 10424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81134 payload_rows_remote=72524 dup_rows=17170 dup_rows_remote=16232 dedup_ratio=0.174662 router_tokens_per_expert=[113, 170, 488, 361, 245, 3329, 430, 685, 5, 141, 57, 106, 307, 666, 102, 153, 433, 0, 41, 0, 54, 938, 124, 1596, 108, 489, 507, 817, 219, 140, 12, 142, 5005, 326, 77, 531, 10, 427, 154, 1568, 282, 479, 899, 1203, 99, 74, 595, 402, 391, 548, 3240, 242, 3237, 54, 3515, 32, 216, 3678, 3386, 605, 1402, 2897, 153, 447] payload_rows_per_dst=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] dup_rows_per_dst=[1246, 938, 2324, 1396, 3434, 6842, 582, 408] combine_rows_per_src=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88284 payload_rows_remote=75204 dup_rows=10020 dup_rows_remote=7210 dedup_ratio=0.101929 router_tokens_per_expert=[2, 61, 243, 113, 16, 4284, 4, 395, 0, 2, 82, 9, 78, 12, 0, 0, 2343, 1, 0, 0, 10, 1863, 5, 1435, 0, 0, 47, 3404, 0, 1, 0, 68, 5232, 36, 82, 353, 1, 18, 164, 805, 225, 14, 24, 1267, 88, 2, 22, 0, 555, 0, 4425, 76, 6407, 61, 3975, 7, 10, 2206, 3575, 619, 1706, 2519, 7, 193] payload_rows_per_dst=[10198, 13014, 9878, 13080, 11800, 9668, 9020, 11626] dup_rows_per_dst=[38, 166, 1436, 2810, 1582, 766, 328, 2894] combine_rows_per_src=[10198, 13014, 9878, 13080, 11800, 9668, 9020, 11626] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89684 payload_rows_remote=79460 dup_rows=8620 dup_rows_remote=7982 dedup_ratio=0.087687 router_tokens_per_expert=[0, 55, 248, 67, 24, 4858, 0, 468, 0, 0, 38, 0, 100, 0, 0, 0, 1551, 0, 0, 0, 0, 1563, 0, 1480, 0, 50, 7, 1684, 0, 0, 0, 55, 5077, 38, 100, 451, 0, 0, 166, 1093, 242, 0, 0, 1156, 67, 0, 0, 0, 357, 0, 4654, 102, 6164, 55, 4679, 0, 17, 2861, 3966, 685, 2203, 2513, 0, 258] payload_rows_per_dst=[11402, 12530, 8238, 11916, 11800, 10224, 10370, 13204] dup_rows_per_dst=[38, 74, 950, 984, 2050, 638, 16, 3870] combine_rows_per_src=[11402, 12530, 8238, 11916, 11800, 10224, 10370, 13204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343144 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=310287464 ingress_bytes=413753344 link_bytes=413753344 token_bytes=620231784 weight_movement_bytes=103809024 total_nvlink_bytes=724040808 weight_to_token_ratio=0.167371 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556920524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86356 payload_rows_remote=74348 dup_rows=11948 dup_rows_remote=9306 dedup_ratio=0.121541 router_tokens_per_expert=[0, 53, 271, 80, 7, 4465, 0, 432, 0, 0, 64, 0, 90, 0, 0, 0, 2152, 0, 0, 0, 0, 1917, 0, 1446, 0, 0, 15, 4506, 0, 0, 0, 53, 5032, 16, 90, 293, 0, 0, 163, 742, 215, 0, 0, 1536, 52, 0, 0, 0, 583, 0, 4408, 88, 6396, 53, 3375, 0, 5, 1821, 3500, 681, 1686, 2640, 0, 226] payload_rows_per_dst=[10586, 12972, 12008, 13818, 11268, 10952, 7834, 6918] dup_rows_per_dst=[30, 128, 2642, 4146, 1404, 2034, 364, 1200] combine_rows_per_src=[10586, 12972, 12008, 13818, 11268, 10952, 7834, 6918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308035584 dispatch_meta_bytes=329656 combine_payload_bytes=308035584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308365240 ingress_bytes=342638592 link_bytes=342638592 token_bytes=616400824 weight_movement_bytes=34603008 total_nvlink_bytes=651003832 weight_to_token_ratio=0.056137 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289321437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84774 payload_rows_remote=72286 dup_rows=13530 dup_rows_remote=13444 dedup_ratio=0.137634 router_tokens_per_expert=[0, 1, 2, 286, 1, 0, 0, 9, 367, 51, 1160, 12, 695, 330, 6, 85, 75, 2, 3, 0, 6, 64, 0, 3, 1, 4948, 3, 243, 224, 1182, 2, 291, 1002, 3048, 0, 4, 1, 2074, 3, 914, 1967, 1652, 4096, 0, 558, 438, 1042, 1, 0, 62, 11, 2726, 120, 7, 76, 4, 896, 10, 5988, 6166, 5063, 1025, 0, 146] payload_rows_per_dst=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] dup_rows_per_dst=[86, 3202, 1078, 358, 2578, 3138, 1904, 1186] combine_rows_per_src=[12488, 12106, 12892, 9630, 9352, 11526, 10544, 6236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81134 payload_rows_remote=72524 dup_rows=17170 dup_rows_remote=16232 dedup_ratio=0.174662 router_tokens_per_expert=[113, 170, 488, 361, 245, 3329, 430, 685, 5, 141, 57, 106, 307, 666, 102, 153, 433, 0, 41, 0, 54, 938, 124, 1596, 108, 489, 507, 817, 219, 140, 12, 142, 5005, 326, 77, 531, 10, 427, 154, 1568, 282, 479, 899, 1203, 99, 74, 595, 402, 391, 548, 3240, 242, 3237, 54, 3515, 32, 216, 3678, 3386, 605, 1402, 2897, 153, 447] payload_rows_per_dst=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] dup_rows_per_dst=[1246, 938, 2324, 1396, 3434, 6842, 582, 408] combine_rows_per_src=[10396, 8610, 11404, 9952, 12762, 13790, 8982, 5238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=321192 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=274425512 ingress_bytes=412516352 link_bytes=412516352 token_bytes=548529832 weight_movement_bytes=138412032 total_nvlink_bytes=686941864 weight_to_token_ratio=0.252333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.552265822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325427200 dispatch_meta_bytes=353216 combine_payload_bytes=325427200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325780416 ingress_bytes=360030208 link_bytes=360030208 token_bytes=651207616 weight_movement_bytes=34603008 total_nvlink_bytes=685810624 weight_to_token_ratio=0.053137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354764688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308035584 dispatch_meta_bytes=329656 combine_payload_bytes=308035584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=308365240 ingress_bytes=342638592 link_bytes=342638592 token_bytes=616400824 weight_movement_bytes=34603008 total_nvlink_bytes=651003832 weight_to_token_ratio=0.056137 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289321437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=309944320 dispatch_meta_bytes=343144 combine_payload_bytes=309944320 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=310287464 ingress_bytes=413753344 link_bytes=413753344 token_bytes=620231784 weight_movement_bytes=103809024 total_nvlink_bytes=724040808 weight_to_token_ratio=0.167371 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556920524e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=297058304 dispatch_meta_bytes=355024 combine_payload_bytes=297058304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=297413328 ingress_bytes=366264320 link_bytes=366264320 token_bytes=594471632 weight_movement_bytes=69206016 total_nvlink_bytes=663677648 weight_to_token_ratio=0.116416 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378223149e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84558 payload_rows_remote=70242 dup_rows=13746 dup_rows_remote=8884 dedup_ratio=0.139832 router_tokens_per_expert=[0, 47, 333, 68, 17, 5388, 4, 449, 0, 0, 42, 0, 73, 12, 3, 3, 1883, 2, 0, 0, 4, 1217, 4, 1518, 1, 49, 10, 1363, 2, 1, 4, 51, 5591, 56, 73, 289, 0, 5, 136, 1133, 247, 12, 14, 1100, 64, 0, 12, 4, 406, 1, 5005, 76, 6069, 47, 4344, 2, 27, 3318, 3199, 673, 2165, 2393, 9, 134] payload_rows_per_dst=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] dup_rows_per_dst=[4862, 434, 3264, 680, 1142, 2608, 82, 674] combine_rows_per_src=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=334616 combine_payload_bytes=304529408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304864024 ingress_bytes=339132416 link_bytes=339132416 token_bytes=609393432 weight_movement_bytes=34603008 total_nvlink_bytes=643996440 weight_to_token_ratio=0.056783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276127979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=325468160 dispatch_meta_bytes=349768 combine_payload_bytes=325468160 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=325817928 ingress_bytes=394674176 link_bytes=394674176 token_bytes=651286088 weight_movement_bytes=69206016 total_nvlink_bytes=720492104 weight_to_token_ratio=0.106261 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=27392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.485127150e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88182 payload_rows_remote=72468 dup_rows=10122 dup_rows_remote=7348 dedup_ratio=0.102966 router_tokens_per_expert=[0, 0, 70, 0, 70, 0, 0, 0, 0, 2480, 0, 158, 75, 467, 52, 1352, 0, 7214, 0, 185, 0, 0, 68, 0, 7418, 0, 275, 251, 0, 1014, 74, 5548, 7556, 3638, 0, 358, 0, 0, 0, 0, 95, 67, 3836, 0, 0, 1918, 251, 0, 89, 0, 0, 1599, 0, 0, 0, 0, 0, 2825, 82, 0, 0, 0, 0, 67] payload_rows_per_dst=[10318, 7798, 14796, 6840, 7130, 11484, 15714, 14102] dup_rows_per_dst=[140, 1370, 138, 1046, 0, 1712, 2774, 2942] combine_rows_per_src=[10318, 7798, 14796, 6840, 7130, 11484, 15714, 14102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=334616 combine_payload_bytes=304529408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304864024 ingress_bytes=339132416 link_bytes=339132416 token_bytes=609393432 weight_movement_bytes=34603008 total_nvlink_bytes=643996440 weight_to_token_ratio=0.056783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276127979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296083456 dispatch_meta_bytes=342920 combine_payload_bytes=296083456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296426376 ingress_bytes=330686464 link_bytes=330686464 token_bytes=592509832 weight_movement_bytes=34603008 total_nvlink_bytes=627112840 weight_to_token_ratio=0.058401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244346542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=297058304 dispatch_meta_bytes=355024 combine_payload_bytes=297058304 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=297413328 ingress_bytes=366264320 link_bytes=366264320 token_bytes=594471632 weight_movement_bytes=69206016 total_nvlink_bytes=663677648 weight_to_token_ratio=0.116416 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.378223149e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287711232 dispatch_meta_bytes=316504 combine_payload_bytes=287711232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288027736 ingress_bytes=322314240 link_bytes=322314240 token_bytes=575738968 weight_movement_bytes=34603008 total_nvlink_bytes=610341976 weight_to_token_ratio=0.060102 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212842537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87806 payload_rows_remote=78788 dup_rows=10498 dup_rows_remote=10498 dedup_ratio=0.106791 router_tokens_per_expert=[50, 0, 68, 0, 68, 0, 0, 0, 0, 3085, 0, 151, 36, 99, 337, 1419, 3, 7241, 0, 80, 0, 0, 59, 0, 7427, 0, 356, 247, 0, 1452, 80, 5668, 7122, 4265, 0, 244, 0, 0, 0, 0, 88, 0, 2051, 0, 0, 1780, 247, 0, 101, 0, 0, 2301, 0, 0, 0, 0, 0, 2887, 81, 0, 0, 0, 0, 59] payload_rows_per_dst=[15042, 8316, 14550, 4094, 9018, 7602, 14952, 14232] dup_rows_per_dst=[184, 1938, 216, 176, 0, 730, 4096, 3158] combine_rows_per_src=[15042, 8316, 14550, 4094, 9018, 7602, 14952, 14232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=296828928 dispatch_meta_bytes=319264 combine_payload_bytes=296828928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297148192 ingress_bytes=331431936 link_bytes=331431936 token_bytes=593977120 weight_movement_bytes=34603008 total_nvlink_bytes=628580128 weight_to_token_ratio=0.058256 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247151693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87832 payload_rows_remote=75000 dup_rows=10472 dup_rows_remote=8642 dedup_ratio=0.106527 router_tokens_per_expert=[61, 1, 61, 1, 61, 0, 0, 0, 3, 3258, 0, 192, 77, 77, 89, 1419, 6, 7170, 12, 85, 0, 0, 65, 2, 7421, 18, 313, 239, 10, 1243, 76, 5738, 7421, 3967, 4, 238, 18, 12, 0, 1, 132, 37, 1888, 15, 1, 1955, 245, 18, 130, 0, 0, 2258, 1, 0, 1, 1, 10, 2878, 135, 0, 10, 17, 0, 61] payload_rows_per_dst=[15014, 8766, 14536, 12832, 12388, 7798, 9864, 6634] dup_rows_per_dst=[198, 1464, 144, 1830, 4296, 784, 1556, 200] combine_rows_per_src=[15014, 8766, 14536, 12832, 12388, 7798, 9864, 6634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81070 payload_rows_remote=68882 dup_rows=17234 dup_rows_remote=12824 dedup_ratio=0.175313 router_tokens_per_expert=[0, 0, 160, 1238, 0, 0, 984, 0, 0, 5280, 837, 0, 0, 0, 752, 0, 208, 322, 0, 1427, 11, 54, 0, 0, 5485, 58, 0, 117, 0, 0, 0, 0, 58, 347, 0, 0, 2409, 776, 0, 99, 7, 0, 5086, 0, 0, 0, 0, 0, 610, 0, 99, 0, 7486, 0, 3, 27, 6748, 0, 0, 0, 7142, 543, 779, 0] payload_rows_per_dst=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] dup_rows_per_dst=[136, 1330, 270, 1238, 558, 4410, 1074, 8218] combine_rows_per_src=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82886 payload_rows_remote=68946 dup_rows=15418 dup_rows_remote=12014 dedup_ratio=0.156840 router_tokens_per_expert=[14, 1, 64, 0, 64, 0, 0, 0, 1, 2380, 0, 157, 74, 414, 2, 1310, 4, 7139, 5, 132, 0, 0, 71, 1, 7417, 8, 186, 249, 1, 771, 73, 5103, 7546, 4568, 6, 268, 5, 0, 0, 2, 368, 65, 3470, 4, 0, 1906, 243, 9, 143, 0, 0, 1339, 0, 0, 0, 0, 2, 3427, 62, 0, 4, 6, 0, 68] payload_rows_per_dst=[158, 7546, 14498, 15370, 554, 15012, 15808, 13940] dup_rows_per_dst=[128, 1130, 206, 2040, 8, 6254, 2248, 3404] combine_rows_per_src=[158, 7546, 14498, 15370, 554, 15012, 15808, 13940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87806 payload_rows_remote=78788 dup_rows=10498 dup_rows_remote=10498 dedup_ratio=0.106791 router_tokens_per_expert=[50, 0, 68, 0, 68, 0, 0, 0, 0, 3085, 0, 151, 36, 99, 337, 1419, 3, 7241, 0, 80, 0, 0, 59, 0, 7427, 0, 356, 247, 0, 1452, 80, 5668, 7122, 4265, 0, 244, 0, 0, 0, 0, 88, 0, 2051, 0, 0, 1780, 247, 0, 101, 0, 0, 2301, 0, 0, 0, 0, 0, 2887, 81, 0, 0, 0, 0, 59] payload_rows_per_dst=[15042, 8316, 14550, 4094, 9018, 7602, 14952, 14232] dup_rows_per_dst=[184, 1938, 216, 176, 0, 730, 4096, 3158] combine_rows_per_src=[15042, 8316, 14550, 4094, 9018, 7602, 14952, 14232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80858 payload_rows_remote=70268 dup_rows=17446 dup_rows_remote=13140 dedup_ratio=0.177470 router_tokens_per_expert=[107, 78, 74, 90, 70, 78, 18, 1, 156, 1984, 13, 218, 168, 1405, 401, 579, 29, 5411, 268, 524, 31, 0, 558, 71, 5405, 467, 242, 418, 78, 1736, 148, 4357, 5889, 1798, 556, 472, 173, 156, 2, 208, 794, 186, 3665, 598, 26, 2283, 466, 873, 121, 62, 14, 1343, 80, 14, 54, 107, 289, 2544, 122, 4, 283, 624, 34, 129] payload_rows_per_dst=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] dup_rows_per_dst=[194, 4306, 244, 2758, 4288, 4640, 190, 826] combine_rows_per_src=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87832 payload_rows_remote=75000 dup_rows=10472 dup_rows_remote=8642 dedup_ratio=0.106527 router_tokens_per_expert=[61, 1, 61, 1, 61, 0, 0, 0, 3, 3258, 0, 192, 77, 77, 89, 1419, 6, 7170, 12, 85, 0, 0, 65, 2, 7421, 18, 313, 239, 10, 1243, 76, 5738, 7421, 3967, 4, 238, 18, 12, 0, 1, 132, 37, 1888, 15, 1, 1955, 245, 18, 130, 0, 0, 2258, 1, 0, 1, 1, 10, 2878, 135, 0, 10, 17, 0, 61] payload_rows_per_dst=[15014, 8766, 14536, 12832, 12388, 7798, 9864, 6634] dup_rows_per_dst=[198, 1464, 144, 1830, 4296, 784, 1556, 200] combine_rows_per_src=[15014, 8766, 14536, 12832, 12388, 7798, 9864, 6634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=322715648 dispatch_meta_bytes=357144 combine_payload_bytes=322715648 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=323072792 ingress_bytes=461127680 link_bytes=461127680 token_bytes=645788440 weight_movement_bytes=138412032 total_nvlink_bytes=784200472 weight_to_token_ratio=0.214330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735186336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=307200000 dispatch_meta_bytes=334568 combine_payload_bytes=307200000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307534568 ingress_bytes=411009024 link_bytes=411009024 token_bytes=614734568 weight_movement_bytes=103809024 total_nvlink_bytes=718543592 weight_to_token_ratio=0.168868 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546593868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88410 payload_rows_remote=79450 dup_rows=9894 dup_rows_remote=8854 dedup_ratio=0.100647 router_tokens_per_expert=[0, 0, 55, 0, 55, 0, 0, 0, 0, 2972, 0, 259, 100, 290, 30, 1523, 1, 7111, 0, 142, 0, 0, 54, 0, 7386, 0, 205, 247, 0, 1140, 99, 5597, 7529, 3751, 0, 202, 0, 0, 0, 0, 139, 48, 2646, 0, 0, 1920, 247, 0, 205, 0, 0, 1875, 0, 0, 0, 0, 0, 3117, 160, 0, 0, 0, 0, 47] payload_rows_per_dst=[14882, 8780, 14460, 3184, 7906, 8960, 15952, 14286] dup_rows_per_dst=[110, 1568, 156, 198, 0, 1040, 3266, 3556] combine_rows_per_src=[14882, 8780, 14460, 3184, 7906, 8960, 15952, 14286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84558 payload_rows_remote=70242 dup_rows=13746 dup_rows_remote=8884 dedup_ratio=0.139832 router_tokens_per_expert=[0, 47, 333, 68, 17, 5388, 4, 449, 0, 0, 42, 0, 73, 12, 3, 3, 1883, 2, 0, 0, 4, 1217, 4, 1518, 1, 49, 10, 1363, 2, 1, 4, 51, 5591, 56, 73, 289, 0, 5, 136, 1133, 247, 12, 14, 1100, 64, 0, 12, 4, 406, 1, 5005, 76, 6069, 47, 4344, 2, 27, 3318, 3199, 673, 2165, 2393, 9, 134] payload_rows_per_dst=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] dup_rows_per_dst=[4862, 434, 3264, 680, 1142, 2608, 82, 674] combine_rows_per_src=[14316, 12444, 12628, 12292, 6858, 11482, 9196, 5342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86012 payload_rows_remote=71522 dup_rows=12292 dup_rows_remote=12038 dedup_ratio=0.125041 router_tokens_per_expert=[40, 0, 53, 0, 53, 0, 0, 0, 0, 3787, 0, 217, 67, 46, 27, 1450, 2, 7208, 0, 90, 0, 0, 72, 0, 7418, 0, 229, 265, 0, 1194, 89, 5738, 7399, 4125, 0, 28, 0, 0, 0, 0, 133, 1, 1646, 0, 0, 1829, 265, 0, 319, 0, 0, 2114, 0, 0, 0, 0, 0, 3037, 140, 0, 0, 0, 0, 71] payload_rows_per_dst=[14986, 9662, 14490, 13266, 15170, 7078, 4866, 6494] dup_rows_per_dst=[142, 1526, 254, 1764, 7934, 670, 0, 2] combine_rows_per_src=[14986, 9662, 14490, 13266, 15170, 7078, 4866, 6494] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=282140672 dispatch_meta_bytes=326824 combine_payload_bytes=282140672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282467496 ingress_bytes=316743680 link_bytes=316743680 token_bytes=564608168 weight_movement_bytes=34603008 total_nvlink_bytes=599211176 weight_to_token_ratio=0.061287 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191880968e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=282402816 dispatch_meta_bytes=323840 combine_payload_bytes=282402816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282726656 ingress_bytes=317005824 link_bytes=317005824 token_bytes=565129472 weight_movement_bytes=34603008 total_nvlink_bytes=599732480 weight_to_token_ratio=0.061230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.192867395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80858 payload_rows_remote=70268 dup_rows=17446 dup_rows_remote=13140 dedup_ratio=0.177470 router_tokens_per_expert=[107, 78, 74, 90, 70, 78, 18, 1, 156, 1984, 13, 218, 168, 1405, 401, 579, 29, 5411, 268, 524, 31, 0, 558, 71, 5405, 467, 242, 418, 78, 1736, 148, 4357, 5889, 1798, 556, 472, 173, 156, 2, 208, 794, 186, 3665, 598, 26, 2283, 466, 873, 121, 62, 14, 1343, 80, 14, 54, 107, 289, 2544, 122, 4, 283, 624, 34, 129] payload_rows_per_dst=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] dup_rows_per_dst=[194, 4306, 244, 2758, 4288, 4640, 190, 826] combine_rows_per_src=[11648, 10590, 8492, 12134, 14220, 13142, 3400, 7232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=307200000 dispatch_meta_bytes=334568 combine_payload_bytes=307200000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=307534568 ingress_bytes=411009024 link_bytes=411009024 token_bytes=614734568 weight_movement_bytes=103809024 total_nvlink_bytes=718543592 weight_to_token_ratio=0.168868 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.546593868e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78456 payload_rows_remote=63398 dup_rows=19848 dup_rows_remote=19746 dedup_ratio=0.201904 router_tokens_per_expert=[10, 1, 47, 1, 47, 0, 0, 0, 5, 2779, 0, 137, 74, 214, 52, 1291, 0, 7220, 8, 75, 0, 0, 81, 6, 7474, 15, 225, 256, 7, 1113, 72, 5553, 7541, 3508, 11, 167, 5, 0, 0, 1, 247, 79, 3314, 11, 0, 1779, 259, 10, 256, 0, 0, 1551, 1, 0, 0, 1, 6, 3488, 63, 0, 1, 13, 0, 77] payload_rows_per_dst=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] dup_rows_per_dst=[102, 7824, 10, 6484, 4282, 1130, 0, 16] combine_rows_per_src=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=322715648 dispatch_meta_bytes=357144 combine_payload_bytes=322715648 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=323072792 ingress_bytes=461127680 link_bytes=461127680 token_bytes=645788440 weight_movement_bytes=138412032 total_nvlink_bytes=784200472 weight_to_token_ratio=0.214330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=24320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.735186336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287817728 dispatch_meta_bytes=333632 combine_payload_bytes=287817728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288151360 ingress_bytes=322420736 link_bytes=322420736 token_bytes=575969088 weight_movement_bytes=34603008 total_nvlink_bytes=610572096 weight_to_token_ratio=0.060078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213243273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80436 payload_rows_remote=65418 dup_rows=17868 dup_rows_remote=16718 dedup_ratio=0.181763 router_tokens_per_expert=[0, 0, 82, 1336, 0, 0, 1011, 0, 0, 5548, 850, 0, 0, 0, 727, 0, 256, 335, 0, 1359, 144, 68, 0, 0, 6024, 30, 0, 76, 0, 0, 0, 0, 78, 326, 0, 0, 1729, 1207, 0, 74, 45, 0, 4551, 0, 0, 0, 0, 0, 619, 0, 74, 0, 7359, 0, 1, 31, 6826, 0, 0, 0, 7098, 558, 730, 0] payload_rows_per_dst=[4580, 12872, 3974, 12260, 6414, 9102, 15018, 16216] dup_rows_per_dst=[278, 1378, 350, 0, 414, 90, 1150, 14208] combine_rows_per_src=[4580, 12872, 3974, 12260, 6414, 9102, 15018, 16216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=325427200 dispatch_meta_bytes=353216 combine_payload_bytes=325427200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325780416 ingress_bytes=360030208 link_bytes=360030208 token_bytes=651207616 weight_movement_bytes=34603008 total_nvlink_bytes=685810624 weight_to_token_ratio=0.053137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354764688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=287711232 dispatch_meta_bytes=316504 combine_payload_bytes=287711232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288027736 ingress_bytes=322314240 link_bytes=322314240 token_bytes=575738968 weight_movement_bytes=34603008 total_nvlink_bytes=610341976 weight_to_token_ratio=0.060102 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=37248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212842537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=292954112 dispatch_meta_bytes=334240 combine_payload_bytes=292954112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=293288352 ingress_bytes=362160128 link_bytes=362160128 token_bytes=586242464 weight_movement_bytes=69206016 total_nvlink_bytes=655448480 weight_to_token_ratio=0.118050 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.362779405e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=259678208 dispatch_meta_bytes=332576 combine_payload_bytes=259678208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260010784 ingress_bytes=294281216 link_bytes=294281216 token_bytes=519688992 weight_movement_bytes=34603008 total_nvlink_bytes=554292000 weight_to_token_ratio=0.066584 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107356524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=287817728 dispatch_meta_bytes=333632 combine_payload_bytes=287817728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288151360 ingress_bytes=322420736 link_bytes=322420736 token_bytes=575969088 weight_movement_bytes=34603008 total_nvlink_bytes=610572096 weight_to_token_ratio=0.060078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.213243273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81902 payload_rows_remote=73758 dup_rows=16402 dup_rows_remote=15638 dedup_ratio=0.166850 router_tokens_per_expert=[0, 0, 81, 874, 0, 0, 1956, 0, 0, 3103, 1116, 0, 0, 0, 815, 0, 217, 423, 0, 957, 0, 59, 0, 0, 5163, 0, 0, 196, 0, 0, 0, 0, 59, 327, 0, 0, 2502, 1486, 0, 80, 76, 0, 5646, 0, 0, 0, 0, 0, 617, 0, 80, 0, 7544, 0, 93, 0, 7018, 0, 0, 0, 7176, 589, 899, 0] payload_rows_per_dst=[10610, 8676, 10652, 12734, 8144, 12516, 15594, 2976] dup_rows_per_dst=[2600, 1392, 1766, 4632, 764, 4174, 1074, 0] combine_rows_per_src=[10610, 8676, 10652, 12734, 8144, 12516, 15594, 2976] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267952128 dispatch_meta_bytes=328544 combine_payload_bytes=267952128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268280672 ingress_bytes=337158144 link_bytes=337158144 token_bytes=536232800 weight_movement_bytes=69206016 total_nvlink_bytes=605438816 weight_to_token_ratio=0.129060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268698952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88174 payload_rows_remote=76988 dup_rows=10130 dup_rows_remote=10128 dedup_ratio=0.103048 router_tokens_per_expert=[0, 1, 117, 964, 0, 9, 1601, 0, 0, 3860, 944, 1, 13, 28, 745, 1, 160, 427, 18, 1156, 25, 63, 3, 0, 5367, 4, 0, 203, 3, 2, 0, 15, 80, 315, 8, 6, 2616, 1043, 1, 76, 92, 11, 5563, 0, 1, 6, 0, 24, 620, 22, 76, 0, 7529, 17, 31, 4, 6834, 0, 2, 0, 7143, 509, 793, 0] payload_rows_per_dst=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] dup_rows_per_dst=[3780, 1266, 3114, 2, 654, 182, 1120, 12] combine_rows_per_src=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89500 payload_rows_remote=82544 dup_rows=8804 dup_rows_remote=8458 dedup_ratio=0.089559 router_tokens_per_expert=[3665, 0, 0, 174, 788, 2375, 0, 0, 1595, 67, 2067, 0, 0, 0, 0, 0, 0, 99, 122, 0, 0, 99, 0, 0, 78, 0, 0, 0, 0, 100, 0, 6, 0, 0, 6089, 0, 0, 0, 1922, 462, 0, 6746, 2010, 7753, 0, 0, 4597, 90, 0, 0, 0, 100, 0, 0, 2, 6353, 326, 0, 0, 360, 610, 249, 247, 1] payload_rows_per_dst=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] dup_rows_per_dst=[1936, 1012, 308, 356, 3462, 346, 4, 1380] combine_rows_per_src=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78500 payload_rows_remote=62326 dup_rows=19804 dup_rows_remote=5448 dedup_ratio=0.201457 router_tokens_per_expert=[0, 1, 52, 1367, 1, 11, 1123, 0, 0, 5237, 803, 1, 5, 9, 737, 0, 259, 341, 10, 1454, 227, 71, 0, 0, 5703, 82, 0, 89, 1, 0, 1, 0, 81, 314, 8, 4, 2332, 1012, 0, 73, 9, 0, 4628, 0, 0, 4, 0, 9, 619, 9, 73, 0, 7149, 6, 3, 28, 6679, 0, 0, 0, 7261, 528, 738, 0] payload_rows_per_dst=[4910, 10076, 4374, 11752, 7112, 9280, 14822, 16174] dup_rows_per_dst=[200, 1140, 350, 0, 536, 20, 3202, 14356] combine_rows_per_src=[4910, 10076, 4374, 11752, 7112, 9280, 14822, 16174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315342848 dispatch_meta_bytes=348464 combine_payload_bytes=315342848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315691312 ingress_bytes=349945856 link_bytes=349945856 token_bytes=631034160 weight_movement_bytes=34603008 total_nvlink_bytes=665637168 weight_to_token_ratio=0.054835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316818083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88174 payload_rows_remote=76988 dup_rows=10130 dup_rows_remote=10128 dedup_ratio=0.103048 router_tokens_per_expert=[0, 1, 117, 964, 0, 9, 1601, 0, 0, 3860, 944, 1, 13, 28, 745, 1, 160, 427, 18, 1156, 25, 63, 3, 0, 5367, 4, 0, 203, 3, 2, 0, 15, 80, 315, 8, 6, 2616, 1043, 1, 76, 92, 11, 5563, 0, 1, 6, 0, 24, 620, 22, 76, 0, 7529, 17, 31, 4, 6834, 0, 2, 0, 7143, 509, 793, 0] payload_rows_per_dst=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] dup_rows_per_dst=[3780, 1266, 3114, 2, 654, 182, 1120, 12] combine_rows_per_src=[15272, 9918, 14876, 11186, 7636, 11212, 15478, 2596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302112768 dispatch_meta_bytes=357584 combine_payload_bytes=302112768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302470352 ingress_bytes=336715776 link_bytes=336715776 token_bytes=604583120 weight_movement_bytes=34603008 total_nvlink_bytes=639186128 weight_to_token_ratio=0.057234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267034357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79740 payload_rows_remote=68164 dup_rows=18564 dup_rows_remote=15888 dedup_ratio=0.188843 router_tokens_per_expert=[51, 94, 210, 866, 212, 342, 1096, 0, 0, 3595, 1278, 38, 700, 937, 575, 3, 204, 434, 512, 2212, 335, 70, 6, 55, 4127, 264, 12, 162, 137, 59, 55, 165, 228, 401, 767, 457, 1173, 1113, 299, 94, 426, 261, 4275, 33, 104, 139, 275, 897, 628, 247, 98, 16, 5931, 325, 224, 117, 4224, 14, 280, 134, 5378, 652, 1106, 30] payload_rows_per_dst=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] dup_rows_per_dst=[2690, 2676, 5122, 342, 5220, 1470, 616, 428] combine_rows_per_src=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81902 payload_rows_remote=73758 dup_rows=16402 dup_rows_remote=15638 dedup_ratio=0.166850 router_tokens_per_expert=[0, 0, 81, 874, 0, 0, 1956, 0, 0, 3103, 1116, 0, 0, 0, 815, 0, 217, 423, 0, 957, 0, 59, 0, 0, 5163, 0, 0, 196, 0, 0, 0, 0, 59, 327, 0, 0, 2502, 1486, 0, 80, 76, 0, 5646, 0, 0, 0, 0, 0, 617, 0, 80, 0, 7544, 0, 93, 0, 7018, 0, 0, 0, 7176, 589, 899, 0] payload_rows_per_dst=[10610, 8676, 10652, 12734, 8144, 12516, 15594, 2976] dup_rows_per_dst=[2600, 1392, 1766, 4632, 764, 4174, 1074, 0] combine_rows_per_src=[10610, 8676, 10652, 12734, 8144, 12516, 15594, 2976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=338100224 dispatch_meta_bytes=364008 combine_payload_bytes=338100224 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=338464232 ingress_bytes=476512256 link_bytes=476512256 token_bytes=676564456 weight_movement_bytes=138412032 total_nvlink_bytes=814976488 weight_to_token_ratio=0.204581 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.793077257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78456 payload_rows_remote=63398 dup_rows=19848 dup_rows_remote=19746 dedup_ratio=0.201904 router_tokens_per_expert=[10, 1, 47, 1, 47, 0, 0, 0, 5, 2779, 0, 137, 74, 214, 52, 1291, 0, 7220, 8, 75, 0, 0, 81, 6, 7474, 15, 225, 256, 7, 1113, 72, 5553, 7541, 3508, 11, 167, 5, 0, 0, 1, 247, 79, 3314, 11, 0, 1779, 259, 10, 256, 0, 0, 1551, 1, 0, 0, 1, 6, 3488, 63, 0, 1, 13, 0, 77] payload_rows_per_dst=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] dup_rows_per_dst=[102, 7824, 10, 6484, 4282, 1130, 0, 16] combine_rows_per_src=[15058, 15720, 1114, 14288, 11110, 10268, 3618, 7280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81070 payload_rows_remote=68882 dup_rows=17234 dup_rows_remote=12824 dedup_ratio=0.175313 router_tokens_per_expert=[0, 0, 160, 1238, 0, 0, 984, 0, 0, 5280, 837, 0, 0, 0, 752, 0, 208, 322, 0, 1427, 11, 54, 0, 0, 5485, 58, 0, 117, 0, 0, 0, 0, 58, 347, 0, 0, 2409, 776, 0, 99, 7, 0, 5086, 0, 0, 0, 0, 0, 610, 0, 99, 0, 7486, 0, 3, 27, 6748, 0, 0, 0, 7142, 543, 779, 0] payload_rows_per_dst=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] dup_rows_per_dst=[136, 1330, 270, 1238, 558, 4410, 1074, 8218] combine_rows_per_src=[4628, 12408, 3774, 11892, 6820, 12188, 15376, 13984] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85052 payload_rows_remote=70220 dup_rows=13252 dup_rows_remote=10188 dedup_ratio=0.134806 router_tokens_per_expert=[0, 0, 138, 784, 0, 0, 1618, 0, 0, 3385, 955, 0, 0, 0, 785, 0, 167, 467, 0, 1044, 0, 72, 0, 0, 5169, 0, 0, 306, 0, 0, 0, 0, 74, 355, 0, 0, 2285, 1656, 0, 89, 83, 0, 5897, 0, 0, 0, 0, 0, 622, 0, 89, 0, 7518, 7, 71, 0, 7012, 0, 0, 0, 7198, 521, 785, 0] payload_rows_per_dst=[15296, 8952, 14832, 10950, 11674, 11796, 8940, 2612] dup_rows_per_dst=[3808, 1298, 3064, 0, 4358, 164, 560, 0] combine_rows_per_src=[15296, 8952, 14832, 10950, 11674, 11796, 8940, 2612] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=255287296 dispatch_meta_bytes=271096 combine_payload_bytes=255287296 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=255558392 ingress_bytes=428302336 link_bytes=428302336 token_bytes=510845688 weight_movement_bytes=173015040 total_nvlink_bytes=683860728 weight_to_token_ratio=0.338684 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611667209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84540 payload_rows_remote=69236 dup_rows=13764 dup_rows_remote=11052 dedup_ratio=0.140015 router_tokens_per_expert=[0, 1, 63, 1226, 3, 7, 991, 0, 0, 5085, 868, 0, 13, 18, 688, 1, 249, 330, 5, 1281, 29, 75, 0, 1, 5698, 52, 0, 89, 0, 1, 2, 6, 87, 328, 8, 5, 1779, 1383, 3, 72, 17, 8, 5125, 0, 5, 3, 1, 16, 620, 12, 73, 0, 7482, 3, 15, 70, 6717, 0, 1, 0, 7278, 537, 722, 0] payload_rows_per_dst=[15304, 11980, 15022, 11692, 16294, 10288, 1440, 2520] dup_rows_per_dst=[2712, 1366, 3474, 4, 6000, 62, 146, 0] combine_rows_per_src=[15304, 11980, 15022, 11692, 16294, 10288, 1440, 2520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=315342848 dispatch_meta_bytes=348464 combine_payload_bytes=315342848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315691312 ingress_bytes=349945856 link_bytes=349945856 token_bytes=631034160 weight_movement_bytes=34603008 total_nvlink_bytes=665637168 weight_to_token_ratio=0.054835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316818083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79740 payload_rows_remote=68164 dup_rows=18564 dup_rows_remote=15888 dedup_ratio=0.188843 router_tokens_per_expert=[51, 94, 210, 866, 212, 342, 1096, 0, 0, 3595, 1278, 38, 700, 937, 575, 3, 204, 434, 512, 2212, 335, 70, 6, 55, 4127, 264, 12, 162, 137, 59, 55, 165, 228, 401, 767, 457, 1173, 1113, 299, 94, 426, 261, 4275, 33, 104, 139, 275, 897, 628, 247, 98, 16, 5931, 325, 224, 117, 4224, 14, 280, 134, 5378, 652, 1106, 30] payload_rows_per_dst=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] dup_rows_per_dst=[2690, 2676, 5122, 342, 5220, 1470, 616, 428] combine_rows_per_src=[11500, 11576, 13290, 9620, 15706, 11350, 2694, 4004] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79954 payload_rows_remote=66582 dup_rows=18350 dup_rows_remote=18346 dedup_ratio=0.186666 router_tokens_per_expert=[3808, 0, 0, 172, 738, 2178, 0, 0, 2260, 84, 2083, 0, 0, 0, 0, 0, 0, 75, 154, 0, 0, 74, 0, 0, 73, 0, 0, 0, 1, 75, 0, 0, 0, 0, 5953, 0, 0, 0, 1794, 403, 0, 6661, 1571, 7862, 0, 0, 4604, 74, 0, 0, 0, 75, 1, 0, 2, 6610, 328, 0, 0, 363, 642, 183, 251, 0] payload_rows_per_dst=[2658, 7730, 11892, 5664, 14678, 12612, 13372, 11348] dup_rows_per_dst=[288, 1124, 254, 106, 8112, 7068, 4, 1394] combine_rows_per_src=[2658, 7730, 11892, 5664, 14678, 12612, 13372, 11348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279199744 dispatch_meta_bytes=336208 combine_payload_bytes=279199744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279535952 ingress_bytes=348405760 link_bytes=348405760 token_bytes=558735696 weight_movement_bytes=69206016 total_nvlink_bytes=627941712 weight_to_token_ratio=0.123862 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311022826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302112768 dispatch_meta_bytes=357584 combine_payload_bytes=302112768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302470352 ingress_bytes=336715776 link_bytes=336715776 token_bytes=604583120 weight_movement_bytes=34603008 total_nvlink_bytes=639186128 weight_to_token_ratio=0.057234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267034357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=259678208 dispatch_meta_bytes=332576 combine_payload_bytes=259678208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260010784 ingress_bytes=294281216 link_bytes=294281216 token_bytes=519688992 weight_movement_bytes=34603008 total_nvlink_bytes=554292000 weight_to_token_ratio=0.066584 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107356524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=282140672 dispatch_meta_bytes=326824 combine_payload_bytes=282140672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282467496 ingress_bytes=316743680 link_bytes=316743680 token_bytes=564608168 weight_movement_bytes=34603008 total_nvlink_bytes=599211176 weight_to_token_ratio=0.061287 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191880968e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=287621120 dispatch_meta_bytes=321632 combine_payload_bytes=287621120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287942752 ingress_bytes=322224128 link_bytes=322224128 token_bytes=575563872 weight_movement_bytes=34603008 total_nvlink_bytes=610166880 weight_to_token_ratio=0.060120 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=31616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212503452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=283590656 dispatch_meta_bytes=321152 combine_payload_bytes=283590656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283911808 ingress_bytes=318193664 link_bytes=318193664 token_bytes=567502464 weight_movement_bytes=34603008 total_nvlink_bytes=602105472 weight_to_token_ratio=0.060974 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197337141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88940 payload_rows_remote=73488 dup_rows=9364 dup_rows_remote=8902 dedup_ratio=0.095256 router_tokens_per_expert=[3189, 1, 32, 130, 728, 3753, 5, 3, 1817, 92, 1184, 0, 2, 3, 0, 2, 0, 76, 184, 23, 4, 76, 16, 0, 156, 0, 0, 14, 0, 76, 0, 35, 6, 0, 5620, 0, 0, 6, 1552, 416, 0, 6809, 2783, 7676, 0, 22, 4281, 18, 8, 3, 3, 76, 0, 1, 13, 6266, 528, 28, 0, 325, 588, 181, 240, 102] payload_rows_per_dst=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] dup_rows_per_dst=[3074, 3590, 338, 462, 236, 4, 16, 1644] combine_rows_per_src=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279199744 dispatch_meta_bytes=336208 combine_payload_bytes=279199744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=279535952 ingress_bytes=348405760 link_bytes=348405760 token_bytes=558735696 weight_movement_bytes=69206016 total_nvlink_bytes=627941712 weight_to_token_ratio=0.123862 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.311022826e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89134 payload_rows_remote=77424 dup_rows=9170 dup_rows_remote=7194 dedup_ratio=0.093282 router_tokens_per_expert=[3201, 0, 0, 101, 776, 4780, 0, 0, 1340, 96, 741, 0, 0, 0, 0, 0, 0, 80, 165, 0, 0, 80, 0, 0, 172, 0, 0, 0, 6, 80, 0, 66, 0, 0, 5656, 0, 0, 0, 1228, 396, 0, 6823, 3090, 7982, 0, 0, 3523, 0, 0, 0, 0, 80, 0, 0, 0, 6457, 734, 0, 0, 380, 686, 140, 247, 46] payload_rows_per_dst=[13736, 4966, 13992, 16108, 11710, 6180, 13074, 9368] dup_rows_per_dst=[3980, 262, 304, 504, 1976, 0, 0, 2144] combine_rows_per_src=[13736, 4966, 13992, 16108, 11710, 6180, 13074, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=272719872 dispatch_meta_bytes=339712 combine_payload_bytes=272719872 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=273059584 ingress_bytes=307322880 link_bytes=307322880 token_bytes=545779456 weight_movement_bytes=34603008 total_nvlink_bytes=580382464 weight_to_token_ratio=0.063401 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156431256e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82780 payload_rows_remote=69684 dup_rows=15524 dup_rows_remote=10178 dedup_ratio=0.157918 router_tokens_per_expert=[4607, 491, 0, 0, 0, 0, 1423, 2, 0, 2855, 0, 4286, 0, 0, 648, 0, 0, 199, 784, 0, 0, 357, 0, 0, 0, 0, 4284, 623, 1591, 0, 154, 1194, 0, 0, 0, 0, 0, 0, 0, 4581, 0, 0, 418, 122, 4487, 199, 7326, 0, 6888, 54, 0, 157, 0, 99, 0, 0, 67, 0, 0, 0, 288, 86, 0, 882] payload_rows_per_dst=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] dup_rows_per_dst=[2162, 158, 554, 42, 4874, 5346, 616, 1772] combine_rows_per_src=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66604 payload_rows_remote=54916 dup_rows=31700 dup_rows_remote=30130 dedup_ratio=0.322469 router_tokens_per_expert=[3595, 0, 12, 186, 788, 1585, 1, 1, 2039, 85, 2437, 0, 1, 3, 0, 0, 0, 74, 154, 8, 1, 73, 6, 0, 82, 0, 0, 5, 0, 74, 0, 0, 2, 0, 6416, 0, 0, 1, 1998, 357, 0, 6752, 1527, 7699, 0, 7, 4802, 71, 2, 0, 0, 74, 6, 1, 84, 6316, 385, 12, 0, 348, 613, 192, 240, 37] payload_rows_per_dst=[1824, 8064, 620, 322, 15086, 16194, 12806, 11688] dup_rows_per_dst=[152, 1066, 12, 0, 9652, 19088, 160, 1570] combine_rows_per_src=[1824, 8064, 620, 322, 15086, 16194, 12806, 11688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301006848 dispatch_meta_bytes=329560 combine_payload_bytes=301006848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301336408 ingress_bytes=335609856 link_bytes=335609856 token_bytes=602343256 weight_movement_bytes=34603008 total_nvlink_bytes=636946264 weight_to_token_ratio=0.057447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=285425664 dispatch_meta_bytes=319448 combine_payload_bytes=285425664 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=285745112 ingress_bytes=423837696 link_bytes=423837696 token_bytes=571170776 weight_movement_bytes=138412032 total_nvlink_bytes=709582808 weight_to_token_ratio=0.242330 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594867128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88940 payload_rows_remote=73488 dup_rows=9364 dup_rows_remote=8902 dedup_ratio=0.095256 router_tokens_per_expert=[3189, 1, 32, 130, 728, 3753, 5, 3, 1817, 92, 1184, 0, 2, 3, 0, 2, 0, 76, 184, 23, 4, 76, 16, 0, 156, 0, 0, 14, 0, 76, 0, 35, 6, 0, 5620, 0, 0, 6, 1552, 416, 0, 6809, 2783, 7676, 0, 22, 4281, 18, 8, 3, 3, 76, 0, 1, 13, 6266, 528, 28, 0, 325, 588, 181, 240, 102] payload_rows_per_dst=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] dup_rows_per_dst=[3074, 3590, 338, 462, 236, 4, 16, 1644] combine_rows_per_src=[12608, 13850, 14038, 15452, 3724, 5642, 12724, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=317128704 dispatch_meta_bytes=338472 combine_payload_bytes=317128704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317467176 ingress_bytes=386334720 link_bytes=386334720 token_bytes=634595880 weight_movement_bytes=69206016 total_nvlink_bytes=703801896 weight_to_token_ratio=0.109055 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453746449e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89134 payload_rows_remote=77424 dup_rows=9170 dup_rows_remote=7194 dedup_ratio=0.093282 router_tokens_per_expert=[3201, 0, 0, 101, 776, 4780, 0, 0, 1340, 96, 741, 0, 0, 0, 0, 0, 0, 80, 165, 0, 0, 80, 0, 0, 172, 0, 0, 0, 6, 80, 0, 66, 0, 0, 5656, 0, 0, 0, 1228, 396, 0, 6823, 3090, 7982, 0, 0, 3523, 0, 0, 0, 0, 80, 0, 0, 0, 6457, 734, 0, 0, 380, 686, 140, 247, 46] payload_rows_per_dst=[13736, 4966, 13992, 16108, 11710, 6180, 13074, 9368] dup_rows_per_dst=[3980, 262, 304, 504, 1976, 0, 0, 2144] combine_rows_per_src=[13736, 4966, 13992, 16108, 11710, 6180, 13074, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83708 payload_rows_remote=72500 dup_rows=14596 dup_rows_remote=10520 dedup_ratio=0.148478 router_tokens_per_expert=[1367, 39, 1166, 196, 808, 3213, 326, 181, 2495, 163, 1012, 0, 112, 63, 50, 60, 3, 53, 451, 846, 113, 96, 395, 228, 213, 58, 160, 192, 59, 109, 73, 179, 666, 10, 3687, 4, 119, 175, 2834, 664, 51, 4985, 1855, 4797, 87, 561, 4509, 154, 79, 487, 131, 92, 97, 112, 433, 3534, 1227, 1044, 40, 575, 635, 245, 415, 369] payload_rows_per_dst=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] dup_rows_per_dst=[2402, 4076, 1536, 666, 1042, 62, 250, 4562] combine_rows_per_src=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89500 payload_rows_remote=82544 dup_rows=8804 dup_rows_remote=8458 dedup_ratio=0.089559 router_tokens_per_expert=[3665, 0, 0, 174, 788, 2375, 0, 0, 1595, 67, 2067, 0, 0, 0, 0, 0, 0, 99, 122, 0, 0, 99, 0, 0, 78, 0, 0, 0, 0, 100, 0, 6, 0, 0, 6089, 0, 0, 0, 1922, 462, 0, 6746, 2010, 7753, 0, 0, 4597, 90, 0, 0, 0, 100, 0, 0, 2, 6353, 326, 0, 0, 360, 610, 249, 247, 1] payload_rows_per_dst=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] dup_rows_per_dst=[1936, 1012, 308, 356, 3462, 346, 4, 1380] combine_rows_per_src=[8966, 6446, 13824, 15518, 13484, 6956, 12906, 11400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89318 payload_rows_remote=75414 dup_rows=8986 dup_rows_remote=8676 dedup_ratio=0.091410 router_tokens_per_expert=[3407, 0, 0, 102, 762, 4100, 0, 0, 1317, 74, 735, 0, 0, 0, 0, 0, 0, 86, 128, 0, 0, 88, 0, 0, 229, 0, 0, 0, 0, 90, 0, 50, 0, 0, 5564, 0, 0, 0, 1234, 457, 0, 6805, 3009, 7916, 0, 0, 4180, 0, 0, 0, 0, 90, 0, 0, 0, 6575, 701, 0, 0, 338, 595, 209, 265, 46] payload_rows_per_dst=[13182, 13144, 13904, 15932, 3104, 6018, 13330, 10704] dup_rows_per_dst=[3560, 2236, 310, 638, 278, 0, 0, 1964] combine_rows_per_src=[13182, 13144, 13904, 15932, 3104, 6018, 13330, 10704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84540 payload_rows_remote=69236 dup_rows=13764 dup_rows_remote=11052 dedup_ratio=0.140015 router_tokens_per_expert=[0, 1, 63, 1226, 3, 7, 991, 0, 0, 5085, 868, 0, 13, 18, 688, 1, 249, 330, 5, 1281, 29, 75, 0, 1, 5698, 52, 0, 89, 0, 1, 2, 6, 87, 328, 8, 5, 1779, 1383, 3, 72, 17, 8, 5125, 0, 5, 3, 1, 16, 620, 12, 73, 0, 7482, 3, 15, 70, 6717, 0, 1, 0, 7278, 537, 722, 0] payload_rows_per_dst=[15304, 11980, 15022, 11692, 16294, 10288, 1440, 2520] dup_rows_per_dst=[2712, 1366, 3474, 4, 6000, 62, 146, 0] combine_rows_per_src=[15304, 11980, 15022, 11692, 16294, 10288, 1440, 2520] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=224935936 dispatch_meta_bytes=340184 combine_payload_bytes=224935936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=225276120 ingress_bytes=259538944 link_bytes=259538944 token_bytes=450212056 weight_movement_bytes=34603008 total_nvlink_bytes=484815064 weight_to_token_ratio=0.076859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.766241517e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83744 payload_rows_remote=71296 dup_rows=14560 dup_rows_remote=9260 dedup_ratio=0.148112 router_tokens_per_expert=[3414, 0, 20, 150, 762, 2879, 5, 2, 1851, 71, 1839, 0, 1, 3, 0, 3, 0, 72, 119, 17, 3, 72, 6, 5, 89, 0, 0, 5, 0, 73, 0, 3, 7, 0, 5997, 0, 0, 1, 1556, 365, 0, 6790, 1833, 7611, 2, 13, 4875, 72, 2, 2, 1, 72, 0, 4, 32, 6518, 513, 16, 0, 367, 605, 151, 256, 27] payload_rows_per_dst=[12448, 14310, 13900, 15244, 3728, 11134, 9976, 3004] dup_rows_per_dst=[5300, 5220, 268, 318, 130, 1646, 2, 1676] combine_rows_per_src=[12448, 14310, 13900, 15244, 3728, 11134, 9976, 3004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=301006848 dispatch_meta_bytes=329560 combine_payload_bytes=301006848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301336408 ingress_bytes=335609856 link_bytes=335609856 token_bytes=602343256 weight_movement_bytes=34603008 total_nvlink_bytes=636946264 weight_to_token_ratio=0.057447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=317128704 dispatch_meta_bytes=338472 combine_payload_bytes=317128704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317467176 ingress_bytes=386334720 link_bytes=386334720 token_bytes=634595880 weight_movement_bytes=69206016 total_nvlink_bytes=703801896 weight_to_token_ratio=0.109055 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453746449e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83708 payload_rows_remote=72500 dup_rows=14596 dup_rows_remote=10520 dedup_ratio=0.148478 router_tokens_per_expert=[1367, 39, 1166, 196, 808, 3213, 326, 181, 2495, 163, 1012, 0, 112, 63, 50, 60, 3, 53, 451, 846, 113, 96, 395, 228, 213, 58, 160, 192, 59, 109, 73, 179, 666, 10, 3687, 4, 119, 175, 2834, 664, 51, 4985, 1855, 4797, 87, 561, 4509, 154, 79, 487, 131, 92, 97, 112, 433, 3534, 1227, 1044, 40, 575, 635, 245, 415, 369] payload_rows_per_dst=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] dup_rows_per_dst=[2402, 4076, 1536, 666, 1042, 62, 250, 4562] combine_rows_per_src=[12190, 11208, 12804, 11014, 7902, 5354, 9680, 13556] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79836 payload_rows_remote=65806 dup_rows=18468 dup_rows_remote=17878 dedup_ratio=0.187866 router_tokens_per_expert=[4063, 422, 0, 0, 0, 0, 1414, 0, 0, 3368, 0, 4237, 0, 0, 723, 0, 0, 149, 864, 0, 0, 365, 0, 0, 0, 0, 3762, 609, 1293, 0, 146, 1082, 0, 0, 0, 0, 0, 0, 0, 5454, 0, 0, 429, 154, 4690, 132, 7320, 0, 7015, 67, 0, 154, 0, 74, 0, 0, 84, 0, 0, 0, 227, 59, 0, 796] payload_rows_per_dst=[9864, 7996, 2756, 6246, 13116, 15734, 14030, 10094] dup_rows_per_dst=[1934, 186, 0, 14, 6266, 7860, 590, 1618] combine_rows_per_src=[9864, 7996, 2756, 6246, 13116, 15734, 14030, 10094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=332080 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297292080 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594252080 weight_movement_bytes=34603008 total_nvlink_bytes=628855088 weight_to_token_ratio=0.058230 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=338100224 dispatch_meta_bytes=364008 combine_payload_bytes=338100224 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=338464232 ingress_bytes=476512256 link_bytes=476512256 token_bytes=676564456 weight_movement_bytes=138412032 total_nvlink_bytes=814976488 weight_to_token_ratio=0.204581 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.793077257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=308895744 dispatch_meta_bytes=336360 combine_payload_bytes=308895744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=309232104 ingress_bytes=343498752 link_bytes=343498752 token_bytes=618127848 weight_movement_bytes=34603008 total_nvlink_bytes=652730856 weight_to_token_ratio=0.055980 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.292558150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88204 payload_rows_remote=78900 dup_rows=10100 dup_rows_remote=8764 dedup_ratio=0.102743 router_tokens_per_expert=[4782, 588, 0, 0, 1, 5, 1460, 31, 0, 1934, 7, 4173, 3, 0, 462, 0, 5, 153, 606, 0, 12, 439, 0, 9, 1, 1, 4965, 612, 1733, 15, 138, 1487, 29, 20, 2, 0, 20, 2, 0, 3457, 4, 26, 929, 149, 5366, 86, 7285, 13, 6703, 0, 3, 159, 0, 76, 0, 5, 88, 0, 0, 33, 289, 31, 0, 755] payload_rows_per_dst=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] dup_rows_per_dst=[2314, 2172, 1170, 1336, 48, 1060, 396, 1604] combine_rows_per_src=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292028416 dispatch_meta_bytes=322224 combine_payload_bytes=292028416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292350640 ingress_bytes=395837440 link_bytes=395837440 token_bytes=584379056 weight_movement_bytes=103809024 total_nvlink_bytes=688188080 weight_to_token_ratio=0.177640 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.489504419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92310 payload_rows_remote=77158 dup_rows=5994 dup_rows_remote=4266 dedup_ratio=0.060974 router_tokens_per_expert=[0, 0, 172, 0, 0, 0, 1551, 0, 0, 0, 154, 146, 7348, 0, 0, 4813, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 8091, 0, 0, 5964, 0, 0, 0, 154, 0, 0, 6604, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 633, 0, 100, 0, 5905, 0, 0, 842, 113, 0, 0] payload_rows_per_dst=[7464, 8160, 11338, 6800, 13516, 15152, 16186, 13694] dup_rows_per_dst=[1110, 0, 1210, 0, 0, 1728, 1920, 26] combine_rows_per_src=[7464, 8160, 11338, 6800, 13516, 15152, 16186, 13694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=283590656 dispatch_meta_bytes=321152 combine_payload_bytes=283590656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283911808 ingress_bytes=318193664 link_bytes=318193664 token_bytes=567502464 weight_movement_bytes=34603008 total_nvlink_bytes=602105472 weight_to_token_ratio=0.060974 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197337141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=296960000 dispatch_meta_bytes=332080 combine_payload_bytes=296960000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297292080 ingress_bytes=331563008 link_bytes=331563008 token_bytes=594252080 weight_movement_bytes=34603008 total_nvlink_bytes=628855088 weight_to_token_ratio=0.058230 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247644906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89370 payload_rows_remote=82336 dup_rows=8934 dup_rows_remote=8934 dedup_ratio=0.090881 router_tokens_per_expert=[4760, 716, 0, 0, 0, 0, 1538, 55, 0, 1313, 0, 4377, 0, 0, 334, 0, 0, 160, 511, 0, 0, 486, 0, 0, 0, 0, 5757, 666, 1087, 0, 151, 1585, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 951, 163, 5516, 84, 7271, 0, 6794, 0, 0, 153, 0, 80, 0, 0, 97, 0, 0, 0, 259, 0, 0, 771] payload_rows_per_dst=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] dup_rows_per_dst=[2502, 1566, 992, 68, 0, 1882, 354, 1570] combine_rows_per_src=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=269541376 dispatch_meta_bytes=334736 combine_payload_bytes=269541376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=269876112 ingress_bytes=338747392 link_bytes=338747392 token_bytes=539417488 weight_movement_bytes=69206016 total_nvlink_bytes=608623504 weight_to_token_ratio=0.128298 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274679165e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78692 payload_rows_remote=69220 dup_rows=19612 dup_rows_remote=17506 dedup_ratio=0.199504 router_tokens_per_expert=[4680, 352, 0, 0, 2, 0, 1292, 8, 2, 4052, 3, 3888, 5, 0, 862, 0, 2, 147, 916, 0, 1, 349, 0, 4, 0, 1, 3399, 627, 1533, 7, 138, 1017, 11, 3, 0, 0, 8, 0, 0, 5410, 0, 10, 404, 149, 3921, 184, 7316, 3, 6897, 81, 1, 176, 0, 73, 0, 3, 85, 0, 0, 12, 335, 74, 1, 708] payload_rows_per_dst=[4918, 12826, 2836, 6610, 13136, 15064, 13830, 9472] dup_rows_per_dst=[972, 3800, 2, 36, 5504, 6560, 632, 2106] combine_rows_per_src=[4918, 12826, 2836, 6610, 13136, 15064, 13830, 9472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=323174400 dispatch_meta_bytes=350656 combine_payload_bytes=323174400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323525056 ingress_bytes=392380416 link_bytes=392380416 token_bytes=646699456 weight_movement_bytes=69206016 total_nvlink_bytes=715905472 weight_to_token_ratio=0.107014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=325696 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316364864 ingress_bytes=350642176 link_bytes=350642176 token_bytes=632404032 weight_movement_bytes=34603008 total_nvlink_bytes=667007040 weight_to_token_ratio=0.054717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319438279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89370 payload_rows_remote=82336 dup_rows=8934 dup_rows_remote=8934 dedup_ratio=0.090881 router_tokens_per_expert=[4760, 716, 0, 0, 0, 0, 1538, 55, 0, 1313, 0, 4377, 0, 0, 334, 0, 0, 160, 511, 0, 0, 486, 0, 0, 0, 0, 5757, 666, 1087, 0, 151, 1585, 0, 0, 0, 0, 0, 0, 0, 3517, 0, 0, 951, 163, 5516, 84, 7271, 0, 6794, 0, 0, 153, 0, 80, 0, 0, 97, 0, 0, 0, 259, 0, 0, 771] payload_rows_per_dst=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] dup_rows_per_dst=[2502, 1566, 992, 68, 0, 1882, 354, 1570] combine_rows_per_src=[11636, 10482, 15864, 6910, 7034, 12028, 13700, 11716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88204 payload_rows_remote=78900 dup_rows=10100 dup_rows_remote=8764 dedup_ratio=0.102743 router_tokens_per_expert=[4782, 588, 0, 0, 1, 5, 1460, 31, 0, 1934, 7, 4173, 3, 0, 462, 0, 5, 153, 606, 0, 12, 439, 0, 9, 1, 1, 4965, 612, 1733, 15, 138, 1487, 29, 20, 2, 0, 20, 2, 0, 3457, 4, 26, 929, 149, 5366, 86, 7285, 13, 6703, 0, 3, 159, 0, 76, 0, 5, 88, 0, 0, 33, 289, 31, 0, 755] payload_rows_per_dst=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] dup_rows_per_dst=[2314, 2172, 1170, 1336, 48, 1060, 396, 1604] combine_rows_per_src=[11420, 10986, 15848, 9304, 7012, 8618, 13496, 11520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337248256 dispatch_meta_bytes=365080 combine_payload_bytes=337248256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337613336 ingress_bytes=371851264 link_bytes=371851264 token_bytes=674861592 weight_movement_bytes=34603008 total_nvlink_bytes=709464600 weight_to_token_ratio=0.051274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399246370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82780 payload_rows_remote=69684 dup_rows=15524 dup_rows_remote=10178 dedup_ratio=0.157918 router_tokens_per_expert=[4607, 491, 0, 0, 0, 0, 1423, 2, 0, 2855, 0, 4286, 0, 0, 648, 0, 0, 199, 784, 0, 0, 357, 0, 0, 0, 0, 4284, 623, 1591, 0, 154, 1194, 0, 0, 0, 0, 0, 0, 0, 4581, 0, 0, 418, 122, 4487, 199, 7326, 0, 6888, 54, 0, 157, 0, 99, 0, 0, 67, 0, 0, 0, 288, 86, 0, 882] payload_rows_per_dst=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] dup_rows_per_dst=[2162, 158, 554, 42, 4874, 5346, 616, 1772] combine_rows_per_src=[10884, 7296, 8382, 7082, 12412, 13096, 13780, 9848] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86850 payload_rows_remote=71022 dup_rows=11454 dup_rows_remote=10556 dedup_ratio=0.116516 router_tokens_per_expert=[4845, 758, 0, 0, 0, 0, 1469, 55, 0, 1377, 0, 4302, 0, 0, 382, 0, 0, 179, 448, 0, 0, 416, 0, 0, 0, 0, 6000, 635, 1207, 0, 141, 1475, 0, 0, 0, 0, 0, 0, 0, 3207, 0, 0, 927, 126, 5627, 102, 7320, 0, 6767, 0, 0, 181, 0, 89, 0, 0, 74, 0, 0, 0, 230, 2, 0, 811] payload_rows_per_dst=[11714, 10446, 15828, 14772, 6414, 2164, 13646, 11866] dup_rows_per_dst=[2540, 1676, 898, 4144, 0, 146, 428, 1622] combine_rows_per_src=[11714, 10446, 15828, 14772, 6414, 2164, 13646, 11866] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81812 payload_rows_remote=71150 dup_rows=16492 dup_rows_remote=13402 dedup_ratio=0.167765 router_tokens_per_expert=[3678, 1213, 54, 6, 154, 250, 949, 113, 203, 2899, 498, 2160, 477, 77, 561, 1, 217, 248, 971, 230, 185, 520, 76, 163, 4, 221, 1759, 747, 1855, 453, 172, 952, 1143, 328, 163, 0, 1004, 94, 20, 3243, 69, 779, 1785, 71, 2526, 346, 5305, 502, 4983, 127, 69, 368, 0, 88, 109, 371, 61, 37, 11, 1215, 1238, 153, 186, 692] payload_rows_per_dst=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] dup_rows_per_dst=[2852, 3090, 2466, 2078, 2430, 1022, 564, 1990] combine_rows_per_src=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283525120 dispatch_meta_bytes=346904 combine_payload_bytes=283525120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=283872024 ingress_bytes=352731136 link_bytes=352731136 token_bytes=567397144 weight_movement_bytes=69206016 total_nvlink_bytes=636603160 weight_to_token_ratio=0.121971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=36608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327298868e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77118 payload_rows_remote=66486 dup_rows=21186 dup_rows_remote=19208 dedup_ratio=0.215515 router_tokens_per_expert=[4474, 468, 0, 0, 4, 2, 1327, 11, 2, 2436, 10, 4282, 3, 0, 748, 0, 5, 145, 774, 7, 1, 359, 1, 1, 4, 4, 4274, 639, 1527, 5, 113, 1120, 21, 10, 3, 0, 18, 3, 0, 5194, 1, 13, 542, 112, 4433, 142, 7379, 11, 7019, 48, 3, 159, 0, 72, 0, 27, 64, 0, 0, 19, 275, 72, 2, 764] payload_rows_per_dst=[10632, 12022, 15830, 16336, 10452, 1584, 474, 9788] dup_rows_per_dst=[1978, 2940, 1514, 13036, 46, 58, 144, 1470] combine_rows_per_src=[10632, 12022, 15830, 16336, 10452, 1584, 474, 9788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83744 payload_rows_remote=71296 dup_rows=14560 dup_rows_remote=9260 dedup_ratio=0.148112 router_tokens_per_expert=[3414, 0, 20, 150, 762, 2879, 5, 2, 1851, 71, 1839, 0, 1, 3, 0, 3, 0, 72, 119, 17, 3, 72, 6, 5, 89, 0, 0, 5, 0, 73, 0, 3, 7, 0, 5997, 0, 0, 1, 1556, 365, 0, 6790, 1833, 7611, 2, 13, 4875, 72, 2, 2, 1, 72, 0, 4, 32, 6518, 513, 16, 0, 367, 605, 151, 256, 27] payload_rows_per_dst=[12448, 14310, 13900, 15244, 3728, 11134, 9976, 3004] dup_rows_per_dst=[5300, 5220, 268, 318, 130, 1646, 2, 1676] combine_rows_per_src=[12448, 14310, 13900, 15244, 3728, 11134, 9976, 3004] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=337248256 dispatch_meta_bytes=365080 combine_payload_bytes=337248256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=337613336 ingress_bytes=371851264 link_bytes=371851264 token_bytes=674861592 weight_movement_bytes=34603008 total_nvlink_bytes=709464600 weight_to_token_ratio=0.051274 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399246370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=323174400 dispatch_meta_bytes=350656 combine_payload_bytes=323174400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=323525056 ingress_bytes=392380416 link_bytes=392380416 token_bytes=646699456 weight_movement_bytes=69206016 total_nvlink_bytes=715905472 weight_to_token_ratio=0.107014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13440 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.476495916e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81812 payload_rows_remote=71150 dup_rows=16492 dup_rows_remote=13402 dedup_ratio=0.167765 router_tokens_per_expert=[3678, 1213, 54, 6, 154, 250, 949, 113, 203, 2899, 498, 2160, 477, 77, 561, 1, 217, 248, 971, 230, 185, 520, 76, 163, 4, 221, 1759, 747, 1855, 453, 172, 952, 1143, 328, 163, 0, 1004, 94, 20, 3243, 69, 779, 1785, 71, 2526, 346, 5305, 502, 4983, 127, 69, 368, 0, 88, 109, 371, 61, 37, 11, 1215, 1238, 153, 186, 692] payload_rows_per_dst=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] dup_rows_per_dst=[2852, 3090, 2466, 2078, 2430, 1022, 564, 1990] combine_rows_per_src=[9982, 10662, 13364, 10290, 9560, 6082, 11624, 10248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88982 payload_rows_remote=72746 dup_rows=9322 dup_rows_remote=7452 dedup_ratio=0.094828 router_tokens_per_expert=[0, 0, 84, 0, 0, 0, 1599, 0, 0, 0, 144, 137, 7384, 0, 0, 5018, 2, 0, 0, 0, 0, 5497, 0, 0, 0, 0, 0, 8118, 0, 0, 5834, 0, 0, 0, 144, 0, 0, 6559, 0, 0, 0, 0, 860, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 601, 0, 75, 0, 5915, 0, 0, 837, 85, 0, 0] payload_rows_per_dst=[3230, 562, 15070, 11668, 13406, 15150, 16236, 13660] dup_rows_per_dst=[136, 0, 5964, 0, 0, 1338, 1870, 14] combine_rows_per_src=[3230, 562, 15070, 11668, 13406, 15150, 16236, 13660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=285425664 dispatch_meta_bytes=319448 combine_payload_bytes=285425664 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=285745112 ingress_bytes=423837696 link_bytes=423837696 token_bytes=571170776 weight_movement_bytes=138412032 total_nvlink_bytes=709582808 weight_to_token_ratio=0.242330 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20864 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.594867128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=290906112 dispatch_meta_bytes=326312 combine_payload_bytes=290906112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291232424 ingress_bytes=325509120 link_bytes=325509120 token_bytes=582138536 weight_movement_bytes=34603008 total_nvlink_bytes=616741544 weight_to_token_ratio=0.059441 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224864613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=291430400 dispatch_meta_bytes=338208 combine_payload_bytes=291430400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291768608 ingress_bytes=360636416 link_bytes=360636416 token_bytes=583199008 weight_movement_bytes=69206016 total_nvlink_bytes=652405024 weight_to_token_ratio=0.118666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357045800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272326656 dispatch_meta_bytes=342776 combine_payload_bytes=272326656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272669432 ingress_bytes=341532672 link_bytes=341532672 token_bytes=544996088 weight_movement_bytes=69206016 total_nvlink_bytes=614202104 weight_to_token_ratio=0.126984 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285159949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92218 payload_rows_remote=92150 dup_rows=6086 dup_rows_remote=6078 dedup_ratio=0.061910 router_tokens_per_expert=[1, 6, 622, 2, 0, 1, 1319, 0, 0, 0, 137, 192, 7214, 2, 30, 5055, 3, 0, 3, 2, 2, 4289, 7, 20, 4, 0, 1, 7996, 1, 31, 6250, 1, 1, 1, 136, 0, 1, 7466, 7, 6, 2, 0, 974, 0, 1, 34, 27, 0, 0, 21, 82, 1, 0, 587, 21, 76, 4, 5191, 0, 3, 755, 546, 14, 4] payload_rows_per_dst=[13632, 10830, 8642, 68, 15222, 14998, 16190, 12636] dup_rows_per_dst=[2770, 2, 10, 8, 14, 1506, 1378, 398] combine_rows_per_src=[13632, 10830, 8642, 68, 15222, 14998, 16190, 12636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92266 payload_rows_remote=77296 dup_rows=6038 dup_rows_remote=5930 dedup_ratio=0.061422 router_tokens_per_expert=[3, 0, 0, 0, 0, 92, 7, 0, 0, 0, 113, 0, 0, 55, 1047, 0, 3716, 0, 199, 7485, 7544, 0, 0, 0, 0, 125, 493, 0, 7652, 0, 55, 0, 3283, 0, 0, 0, 247, 6568, 7064, 100, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1619, 0, 596, 0, 0, 0, 0, 0, 0, 0, 164, 177, 0, 336, 358] payload_rows_per_dst=[14332, 10256, 6808, 8658, 6766, 14970, 16074, 14402] dup_rows_per_dst=[0, 852, 0, 120, 494, 108, 3660, 804] combine_rows_per_src=[14332, 10256, 6808, 8658, 6766, 14970, 16074, 14402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=292028416 dispatch_meta_bytes=322224 combine_payload_bytes=292028416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=292350640 ingress_bytes=395837440 link_bytes=395837440 token_bytes=584379056 weight_movement_bytes=103809024 total_nvlink_bytes=688188080 weight_to_token_ratio=0.177640 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.489504419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=291430400 dispatch_meta_bytes=338208 combine_payload_bytes=291430400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=291768608 ingress_bytes=360636416 link_bytes=360636416 token_bytes=583199008 weight_movement_bytes=69206016 total_nvlink_bytes=652405024 weight_to_token_ratio=0.118666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.357045800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93070 payload_rows_remote=77454 dup_rows=5234 dup_rows_remote=5234 dedup_ratio=0.053243 router_tokens_per_expert=[0, 0, 674, 0, 0, 0, 1283, 0, 0, 0, 148, 174, 7236, 0, 0, 5649, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 8111, 0, 0, 7085, 0, 0, 0, 128, 0, 0, 7680, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 80, 0, 5226, 0, 0, 505, 674, 0, 0] payload_rows_per_dst=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] dup_rows_per_dst=[3014, 0, 0, 0, 0, 770, 1290, 160] combine_rows_per_src=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=297967616 dispatch_meta_bytes=320792 combine_payload_bytes=297967616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298288408 ingress_bytes=332570624 link_bytes=332570624 token_bytes=596256024 weight_movement_bytes=34603008 total_nvlink_bytes=630859032 weight_to_token_ratio=0.058034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251436484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78992 payload_rows_remote=63310 dup_rows=19312 dup_rows_remote=10490 dedup_ratio=0.196452 router_tokens_per_expert=[0, 0, 203, 0, 0, 1, 1520, 0, 0, 0, 139, 89, 7429, 0, 10, 5192, 0, 0, 1, 0, 1, 5592, 1, 6, 1, 0, 0, 8076, 0, 12, 5425, 0, 0, 0, 137, 0, 0, 6077, 7, 3, 1, 0, 1111, 0, 2, 12, 9, 1, 0, 2, 267, 1, 0, 600, 6, 74, 0, 6077, 0, 0, 859, 201, 7, 0] payload_rows_per_dst=[3266, 476, 15076, 10874, 2226, 15178, 16214, 15682] dup_rows_per_dst=[182, 0, 6510, 2, 6, 1952, 1838, 8822] combine_rows_per_src=[3266, 476, 15076, 10874, 2226, 15178, 16214, 15682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=377446400 dispatch_meta_bytes=392912 combine_payload_bytes=377446400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377839312 ingress_bytes=481255424 link_bytes=481255424 token_bytes=755285712 weight_movement_bytes=103809024 total_nvlink_bytes=859094736 weight_to_token_ratio=0.137443 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.810925416e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=316604416 dispatch_meta_bytes=332904 combine_payload_bytes=316604416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316937320 ingress_bytes=351207424 link_bytes=351207424 token_bytes=633541736 weight_movement_bytes=34603008 total_nvlink_bytes=668144744 weight_to_token_ratio=0.054618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321565262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317251584 dispatch_meta_bytes=330752 combine_payload_bytes=317251584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=317582336 ingress_bytes=386457600 link_bytes=386457600 token_bytes=634833920 weight_movement_bytes=69206016 total_nvlink_bytes=704039936 weight_to_token_ratio=0.109014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454208836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93070 payload_rows_remote=77454 dup_rows=5234 dup_rows_remote=5234 dedup_ratio=0.053243 router_tokens_per_expert=[0, 0, 674, 0, 0, 0, 1283, 0, 0, 0, 148, 174, 7236, 0, 0, 5649, 0, 0, 0, 0, 0, 3331, 0, 0, 0, 0, 0, 8111, 0, 0, 7085, 0, 0, 0, 128, 0, 0, 7680, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 566, 0, 80, 0, 5226, 0, 0, 505, 674, 0, 0] payload_rows_per_dst=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] dup_rows_per_dst=[3014, 0, 0, 0, 0, 770, 1290, 160] combine_rows_per_src=[15070, 11942, 6662, 0, 15616, 14906, 16224, 12650] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92218 payload_rows_remote=92150 dup_rows=6086 dup_rows_remote=6078 dedup_ratio=0.061910 router_tokens_per_expert=[1, 6, 622, 2, 0, 1, 1319, 0, 0, 0, 137, 192, 7214, 2, 30, 5055, 3, 0, 3, 2, 2, 4289, 7, 20, 4, 0, 1, 7996, 1, 31, 6250, 1, 1, 1, 136, 0, 1, 7466, 7, 6, 2, 0, 974, 0, 1, 34, 27, 0, 0, 21, 82, 1, 0, 587, 21, 76, 4, 5191, 0, 3, 755, 546, 14, 4] payload_rows_per_dst=[13632, 10830, 8642, 68, 15222, 14998, 16190, 12636] dup_rows_per_dst=[2770, 2, 10, 8, 14, 1506, 1378, 398] combine_rows_per_src=[13632, 10830, 8642, 68, 15222, 14998, 16190, 12636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85564 payload_rows_remote=78200 dup_rows=12740 dup_rows_remote=12740 dedup_ratio=0.129598 router_tokens_per_expert=[0, 0, 746, 0, 0, 0, 1228, 0, 0, 0, 142, 182, 7239, 0, 0, 5062, 0, 0, 0, 0, 0, 3682, 0, 0, 0, 0, 0, 8100, 0, 0, 7018, 0, 0, 0, 129, 0, 0, 7612, 0, 0, 0, 0, 800, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 602, 0, 90, 0, 5365, 0, 0, 495, 660, 0, 0] payload_rows_per_dst=[14936, 13610, 7364, 15074, 15482, 6882, 2236, 9980] dup_rows_per_dst=[3048, 5766, 0, 3022, 0, 592, 280, 32] combine_rows_per_src=[14936, 13610, 7364, 15074, 15482, 6882, 2236, 9980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92310 payload_rows_remote=77158 dup_rows=5994 dup_rows_remote=4266 dedup_ratio=0.060974 router_tokens_per_expert=[0, 0, 172, 0, 0, 0, 1551, 0, 0, 0, 154, 146, 7348, 0, 0, 4813, 0, 0, 0, 0, 0, 5241, 0, 0, 0, 0, 0, 8091, 0, 0, 5964, 0, 0, 0, 154, 0, 0, 6604, 0, 0, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 229, 0, 0, 633, 0, 100, 0, 5905, 0, 0, 842, 113, 0, 0] payload_rows_per_dst=[7464, 8160, 11338, 6800, 13516, 15152, 16186, 13694] dup_rows_per_dst=[1110, 0, 1210, 0, 0, 1728, 1920, 26] combine_rows_per_src=[7464, 8160, 11338, 6800, 13516, 15152, 16186, 13694] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=259317760 dispatch_meta_bytes=295200 combine_payload_bytes=259317760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=259612960 ingress_bytes=328523776 link_bytes=328523776 token_bytes=518930720 weight_movement_bytes=69206016 total_nvlink_bytes=588136736 weight_to_token_ratio=0.133363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236208521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=60536 dup_rows=23814 dup_rows_remote=16310 dedup_ratio=0.242249 router_tokens_per_expert=[32, 196, 1750, 206, 2, 18, 1533, 0, 0, 55, 141, 66, 5588, 1, 908, 3970, 201, 16, 13, 132, 159, 3441, 165, 767, 52, 106, 57, 5933, 71, 1281, 3881, 120, 223, 27, 138, 5, 45, 3678, 443, 118, 143, 0, 561, 1, 481, 1405, 539, 55, 1, 286, 216, 202, 159, 301, 915, 121, 86, 3874, 0, 257, 857, 1849, 1130, 175] payload_rows_per_dst=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] dup_rows_per_dst=[4246, 7504, 146, 7480, 604, 1776, 504, 1554] combine_rows_per_src=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=61106 dup_rows=24332 dup_rows_remote=21988 dedup_ratio=0.247518 router_tokens_per_expert=[4, 6, 201, 0, 0, 0, 1357, 0, 0, 0, 119, 83, 7415, 1, 22, 4898, 8, 0, 1, 0, 3, 5307, 2, 14, 1, 1, 0, 8067, 1, 23, 6037, 1, 2, 0, 119, 0, 0, 6816, 2, 3, 1, 0, 850, 0, 3, 24, 13, 0, 0, 8, 290, 2, 1, 633, 14, 72, 2, 5806, 0, 6, 790, 111, 10, 2] payload_rows_per_dst=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] dup_rows_per_dst=[2344, 9726, 2, 11586, 0, 24, 636, 14] combine_rows_per_src=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=317251584 dispatch_meta_bytes=330752 combine_payload_bytes=317251584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=317582336 ingress_bytes=386457600 link_bytes=386457600 token_bytes=634833920 weight_movement_bytes=69206016 total_nvlink_bytes=704039936 weight_to_token_ratio=0.109014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454208836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77118 payload_rows_remote=66486 dup_rows=21186 dup_rows_remote=19208 dedup_ratio=0.215515 router_tokens_per_expert=[4474, 468, 0, 0, 4, 2, 1327, 11, 2, 2436, 10, 4282, 3, 0, 748, 0, 5, 145, 774, 7, 1, 359, 1, 1, 4, 4, 4274, 639, 1527, 5, 113, 1120, 21, 10, 3, 0, 18, 3, 0, 5194, 1, 13, 542, 112, 4433, 142, 7379, 11, 7019, 48, 3, 159, 0, 72, 0, 27, 64, 0, 0, 19, 275, 72, 2, 764] payload_rows_per_dst=[10632, 12022, 15830, 16336, 10452, 1584, 474, 9788] dup_rows_per_dst=[1978, 2940, 1514, 13036, 46, 58, 144, 1470] combine_rows_per_src=[10632, 12022, 15830, 16336, 10452, 1584, 474, 9788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=60536 dup_rows=23814 dup_rows_remote=16310 dedup_ratio=0.242249 router_tokens_per_expert=[32, 196, 1750, 206, 2, 18, 1533, 0, 0, 55, 141, 66, 5588, 1, 908, 3970, 201, 16, 13, 132, 159, 3441, 165, 767, 52, 106, 57, 5933, 71, 1281, 3881, 120, 223, 27, 138, 5, 45, 3678, 443, 118, 143, 0, 561, 1, 481, 1405, 539, 55, 1, 286, 216, 202, 159, 301, 915, 121, 86, 3874, 0, 257, 857, 1849, 1130, 175] payload_rows_per_dst=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] dup_rows_per_dst=[4246, 7504, 146, 7480, 604, 1776, 504, 1554] combine_rows_per_src=[10990, 13954, 9642, 15508, 8750, 4594, 3898, 7154] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90606 payload_rows_remote=74512 dup_rows=7698 dup_rows_remote=3596 dedup_ratio=0.078308 router_tokens_per_expert=[0, 0, 0, 0, 0, 47, 143, 0, 0, 6, 88, 0, 0, 70, 918, 0, 3351, 0, 124, 7629, 7641, 0, 0, 0, 0, 128, 511, 0, 7677, 0, 70, 0, 3437, 0, 0, 0, 251, 6265, 7038, 75, 0, 0, 0, 0, 0, 0, 0, 78, 0, 1881, 0, 540, 0, 0, 0, 0, 0, 0, 0, 166, 404, 0, 339, 275] payload_rows_per_dst=[9546, 2160, 15530, 8088, 9778, 15276, 16094, 14134] dup_rows_per_dst=[52, 4, 0, 32, 2606, 138, 4102, 764] combine_rows_per_src=[9546, 2160, 15530, 8088, 9778, 15276, 16094, 14134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=377446400 dispatch_meta_bytes=392912 combine_payload_bytes=377446400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377839312 ingress_bytes=481255424 link_bytes=481255424 token_bytes=755285712 weight_movement_bytes=103809024 total_nvlink_bytes=859094736 weight_to_token_ratio=0.137443 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.810925416e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=320307200 dispatch_meta_bytes=363760 combine_payload_bytes=320307200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=320670960 ingress_bytes=354910208 link_bytes=354910208 token_bytes=640978160 weight_movement_bytes=34603008 total_nvlink_bytes=675581168 weight_to_token_ratio=0.053985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335498540e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=247955456 dispatch_meta_bytes=307384 combine_payload_bytes=247955456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=248262840 ingress_bytes=351764480 link_bytes=351764480 token_bytes=496218296 weight_movement_bytes=103809024 total_nvlink_bytes=600027320 weight_to_token_ratio=0.209200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323661419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250290176 dispatch_meta_bytes=332376 combine_payload_bytes=250290176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250622552 ingress_bytes=284893184 link_bytes=284893184 token_bytes=500912728 weight_movement_bytes=34603008 total_nvlink_bytes=535515736 weight_to_token_ratio=0.069080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072030116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93876 payload_rows_remote=78692 dup_rows=4428 dup_rows_remote=3618 dedup_ratio=0.045044 router_tokens_per_expert=[21, 0, 30, 0, 31, 1, 58, 1, 0, 0, 152, 6, 0, 63, 1026, 0, 2850, 11, 158, 7557, 7554, 0, 21, 6, 0, 123, 644, 0, 7626, 6, 74, 0, 4795, 14, 25, 2, 239, 5597, 6944, 76, 0, 15, 0, 27, 3, 0, 0, 62, 0, 1313, 1, 565, 0, 27, 0, 0, 6, 0, 31, 214, 449, 30, 363, 335] payload_rows_per_dst=[14134, 16120, 6084, 15184, 9792, 15190, 3992, 13380] dup_rows_per_dst=[38, 1482, 8, 810, 510, 138, 772, 670] combine_rows_per_src=[14134, 16120, 6084, 15184, 9792, 15190, 3992, 13380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=316039168 dispatch_meta_bytes=325696 combine_payload_bytes=316039168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316364864 ingress_bytes=350642176 link_bytes=350642176 token_bytes=632404032 weight_movement_bytes=34603008 total_nvlink_bytes=667007040 weight_to_token_ratio=0.054717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319438279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90890 payload_rows_remote=76048 dup_rows=7414 dup_rows_remote=5436 dedup_ratio=0.075419 router_tokens_per_expert=[0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 34, 100, 0, 100, 787, 0, 0, 0, 2281, 2423, 7558, 868, 0, 59, 7584, 0, 0, 7248, 180, 55, 0, 0, 0, 0, 260, 0, 28, 0, 0, 6710, 0, 1063, 0, 0, 0, 0, 99, 0, 0, 0, 0, 363, 4898, 0, 335, 167, 1658, 99, 0, 0, 0, 3649, 2, 363] payload_rows_per_dst=[15168, 13376, 7216, 5166, 13752, 14842, 10652, 10718] dup_rows_per_dst=[362, 1250, 1732, 150, 244, 1978, 874, 824] combine_rows_per_src=[15168, 13376, 7216, 5166, 13752, 14842, 10652, 10718] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272326656 dispatch_meta_bytes=342776 combine_payload_bytes=272326656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272669432 ingress_bytes=341532672 link_bytes=341532672 token_bytes=544996088 weight_movement_bytes=69206016 total_nvlink_bytes=614202104 weight_to_token_ratio=0.126984 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285159949e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=247955456 dispatch_meta_bytes=307384 combine_payload_bytes=247955456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=248262840 ingress_bytes=351764480 link_bytes=351764480 token_bytes=496218296 weight_movement_bytes=103809024 total_nvlink_bytes=600027320 weight_to_token_ratio=0.209200 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323661419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=305201152 dispatch_meta_bytes=312432 combine_payload_bytes=305201152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305513584 ingress_bytes=339804160 link_bytes=339804160 token_bytes=610714736 weight_movement_bytes=34603008 total_nvlink_bytes=645317744 weight_to_token_ratio=0.056660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278655698e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92640 payload_rows_remote=80974 dup_rows=5664 dup_rows_remote=5168 dedup_ratio=0.057617 router_tokens_per_expert=[34, 0, 0, 0, 0, 0, 46, 0, 0, 0, 167, 0, 0, 68, 1012, 0, 3039, 0, 160, 7624, 7646, 0, 0, 0, 0, 38, 681, 0, 7635, 0, 68, 0, 5753, 0, 8, 0, 240, 4443, 6950, 80, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1325, 0, 600, 0, 0, 0, 0, 0, 0, 0, 236, 500, 0, 360, 380] payload_rows_per_dst=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] dup_rows_per_dst=[0, 1522, 0, 42, 496, 118, 3032, 454] combine_rows_per_src=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88102 payload_rows_remote=73842 dup_rows=10202 dup_rows_remote=9168 dedup_ratio=0.103780 router_tokens_per_expert=[4, 0, 9, 0, 11, 71, 276, 0, 0, 32, 102, 3, 0, 64, 786, 0, 3669, 4, 93, 7664, 7661, 0, 9, 7, 0, 185, 522, 0, 7682, 3, 71, 0, 3012, 3, 6, 1, 240, 6725, 7032, 74, 0, 3, 0, 7, 1, 0, 0, 99, 0, 1623, 1, 464, 0, 11, 0, 0, 2, 0, 12, 173, 269, 11, 325, 130] payload_rows_per_dst=[620, 1930, 15544, 8730, 15530, 15402, 16086, 14260] dup_rows_per_dst=[122, 44, 4, 170, 5206, 146, 3476, 1034] combine_rows_per_src=[620, 1930, 15544, 8730, 15530, 15402, 16086, 14260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322322432 dispatch_meta_bytes=329240 combine_payload_bytes=322322432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322651672 ingress_bytes=391528448 link_bytes=391528448 token_bytes=644974104 weight_movement_bytes=69206016 total_nvlink_bytes=714180120 weight_to_token_ratio=0.107300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473290029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=311492608 dispatch_meta_bytes=325936 combine_payload_bytes=311492608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311818544 ingress_bytes=346095616 link_bytes=346095616 token_bytes=623311152 weight_movement_bytes=34603008 total_nvlink_bytes=657914160 weight_to_token_ratio=0.055515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302329940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92640 payload_rows_remote=80974 dup_rows=5664 dup_rows_remote=5168 dedup_ratio=0.057617 router_tokens_per_expert=[34, 0, 0, 0, 0, 0, 46, 0, 0, 0, 167, 0, 0, 68, 1012, 0, 3039, 0, 160, 7624, 7646, 0, 0, 0, 0, 38, 681, 0, 7635, 0, 68, 0, 5753, 0, 8, 0, 240, 4443, 6950, 80, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1325, 0, 600, 0, 0, 0, 0, 0, 0, 0, 236, 500, 0, 360, 380] payload_rows_per_dst=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] dup_rows_per_dst=[0, 1522, 0, 42, 496, 118, 3032, 454] combine_rows_per_src=[14060, 16264, 2242, 5688, 11666, 15248, 16088, 11384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=344568 combine_payload_bytes=331669504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332014072 ingress_bytes=435478528 link_bytes=435478528 token_bytes=663683576 weight_movement_bytes=103809024 total_nvlink_bytes=767492600 weight_to_token_ratio=0.156413 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.638670642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=94264 payload_rows_remote=88506 dup_rows=4040 dup_rows_remote=4040 dedup_ratio=0.041097 router_tokens_per_expert=[15, 0, 0, 0, 0, 0, 73, 0, 0, 0, 191, 0, 0, 53, 1052, 0, 2700, 0, 179, 7523, 7544, 0, 0, 0, 0, 48, 721, 0, 7674, 0, 53, 0, 5705, 0, 14, 0, 248, 4928, 7072, 90, 0, 0, 0, 0, 0, 0, 0, 72, 0, 1257, 0, 611, 0, 0, 0, 0, 0, 0, 0, 196, 468, 0, 322, 343] payload_rows_per_dst=[14320, 16238, 5758, 16154, 11590, 15046, 3050, 12108] dup_rows_per_dst=[0, 1442, 0, 838, 524, 144, 686, 406] combine_rows_per_src=[14320, 16238, 5758, 16154, 11590, 15046, 3050, 12108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93876 payload_rows_remote=78692 dup_rows=4428 dup_rows_remote=3618 dedup_ratio=0.045044 router_tokens_per_expert=[21, 0, 30, 0, 31, 1, 58, 1, 0, 0, 152, 6, 0, 63, 1026, 0, 2850, 11, 158, 7557, 7554, 0, 21, 6, 0, 123, 644, 0, 7626, 6, 74, 0, 4795, 14, 25, 2, 239, 5597, 6944, 76, 0, 15, 0, 27, 3, 0, 0, 62, 0, 1313, 1, 565, 0, 27, 0, 0, 6, 0, 31, 214, 449, 30, 363, 335] payload_rows_per_dst=[14134, 16120, 6084, 15184, 9792, 15190, 3992, 13380] dup_rows_per_dst=[38, 1482, 8, 810, 510, 138, 772, 670] combine_rows_per_src=[14134, 16120, 6084, 15184, 9792, 15190, 3992, 13380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92266 payload_rows_remote=77296 dup_rows=6038 dup_rows_remote=5930 dedup_ratio=0.061422 router_tokens_per_expert=[3, 0, 0, 0, 0, 92, 7, 0, 0, 0, 113, 0, 0, 55, 1047, 0, 3716, 0, 199, 7485, 7544, 0, 0, 0, 0, 125, 493, 0, 7652, 0, 55, 0, 3283, 0, 0, 0, 247, 6568, 7064, 100, 0, 0, 0, 0, 0, 0, 0, 54, 0, 1619, 0, 596, 0, 0, 0, 0, 0, 0, 0, 164, 177, 0, 336, 358] payload_rows_per_dst=[14332, 10256, 6808, 8658, 6766, 14970, 16074, 14402] dup_rows_per_dst=[0, 852, 0, 120, 494, 108, 3660, 804] combine_rows_per_src=[14332, 10256, 6808, 8658, 6766, 14970, 16074, 14402] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=302456832 dispatch_meta_bytes=332040 combine_payload_bytes=302456832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302788872 ingress_bytes=337059840 link_bytes=337059840 token_bytes=605245704 weight_movement_bytes=34603008 total_nvlink_bytes=639848712 weight_to_token_ratio=0.057172 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268329042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81606 payload_rows_remote=70442 dup_rows=16698 dup_rows_remote=15174 dedup_ratio=0.169861 router_tokens_per_expert=[19, 5, 1266, 1, 1413, 93, 502, 363, 39, 42, 544, 68, 16, 198, 356, 106, 3235, 523, 223, 5132, 4975, 11, 584, 188, 0, 1263, 749, 0, 5476, 536, 471, 0, 1824, 175, 422, 148, 219, 3120, 4687, 83, 3, 288, 2, 1052, 29, 66, 0, 138, 91, 547, 403, 1391, 0, 1090, 0, 4, 472, 72, 634, 259, 421, 1286, 1488, 341] payload_rows_per_dst=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] dup_rows_per_dst=[2736, 1524, 4296, 3290, 1324, 230, 1002, 2296] combine_rows_per_src=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=67694 dup_rows=16266 dup_rows_remote=16228 dedup_ratio=0.165466 router_tokens_per_expert=[1, 0, 19, 0, 23, 68, 44, 2, 0, 1, 105, 1, 0, 51, 887, 0, 3208, 11, 149, 7595, 7606, 0, 12, 2, 0, 132, 545, 0, 7667, 6, 50, 0, 3630, 7, 7, 4, 256, 6599, 7034, 72, 0, 2, 0, 17, 4, 1, 0, 75, 0, 1529, 3, 518, 0, 20, 0, 0, 15, 0, 15, 198, 290, 23, 297, 351] payload_rows_per_dst=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] dup_rows_per_dst=[38, 1276, 12898, 614, 656, 0, 730, 54] combine_rows_per_src=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=331669504 dispatch_meta_bytes=344568 combine_payload_bytes=331669504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332014072 ingress_bytes=435478528 link_bytes=435478528 token_bytes=663683576 weight_movement_bytes=103809024 total_nvlink_bytes=767492600 weight_to_token_ratio=0.156413 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.638670642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=362520576 dispatch_meta_bytes=370184 combine_payload_bytes=362520576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362890760 ingress_bytes=500932608 link_bytes=500932608 token_bytes=725411336 weight_movement_bytes=138412032 total_nvlink_bytes=863823368 weight_to_token_ratio=0.190805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.884969076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=61106 dup_rows=24332 dup_rows_remote=21988 dedup_ratio=0.247518 router_tokens_per_expert=[4, 6, 201, 0, 0, 0, 1357, 0, 0, 0, 119, 83, 7415, 1, 22, 4898, 8, 0, 1, 0, 3, 5307, 2, 14, 1, 1, 0, 8067, 1, 23, 6037, 1, 2, 0, 119, 0, 0, 6816, 2, 3, 1, 0, 850, 0, 3, 24, 13, 0, 0, 8, 290, 2, 1, 633, 14, 72, 2, 5806, 0, 6, 790, 111, 10, 2] payload_rows_per_dst=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] dup_rows_per_dst=[2344, 9726, 2, 11586, 0, 24, 636, 14] combine_rows_per_src=[12866, 15772, 10668, 16214, 13462, 1758, 1404, 1828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81606 payload_rows_remote=70442 dup_rows=16698 dup_rows_remote=15174 dedup_ratio=0.169861 router_tokens_per_expert=[19, 5, 1266, 1, 1413, 93, 502, 363, 39, 42, 544, 68, 16, 198, 356, 106, 3235, 523, 223, 5132, 4975, 11, 584, 188, 0, 1263, 749, 0, 5476, 536, 471, 0, 1824, 175, 422, 148, 219, 3120, 4687, 83, 3, 288, 2, 1052, 29, 66, 0, 138, 91, 547, 403, 1391, 0, 1090, 0, 4, 472, 72, 634, 259, 421, 1286, 1488, 341] payload_rows_per_dst=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] dup_rows_per_dst=[2736, 1524, 4296, 3290, 1324, 230, 1002, 2296] combine_rows_per_src=[13962, 11164, 12816, 13700, 8132, 5952, 6050, 9830] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84532 payload_rows_remote=74044 dup_rows=13772 dup_rows_remote=12852 dedup_ratio=0.140096 router_tokens_per_expert=[0, 0, 0, 194, 0, 0, 0, 0, 0, 0, 3, 75, 0, 75, 742, 0, 0, 0, 1859, 2579, 7600, 953, 0, 72, 7918, 0, 0, 7342, 92, 70, 0, 0, 0, 0, 154, 0, 58, 0, 0, 6361, 0, 972, 0, 0, 0, 0, 77, 0, 0, 0, 0, 361, 4893, 0, 365, 85, 1446, 74, 0, 0, 0, 4350, 1, 381] payload_rows_per_dst=[2554, 1634, 15774, 14864, 12972, 14968, 10488, 11278] dup_rows_per_dst=[50, 156, 5194, 4238, 174, 1814, 920, 1226] combine_rows_per_src=[2554, 1634, 15774, 14864, 12972, 14968, 10488, 11278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93130 payload_rows_remote=78582 dup_rows=5174 dup_rows_remote=4232 dedup_ratio=0.052633 router_tokens_per_expert=[0, 1813, 657, 1163, 7626, 6984, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2160, 0, 0, 0, 0, 997, 5081, 99, 0, 0, 0, 0, 0, 437, 0, 0, 6356, 0, 122, 0, 0, 761, 0, 0, 0, 0, 0, 0, 1205, 1, 0, 0, 0, 67, 7376, 203, 5738, 0, 0, 98, 0, 0, 0] payload_rows_per_dst=[6850, 15252, 4320, 11886, 13516, 14548, 15084, 11674] dup_rows_per_dst=[416, 416, 0, 468, 314, 942, 2214, 404] combine_rows_per_src=[6850, 15252, 4320, 11886, 13516, 14548, 15084, 11674] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91564 payload_rows_remote=86344 dup_rows=6740 dup_rows_remote=6608 dedup_ratio=0.068563 router_tokens_per_expert=[0, 0, 25, 202, 7, 21, 0, 0, 1, 0, 15, 75, 7, 75, 803, 31, 0, 0, 1724, 2469, 7421, 714, 7, 46, 7583, 18, 0, 7144, 116, 72, 1, 0, 0, 27, 233, 25, 0, 1, 0, 6872, 17, 1257, 1, 33, 1, 0, 75, 0, 9, 1, 5, 286, 5609, 10, 596, 182, 1909, 80, 13, 0, 30, 2893, 26, 384] payload_rows_per_dst=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] dup_rows_per_dst=[378, 1308, 500, 132, 186, 2226, 1252, 758] combine_rows_per_src=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288530432 dispatch_meta_bytes=342464 combine_payload_bytes=288530432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288872896 ingress_bytes=323133440 link_bytes=323133440 token_bytes=577403328 weight_movement_bytes=34603008 total_nvlink_bytes=612006336 weight_to_token_ratio=0.059929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215925120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=277274624 dispatch_meta_bytes=335688 combine_payload_bytes=277274624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277610312 ingress_bytes=311877632 link_bytes=311877632 token_bytes=554884936 weight_movement_bytes=34603008 total_nvlink_bytes=589487944 weight_to_token_ratio=0.062361 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173570421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=362520576 dispatch_meta_bytes=370184 combine_payload_bytes=362520576 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=362890760 ingress_bytes=500932608 link_bytes=500932608 token_bytes=725411336 weight_movement_bytes=138412032 total_nvlink_bytes=863823368 weight_to_token_ratio=0.190805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.884969076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322322432 dispatch_meta_bytes=329240 combine_payload_bytes=322322432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=322651672 ingress_bytes=391528448 link_bytes=391528448 token_bytes=644974104 weight_movement_bytes=69206016 total_nvlink_bytes=714180120 weight_to_token_ratio=0.107300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.473290029e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=316604416 dispatch_meta_bytes=332904 combine_payload_bytes=316604416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316937320 ingress_bytes=351207424 link_bytes=351207424 token_bytes=633541736 weight_movement_bytes=34603008 total_nvlink_bytes=668144744 weight_to_token_ratio=0.054618 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321565262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91702 payload_rows_remote=77736 dup_rows=6602 dup_rows_remote=6442 dedup_ratio=0.067159 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 840, 0, 0, 0, 1340, 2343, 7405, 712, 0, 0, 7587, 0, 0, 7228, 81, 68, 0, 0, 0, 0, 161, 0, 0, 0, 0, 6902, 0, 1602, 0, 0, 0, 0, 80, 0, 0, 0, 0, 215, 5661, 0, 542, 148, 2642, 80, 0, 0, 0, 2659, 4, 483] payload_rows_per_dst=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] dup_rows_per_dst=[418, 1348, 46, 136, 160, 2374, 1128, 992] combine_rows_per_src=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=250290176 dispatch_meta_bytes=332376 combine_payload_bytes=250290176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250622552 ingress_bytes=284893184 link_bytes=284893184 token_bytes=500912728 weight_movement_bytes=34603008 total_nvlink_bytes=535515736 weight_to_token_ratio=0.069080 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.072030116e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=288530432 dispatch_meta_bytes=342464 combine_payload_bytes=288530432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=288872896 ingress_bytes=323133440 link_bytes=323133440 token_bytes=577403328 weight_movement_bytes=34603008 total_nvlink_bytes=612006336 weight_to_token_ratio=0.059929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.215925120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=303284224 dispatch_meta_bytes=347584 combine_payload_bytes=303284224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=303631808 ingress_bytes=372490240 link_bytes=372490240 token_bytes=606916032 weight_movement_bytes=69206016 total_nvlink_bytes=676122048 weight_to_token_ratio=0.114029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=38656 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401650785e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81712 payload_rows_remote=68878 dup_rows=16592 dup_rows_remote=13214 dedup_ratio=0.168783 router_tokens_per_expert=[0, 0, 9, 214, 0, 9, 0, 0, 0, 0, 4, 74, 3, 74, 673, 9, 0, 1, 2357, 2192, 7740, 865, 1, 70, 7731, 7, 0, 7305, 89, 64, 0, 0, 0, 9, 126, 12, 55, 0, 0, 6445, 4, 997, 1, 11, 0, 0, 73, 0, 4, 1, 1, 412, 4674, 2, 354, 62, 1568, 75, 2, 0, 13, 4344, 27, 389] payload_rows_per_dst=[446, 1514, 16046, 15902, 13106, 14956, 6908, 12834] dup_rows_per_dst=[18, 160, 6022, 4264, 188, 1826, 736, 3378] combine_rows_per_src=[446, 1514, 16046, 15902, 13106, 14956, 6908, 12834] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321871872 dispatch_meta_bytes=331256 combine_payload_bytes=321871872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322203128 ingress_bytes=356474880 link_bytes=356474880 token_bytes=644075000 weight_movement_bytes=34603008 total_nvlink_bytes=678678008 weight_to_token_ratio=0.053725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341386275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=353665024 dispatch_meta_bytes=371808 combine_payload_bytes=353665024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354036832 ingress_bytes=457474048 link_bytes=457474048 token_bytes=707701856 weight_movement_bytes=103809024 total_nvlink_bytes=811510880 weight_to_token_ratio=0.146685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.721438013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91702 payload_rows_remote=77736 dup_rows=6602 dup_rows_remote=6442 dedup_ratio=0.067159 router_tokens_per_expert=[0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 80, 0, 80, 840, 0, 0, 0, 1340, 2343, 7405, 712, 0, 0, 7587, 0, 0, 7228, 81, 68, 0, 0, 0, 0, 161, 0, 0, 0, 0, 6902, 0, 1602, 0, 0, 0, 0, 80, 0, 0, 0, 0, 215, 5661, 0, 542, 148, 2642, 80, 0, 0, 0, 2659, 4, 483] payload_rows_per_dst=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] dup_rows_per_dst=[418, 1348, 46, 136, 160, 2374, 1128, 992] combine_rows_per_src=[15174, 15462, 4058, 4848, 13966, 15446, 12004, 10744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90164 payload_rows_remote=85866 dup_rows=8140 dup_rows_remote=7910 dedup_ratio=0.082804 router_tokens_per_expert=[0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5, 90, 0, 88, 764, 0, 0, 0, 1599, 2661, 7465, 657, 0, 8, 7688, 0, 0, 7223, 138, 53, 0, 0, 0, 0, 243, 0, 0, 0, 0, 6921, 0, 1352, 0, 0, 0, 0, 89, 0, 0, 0, 0, 239, 5476, 0, 524, 189, 2182, 89, 0, 0, 0, 2764, 2, 463] payload_rows_per_dst=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] dup_rows_per_dst=[356, 1188, 230, 4050, 178, 212, 1068, 858] combine_rows_per_src=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=318406656 dispatch_meta_bytes=336712 combine_payload_bytes=318406656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318743368 ingress_bytes=387612672 link_bytes=387612672 token_bytes=637150024 weight_movement_bytes=69206016 total_nvlink_bytes=706356040 weight_to_token_ratio=0.108618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.458555279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=282124288 dispatch_meta_bytes=328368 combine_payload_bytes=282124288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282452656 ingress_bytes=316727296 link_bytes=316727296 token_bytes=564576944 weight_movement_bytes=34603008 total_nvlink_bytes=599179952 weight_to_token_ratio=0.061290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191819316e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81442 payload_rows_remote=69796 dup_rows=16862 dup_rows_remote=15718 dedup_ratio=0.171529 router_tokens_per_expert=[12, 1, 887, 1464, 121, 350, 0, 0, 21, 27, 111, 69, 520, 1, 728, 607, 23, 303, 617, 1462, 4311, 666, 47, 91, 5440, 349, 11, 3659, 319, 72, 30, 56, 0, 1015, 636, 1469, 50, 0, 0, 4145, 828, 693, 152, 1289, 25, 0, 236, 0, 167, 329, 346, 867, 5542, 235, 2002, 293, 1892, 276, 247, 7, 1411, 1674, 187, 764] payload_rows_per_dst=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] dup_rows_per_dst=[2524, 1144, 1284, 550, 2028, 1850, 6012, 1470] combine_rows_per_src=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90164 payload_rows_remote=85866 dup_rows=8140 dup_rows_remote=7910 dedup_ratio=0.082804 router_tokens_per_expert=[0, 0, 0, 180, 0, 0, 0, 0, 0, 0, 5, 90, 0, 88, 764, 0, 0, 0, 1599, 2661, 7465, 657, 0, 8, 7688, 0, 0, 7223, 138, 53, 0, 0, 0, 0, 243, 0, 0, 0, 0, 6921, 0, 1352, 0, 0, 0, 0, 89, 0, 0, 0, 0, 239, 5476, 0, 524, 189, 2182, 89, 0, 0, 0, 2764, 2, 463] payload_rows_per_dst=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] dup_rows_per_dst=[356, 1188, 230, 4050, 178, 212, 1068, 858] combine_rows_per_src=[15380, 15636, 4298, 14634, 14150, 4136, 11788, 10142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90890 payload_rows_remote=76048 dup_rows=7414 dup_rows_remote=5436 dedup_ratio=0.075419 router_tokens_per_expert=[0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 34, 100, 0, 100, 787, 0, 0, 0, 2281, 2423, 7558, 868, 0, 59, 7584, 0, 0, 7248, 180, 55, 0, 0, 0, 0, 260, 0, 28, 0, 0, 6710, 0, 1063, 0, 0, 0, 0, 99, 0, 0, 0, 0, 363, 4898, 0, 335, 167, 1658, 99, 0, 0, 0, 3649, 2, 363] payload_rows_per_dst=[15168, 13376, 7216, 5166, 13752, 14842, 10652, 10718] dup_rows_per_dst=[362, 1250, 1732, 150, 244, 1978, 874, 824] combine_rows_per_src=[15168, 13376, 7216, 5166, 13752, 14842, 10652, 10718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91564 payload_rows_remote=86344 dup_rows=6740 dup_rows_remote=6608 dedup_ratio=0.068563 router_tokens_per_expert=[0, 0, 25, 202, 7, 21, 0, 0, 1, 0, 15, 75, 7, 75, 803, 31, 0, 0, 1724, 2469, 7421, 714, 7, 46, 7583, 18, 0, 7144, 116, 72, 1, 0, 0, 27, 233, 25, 0, 1, 0, 6872, 17, 1257, 1, 33, 1, 0, 75, 0, 9, 1, 5, 286, 5609, 10, 596, 182, 1909, 80, 13, 0, 30, 2893, 26, 384] payload_rows_per_dst=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] dup_rows_per_dst=[378, 1308, 500, 132, 186, 2226, 1252, 758] combine_rows_per_src=[15298, 15548, 4482, 5220, 14130, 14830, 12144, 9912] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86672 payload_rows_remote=71016 dup_rows=11632 dup_rows_remote=11136 dedup_ratio=0.118327 router_tokens_per_expert=[0, 0, 15, 240, 8, 14, 0, 0, 0, 0, 6, 72, 5, 70, 832, 7, 0, 0, 1775, 2498, 7475, 1008, 3, 70, 7799, 14, 0, 7298, 97, 49, 0, 0, 0, 19, 152, 26, 22, 0, 0, 6533, 14, 1010, 2, 22, 0, 0, 77, 0, 1, 5, 7, 426, 5236, 3, 303, 77, 1530, 76, 0, 0, 24, 3901, 4, 327] payload_rows_per_dst=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] dup_rows_per_dst=[496, 1330, 8286, 50, 72, 32, 728, 638] combine_rows_per_src=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=318406656 dispatch_meta_bytes=336712 combine_payload_bytes=318406656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318743368 ingress_bytes=387612672 link_bytes=387612672 token_bytes=637150024 weight_movement_bytes=69206016 total_nvlink_bytes=706356040 weight_to_token_ratio=0.108618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.458555279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=351707136 dispatch_meta_bytes=375104 combine_payload_bytes=351707136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=352082240 ingress_bytes=455516160 link_bytes=455516160 token_bytes=703789376 weight_movement_bytes=103809024 total_nvlink_bytes=807598400 weight_to_token_ratio=0.147500 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.714070638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87922 payload_rows_remote=71974 dup_rows=10382 dup_rows_remote=10106 dedup_ratio=0.105611 router_tokens_per_expert=[65, 0, 0, 0, 0, 0, 0, 2712, 543, 0, 202, 0, 0, 0, 2, 3807, 0, 0, 0, 6177, 188, 1294, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 0, 0, 0, 0, 0, 0, 7017, 0, 0, 0, 7947, 0, 0, 165, 0, 0, 0, 0, 2777, 0, 7924, 0, 0, 0, 0, 0, 3490, 166, 462, 0, 100] payload_rows_per_dst=[5490, 8100, 14202, 15848, 14638, 15948, 5554, 8142] dup_rows_per_dst=[64, 1008, 2842, 0, 5898, 276, 0, 294] combine_rows_per_src=[5490, 8100, 14202, 15848, 14638, 15948, 5554, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82038 payload_rows_remote=67694 dup_rows=16266 dup_rows_remote=16228 dedup_ratio=0.165466 router_tokens_per_expert=[1, 0, 19, 0, 23, 68, 44, 2, 0, 1, 105, 1, 0, 51, 887, 0, 3208, 11, 149, 7595, 7606, 0, 12, 2, 0, 132, 545, 0, 7667, 6, 50, 0, 3630, 7, 7, 4, 256, 6599, 7034, 72, 0, 2, 0, 17, 4, 1, 0, 75, 0, 1529, 3, 518, 0, 20, 0, 0, 15, 0, 15, 198, 290, 23, 297, 351] payload_rows_per_dst=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] dup_rows_per_dst=[38, 1276, 12898, 614, 656, 0, 730, 54] combine_rows_per_src=[14344, 16026, 16316, 16186, 13234, 198, 3410, 2324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81442 payload_rows_remote=69796 dup_rows=16862 dup_rows_remote=15718 dedup_ratio=0.171529 router_tokens_per_expert=[12, 1, 887, 1464, 121, 350, 0, 0, 21, 27, 111, 69, 520, 1, 728, 607, 23, 303, 617, 1462, 4311, 666, 47, 91, 5440, 349, 11, 3659, 319, 72, 30, 56, 0, 1015, 636, 1469, 50, 0, 0, 4145, 828, 693, 152, 1289, 25, 0, 236, 0, 167, 329, 346, 867, 5542, 235, 2002, 293, 1892, 276, 247, 7, 1411, 1674, 187, 764] payload_rows_per_dst=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] dup_rows_per_dst=[2524, 1144, 1284, 550, 2028, 1850, 6012, 1470] combine_rows_per_src=[14026, 11646, 7648, 9234, 9296, 4596, 13550, 11446] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91628 payload_rows_remote=76276 dup_rows=6676 dup_rows_remote=2968 dedup_ratio=0.067912 router_tokens_per_expert=[0, 2170, 567, 1068, 7467, 7215, 0, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1859, 0, 0, 0, 0, 1000, 5324, 74, 0, 0, 0, 0, 0, 486, 0, 0, 6474, 0, 154, 0, 0, 686, 0, 0, 0, 0, 0, 0, 1312, 0, 0, 0, 0, 84, 7453, 208, 5237, 0, 0, 74, 0, 0, 0] payload_rows_per_dst=[7310, 14934, 3718, 12364, 12492, 14836, 15352, 10622] dup_rows_per_dst=[300, 480, 0, 432, 374, 966, 3708, 416] combine_rows_per_src=[7310, 14934, 3718, 12364, 12492, 14836, 15352, 10622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91064 payload_rows_remote=77504 dup_rows=7240 dup_rows_remote=6618 dedup_ratio=0.073649 router_tokens_per_expert=[0, 1200, 509, 1097, 7796, 6384, 3, 0, 1, 1, 9, 118, 6, 2, 19, 10, 36, 1, 11, 0, 20, 2777, 2, 0, 0, 0, 987, 6002, 76, 0, 26, 0, 1, 1, 442, 0, 1, 5862, 8, 149, 1, 0, 673, 0, 21, 2, 22, 31, 3, 1133, 60, 0, 1, 36, 88, 7061, 152, 6188, 5, 28, 76, 3, 3, 8] payload_rows_per_dst=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] dup_rows_per_dst=[422, 256, 4306, 622, 496, 736, 74, 328] combine_rows_per_src=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285884416 dispatch_meta_bytes=342056 combine_payload_bytes=285884416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286226472 ingress_bytes=320487424 link_bytes=320487424 token_bytes=572110888 weight_movement_bytes=34603008 total_nvlink_bytes=606713896 weight_to_token_ratio=0.060483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205968375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290881536 dispatch_meta_bytes=328608 combine_payload_bytes=290881536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291210144 ingress_bytes=325484544 link_bytes=325484544 token_bytes=582091680 weight_movement_bytes=34603008 total_nvlink_bytes=616694688 weight_to_token_ratio=0.059446 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224772135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=351707136 dispatch_meta_bytes=375104 combine_payload_bytes=351707136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=352082240 ingress_bytes=455516160 link_bytes=455516160 token_bytes=703789376 weight_movement_bytes=103809024 total_nvlink_bytes=807598400 weight_to_token_ratio=0.147500 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.714070638e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=311492608 dispatch_meta_bytes=325936 combine_payload_bytes=311492608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=311818544 ingress_bytes=346095616 link_bytes=346095616 token_bytes=623311152 weight_movement_bytes=34603008 total_nvlink_bytes=657914160 weight_to_token_ratio=0.055515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=37120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.302329940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=353665024 dispatch_meta_bytes=371808 combine_payload_bytes=353665024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354036832 ingress_bytes=457474048 link_bytes=457474048 token_bytes=707701856 weight_movement_bytes=103809024 total_nvlink_bytes=811510880 weight_to_token_ratio=0.146685 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.721438013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=294805504 dispatch_meta_bytes=328320 combine_payload_bytes=294805504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295133824 ingress_bytes=364011520 link_bytes=364011520 token_bytes=589939328 weight_movement_bytes=69206016 total_nvlink_bytes=659145344 weight_to_token_ratio=0.117310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369746044e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312426496 dispatch_meta_bytes=316976 combine_payload_bytes=312426496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312743472 ingress_bytes=381632512 link_bytes=381632512 token_bytes=625169968 weight_movement_bytes=69206016 total_nvlink_bytes=694375984 weight_to_token_ratio=0.110700 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436052419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93196 payload_rows_remote=81158 dup_rows=5108 dup_rows_remote=4614 dedup_ratio=0.051961 router_tokens_per_expert=[0, 1031, 500, 1104, 7974, 6776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3517, 0, 0, 0, 0, 818, 6048, 80, 0, 0, 0, 0, 0, 623, 0, 0, 5479, 0, 164, 0, 0, 512, 0, 0, 0, 12, 0, 0, 1209, 12, 0, 0, 0, 97, 6985, 139, 6000, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] dup_rows_per_dst=[316, 0, 786, 584, 494, 608, 2064, 256] combine_rows_per_src=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=277274624 dispatch_meta_bytes=335688 combine_payload_bytes=277274624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277610312 ingress_bytes=311877632 link_bytes=311877632 token_bytes=554884936 weight_movement_bytes=34603008 total_nvlink_bytes=589487944 weight_to_token_ratio=0.062361 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173570421e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80148 payload_rows_remote=68520 dup_rows=18156 dup_rows_remote=17052 dedup_ratio=0.184692 router_tokens_per_expert=[0, 2612, 530, 1176, 7570, 7125, 1, 0, 0, 0, 4, 211, 5, 1, 8, 10, 11, 0, 7, 0, 6, 1464, 0, 0, 0, 0, 994, 4815, 73, 0, 6, 0, 0, 0, 393, 0, 0, 6483, 17, 149, 0, 0, 682, 0, 8, 0, 2, 12, 1, 1209, 21, 0, 0, 12, 85, 7606, 207, 5550, 0, 11, 73, 0, 0, 2] payload_rows_per_dst=[5548, 14160, 2968, 6768, 8302, 14622, 16152, 11628] dup_rows_per_dst=[348, 412, 8, 318, 248, 1036, 14682, 1104] combine_rows_per_src=[5548, 14160, 2968, 6768, 8302, 14622, 16152, 11628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317456384 dispatch_meta_bytes=336488 combine_payload_bytes=317456384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317792872 ingress_bytes=386662400 link_bytes=386662400 token_bytes=635249256 weight_movement_bytes=69206016 total_nvlink_bytes=704455272 weight_to_token_ratio=0.108943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454979482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=285884416 dispatch_meta_bytes=342056 combine_payload_bytes=285884416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286226472 ingress_bytes=320487424 link_bytes=320487424 token_bytes=572110888 weight_movement_bytes=34603008 total_nvlink_bytes=606713896 weight_to_token_ratio=0.060483 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.205968375e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=312426496 dispatch_meta_bytes=316976 combine_payload_bytes=312426496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=312743472 ingress_bytes=381632512 link_bytes=381632512 token_bytes=625169968 weight_movement_bytes=69206016 total_nvlink_bytes=694375984 weight_to_token_ratio=0.110700 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.436052419e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93196 payload_rows_remote=81158 dup_rows=5108 dup_rows_remote=4614 dedup_ratio=0.051961 router_tokens_per_expert=[0, 1031, 500, 1104, 7974, 6776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3517, 0, 0, 0, 0, 818, 6048, 80, 0, 0, 0, 0, 0, 623, 0, 0, 5479, 0, 164, 0, 0, 512, 0, 0, 0, 12, 0, 0, 1209, 12, 0, 0, 0, 97, 6985, 139, 6000, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] dup_rows_per_dst=[316, 0, 786, 584, 494, 608, 2064, 256] combine_rows_per_src=[4954, 15948, 7604, 13308, 12038, 13992, 13192, 12160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90532 payload_rows_remote=74888 dup_rows=7772 dup_rows_remote=2540 dedup_ratio=0.079061 router_tokens_per_expert=[0, 994, 531, 1165, 7902, 6745, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 846, 6065, 89, 0, 0, 0, 0, 0, 509, 0, 0, 5687, 16, 127, 0, 0, 550, 0, 0, 0, 20, 0, 0, 1163, 31, 0, 0, 0, 74, 6997, 161, 5965, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] dup_rows_per_dst=[540, 10, 5232, 652, 406, 588, 62, 282] combine_rows_per_src=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=332423168 dispatch_meta_bytes=343088 combine_payload_bytes=332423168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=332766256 ingress_bytes=401629184 link_bytes=401629184 token_bytes=665189424 weight_movement_bytes=69206016 total_nvlink_bytes=734395440 weight_to_token_ratio=0.104040 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511298286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=280657920 dispatch_meta_bytes=342288 combine_payload_bytes=280657920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281000208 ingress_bytes=315260928 link_bytes=315260928 token_bytes=561658128 weight_movement_bytes=34603008 total_nvlink_bytes=596261136 weight_to_token_ratio=0.061609 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.186301491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74912 payload_rows_remote=71130 dup_rows=23392 dup_rows_remote=22718 dedup_ratio=0.237956 router_tokens_per_expert=[0, 1221, 465, 907, 4963, 5066, 117, 0, 78, 57, 114, 221, 187, 295, 505, 594, 1439, 319, 33, 41, 1266, 1733, 1, 0, 10, 0, 1002, 3891, 156, 2, 610, 56, 50, 52, 674, 27, 7, 4262, 187, 85, 340, 208, 716, 29, 844, 177, 147, 794, 500, 1233, 491, 0, 34, 1587, 76, 4783, 255, 4173, 114, 1208, 69, 80, 139, 462] payload_rows_per_dst=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] dup_rows_per_dst=[12486, 674, 4648, 588, 822, 1286, 1488, 1400] combine_rows_per_src=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90532 payload_rows_remote=74888 dup_rows=7772 dup_rows_remote=2540 dedup_ratio=0.079061 router_tokens_per_expert=[0, 994, 531, 1165, 7902, 6745, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3441, 0, 0, 0, 0, 846, 6065, 89, 0, 0, 0, 0, 0, 509, 0, 0, 5687, 16, 127, 0, 0, 550, 0, 0, 0, 20, 0, 0, 1163, 31, 0, 0, 0, 74, 6997, 161, 5965, 0, 0, 69, 0, 0, 0] payload_rows_per_dst=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] dup_rows_per_dst=[540, 10, 5232, 652, 406, 588, 62, 282] combine_rows_per_src=[4876, 15804, 15644, 13348, 12272, 14006, 2474, 12108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93130 payload_rows_remote=78582 dup_rows=5174 dup_rows_remote=4232 dedup_ratio=0.052633 router_tokens_per_expert=[0, 1813, 657, 1163, 7626, 6984, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2160, 0, 0, 0, 0, 997, 5081, 99, 0, 0, 0, 0, 0, 437, 0, 0, 6356, 0, 122, 0, 0, 761, 0, 0, 0, 0, 0, 0, 1205, 1, 0, 0, 0, 67, 7376, 203, 5738, 0, 0, 98, 0, 0, 0] payload_rows_per_dst=[6850, 15252, 4320, 11886, 13516, 14548, 15084, 11674] dup_rows_per_dst=[416, 416, 0, 468, 314, 942, 2214, 404] combine_rows_per_src=[6850, 15252, 4320, 11886, 13516, 14548, 15084, 11674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91064 payload_rows_remote=77504 dup_rows=7240 dup_rows_remote=6618 dedup_ratio=0.073649 router_tokens_per_expert=[0, 1200, 509, 1097, 7796, 6384, 3, 0, 1, 1, 9, 118, 6, 2, 19, 10, 36, 1, 11, 0, 20, 2777, 2, 0, 0, 0, 987, 6002, 76, 0, 26, 0, 1, 1, 442, 0, 1, 5862, 8, 149, 1, 0, 673, 0, 21, 2, 22, 31, 3, 1133, 60, 0, 1, 36, 88, 7061, 152, 6188, 5, 28, 76, 3, 3, 8] payload_rows_per_dst=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] dup_rows_per_dst=[422, 256, 4306, 622, 496, 736, 74, 328] combine_rows_per_src=[5196, 15668, 15510, 13560, 12432, 13532, 2568, 12598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=57680 dup_rows=24296 dup_rows_remote=4520 dedup_ratio=0.247152 router_tokens_per_expert=[0, 1788, 528, 1142, 7466, 7125, 3, 0, 1, 0, 3, 227, 3, 4, 8, 8, 23, 4, 28, 0, 15, 1855, 0, 0, 0, 0, 963, 5219, 72, 0, 16, 0, 0, 1, 538, 0, 0, 6458, 8, 111, 4, 0, 653, 0, 14, 2, 3, 15, 8, 1303, 16, 0, 0, 27, 64, 7564, 190, 5567, 0, 23, 72, 1, 0, 9] payload_rows_per_dst=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] dup_rows_per_dst=[19776, 8, 3400, 238, 428, 12, 38, 396] combine_rows_per_src=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=332423168 dispatch_meta_bytes=343088 combine_payload_bytes=332423168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=332766256 ingress_bytes=401629184 link_bytes=401629184 token_bytes=665189424 weight_movement_bytes=69206016 total_nvlink_bytes=734395440 weight_to_token_ratio=0.104040 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.511298286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=306741248 dispatch_meta_bytes=309712 combine_payload_bytes=306741248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307050960 ingress_bytes=341344256 link_bytes=341344256 token_bytes=613792208 weight_movement_bytes=34603008 total_nvlink_bytes=648395216 weight_to_token_ratio=0.056376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284450955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93478 payload_rows_remote=91238 dup_rows=4826 dup_rows_remote=4502 dedup_ratio=0.049093 router_tokens_per_expert=[0, 0, 4159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 0, 658, 0, 0, 362, 55, 0, 0, 0, 56, 0, 442, 0, 0, 0, 0, 0, 0, 6569, 7770, 0, 151, 0, 0, 0, 154, 0, 511, 771, 0, 0, 0, 7326, 7753, 0, 0, 487, 0, 0, 247, 0, 100, 0, 0, 0, 3803, 0, 0, 0, 0, 0] payload_rows_per_dst=[8318, 13496, 16186, 13254, 15540, 2240, 15256, 9188] dup_rows_per_dst=[0, 622, 266, 768, 610, 324, 1064, 1172] combine_rows_per_src=[8318, 13496, 16186, 13254, 15540, 2240, 15256, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89006 payload_rows_remote=73476 dup_rows=9298 dup_rows_remote=9298 dedup_ratio=0.094584 router_tokens_per_expert=[802, 30, 0, 2, 0, 0, 2, 2687, 535, 37, 437, 2, 0, 0, 28, 3155, 0, 11, 1, 5071, 256, 522, 7, 1331, 0, 0, 0, 0, 0, 2, 0, 10, 3028, 21, 3, 0, 0, 0, 0, 6981, 0, 34, 0, 7259, 18, 32, 194, 1, 0, 0, 0, 3599, 17, 7753, 0, 25, 0, 6, 2, 4614, 165, 395, 1, 76] payload_rows_per_dst=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] dup_rows_per_dst=[3656, 968, 1320, 0, 2258, 270, 36, 790] combine_rows_per_src=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86672 payload_rows_remote=71016 dup_rows=11632 dup_rows_remote=11136 dedup_ratio=0.118327 router_tokens_per_expert=[0, 0, 15, 240, 8, 14, 0, 0, 0, 0, 6, 72, 5, 70, 832, 7, 0, 0, 1775, 2498, 7475, 1008, 3, 70, 7799, 14, 0, 7298, 97, 49, 0, 0, 0, 19, 152, 26, 22, 0, 0, 6533, 14, 1010, 2, 22, 0, 0, 77, 0, 1, 5, 7, 426, 5236, 3, 303, 77, 1530, 76, 0, 0, 24, 3901, 4, 327] payload_rows_per_dst=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] dup_rows_per_dst=[496, 1330, 8286, 50, 72, 32, 728, 638] combine_rows_per_src=[15656, 15604, 15488, 14866, 3702, 2218, 8052, 11086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74912 payload_rows_remote=71130 dup_rows=23392 dup_rows_remote=22718 dedup_ratio=0.237956 router_tokens_per_expert=[0, 1221, 465, 907, 4963, 5066, 117, 0, 78, 57, 114, 221, 187, 295, 505, 594, 1439, 319, 33, 41, 1266, 1733, 1, 0, 10, 0, 1002, 3891, 156, 2, 610, 56, 50, 52, 674, 27, 7, 4262, 187, 85, 340, 208, 716, 29, 844, 177, 147, 794, 500, 1233, 491, 0, 34, 1587, 76, 4783, 255, 4173, 114, 1208, 69, 80, 139, 462] payload_rows_per_dst=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] dup_rows_per_dst=[12486, 674, 4648, 588, 822, 1286, 1488, 1400] combine_rows_per_src=[12638, 3782, 14582, 10866, 9866, 5224, 6354, 11600] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77062 payload_rows_remote=62348 dup_rows=21242 dup_rows_remote=11066 dedup_ratio=0.216085 router_tokens_per_expert=[29, 0, 0, 0, 0, 0, 0, 3092, 522, 0, 151, 0, 0, 0, 10, 3647, 0, 0, 0, 6415, 200, 1354, 0, 902, 0, 0, 0, 0, 0, 0, 0, 0, 3418, 0, 0, 0, 0, 0, 0, 7000, 0, 0, 0, 7939, 0, 0, 186, 0, 0, 0, 0, 1717, 0, 8037, 0, 0, 0, 0, 0, 3861, 158, 439, 0, 75] payload_rows_per_dst=[6242, 7742, 14468, 8558, 11382, 2276, 14714, 11680] dup_rows_per_dst=[0, 918, 3274, 0, 3616, 34, 10176, 3224] combine_rows_per_src=[6242, 7742, 14468, 8558, 11382, 2276, 14714, 11680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291348480 dispatch_meta_bytes=375392 combine_payload_bytes=291348480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291723872 ingress_bytes=325951488 link_bytes=325951488 token_bytes=583072352 weight_movement_bytes=34603008 total_nvlink_bytes=617675360 weight_to_token_ratio=0.059346 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226529208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=236257280 dispatch_meta_bytes=248800 combine_payload_bytes=236257280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=236506080 ingress_bytes=374669312 link_bytes=374669312 token_bytes=472763360 weight_movement_bytes=138412032 total_nvlink_bytes=611175392 weight_to_token_ratio=0.292772 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409850458e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=321871872 dispatch_meta_bytes=331256 combine_payload_bytes=321871872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=322203128 ingress_bytes=356474880 link_bytes=356474880 token_bytes=644075000 weight_movement_bytes=34603008 total_nvlink_bytes=678678008 weight_to_token_ratio=0.053725 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341386275e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=306741248 dispatch_meta_bytes=309712 combine_payload_bytes=306741248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307050960 ingress_bytes=341344256 link_bytes=341344256 token_bytes=613792208 weight_movement_bytes=34603008 total_nvlink_bytes=648395216 weight_to_token_ratio=0.056376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284450955e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=373710848 dispatch_meta_bytes=382960 combine_payload_bytes=373710848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=374093808 ingress_bytes=477519872 link_bytes=477519872 token_bytes=747804656 weight_movement_bytes=103809024 total_nvlink_bytes=851613680 weight_to_token_ratio=0.138818 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.796868835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300957696 dispatch_meta_bytes=331096 combine_payload_bytes=300957696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301288792 ingress_bytes=335560704 link_bytes=335560704 token_bytes=602246488 weight_movement_bytes=34603008 total_nvlink_bytes=636849496 weight_to_token_ratio=0.057457 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262687914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90170 payload_rows_remote=75492 dup_rows=8134 dup_rows_remote=4502 dedup_ratio=0.082743 router_tokens_per_expert=[698, 0, 0, 0, 0, 0, 0, 2885, 587, 0, 438, 0, 0, 0, 52, 3222, 0, 0, 0, 5153, 289, 426, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 0, 7312, 0, 0, 191, 0, 0, 0, 0, 3736, 0, 7842, 0, 0, 0, 0, 0, 5026, 177, 460, 0, 80] payload_rows_per_dst=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] dup_rows_per_dst=[890, 1130, 1444, 0, 3632, 118, 0, 920] combine_rows_per_src=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69158 payload_rows_remote=53454 dup_rows=29146 dup_rows_remote=23406 dedup_ratio=0.296488 router_tokens_per_expert=[184, 9, 0, 0, 0, 0, 0, 2988, 545, 13, 244, 1, 0, 0, 49, 3037, 0, 6, 0, 6308, 184, 1774, 0, 869, 0, 0, 0, 0, 0, 0, 0, 4, 4321, 6, 0, 0, 0, 0, 0, 6951, 0, 12, 0, 7995, 7, 12, 207, 0, 0, 0, 0, 1595, 5, 8047, 0, 8, 0, 2, 0, 3118, 158, 419, 0, 74] payload_rows_per_dst=[6256, 6800, 12826, 8, 8724, 2530, 16310, 15704] dup_rows_per_dst=[106, 978, 3164, 0, 102, 66, 18990, 5740] combine_rows_per_src=[6256, 6800, 12826, 8, 8724, 2530, 16310, 15704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317456384 dispatch_meta_bytes=336488 combine_payload_bytes=317456384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317792872 ingress_bytes=386662400 link_bytes=386662400 token_bytes=635249256 weight_movement_bytes=69206016 total_nvlink_bytes=704455272 weight_to_token_ratio=0.108943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=38784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.454979482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=290881536 dispatch_meta_bytes=328608 combine_payload_bytes=290881536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291210144 ingress_bytes=325484544 link_bytes=325484544 token_bytes=582091680 weight_movement_bytes=34603008 total_nvlink_bytes=616694688 weight_to_token_ratio=0.059446 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224772135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291348480 dispatch_meta_bytes=375392 combine_payload_bytes=291348480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291723872 ingress_bytes=325951488 link_bytes=325951488 token_bytes=583072352 weight_movement_bytes=34603008 total_nvlink_bytes=617675360 weight_to_token_ratio=0.059346 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46080 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226529208e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=255377408 dispatch_meta_bytes=293656 combine_payload_bytes=255377408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=255671064 ingress_bytes=324583424 link_bytes=324583424 token_bytes=511048472 weight_movement_bytes=69206016 total_nvlink_bytes=580254488 weight_to_token_ratio=0.135420 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.221381293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90170 payload_rows_remote=75492 dup_rows=8134 dup_rows_remote=4502 dedup_ratio=0.082743 router_tokens_per_expert=[698, 0, 0, 0, 0, 0, 0, 2885, 587, 0, 438, 0, 0, 0, 52, 3222, 0, 0, 0, 5153, 289, 426, 0, 1423, 0, 0, 0, 0, 0, 0, 0, 0, 2107, 0, 0, 0, 0, 0, 0, 7048, 0, 0, 0, 7312, 0, 0, 191, 0, 0, 0, 0, 3736, 0, 7842, 0, 0, 0, 0, 0, 5026, 177, 460, 0, 80] payload_rows_per_dst=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] dup_rows_per_dst=[890, 1130, 1444, 0, 3632, 118, 0, 920] combine_rows_per_src=[6276, 7468, 13138, 15684, 14678, 14888, 7472, 10566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87526 payload_rows_remote=79314 dup_rows=10778 dup_rows_remote=10210 dedup_ratio=0.109639 router_tokens_per_expert=[589, 0, 0, 0, 0, 0, 0, 3056, 527, 0, 438, 0, 0, 0, 57, 3157, 0, 0, 0, 5336, 267, 489, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 7076, 0, 0, 0, 7353, 0, 0, 183, 0, 0, 0, 0, 4153, 0, 7807, 0, 0, 0, 0, 0, 4653, 163, 433, 0, 90] payload_rows_per_dst=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] dup_rows_per_dst=[6036, 3164, 568, 0, 0, 146, 0, 864] combine_rows_per_src=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=319976 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309535208 ingress_bytes=413024256 link_bytes=413024256 token_bytes=618750440 weight_movement_bytes=103809024 total_nvlink_bytes=722559464 weight_to_token_ratio=0.167772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554177024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=218947584 dispatch_meta_bytes=307440 combine_payload_bytes=218947584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219255024 ingress_bytes=253550592 link_bytes=253550592 token_bytes=438202608 weight_movement_bytes=34603008 total_nvlink_bytes=472805616 weight_to_token_ratio=0.078966 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.540904652e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81900 payload_rows_remote=67032 dup_rows=16404 dup_rows_remote=11382 dedup_ratio=0.166870 router_tokens_per_expert=[2114, 1259, 36, 87, 43, 54, 268, 956, 388, 1626, 2510, 89, 298, 0, 257, 1550, 34, 447, 90, 3227, 321, 818, 69, 804, 4, 44, 0, 55, 33, 109, 0, 107, 4615, 1580, 234, 0, 27, 0, 1, 3162, 0, 1580, 37, 3821, 903, 1384, 212, 36, 40, 76, 11, 1231, 315, 5203, 0, 815, 54, 280, 76, 4843, 302, 508, 40, 69] payload_rows_per_dst=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] dup_rows_per_dst=[3118, 5022, 318, 76, 1216, 4550, 688, 1416] combine_rows_per_src=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87922 payload_rows_remote=71974 dup_rows=10382 dup_rows_remote=10106 dedup_ratio=0.105611 router_tokens_per_expert=[65, 0, 0, 0, 0, 0, 0, 2712, 543, 0, 202, 0, 0, 0, 2, 3807, 0, 0, 0, 6177, 188, 1294, 0, 863, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 0, 0, 0, 0, 0, 0, 7017, 0, 0, 0, 7947, 0, 0, 165, 0, 0, 0, 0, 2777, 0, 7924, 0, 0, 0, 0, 0, 3490, 166, 462, 0, 100] payload_rows_per_dst=[5490, 8100, 14202, 15848, 14638, 15948, 5554, 8142] dup_rows_per_dst=[64, 1008, 2842, 0, 5898, 276, 0, 294] combine_rows_per_src=[5490, 8100, 14202, 15848, 14638, 15948, 5554, 8142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87526 payload_rows_remote=79314 dup_rows=10778 dup_rows_remote=10210 dedup_ratio=0.109639 router_tokens_per_expert=[589, 0, 0, 0, 0, 0, 0, 3056, 527, 0, 438, 0, 0, 0, 57, 3157, 0, 0, 0, 5336, 267, 489, 0, 1336, 0, 0, 0, 0, 0, 0, 0, 0, 1989, 0, 0, 0, 0, 0, 0, 7076, 0, 0, 0, 7353, 0, 0, 183, 0, 0, 0, 0, 4153, 0, 7807, 0, 0, 0, 0, 0, 4653, 163, 433, 0, 90] payload_rows_per_dst=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] dup_rows_per_dst=[6036, 3164, 568, 0, 0, 146, 0, 864] combine_rows_per_src=[15406, 11270, 8212, 15614, 3978, 14926, 8306, 9814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86888 payload_rows_remote=71576 dup_rows=11416 dup_rows_remote=6456 dedup_ratio=0.116130 router_tokens_per_expert=[170, 17, 0, 0, 0, 0, 1, 3057, 516, 27, 220, 2, 2, 0, 4, 3569, 0, 7, 3, 6384, 182, 1183, 1, 925, 0, 1, 0, 0, 0, 2, 0, 2, 3445, 22, 2, 0, 0, 0, 0, 6891, 0, 27, 0, 7814, 13, 25, 158, 0, 1, 0, 0, 2237, 8, 7971, 0, 16, 0, 1, 1, 3597, 136, 440, 0, 72] payload_rows_per_dst=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] dup_rows_per_dst=[4960, 5966, 8, 2, 14, 246, 16, 204] combine_rows_per_src=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=309215232 dispatch_meta_bytes=319976 combine_payload_bytes=309215232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309535208 ingress_bytes=413024256 link_bytes=413024256 token_bytes=618750440 weight_movement_bytes=103809024 total_nvlink_bytes=722559464 weight_to_token_ratio=0.167772 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554177024e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324870144 dispatch_meta_bytes=358096 combine_payload_bytes=324870144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325228240 ingress_bytes=428679168 link_bytes=428679168 token_bytes=650098384 weight_movement_bytes=103809024 total_nvlink_bytes=753907408 weight_to_token_ratio=0.159682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613085198e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90312 payload_rows_remote=74744 dup_rows=7992 dup_rows_remote=6400 dedup_ratio=0.081299 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 999, 0, 99, 7511, 829, 0, 0, 99, 0, 0, 3, 0, 0, 55, 0, 199, 0, 0, 0, 0, 0, 0, 642, 7772, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 8093, 100, 0, 0, 804, 6, 0, 99, 0, 247, 0, 0, 0, 0, 5510, 0, 0, 0, 0, 347, 7905, 0, 0, 0, 261] payload_rows_per_dst=[5102, 12138, 9520, 13996, 6888, 15568, 11090, 16010] dup_rows_per_dst=[372, 1464, 322, 1124, 1678, 1592, 424, 1016] combine_rows_per_src=[5102, 12138, 9520, 13996, 6888, 15568, 11090, 16010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93492 payload_rows_remote=80056 dup_rows=4812 dup_rows_remote=3774 dedup_ratio=0.048950 router_tokens_per_expert=[3, 0, 4117, 12, 0, 0, 2, 0, 35, 2, 1, 0, 7689, 1, 602, 0, 4, 330, 61, 0, 0, 0, 74, 1, 704, 0, 13, 0, 0, 0, 0, 6520, 7168, 0, 136, 0, 38, 0, 138, 0, 502, 707, 4, 6, 35, 7449, 7991, 0, 38, 491, 0, 2, 243, 0, 76, 0, 0, 37, 3894, 25, 0, 0, 0, 1] payload_rows_per_dst=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] dup_rows_per_dst=[14, 730, 766, 1038, 548, 374, 1296, 46] combine_rows_per_src=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89006 payload_rows_remote=73476 dup_rows=9298 dup_rows_remote=9298 dedup_ratio=0.094584 router_tokens_per_expert=[802, 30, 0, 2, 0, 0, 2, 2687, 535, 37, 437, 2, 0, 0, 28, 3155, 0, 11, 1, 5071, 256, 522, 7, 1331, 0, 0, 0, 0, 0, 2, 0, 10, 3028, 21, 3, 0, 0, 0, 0, 6981, 0, 34, 0, 7259, 18, 32, 194, 1, 0, 0, 0, 3599, 17, 7753, 0, 25, 0, 6, 2, 4614, 165, 395, 1, 76] payload_rows_per_dst=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] dup_rows_per_dst=[3656, 968, 1320, 0, 2258, 270, 36, 790] combine_rows_per_src=[11652, 7420, 13078, 15530, 9546, 14806, 7246, 9728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=57680 dup_rows=24296 dup_rows_remote=4520 dedup_ratio=0.247152 router_tokens_per_expert=[0, 1788, 528, 1142, 7466, 7125, 3, 0, 1, 0, 3, 227, 3, 4, 8, 8, 23, 4, 28, 0, 15, 1855, 0, 0, 0, 0, 963, 5219, 72, 0, 16, 0, 0, 1, 538, 0, 0, 6458, 8, 111, 4, 0, 653, 0, 14, 2, 3, 15, 8, 1303, 16, 0, 0, 27, 64, 7564, 190, 5567, 0, 23, 72, 1, 0, 9] payload_rows_per_dst=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] dup_rows_per_dst=[19776, 8, 3400, 238, 428, 12, 38, 396] combine_rows_per_src=[16328, 500, 15578, 12302, 13804, 1370, 2798, 11328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86980 payload_rows_remote=71476 dup_rows=11324 dup_rows_remote=10402 dedup_ratio=0.115194 router_tokens_per_expert=[0, 0, 4144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7794, 0, 659, 0, 0, 365, 70, 0, 0, 0, 71, 0, 459, 0, 0, 0, 0, 17, 0, 6603, 7942, 0, 142, 0, 0, 0, 143, 0, 261, 719, 0, 0, 0, 7450, 7697, 0, 0, 437, 0, 0, 251, 0, 75, 0, 0, 0, 3853, 0, 0, 0, 0, 0] payload_rows_per_dst=[8288, 1318, 990, 13346, 15884, 15560, 15504, 16090] dup_rows_per_dst=[0, 0, 148, 812, 570, 1668, 922, 7204] combine_rows_per_src=[8288, 1318, 990, 13346, 15884, 15560, 15504, 16090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=274563072 dispatch_meta_bytes=313656 combine_payload_bytes=274563072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274876728 ingress_bytes=309166080 link_bytes=309166080 token_bytes=549439800 weight_movement_bytes=34603008 total_nvlink_bytes=584042808 weight_to_token_ratio=0.062979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163367069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=293175296 dispatch_meta_bytes=312128 combine_payload_bytes=293175296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293487424 ingress_bytes=327778304 link_bytes=327778304 token_bytes=586662720 weight_movement_bytes=34603008 total_nvlink_bytes=621265728 weight_to_token_ratio=0.058983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81900 payload_rows_remote=67032 dup_rows=16404 dup_rows_remote=11382 dedup_ratio=0.166870 router_tokens_per_expert=[2114, 1259, 36, 87, 43, 54, 268, 956, 388, 1626, 2510, 89, 298, 0, 257, 1550, 34, 447, 90, 3227, 321, 818, 69, 804, 4, 44, 0, 55, 33, 109, 0, 107, 4615, 1580, 234, 0, 27, 0, 1, 3162, 0, 1580, 37, 3821, 903, 1384, 212, 36, 40, 76, 11, 1231, 315, 5203, 0, 815, 54, 280, 76, 4843, 302, 508, 40, 69] payload_rows_per_dst=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] dup_rows_per_dst=[3118, 5022, 318, 76, 1216, 4550, 688, 1416] combine_rows_per_src=[12840, 14868, 4848, 11034, 11698, 11396, 4288, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=294805504 dispatch_meta_bytes=328320 combine_payload_bytes=294805504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=295133824 ingress_bytes=364011520 link_bytes=364011520 token_bytes=589939328 weight_movement_bytes=69206016 total_nvlink_bytes=659145344 weight_to_token_ratio=0.117310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.369746044e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324870144 dispatch_meta_bytes=358096 combine_payload_bytes=324870144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325228240 ingress_bytes=428679168 link_bytes=428679168 token_bytes=650098384 weight_movement_bytes=103809024 total_nvlink_bytes=753907408 weight_to_token_ratio=0.159682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613085198e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93640 payload_rows_remote=79218 dup_rows=4664 dup_rows_remote=4134 dedup_ratio=0.047445 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 530, 0, 0, 380, 68, 0, 0, 0, 69, 0, 522, 0, 0, 0, 0, 0, 0, 6510, 7211, 0, 126, 0, 0, 0, 139, 0, 391, 743, 0, 0, 0, 7568, 8044, 0, 0, 552, 0, 0, 247, 0, 80, 0, 0, 0, 4114, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] dup_rows_per_dst=[0, 566, 898, 822, 530, 336, 1512, 0] combine_rows_per_src=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=306151424 dispatch_meta_bytes=324576 combine_payload_bytes=306151424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306476000 ingress_bytes=340754432 link_bytes=340754432 token_bytes=612627424 weight_movement_bytes=34603008 total_nvlink_bytes=647230432 weight_to_token_ratio=0.056483 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282231495e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78628 payload_rows_remote=62576 dup_rows=19676 dup_rows_remote=13398 dedup_ratio=0.200155 router_tokens_per_expert=[1, 0, 4712, 4, 1, 0, 0, 0, 11, 0, 1, 0, 7761, 0, 542, 0, 0, 345, 64, 0, 0, 0, 81, 0, 344, 0, 4, 0, 0, 55, 0, 6582, 7953, 0, 144, 0, 13, 0, 147, 0, 293, 720, 0, 3, 12, 7415, 7725, 0, 14, 479, 0, 1, 247, 0, 74, 0, 0, 13, 3382, 9, 0, 0, 0, 0] payload_rows_per_dst=[9434, 1106, 852, 3936, 16030, 15756, 15462, 16052] dup_rows_per_dst=[2, 2, 128, 282, 7430, 4556, 998, 6278] combine_rows_per_src=[9434, 1106, 852, 3936, 16030, 15756, 15462, 16052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=327909376 dispatch_meta_bytes=335320 combine_payload_bytes=327909376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=328244696 ingress_bytes=431718400 link_bytes=431718400 token_bytes=656154072 weight_movement_bytes=103809024 total_nvlink_bytes=759963096 weight_to_token_ratio=0.158208 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.624521583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=300957696 dispatch_meta_bytes=331096 combine_payload_bytes=300957696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301288792 ingress_bytes=335560704 link_bytes=335560704 token_bytes=602246488 weight_movement_bytes=34603008 total_nvlink_bytes=636849496 weight_to_token_ratio=0.057457 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262687914e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=236257280 dispatch_meta_bytes=248800 combine_payload_bytes=236257280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=236506080 ingress_bytes=374669312 link_bytes=374669312 token_bytes=472763360 weight_movement_bytes=138412032 total_nvlink_bytes=611175392 weight_to_token_ratio=0.292772 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.409850458e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=292765696 dispatch_meta_bytes=327512 combine_payload_bytes=292765696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293093208 ingress_bytes=327368704 link_bytes=327368704 token_bytes=585858904 weight_movement_bytes=34603008 total_nvlink_bytes=620461912 weight_to_token_ratio=0.059064 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231862078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=274563072 dispatch_meta_bytes=313656 combine_payload_bytes=274563072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274876728 ingress_bytes=309166080 link_bytes=309166080 token_bytes=549439800 weight_movement_bytes=34603008 total_nvlink_bytes=584042808 weight_to_token_ratio=0.062979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.163367069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93640 payload_rows_remote=79218 dup_rows=4664 dup_rows_remote=4134 dedup_ratio=0.047445 router_tokens_per_expert=[0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7781, 0, 530, 0, 0, 380, 68, 0, 0, 0, 69, 0, 522, 0, 0, 0, 0, 0, 0, 6510, 7211, 0, 126, 0, 0, 0, 139, 0, 391, 743, 0, 0, 0, 7568, 8044, 0, 0, 552, 0, 0, 247, 0, 80, 0, 0, 0, 4114, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] dup_rows_per_dst=[0, 566, 898, 822, 530, 336, 1512, 0] combine_rows_per_src=[8154, 16056, 16224, 13242, 14422, 1932, 15382, 8228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90066 payload_rows_remote=73860 dup_rows=8238 dup_rows_remote=7438 dedup_ratio=0.083801 router_tokens_per_expert=[0, 0, 4102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 627, 0, 0, 343, 53, 0, 0, 0, 57, 0, 544, 0, 0, 0, 0, 0, 0, 6562, 7191, 0, 145, 0, 0, 0, 161, 0, 444, 722, 0, 0, 0, 7548, 8050, 0, 0, 504, 0, 0, 265, 0, 90, 0, 0, 0, 3970, 0, 0, 0, 0, 0] payload_rows_per_dst=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] dup_rows_per_dst=[3610, 724, 800, 876, 456, 1088, 684, 0] combine_rows_per_src=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13794185856 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6901002880 ingress_bytes=8450318336 link_bytes=8450318336 total_nvlink_bytes=15351321216 weight_to_token_ratio=0.112883 prefetch_share_of_ingress=0.098277 dedup_ratio=0.143341 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.382515982e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324476928 dispatch_meta_bytes=333408 combine_payload_bytes=324476928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324810336 ingress_bytes=393682944 link_bytes=393682944 token_bytes=649287264 weight_movement_bytes=69206016 total_nvlink_bytes=718493280 weight_to_token_ratio=0.106588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481397224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256311296 dispatch_meta_bytes=303896 combine_payload_bytes=256311296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=256615192 ingress_bytes=290914304 link_bytes=290914304 token_bytes=512926488 weight_movement_bytes=34603008 total_nvlink_bytes=547529496 weight_to_token_ratio=0.067462 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.094687105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78110 payload_rows_remote=65278 dup_rows=20194 dup_rows_remote=18344 dedup_ratio=0.205424 router_tokens_per_expert=[162, 281, 3318, 157, 60, 48, 31, 0, 1420, 220, 104, 8, 4392, 9, 1070, 118, 276, 651, 86, 21, 0, 0, 79, 136, 613, 229, 140, 2, 253, 41, 10, 3497, 4002, 39, 173, 0, 1397, 3, 434, 0, 2269, 2314, 92, 668, 1671, 4743, 4829, 60, 1723, 456, 14, 379, 953, 0, 72, 46, 0, 1723, 2017, 1411, 108, 72, 41, 11] payload_rows_per_dst=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] dup_rows_per_dst=[4608, 1850, 962, 730, 106, 8270, 932, 2736] combine_rows_per_src=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86036 payload_rows_remote=70030 dup_rows=12268 dup_rows_remote=4158 dedup_ratio=0.124797 router_tokens_per_expert=[0, 0, 4321, 3, 4, 0, 0, 0, 17, 23, 0, 0, 7621, 0, 622, 2, 5, 372, 47, 0, 0, 0, 48, 0, 447, 3, 6, 0, 0, 2, 2, 6545, 7730, 0, 146, 0, 25, 0, 151, 0, 398, 760, 1, 13, 26, 7453, 7792, 1, 31, 457, 0, 2, 268, 0, 72, 0, 0, 30, 3679, 24, 1, 1, 1, 0] payload_rows_per_dst=[16006, 15806, 16174, 12820, 352, 15950, 1506, 7422] dup_rows_per_dst=[8110, 1188, 354, 766, 292, 1354, 154, 50] combine_rows_per_src=[16006, 15806, 16174, 12820, 352, 15950, 1506, 7422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93478 payload_rows_remote=91238 dup_rows=4826 dup_rows_remote=4502 dedup_ratio=0.049093 router_tokens_per_expert=[0, 0, 4159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 0, 658, 0, 0, 362, 55, 0, 0, 0, 56, 0, 442, 0, 0, 0, 0, 0, 0, 6569, 7770, 0, 151, 0, 0, 0, 154, 0, 511, 771, 0, 0, 0, 7326, 7753, 0, 0, 487, 0, 0, 247, 0, 100, 0, 0, 0, 3803, 0, 0, 0, 0, 0] payload_rows_per_dst=[8318, 13496, 16186, 13254, 15540, 2240, 15256, 9188] dup_rows_per_dst=[0, 622, 266, 768, 610, 324, 1064, 1172] combine_rows_per_src=[8318, 13496, 16186, 13254, 15540, 2240, 15256, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90066 payload_rows_remote=73860 dup_rows=8238 dup_rows_remote=7438 dedup_ratio=0.083801 router_tokens_per_expert=[0, 0, 4102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7774, 0, 627, 0, 0, 343, 53, 0, 0, 0, 57, 0, 544, 0, 0, 0, 0, 0, 0, 6562, 7191, 0, 145, 0, 0, 0, 161, 0, 444, 722, 0, 0, 0, 7548, 8050, 0, 0, 504, 0, 0, 265, 0, 90, 0, 0, 0, 3970, 0, 0, 0, 0, 0] payload_rows_per_dst=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] dup_rows_per_dst=[3610, 724, 800, 876, 456, 1088, 684, 0] combine_rows_per_src=[11996, 16078, 16206, 13336, 7136, 11398, 5976, 7940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324476928 dispatch_meta_bytes=333408 combine_payload_bytes=324476928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=324810336 ingress_bytes=393682944 link_bytes=393682944 token_bytes=649287264 weight_movement_bytes=69206016 total_nvlink_bytes=718493280 weight_to_token_ratio=0.106588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481397224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=302530560 dispatch_meta_bytes=325192 combine_payload_bytes=302530560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302855752 ingress_bytes=337133568 link_bytes=337133568 token_bytes=605386312 weight_movement_bytes=34603008 total_nvlink_bytes=639989320 weight_to_token_ratio=0.057159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268606475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90994 payload_rows_remote=75590 dup_rows=7310 dup_rows_remote=5772 dedup_ratio=0.074361 router_tokens_per_expert=[0, 0, 21, 33, 41, 21, 673, 11, 76, 7449, 1248, 0, 0, 77, 2, 0, 20, 3, 0, 65, 35, 249, 29, 0, 1, 8, 0, 0, 795, 7641, 26, 0, 1, 36, 0, 3, 6, 7496, 2, 8076, 76, 0, 3, 797, 47, 3, 163, 1, 290, 2, 0, 0, 0, 4792, 0, 0, 0, 0, 371, 7810, 0, 0, 1, 652] payload_rows_per_dst=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] dup_rows_per_dst=[1508, 152, 726, 1538, 4, 1436, 90, 1856] combine_rows_per_src=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267378688 dispatch_meta_bytes=334488 combine_payload_bytes=267378688 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267713176 ingress_bytes=336584704 link_bytes=336584704 token_bytes=535091864 weight_movement_bytes=69206016 total_nvlink_bytes=604297880 weight_to_token_ratio=0.129335 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266541144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93492 payload_rows_remote=80056 dup_rows=4812 dup_rows_remote=3774 dedup_ratio=0.048950 router_tokens_per_expert=[3, 0, 4117, 12, 0, 0, 2, 0, 35, 2, 1, 0, 7689, 1, 602, 0, 4, 330, 61, 0, 0, 0, 74, 1, 704, 0, 13, 0, 0, 0, 0, 6520, 7168, 0, 136, 0, 38, 0, 138, 0, 502, 707, 4, 6, 35, 7449, 7991, 0, 38, 491, 0, 2, 243, 0, 76, 0, 0, 37, 3894, 25, 0, 0, 0, 1] payload_rows_per_dst=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] dup_rows_per_dst=[14, 730, 766, 1038, 548, 374, 1296, 46] combine_rows_per_src=[8254, 15930, 16156, 13436, 14412, 2134, 15302, 7868] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79466 payload_rows_remote=67246 dup_rows=18838 dup_rows_remote=16028 dedup_ratio=0.191630 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 1169, 0, 74, 7494, 815, 0, 0, 74, 0, 3, 34, 0, 0, 70, 0, 243, 0, 0, 0, 0, 0, 0, 539, 7701, 0, 0, 0, 0, 0, 0, 0, 7698, 0, 8118, 75, 0, 0, 684, 17, 0, 74, 0, 251, 0, 0, 0, 0, 5500, 0, 0, 0, 0, 326, 7906, 0, 0, 0, 280] payload_rows_per_dst=[2338, 15130, 694, 1078, 16350, 15694, 12220, 15962] dup_rows_per_dst=[14, 1790, 0, 0, 11760, 1402, 2810, 1062] combine_rows_per_src=[2338, 15130, 694, 1078, 16350, 15694, 12220, 15962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=286842880 dispatch_meta_bytes=296752 combine_payload_bytes=286842880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287139632 ingress_bytes=321445888 link_bytes=321445888 token_bytes=573982512 weight_movement_bytes=34603008 total_nvlink_bytes=608585520 weight_to_token_ratio=0.060286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209574998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86888 payload_rows_remote=71576 dup_rows=11416 dup_rows_remote=6456 dedup_ratio=0.116130 router_tokens_per_expert=[170, 17, 0, 0, 0, 0, 1, 3057, 516, 27, 220, 2, 2, 0, 4, 3569, 0, 7, 3, 6384, 182, 1183, 1, 925, 0, 1, 0, 0, 0, 2, 0, 2, 3445, 22, 2, 0, 0, 0, 0, 6891, 0, 27, 0, 7814, 13, 25, 158, 0, 1, 0, 0, 2237, 8, 7971, 0, 16, 0, 1, 1, 3597, 136, 440, 0, 72] payload_rows_per_dst=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] dup_rows_per_dst=[4960, 5966, 8, 2, 14, 246, 16, 204] combine_rows_per_src=[15312, 15482, 4594, 15950, 6924, 15828, 4508, 8290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78110 payload_rows_remote=65278 dup_rows=20194 dup_rows_remote=18344 dedup_ratio=0.205424 router_tokens_per_expert=[162, 281, 3318, 157, 60, 48, 31, 0, 1420, 220, 104, 8, 4392, 9, 1070, 118, 276, 651, 86, 21, 0, 0, 79, 136, 613, 229, 140, 2, 253, 41, 10, 3497, 4002, 39, 173, 0, 1397, 3, 434, 0, 2269, 2314, 92, 668, 1671, 4743, 4829, 60, 1723, 456, 14, 379, 953, 0, 72, 46, 0, 1723, 2017, 1411, 108, 72, 41, 11] payload_rows_per_dst=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] dup_rows_per_dst=[4608, 1850, 962, 730, 106, 8270, 932, 2736] combine_rows_per_src=[11510, 12832, 11194, 8840, 3986, 15364, 6354, 8030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=373710848 dispatch_meta_bytes=382960 combine_payload_bytes=373710848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=374093808 ingress_bytes=477519872 link_bytes=477519872 token_bytes=747804656 weight_movement_bytes=103809024 total_nvlink_bytes=851613680 weight_to_token_ratio=0.138818 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.796868835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90786 payload_rows_remote=90786 dup_rows=7518 dup_rows_remote=7518 dedup_ratio=0.076477 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 653, 0, 80, 7428, 1171, 0, 0, 80, 0, 0, 1, 0, 0, 68, 0, 325, 0, 0, 0, 0, 0, 0, 816, 7661, 0, 0, 0, 0, 0, 0, 0, 7634, 0, 8112, 80, 0, 0, 754, 24, 0, 111, 0, 248, 0, 0, 0, 0, 4968, 0, 0, 0, 0, 328, 7879, 0, 0, 0, 731] payload_rows_per_dst=[14856, 2502, 16224, 15324, 0, 15596, 10366, 15918] dup_rows_per_dst=[1306, 160, 788, 1630, 0, 1610, 66, 1958] combine_rows_per_src=[14856, 2502, 16224, 15324, 0, 15596, 10366, 15918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=302530560 dispatch_meta_bytes=325192 combine_payload_bytes=302530560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302855752 ingress_bytes=337133568 link_bytes=337133568 token_bytes=605386312 weight_movement_bytes=34603008 total_nvlink_bytes=639989320 weight_to_token_ratio=0.057159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268606475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=325448 combine_payload_bytes=309616640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=309942088 ingress_bytes=413425664 link_bytes=413425664 token_bytes=619558728 weight_movement_bytes=103809024 total_nvlink_bytes=723367752 weight_to_token_ratio=0.167553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555687490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82602 payload_rows_remote=66590 dup_rows=15702 dup_rows_remote=13646 dedup_ratio=0.159729 router_tokens_per_expert=[0, 2, 6, 11, 14, 9, 944, 1, 73, 7464, 782, 0, 0, 74, 0, 0, 23, 2, 0, 74, 12, 230, 10, 0, 0, 1, 0, 0, 569, 7676, 8, 0, 1, 13, 0, 1, 6, 7691, 1, 8105, 74, 0, 0, 855, 7, 1, 94, 0, 247, 0, 0, 0, 0, 5587, 0, 0, 0, 0, 323, 7905, 0, 0, 0, 256] payload_rows_per_dst=[1918, 15098, 680, 1156, 16238, 15770, 15730, 16012] dup_rows_per_dst=[56, 1688, 22, 0, 16, 1674, 10190, 2056] combine_rows_per_src=[1918, 15098, 680, 1156, 16238, 15770, 15730, 16012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=327909376 dispatch_meta_bytes=335320 combine_payload_bytes=327909376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=328244696 ingress_bytes=431718400 link_bytes=431718400 token_bytes=656154072 weight_movement_bytes=103809024 total_nvlink_bytes=759963096 weight_to_token_ratio=0.158208 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=44416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.624521583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=275439616 dispatch_meta_bytes=333096 combine_payload_bytes=275439616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275772712 ingress_bytes=310042624 link_bytes=310042624 token_bytes=551212328 weight_movement_bytes=34603008 total_nvlink_bytes=585815336 weight_to_token_ratio=0.062776 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.166665434e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90786 payload_rows_remote=90786 dup_rows=7518 dup_rows_remote=7518 dedup_ratio=0.076477 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 653, 0, 80, 7428, 1171, 0, 0, 80, 0, 0, 1, 0, 0, 68, 0, 325, 0, 0, 0, 0, 0, 0, 816, 7661, 0, 0, 0, 0, 0, 0, 0, 7634, 0, 8112, 80, 0, 0, 754, 24, 0, 111, 0, 248, 0, 0, 0, 0, 4968, 0, 0, 0, 0, 328, 7879, 0, 0, 0, 731] payload_rows_per_dst=[14856, 2502, 16224, 15324, 0, 15596, 10366, 15918] dup_rows_per_dst=[1306, 160, 788, 1630, 0, 1610, 66, 1958] combine_rows_per_src=[14856, 2502, 16224, 15324, 0, 15596, 10366, 15918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=293175296 dispatch_meta_bytes=312128 combine_payload_bytes=293175296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293487424 ingress_bytes=327778304 link_bytes=327778304 token_bytes=586662720 weight_movement_bytes=34603008 total_nvlink_bytes=621265728 weight_to_token_ratio=0.058983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.233403370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267378688 dispatch_meta_bytes=334488 combine_payload_bytes=267378688 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267713176 ingress_bytes=336584704 link_bytes=336584704 token_bytes=535091864 weight_movement_bytes=69206016 total_nvlink_bytes=604297880 weight_to_token_ratio=0.129335 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266541144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13945272408 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6976583768 ingress_bytes=8491220992 link_bytes=8491220992 total_nvlink_bytes=15467804760 weight_to_token_ratio=0.109179 prefetch_share_of_ingress=0.089653 dedup_ratio=0.134268 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.389207869e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92378 payload_rows_remote=76172 dup_rows=5926 dup_rows_remote=5310 dedup_ratio=0.060282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 542, 0, 89, 7476, 1042, 0, 0, 89, 0, 0, 5, 0, 0, 53, 0, 250, 0, 0, 0, 0, 0, 0, 761, 7730, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 8103, 90, 0, 0, 687, 15, 0, 113, 0, 273, 0, 0, 0, 0, 5467, 0, 0, 0, 0, 355, 7908, 0, 0, 0, 524] payload_rows_per_dst=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] dup_rows_per_dst=[1084, 1986, 616, 1522, 0, 210, 234, 274] combine_rows_per_src=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12881843768 prefetch_bytes=588251136 reduce_bytes=657457152 weight_movement_bytes=1245708288 egress_bytes=6444709432 ingress_bytes=7682842624 link_bytes=7682842624 total_nvlink_bytes=14127552056 weight_to_token_ratio=0.096703 prefetch_share_of_ingress=0.076567 dedup_ratio=0.171990 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=19 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256952968e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=272752640 dispatch_meta_bytes=320944 combine_payload_bytes=272752640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273073584 ingress_bytes=341958656 link_bytes=341958656 token_bytes=545826224 weight_movement_bytes=69206016 total_nvlink_bytes=615032240 weight_to_token_ratio=0.126791 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.035000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286762893e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81662 payload_rows_remote=70066 dup_rows=16642 dup_rows_remote=14818 dedup_ratio=0.169291 router_tokens_per_expert=[6, 222, 1170, 1670, 1662, 853, 514, 115, 81, 5088, 783, 0, 0, 1158, 144, 108, 62, 336, 0, 212, 1580, 307, 1019, 0, 1, 408, 0, 54, 885, 4495, 975, 0, 56, 1732, 9, 160, 666, 3822, 204, 6221, 69, 31, 1, 2431, 818, 123, 1112, 92, 440, 39, 49, 3, 17, 823, 0, 0, 0, 0, 766, 4436, 43, 242, 85, 754] payload_rows_per_dst=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] dup_rows_per_dst=[8094, 1824, 3534, 912, 430, 1540, 126, 182] combine_rows_per_src=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92090 payload_rows_remote=77118 dup_rows=6214 dup_rows_remote=3978 dedup_ratio=0.063212 router_tokens_per_expert=[0, 17, 15, 28, 27, 16, 1044, 3, 72, 7454, 715, 0, 0, 77, 3, 10, 14, 5, 0, 49, 27, 198, 18, 0, 0, 2, 0, 0, 550, 7650, 16, 0, 1, 28, 0, 3, 18, 7619, 0, 8091, 72, 0, 2, 1052, 40, 0, 106, 0, 261, 1, 0, 0, 0, 5247, 0, 0, 0, 0, 359, 7859, 0, 1, 1, 381] payload_rows_per_dst=[14972, 15814, 16240, 15350, 15332, 2320, 10578, 1484] dup_rows_per_dst=[2236, 1658, 564, 1086, 6, 224, 440, 0] combine_rows_per_src=[14972, 15814, 16240, 15350, 15332, 2320, 10578, 1484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=371859456 dispatch_meta_bytes=393216 combine_payload_bytes=371859456 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372252672 ingress_bytes=475668480 link_bytes=475668480 token_bytes=744112128 weight_movement_bytes=103809024 total_nvlink_bytes=847921152 weight_to_token_ratio=0.139507 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.789902196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=272752640 dispatch_meta_bytes=320944 combine_payload_bytes=272752640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=273073584 ingress_bytes=341958656 link_bytes=341958656 token_bytes=545826224 weight_movement_bytes=69206016 total_nvlink_bytes=615032240 weight_to_token_ratio=0.126791 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.035000 gemm_active_groups=8 gemm_peak_group_rows=41216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286762893e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12881843768 prefetch_bytes=588251136 reduce_bytes=657457152 weight_movement_bytes=1245708288 egress_bytes=6444709432 ingress_bytes=7682842624 link_bytes=7682842624 total_nvlink_bytes=14127552056 weight_to_token_ratio=0.096703 prefetch_share_of_ingress=0.076567 dedup_ratio=0.171990 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=19 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029457 gemm_inner_pad_upper_bound=27305 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256952968e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14115210512 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=7061603600 ingress_bytes=8749154304 link_bytes=8749154304 total_nvlink_bytes=15810757904 weight_to_token_ratio=0.120122 prefetch_share_of_ingress=0.071190 dedup_ratio=0.137880 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.431407099e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90312 payload_rows_remote=74744 dup_rows=7992 dup_rows_remote=6400 dedup_ratio=0.081299 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 999, 0, 99, 7511, 829, 0, 0, 99, 0, 0, 3, 0, 0, 55, 0, 199, 0, 0, 0, 0, 0, 0, 642, 7772, 0, 0, 0, 0, 0, 0, 0, 7571, 0, 8093, 100, 0, 0, 804, 6, 0, 99, 0, 247, 0, 0, 0, 0, 5510, 0, 0, 0, 0, 347, 7905, 0, 0, 0, 261] payload_rows_per_dst=[5102, 12138, 9520, 13996, 6888, 15568, 11090, 16010] dup_rows_per_dst=[372, 1464, 322, 1124, 1678, 1592, 424, 1016] combine_rows_per_src=[5102, 12138, 9520, 13996, 6888, 15568, 11090, 16010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92378 payload_rows_remote=76172 dup_rows=5926 dup_rows_remote=5310 dedup_ratio=0.060282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 542, 0, 89, 7476, 1042, 0, 0, 89, 0, 0, 5, 0, 0, 53, 0, 250, 0, 0, 0, 0, 0, 0, 761, 7730, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 8103, 90, 0, 0, 687, 15, 0, 113, 0, 273, 0, 0, 0, 0, 5467, 0, 0, 0, 0, 355, 7908, 0, 0, 0, 524] payload_rows_per_dst=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] dup_rows_per_dst=[1084, 1986, 616, 1522, 0, 210, 234, 274] combine_rows_per_src=[14952, 12650, 16206, 15460, 15160, 1600, 11246, 5104] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255281648 prefetch_bytes=795869184 reduce_bytes=692060160 weight_movement_bytes=1487929344 egress_bytes=6131360240 ingress_bytes=7611850752 link_bytes=7611850752 total_nvlink_bytes=13743210992 weight_to_token_ratio=0.121411 prefetch_share_of_ingress=0.104557 dedup_ratio=0.208448 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=20 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245338329e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90994 payload_rows_remote=75590 dup_rows=7310 dup_rows_remote=5772 dedup_ratio=0.074361 router_tokens_per_expert=[0, 0, 21, 33, 41, 21, 673, 11, 76, 7449, 1248, 0, 0, 77, 2, 0, 20, 3, 0, 65, 35, 249, 29, 0, 1, 8, 0, 0, 795, 7641, 26, 0, 1, 36, 0, 3, 6, 7496, 2, 8076, 76, 0, 3, 797, 47, 3, 163, 1, 290, 2, 0, 0, 0, 4792, 0, 0, 0, 0, 371, 7810, 0, 0, 1, 652] payload_rows_per_dst=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] dup_rows_per_dst=[1508, 152, 726, 1538, 4, 1436, 90, 1856] combine_rows_per_src=[14990, 2654, 16228, 15404, 1574, 14254, 10078, 15812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86036 payload_rows_remote=70030 dup_rows=12268 dup_rows_remote=4158 dedup_ratio=0.124797 router_tokens_per_expert=[0, 0, 4321, 3, 4, 0, 0, 0, 17, 23, 0, 0, 7621, 0, 622, 2, 5, 372, 47, 0, 0, 0, 48, 0, 447, 3, 6, 0, 0, 2, 2, 6545, 7730, 0, 146, 0, 25, 0, 151, 0, 398, 760, 1, 13, 26, 7453, 7792, 1, 31, 457, 0, 2, 268, 0, 72, 0, 0, 30, 3679, 24, 1, 1, 1, 0] payload_rows_per_dst=[16006, 15806, 16174, 12820, 352, 15950, 1506, 7422] dup_rows_per_dst=[8110, 1188, 354, 766, 292, 1354, 154, 50] combine_rows_per_src=[16006, 15806, 16174, 12820, 352, 15950, 1506, 7422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81662 payload_rows_remote=70066 dup_rows=16642 dup_rows_remote=14818 dedup_ratio=0.169291 router_tokens_per_expert=[6, 222, 1170, 1670, 1662, 853, 514, 115, 81, 5088, 783, 0, 0, 1158, 144, 108, 62, 336, 0, 212, 1580, 307, 1019, 0, 1, 408, 0, 54, 885, 4495, 975, 0, 56, 1732, 9, 160, 666, 3822, 204, 6221, 69, 31, 1, 2431, 818, 123, 1112, 92, 440, 39, 49, 3, 17, 823, 0, 0, 0, 0, 766, 4436, 43, 242, 85, 754] payload_rows_per_dst=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] dup_rows_per_dst=[8094, 1824, 3534, 912, 430, 1540, 126, 182] combine_rows_per_src=[14506, 11596, 15940, 12724, 12868, 7814, 2616, 3598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=306151424 dispatch_meta_bytes=324576 combine_payload_bytes=306151424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306476000 ingress_bytes=340754432 link_bytes=340754432 token_bytes=612627424 weight_movement_bytes=34603008 total_nvlink_bytes=647230432 weight_to_token_ratio=0.056483 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.282231495e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=312000512 dispatch_meta_bytes=325928 combine_payload_bytes=312000512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312326440 ingress_bytes=346603520 link_bytes=346603520 token_bytes=624326952 weight_movement_bytes=34603008 total_nvlink_bytes=658929960 weight_to_token_ratio=0.055424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=43392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.304241142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309616640 dispatch_meta_bytes=325448 combine_payload_bytes=309616640 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=309942088 ingress_bytes=413425664 link_bytes=413425664 token_bytes=619558728 weight_movement_bytes=103809024 total_nvlink_bytes=723367752 weight_to_token_ratio=0.167553 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.555687490e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=286842880 dispatch_meta_bytes=296752 combine_payload_bytes=286842880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287139632 ingress_bytes=321445888 link_bytes=321445888 token_bytes=573982512 weight_movement_bytes=34603008 total_nvlink_bytes=608585520 weight_to_token_ratio=0.060286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.209574998e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=286990336 dispatch_meta_bytes=339536 combine_payload_bytes=286990336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287329872 ingress_bytes=356196352 link_bytes=356196352 token_bytes=574320208 weight_movement_bytes=69206016 total_nvlink_bytes=643526224 weight_to_token_ratio=0.120501 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=20736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.340338196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315875328 dispatch_meta_bytes=324384 combine_payload_bytes=315875328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316199712 ingress_bytes=350478336 link_bytes=350478336 token_bytes=632075040 weight_movement_bytes=34603008 total_nvlink_bytes=666678048 weight_to_token_ratio=0.054745 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318821763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14115210512 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=7061603600 ingress_bytes=8749154304 link_bytes=8749154304 total_nvlink_bytes=15810757904 weight_to_token_ratio=0.120122 prefetch_share_of_ingress=0.071190 dedup_ratio=0.137880 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.431407099e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13794185856 prefetch_bytes=830472192 reduce_bytes=726663168 weight_movement_bytes=1557135360 egress_bytes=6901002880 ingress_bytes=8450318336 link_bytes=8450318336 total_nvlink_bytes=15351321216 weight_to_token_ratio=0.112883 prefetch_share_of_ingress=0.098277 dedup_ratio=0.143341 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=21 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.382515982e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13688605240 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6848211512 ingress_bytes=8259117056 link_bytes=8259117056 total_nvlink_bytes=15107328568 weight_to_token_ratio=0.103643 prefetch_share_of_ingress=0.087983 dedup_ratio=0.144384 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.351234459e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13945272408 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6976583768 ingress_bytes=8491220992 link_bytes=8491220992 total_nvlink_bytes=15467804760 weight_to_token_ratio=0.109179 prefetch_share_of_ingress=0.089653 dedup_ratio=0.134268 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.389207869e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12675550512 prefetch_bytes=830472192 reduce_bytes=449839104 weight_movement_bytes=1280311296 egress_bytes=6341651760 ingress_bytes=7614210048 link_bytes=7614210048 total_nvlink_bytes=13955861808 weight_to_token_ratio=0.101006 prefetch_share_of_ingress=0.109069 dedup_ratio=0.205946 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=13 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245724322e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92090 payload_rows_remote=77118 dup_rows=6214 dup_rows_remote=3978 dedup_ratio=0.063212 router_tokens_per_expert=[0, 17, 15, 28, 27, 16, 1044, 3, 72, 7454, 715, 0, 0, 77, 3, 10, 14, 5, 0, 49, 27, 198, 18, 0, 0, 2, 0, 0, 550, 7650, 16, 0, 1, 28, 0, 3, 18, 7619, 0, 8091, 72, 0, 2, 1052, 40, 0, 106, 0, 261, 1, 0, 0, 0, 5247, 0, 0, 0, 0, 359, 7859, 0, 1, 1, 381] payload_rows_per_dst=[14972, 15814, 16240, 15350, 15332, 2320, 10578, 1484] dup_rows_per_dst=[2236, 1658, 564, 1086, 6, 224, 440, 0] combine_rows_per_src=[14972, 15814, 16240, 15350, 15332, 2320, 10578, 1484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315875328 dispatch_meta_bytes=324384 combine_payload_bytes=315875328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316199712 ingress_bytes=350478336 link_bytes=350478336 token_bytes=632075040 weight_movement_bytes=34603008 total_nvlink_bytes=666678048 weight_to_token_ratio=0.054745 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318821763e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12466860072 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=6237106216 ingress_bytes=7786889216 link_bytes=7786889216 total_nvlink_bytes=14023995432 weight_to_token_ratio=0.124902 prefetch_share_of_ingress=0.093319 dedup_ratio=0.193368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.273975531e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none - [2026-08-26 16:55:28] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 797.2 | throughput per GPU (TFLOP/s/GPU): 160.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.680684E+00 | loss scale: 1.0 | grad norm: 194.973 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:07:36] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 800.9 | throughput per GPU (TFLOP/s/GPU): 159.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.680684E+00 | loss scale: 1.0 | grad norm: 194.973 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75018 payload_rows_remote=64086 dup_rows=23286 dup_rows_remote=20664 dedup_ratio=0.236877 router_tokens_per_expert=[1143, 331, 908, 1182, 728, 704, 813, 999, 1317, 357, 937, 343, 374, 1343, 959, 1033, 746, 1098, 885, 687, 690, 718, 378, 1276, 537, 677, 719, 378, 777, 105, 931, 542, 1031, 838, 1179, 810, 386, 200, 497, 877, 649, 573, 585, 706, 1098, 673, 715, 687, 812, 519, 940, 380, 960, 1275, 931, 802, 1272, 301, 1408, 544, 211, 832, 1012, 834] payload_rows_per_dst=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] dup_rows_per_dst=[2070, 3858, 1744, 3468, 3570, 2612, 3342, 2622] combine_rows_per_src=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75346 payload_rows_remote=66536 dup_rows=22958 dup_rows_remote=20206 dedup_ratio=0.233541 router_tokens_per_expert=[1171, 361, 891, 1215, 700, 732, 809, 1069, 1294, 356, 902, 362, 331, 1297, 908, 1076, 744, 974, 849, 661, 657, 756, 398, 1399, 505, 711, 777, 379, 752, 94, 926, 687, 1003, 840, 1211, 855, 360, 220, 429, 863, 633, 660, 608, 779, 1115, 607, 688, 658, 762, 554, 907, 443, 940, 1290, 757, 779, 1218, 264, 1269, 628, 182, 813, 1084, 990] payload_rows_per_dst=[8648, 9220, 10330, 9194, 8810, 8788, 9914, 10442] dup_rows_per_dst=[2972, 3678, 2546, 2898, 2752, 2708, 2950, 2454] combine_rows_per_src=[8648, 9220, 10330, 9194, 8810, 8788, 9914, 10442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75246 payload_rows_remote=65336 dup_rows=23058 dup_rows_remote=20084 dedup_ratio=0.234558 router_tokens_per_expert=[1160, 324, 860, 1170, 671, 724, 767, 1049, 1229, 318, 901, 339, 383, 1325, 901, 1032, 803, 997, 794, 603, 750, 726, 437, 1366, 557, 661, 806, 330, 737, 93, 909, 817, 1073, 910, 1193, 872, 377, 224, 431, 811, 675, 638, 573, 818, 1105, 591, 775, 769, 750, 584, 863, 434, 954, 1203, 865, 789, 1182, 281, 1126, 616, 193, 856, 1073, 1009] payload_rows_per_dst=[8100, 9276, 9504, 9506, 9692, 9066, 9910, 10192] dup_rows_per_dst=[2616, 3580, 2114, 3048, 3424, 2822, 2974, 2480] combine_rows_per_src=[8100, 9276, 9504, 9506, 9692, 9066, 9910, 10192] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75242 payload_rows_remote=66138 dup_rows=23062 dup_rows_remote=20486 dedup_ratio=0.234599 router_tokens_per_expert=[1090, 342, 874, 1233, 686, 980, 835, 884, 1234, 370, 771, 374, 357, 1188, 841, 1160, 776, 1093, 792, 571, 670, 709, 370, 1425, 414, 640, 866, 303, 756, 95, 987, 839, 1076, 832, 1248, 886, 375, 214, 527, 823, 571, 662, 629, 878, 1095, 619, 740, 646, 633, 521, 790, 493, 1119, 1205, 999, 829, 1163, 281, 1150, 669, 218, 858, 1026, 852] payload_rows_per_dst=[8612, 8974, 10200, 9330, 9010, 9104, 9968, 10044] dup_rows_per_dst=[2770, 3616, 2612, 2936, 2952, 2576, 3210, 2390] combine_rows_per_src=[8612, 8974, 10200, 9330, 9010, 9104, 9968, 10044] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74994 payload_rows_remote=66186 dup_rows=23310 dup_rows_remote=19882 dedup_ratio=0.237122 router_tokens_per_expert=[1105, 349, 854, 1167, 606, 796, 816, 983, 1278, 378, 977, 345, 371, 1352, 923, 972, 760, 1071, 787, 597, 674, 749, 403, 1330, 464, 657, 890, 330, 824, 120, 859, 759, 1171, 836, 1182, 892, 349, 204, 579, 760, 590, 608, 607, 835, 1159, 632, 668, 632, 799, 604, 800, 460, 1001, 1198, 921, 804, 1204, 272, 1156, 638, 243, 843, 1076, 883] payload_rows_per_dst=[9652, 8808, 10092, 9198, 9008, 10414, 8928, 8894] dup_rows_per_dst=[4070, 3428, 2650, 2942, 2938, 3444, 1850, 1988] combine_rows_per_src=[9652, 8808, 10092, 9198, 9008, 10414, 8928, 8894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75246 payload_rows_remote=65336 dup_rows=23058 dup_rows_remote=20084 dedup_ratio=0.234558 router_tokens_per_expert=[1160, 324, 860, 1170, 671, 724, 767, 1049, 1229, 318, 901, 339, 383, 1325, 901, 1032, 803, 997, 794, 603, 750, 726, 437, 1366, 557, 661, 806, 330, 737, 93, 909, 817, 1073, 910, 1193, 872, 377, 224, 431, 811, 675, 638, 573, 818, 1105, 591, 775, 769, 750, 584, 863, 434, 954, 1203, 865, 789, 1182, 281, 1126, 616, 193, 856, 1073, 1009] payload_rows_per_dst=[8100, 9276, 9504, 9506, 9692, 9066, 9910, 10192] dup_rows_per_dst=[2616, 3580, 2114, 3048, 3424, 2822, 2974, 2480] combine_rows_per_src=[8100, 9276, 9504, 9506, 9692, 9066, 9910, 10192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75018 payload_rows_remote=64086 dup_rows=23286 dup_rows_remote=20664 dedup_ratio=0.236877 router_tokens_per_expert=[1143, 331, 908, 1182, 728, 704, 813, 999, 1317, 357, 937, 343, 374, 1343, 959, 1033, 746, 1098, 885, 687, 690, 718, 378, 1276, 537, 677, 719, 378, 777, 105, 931, 542, 1031, 838, 1179, 810, 386, 200, 497, 877, 649, 573, 585, 706, 1098, 673, 715, 687, 812, 519, 940, 380, 960, 1275, 931, 802, 1272, 301, 1408, 544, 211, 832, 1012, 834] payload_rows_per_dst=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] dup_rows_per_dst=[2070, 3858, 1744, 3468, 3570, 2612, 3342, 2622] combine_rows_per_src=[6896, 9468, 8660, 10514, 9510, 8760, 10278, 10932] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74864 payload_rows_remote=65788 dup_rows=23440 dup_rows_remote=20506 dedup_ratio=0.238444 router_tokens_per_expert=[1124, 333, 768, 1145, 617, 755, 768, 1082, 1380, 288, 939, 347, 382, 1350, 916, 1107, 703, 986, 781, 599, 638, 703, 359, 1511, 529, 701, 740, 346, 786, 111, 889, 758, 1092, 809, 1243, 776, 552, 217, 439, 991, 625, 615, 602, 903, 1078, 634, 743, 652, 652, 525, 846, 521, 984, 1324, 866, 761, 1171, 316, 1138, 613, 142, 824, 1023, 1034] payload_rows_per_dst=[9598, 7794, 9998, 9076, 9370, 8938, 9920, 10170] dup_rows_per_dst=[3996, 2924, 2562, 2934, 2868, 2766, 3038, 2352] combine_rows_per_src=[9598, 7794, 9998, 9076, 9370, 8938, 9920, 10170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=64774 dup_rows=23742 dup_rows_remote=21030 dedup_ratio=0.241516 router_tokens_per_expert=[1158, 369, 863, 1175, 666, 999, 862, 817, 1312, 531, 1046, 324, 281, 1440, 826, 876, 763, 1014, 823, 639, 576, 768, 393, 1274, 415, 663, 873, 441, 828, 147, 746, 558, 1062, 814, 1279, 990, 428, 192, 545, 788, 525, 588, 658, 642, 1073, 746, 658, 471, 834, 624, 767, 499, 928, 1353, 896, 694, 1166, 342, 1395, 628, 222, 852, 1187, 840] payload_rows_per_dst=[9800, 7520, 9788, 8840, 9146, 9718, 9200, 10550] dup_rows_per_dst=[4548, 2872, 2712, 2852, 3050, 2732, 2262, 2714] combine_rows_per_src=[9800, 7520, 9788, 8840, 9146, 9718, 9200, 10550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75346 payload_rows_remote=66536 dup_rows=22958 dup_rows_remote=20206 dedup_ratio=0.233541 router_tokens_per_expert=[1171, 361, 891, 1215, 700, 732, 809, 1069, 1294, 356, 902, 362, 331, 1297, 908, 1076, 744, 974, 849, 661, 657, 756, 398, 1399, 505, 711, 777, 379, 752, 94, 926, 687, 1003, 840, 1211, 855, 360, 220, 429, 863, 633, 660, 608, 779, 1115, 607, 688, 658, 762, 554, 907, 443, 940, 1290, 757, 779, 1218, 264, 1269, 628, 182, 813, 1084, 990] payload_rows_per_dst=[8648, 9220, 10330, 9194, 8810, 8788, 9914, 10442] dup_rows_per_dst=[2972, 3678, 2546, 2898, 2752, 2708, 2950, 2454] combine_rows_per_src=[8648, 9220, 10330, 9194, 8810, 8788, 9914, 10442] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262496256 dispatch_meta_bytes=339000 combine_payload_bytes=262496256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262835256 ingress_bytes=331702272 link_bytes=331702272 token_bytes=525331512 weight_movement_bytes=69206016 total_nvlink_bytes=594537528 weight_to_token_ratio=0.131738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.248168945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272531456 dispatch_meta_bytes=346968 combine_payload_bytes=272531456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272878424 ingress_bytes=307134464 link_bytes=307134464 token_bytes=545409880 weight_movement_bytes=34603008 total_nvlink_bytes=580012888 weight_to_token_ratio=0.063444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155722262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267616256 dispatch_meta_bytes=341680 combine_payload_bytes=267616256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267957936 ingress_bytes=336822272 link_bytes=336822272 token_bytes=535574192 weight_movement_bytes=69206016 total_nvlink_bytes=604780208 weight_to_token_ratio=0.129218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267435093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=270901248 dispatch_meta_bytes=346496 combine_payload_bytes=270901248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271247744 ingress_bytes=305504256 link_bytes=305504256 token_bytes=542148992 weight_movement_bytes=34603008 total_nvlink_bytes=576752000 weight_to_token_ratio=0.063826 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.149587920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271097856 dispatch_meta_bytes=344272 combine_payload_bytes=271097856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271442128 ingress_bytes=340303872 link_bytes=340303872 token_bytes=542539984 weight_movement_bytes=69206016 total_nvlink_bytes=611746000 weight_to_token_ratio=0.127559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.280536074e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267616256 dispatch_meta_bytes=341680 combine_payload_bytes=267616256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267957936 ingress_bytes=336822272 link_bytes=336822272 token_bytes=535574192 weight_movement_bytes=69206016 total_nvlink_bytes=604780208 weight_to_token_ratio=0.129218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267435093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262496256 dispatch_meta_bytes=339000 combine_payload_bytes=262496256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262835256 ingress_bytes=331702272 link_bytes=331702272 token_bytes=525331512 weight_movement_bytes=69206016 total_nvlink_bytes=594537528 weight_to_token_ratio=0.131738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.248168945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=269467648 dispatch_meta_bytes=345176 combine_payload_bytes=269467648 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269812824 ingress_bytes=338673664 link_bytes=338673664 token_bytes=539280472 weight_movement_bytes=69206016 total_nvlink_bytes=608486488 weight_to_token_ratio=0.128330 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=9600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274401732e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343216 combine_payload_bytes=265314304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265657520 ingress_bytes=369123328 link_bytes=369123328 token_bytes=530971824 weight_movement_bytes=103809024 total_nvlink_bytes=634780848 weight_to_token_ratio=0.195508 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388981366e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=272531456 dispatch_meta_bytes=346968 combine_payload_bytes=272531456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272878424 ingress_bytes=307134464 link_bytes=307134464 token_bytes=545409880 weight_movement_bytes=34603008 total_nvlink_bytes=580012888 weight_to_token_ratio=0.063444 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155722262e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75276 payload_rows_remote=65468 dup_rows=23028 dup_rows_remote=20138 dedup_ratio=0.234253 router_tokens_per_expert=[707, 880, 1110, 545, 767, 209, 378, 1210, 71, 1050, 1162, 335, 565, 930, 1060, 1318, 1023, 1182, 698, 321, 1394, 206, 1646, 1416, 603, 770, 641, 875, 321, 368, 221, 104, 405, 292, 547, 188, 1063, 1660, 1158, 1005, 2221, 1572, 1116, 1461, 432, 1068, 30, 219, 805, 1101, 997, 12, 131, 1610, 203, 1067, 745, 401, 889, 549, 960, 377, 360, 422] payload_rows_per_dst=[9102, 7766, 9728, 9738, 9116, 9662, 10356, 9808] dup_rows_per_dst=[2510, 2448, 2752, 2510, 1786, 4770, 3362, 2890] combine_rows_per_src=[9102, 7766, 9728, 9738, 9116, 9662, 10356, 9808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76008 payload_rows_remote=66514 dup_rows=22296 dup_rows_remote=20338 dedup_ratio=0.226807 router_tokens_per_expert=[733, 993, 1149, 398, 841, 256, 326, 1046, 77, 959, 1313, 376, 607, 1013, 1031, 1318, 956, 1062, 846, 373, 1534, 161, 1773, 1142, 710, 597, 645, 801, 244, 357, 159, 74, 442, 331, 471, 251, 921, 1902, 1103, 1120, 2038, 1631, 1200, 1300, 441, 1354, 27, 234, 701, 1137, 761, 15, 169, 1509, 213, 1221, 853, 413, 850, 528, 1132, 347, 302, 365] payload_rows_per_dst=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] dup_rows_per_dst=[2516, 2640, 2526, 2060, 2566, 4808, 1958, 3222] combine_rows_per_src=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74970 payload_rows_remote=65332 dup_rows=23334 dup_rows_remote=19016 dedup_ratio=0.237366 router_tokens_per_expert=[1089, 421, 809, 1188, 657, 787, 915, 960, 1315, 408, 894, 367, 293, 1340, 890, 1038, 738, 1046, 830, 670, 603, 764, 401, 1344, 449, 617, 844, 453, 832, 99, 805, 497, 1082, 794, 1322, 893, 391, 216, 589, 815, 647, 546, 558, 805, 1112, 684, 647, 601, 793, 591, 771, 523, 962, 1328, 865, 775, 1157, 338, 1439, 561, 222, 876, 1103, 783] payload_rows_per_dst=[9638, 9634, 10186, 8800, 9116, 10320, 8892, 8384] dup_rows_per_dst=[4318, 3654, 2606, 2768, 3088, 3536, 1668, 1696] combine_rows_per_src=[9638, 9634, 10186, 8800, 9116, 10320, 8892, 8384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=67076 dup_rows=21860 dup_rows_remote=17488 dedup_ratio=0.222371 router_tokens_per_expert=[745, 1028, 1068, 482, 788, 287, 275, 1096, 77, 985, 1247, 287, 542, 987, 1061, 1310, 994, 1296, 764, 316, 1592, 212, 1963, 1141, 601, 629, 629, 761, 243, 370, 197, 97, 477, 361, 539, 199, 887, 1892, 1027, 1094, 2161, 1523, 1282, 1194, 448, 1354, 48, 245, 749, 1156, 727, 21, 146, 1626, 219, 1319, 734, 353, 743, 499, 1116, 317, 264, 362] payload_rows_per_dst=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] dup_rows_per_dst=[2454, 2696, 2646, 2134, 2536, 4372, 2052, 2970] combine_rows_per_src=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76480 payload_rows_remote=65864 dup_rows=21824 dup_rows_remote=19082 dedup_ratio=0.222005 router_tokens_per_expert=[671, 955, 1153, 429, 792, 289, 379, 1041, 77, 911, 1138, 335, 685, 1030, 922, 1346, 1038, 1211, 762, 382, 1502, 199, 1787, 1244, 657, 619, 693, 822, 236, 335, 212, 51, 425, 308, 483, 205, 995, 1836, 1273, 1154, 2041, 1594, 1124, 1422, 456, 1106, 30, 200, 720, 1105, 844, 19, 207, 1697, 188, 1179, 787, 417, 949, 545, 911, 371, 282, 376] payload_rows_per_dst=[8872, 9852, 9912, 9140, 10616, 8364, 9826, 9898] dup_rows_per_dst=[2546, 3036, 2764, 2192, 2742, 3500, 2092, 2952] combine_rows_per_src=[8872, 9852, 9912, 9140, 10616, 8364, 9826, 9898] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76010 payload_rows_remote=65946 dup_rows=22294 dup_rows_remote=18906 dedup_ratio=0.226786 router_tokens_per_expert=[788, 972, 936, 379, 843, 310, 209, 1117, 72, 901, 1541, 363, 641, 946, 892, 1370, 930, 1141, 754, 408, 1500, 191, 1886, 1162, 700, 612, 671, 878, 219, 313, 238, 104, 433, 355, 443, 188, 986, 1577, 1057, 1173, 2134, 1618, 1020, 1269, 432, 1314, 47, 254, 914, 1110, 887, 10, 171, 1842, 190, 1212, 751, 344, 793, 492, 1179, 288, 306, 376] payload_rows_per_dst=[10186, 10064, 8914, 10022, 9924, 7666, 10248, 8986] dup_rows_per_dst=[3922, 3388, 1560, 3082, 2500, 2876, 2424, 2542] combine_rows_per_src=[10186, 10064, 8914, 10022, 9924, 7666, 10248, 8986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76008 payload_rows_remote=66514 dup_rows=22296 dup_rows_remote=20338 dedup_ratio=0.226807 router_tokens_per_expert=[733, 993, 1149, 398, 841, 256, 326, 1046, 77, 959, 1313, 376, 607, 1013, 1031, 1318, 956, 1062, 846, 373, 1534, 161, 1773, 1142, 710, 597, 645, 801, 244, 357, 159, 74, 442, 331, 471, 251, 921, 1902, 1103, 1120, 2038, 1631, 1200, 1300, 441, 1354, 27, 234, 701, 1137, 761, 15, 169, 1509, 213, 1221, 853, 413, 850, 528, 1132, 347, 302, 365] payload_rows_per_dst=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] dup_rows_per_dst=[2516, 2640, 2526, 2060, 2566, 4808, 1958, 3222] combine_rows_per_src=[8968, 8112, 9622, 9190, 10516, 10202, 9494, 9904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75276 payload_rows_remote=65468 dup_rows=23028 dup_rows_remote=20138 dedup_ratio=0.234253 router_tokens_per_expert=[707, 880, 1110, 545, 767, 209, 378, 1210, 71, 1050, 1162, 335, 565, 930, 1060, 1318, 1023, 1182, 698, 321, 1394, 206, 1646, 1416, 603, 770, 641, 875, 321, 368, 221, 104, 405, 292, 547, 188, 1063, 1660, 1158, 1005, 2221, 1572, 1116, 1461, 432, 1068, 30, 219, 805, 1101, 997, 12, 131, 1610, 203, 1067, 745, 401, 889, 549, 960, 377, 360, 422] payload_rows_per_dst=[9102, 7766, 9728, 9738, 9116, 9662, 10356, 9808] dup_rows_per_dst=[2510, 2448, 2752, 2510, 1786, 4770, 3362, 2890] combine_rows_per_src=[9102, 7766, 9728, 9738, 9116, 9662, 10356, 9808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76444 payload_rows_remote=67076 dup_rows=21860 dup_rows_remote=17488 dedup_ratio=0.222371 router_tokens_per_expert=[745, 1028, 1068, 482, 788, 287, 275, 1096, 77, 985, 1247, 287, 542, 987, 1061, 1310, 994, 1296, 764, 316, 1592, 212, 1963, 1141, 601, 629, 629, 761, 243, 370, 197, 97, 477, 361, 539, 199, 887, 1892, 1027, 1094, 2161, 1523, 1282, 1194, 448, 1354, 48, 245, 749, 1156, 727, 21, 146, 1626, 219, 1319, 734, 353, 743, 499, 1116, 317, 264, 362] payload_rows_per_dst=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] dup_rows_per_dst=[2454, 2696, 2646, 2134, 2536, 4372, 2052, 2970] combine_rows_per_src=[9084, 8744, 9984, 9242, 10416, 9368, 9874, 9732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76718 payload_rows_remote=67398 dup_rows=21586 dup_rows_remote=19688 dedup_ratio=0.219584 router_tokens_per_expert=[678, 1074, 1130, 387, 939, 303, 374, 982, 110, 929, 1308, 340, 708, 986, 948, 1331, 985, 1225, 806, 353, 1565, 125, 1761, 1073, 645, 607, 687, 827, 239, 321, 225, 103, 394, 248, 509, 217, 1052, 1809, 1082, 1195, 1955, 1619, 1311, 1394, 490, 1146, 29, 213, 653, 1135, 854, 11, 145, 1546, 218, 1203, 677, 345, 833, 544, 1140, 339, 343, 429] payload_rows_per_dst=[9134, 10128, 9644, 9320, 10548, 8628, 9624, 9692] dup_rows_per_dst=[2600, 3192, 2620, 1898, 2464, 3776, 1906, 3130] combine_rows_per_src=[9134, 10128, 9644, 9320, 10548, 8628, 9624, 9692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=337392 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267937264 ingress_bytes=371408896 link_bytes=371408896 token_bytes=535537136 weight_movement_bytes=103809024 total_nvlink_bytes=639346160 weight_to_token_ratio=0.193841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397581775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76426 payload_rows_remote=66714 dup_rows=21878 dup_rows_remote=19300 dedup_ratio=0.222555 router_tokens_per_expert=[801, 1046, 904, 356, 896, 261, 207, 1128, 73, 867, 1496, 378, 507, 764, 848, 1407, 798, 1350, 778, 439, 1422, 297, 1929, 1332, 753, 770, 791, 883, 224, 351, 197, 96, 433, 386, 402, 187, 968, 1332, 1144, 1004, 2120, 1533, 891, 1212, 464, 1438, 50, 254, 1083, 1083, 950, 12, 148, 1603, 199, 1301, 849, 343, 827, 503, 1046, 331, 365, 342] payload_rows_per_dst=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] dup_rows_per_dst=[2730, 3018, 2578, 2598, 1984, 3308, 2588, 3074] combine_rows_per_src=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76480 payload_rows_remote=65864 dup_rows=21824 dup_rows_remote=19082 dedup_ratio=0.222005 router_tokens_per_expert=[671, 955, 1153, 429, 792, 289, 379, 1041, 77, 911, 1138, 335, 685, 1030, 922, 1346, 1038, 1211, 762, 382, 1502, 199, 1787, 1244, 657, 619, 693, 822, 236, 335, 212, 51, 425, 308, 483, 205, 995, 1836, 1273, 1154, 2041, 1594, 1124, 1422, 456, 1106, 30, 200, 720, 1105, 844, 19, 207, 1697, 188, 1179, 787, 417, 949, 545, 911, 371, 282, 376] payload_rows_per_dst=[8872, 9852, 9912, 9140, 10616, 8364, 9826, 9898] dup_rows_per_dst=[2546, 3036, 2764, 2192, 2742, 3500, 2092, 2952] combine_rows_per_src=[8872, 9852, 9912, 9140, 10616, 8364, 9826, 9898] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274743296 dispatch_meta_bytes=338256 combine_payload_bytes=274743296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275081552 ingress_bytes=378552320 link_bytes=378552320 token_bytes=549824848 weight_movement_bytes=103809024 total_nvlink_bytes=653633872 weight_to_token_ratio=0.188804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424461904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=342424 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268499352 ingress_bytes=302759936 link_bytes=302759936 token_bytes=536656280 weight_movement_bytes=34603008 total_nvlink_bytes=571259288 weight_to_token_ratio=0.064479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139261265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272441344 dispatch_meta_bytes=347408 combine_payload_bytes=272441344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272788752 ingress_bytes=307044352 link_bytes=307044352 token_bytes=545230096 weight_movement_bytes=34603008 total_nvlink_bytes=579833104 weight_to_token_ratio=0.063465 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155383177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269778944 dispatch_meta_bytes=339784 combine_payload_bytes=269778944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270118728 ingress_bytes=373587968 link_bytes=373587968 token_bytes=539897672 weight_movement_bytes=103809024 total_nvlink_bytes=643706696 weight_to_token_ratio=0.192275 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405781447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=339408 combine_payload_bytes=270114816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=270454224 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540569040 weight_movement_bytes=69206016 total_nvlink_bytes=609775056 weight_to_token_ratio=0.128024 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272441344 dispatch_meta_bytes=347408 combine_payload_bytes=272441344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272788752 ingress_bytes=307044352 link_bytes=307044352 token_bytes=545230096 weight_movement_bytes=34603008 total_nvlink_bytes=579833104 weight_to_token_ratio=0.063465 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.155383177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268156928 dispatch_meta_bytes=342424 combine_payload_bytes=268156928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268499352 ingress_bytes=302759936 link_bytes=302759936 token_bytes=536656280 weight_movement_bytes=34603008 total_nvlink_bytes=571259288 weight_to_token_ratio=0.064479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139261265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76426 payload_rows_remote=66714 dup_rows=21878 dup_rows_remote=19300 dedup_ratio=0.222555 router_tokens_per_expert=[801, 1046, 904, 356, 896, 261, 207, 1128, 73, 867, 1496, 378, 507, 764, 848, 1407, 798, 1350, 778, 439, 1422, 297, 1929, 1332, 753, 770, 791, 883, 224, 351, 197, 96, 433, 386, 402, 187, 968, 1332, 1144, 1004, 2120, 1533, 891, 1212, 464, 1438, 50, 254, 1083, 1083, 950, 12, 148, 1603, 199, 1301, 849, 343, 827, 503, 1046, 331, 365, 342] payload_rows_per_dst=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] dup_rows_per_dst=[2730, 3018, 2578, 2598, 1984, 3308, 2588, 3074] combine_rows_per_src=[9010, 9662, 9712, 9772, 9728, 8376, 10170, 9996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274743296 dispatch_meta_bytes=338256 combine_payload_bytes=274743296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275081552 ingress_bytes=378552320 link_bytes=378552320 token_bytes=549824848 weight_movement_bytes=103809024 total_nvlink_bytes=653633872 weight_to_token_ratio=0.188804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=10624 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424461904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=276062208 dispatch_meta_bytes=348344 combine_payload_bytes=276062208 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276410552 ingress_bytes=345268224 link_bytes=345268224 token_bytes=552472760 weight_movement_bytes=69206016 total_nvlink_bytes=621678776 weight_to_token_ratio=0.125266 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=17024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299216531e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=337392 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267937264 ingress_bytes=371408896 link_bytes=371408896 token_bytes=535537136 weight_movement_bytes=103809024 total_nvlink_bytes=639346160 weight_to_token_ratio=0.193841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397581775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=273260544 dispatch_meta_bytes=344056 combine_payload_bytes=273260544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=273604600 ingress_bytes=377069568 link_bytes=377069568 token_bytes=546865144 weight_movement_bytes=103809024 total_nvlink_bytes=650674168 weight_to_token_ratio=0.189826 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.418882428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269778944 dispatch_meta_bytes=339784 combine_payload_bytes=269778944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270118728 ingress_bytes=373587968 link_bytes=373587968 token_bytes=539897672 weight_movement_bytes=103809024 total_nvlink_bytes=643706696 weight_to_token_ratio=0.192275 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.405781447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77098 payload_rows_remote=67400 dup_rows=21206 dup_rows_remote=17164 dedup_ratio=0.215719 router_tokens_per_expert=[238, 848, 378, 419, 91, 851, 1095, 224, 566, 211, 906, 1093, 1099, 1636, 839, 2081, 921, 1323, 254, 689, 54, 403, 1065, 970, 265, 976, 867, 543, 297, 619, 851, 310, 397, 1028, 712, 626, 1758, 1381, 2048, 510, 689, 202, 464, 272, 1587, 1571, 210, 1875, 1266, 702, 748, 854, 96, 286, 332, 769, 795, 291, 98, 870, 220, 1172, 591, 1750] payload_rows_per_dst=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] dup_rows_per_dst=[3038, 1312, 1870, 1220, 4616, 4042, 3564, 1544] combine_rows_per_src=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75834 payload_rows_remote=65452 dup_rows=22470 dup_rows_remote=18182 dedup_ratio=0.228577 router_tokens_per_expert=[724, 933, 940, 539, 920, 235, 321, 1410, 73, 996, 1412, 411, 568, 1034, 904, 1381, 956, 1107, 715, 272, 1313, 235, 1680, 1532, 739, 618, 784, 898, 215, 416, 217, 96, 401, 310, 441, 155, 956, 1314, 1033, 995, 2244, 1476, 917, 1245, 367, 1436, 44, 305, 969, 1165, 1018, 8, 127, 1607, 196, 1075, 888, 375, 933, 528, 831, 410, 374, 415] payload_rows_per_dst=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] dup_rows_per_dst=[4288, 3672, 2674, 4266, 1718, 1932, 2396, 1524] combine_rows_per_src=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=64740 dup_rows=22216 dup_rows_remote=19762 dedup_ratio=0.225993 router_tokens_per_expert=[209, 866, 514, 436, 62, 1100, 958, 247, 776, 299, 953, 1182, 1293, 1243, 571, 1825, 945, 1254, 270, 715, 94, 294, 1017, 1041, 466, 780, 876, 600, 324, 610, 1039, 295, 396, 1175, 858, 659, 1596, 1712, 2135, 472, 699, 155, 319, 255, 1583, 1665, 173, 1487, 1196, 512, 911, 898, 143, 334, 300, 774, 989, 394, 82, 469, 195, 1120, 418, 1924] payload_rows_per_dst=[9230, 7952, 9374, 8592, 11568, 7668, 10356, 11348] dup_rows_per_dst=[3018, 1250, 1886, 1388, 5460, 3330, 3430, 2454] combine_rows_per_src=[9230, 7952, 9374, 8592, 11568, 7668, 10356, 11348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77254 payload_rows_remote=66720 dup_rows=21050 dup_rows_remote=17510 dedup_ratio=0.214132 router_tokens_per_expert=[220, 813, 455, 340, 92, 908, 929, 242, 641, 311, 823, 1257, 1344, 1342, 814, 2086, 890, 1256, 255, 715, 73, 408, 1197, 1033, 293, 836, 798, 533, 290, 640, 1023, 334, 337, 977, 759, 671, 1718, 1632, 2080, 553, 538, 197, 456, 284, 1635, 1431, 165, 1568, 1139, 643, 855, 887, 74, 260, 326, 767, 942, 419, 67, 697, 233, 1281, 502, 1868] payload_rows_per_dst=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] dup_rows_per_dst=[2920, 1540, 1986, 1158, 4608, 3534, 3540, 1764] combine_rows_per_src=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75484 payload_rows_remote=64316 dup_rows=22820 dup_rows_remote=18400 dedup_ratio=0.232137 router_tokens_per_expert=[106, 1008, 436, 359, 72, 929, 1010, 189, 654, 278, 1027, 1290, 1102, 1234, 412, 1797, 882, 1540, 333, 822, 97, 266, 1249, 1055, 342, 926, 997, 613, 396, 591, 984, 202, 390, 983, 663, 514, 1970, 1488, 1770, 504, 737, 137, 362, 272, 1587, 1804, 207, 1720, 1096, 562, 741, 908, 62, 149, 308, 688, 865, 346, 91, 731, 166, 1616, 374, 2143] payload_rows_per_dst=[9022, 11168, 10256, 10800, 7220, 9300, 7326, 10392] dup_rows_per_dst=[2736, 4420, 2776, 3242, 1864, 3808, 1702, 2272] combine_rows_per_src=[9022, 11168, 10256, 10800, 7220, 9300, 7326, 10392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76984 payload_rows_remote=68966 dup_rows=21320 dup_rows_remote=19072 dedup_ratio=0.216878 router_tokens_per_expert=[286, 1012, 380, 442, 90, 966, 869, 241, 699, 324, 898, 1337, 1272, 1519, 880, 1914, 884, 1268, 266, 784, 73, 361, 1250, 968, 290, 825, 772, 429, 409, 643, 931, 322, 351, 1002, 699, 614, 1580, 1510, 2005, 512, 679, 199, 332, 269, 1593, 1540, 195, 1526, 1145, 689, 916, 757, 85, 308, 356, 850, 918, 313, 84, 690, 222, 1276, 536, 1797] payload_rows_per_dst=[9406, 10616, 9704, 9666, 8018, 8768, 10702, 10104] dup_rows_per_dst=[3176, 3084, 2004, 2004, 2248, 3898, 3338, 1568] combine_rows_per_src=[9406, 10616, 9704, 9666, 8018, 8768, 10702, 10104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77098 payload_rows_remote=67400 dup_rows=21206 dup_rows_remote=17164 dedup_ratio=0.215719 router_tokens_per_expert=[238, 848, 378, 419, 91, 851, 1095, 224, 566, 211, 906, 1093, 1099, 1636, 839, 2081, 921, 1323, 254, 689, 54, 403, 1065, 970, 265, 976, 867, 543, 297, 619, 851, 310, 397, 1028, 712, 626, 1758, 1381, 2048, 510, 689, 202, 464, 272, 1587, 1571, 210, 1875, 1266, 702, 748, 854, 96, 286, 332, 769, 795, 291, 98, 870, 220, 1172, 591, 1750] payload_rows_per_dst=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] dup_rows_per_dst=[3038, 1312, 1870, 1220, 4616, 4042, 3564, 1544] combine_rows_per_src=[9346, 8116, 9488, 8236, 11480, 9698, 10704, 10030] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76838 payload_rows_remote=66368 dup_rows=21466 dup_rows_remote=18772 dedup_ratio=0.218363 router_tokens_per_expert=[242, 952, 404, 454, 121, 1055, 994, 228, 771, 326, 933, 1260, 1230, 1406, 839, 1968, 865, 1211, 250, 755, 70, 384, 1150, 898, 370, 852, 775, 508, 278, 727, 996, 329, 381, 1082, 740, 700, 1747, 1586, 2055, 498, 640, 140, 324, 305, 1556, 1607, 148, 1519, 1055, 646, 814, 873, 71, 309, 293, 666, 984, 382, 69, 701, 153, 1289, 475, 1743] payload_rows_per_dst=[9634, 10532, 9256, 10470, 7826, 8772, 10212, 10136] dup_rows_per_dst=[3376, 2998, 1910, 2694, 2148, 3706, 3178, 1456] combine_rows_per_src=[9634, 10532, 9256, 10470, 7826, 8772, 10212, 10136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77254 payload_rows_remote=66720 dup_rows=21050 dup_rows_remote=17510 dedup_ratio=0.214132 router_tokens_per_expert=[220, 813, 455, 340, 92, 908, 929, 242, 641, 311, 823, 1257, 1344, 1342, 814, 2086, 890, 1256, 255, 715, 73, 408, 1197, 1033, 293, 836, 798, 533, 290, 640, 1023, 334, 337, 977, 759, 671, 1718, 1632, 2080, 553, 538, 197, 456, 284, 1635, 1431, 165, 1568, 1139, 643, 855, 887, 74, 260, 326, 767, 942, 419, 67, 697, 233, 1281, 502, 1868] payload_rows_per_dst=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] dup_rows_per_dst=[2920, 1540, 1986, 1158, 4608, 3534, 3540, 1764] combine_rows_per_src=[9238, 8840, 9668, 8336, 11370, 9014, 10534, 10254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=64740 dup_rows=22216 dup_rows_remote=19762 dedup_ratio=0.225993 router_tokens_per_expert=[209, 866, 514, 436, 62, 1100, 958, 247, 776, 299, 953, 1182, 1293, 1243, 571, 1825, 945, 1254, 270, 715, 94, 294, 1017, 1041, 466, 780, 876, 600, 324, 610, 1039, 295, 396, 1175, 858, 659, 1596, 1712, 2135, 472, 699, 155, 319, 255, 1583, 1665, 173, 1487, 1196, 512, 911, 898, 143, 334, 300, 774, 989, 394, 82, 469, 195, 1120, 418, 1924] payload_rows_per_dst=[9230, 7952, 9374, 8592, 11568, 7668, 10356, 11348] dup_rows_per_dst=[3018, 1250, 1886, 1388, 5460, 3330, 3430, 2454] combine_rows_per_src=[9230, 7952, 9374, 8592, 11568, 7668, 10356, 11348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=338008 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265513048 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530688088 weight_movement_bytes=34603008 total_nvlink_bytes=565291096 weight_to_token_ratio=0.065204 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273285120 dispatch_meta_bytes=336920 combine_payload_bytes=273285120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273622040 ingress_bytes=307888128 link_bytes=307888128 token_bytes=546907160 weight_movement_bytes=34603008 total_nvlink_bytes=581510168 weight_to_token_ratio=0.063270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158558239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263438336 dispatch_meta_bytes=330864 combine_payload_bytes=263438336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263769200 ingress_bytes=367247360 link_bytes=367247360 token_bytes=527207536 weight_movement_bytes=103809024 total_nvlink_bytes=631016560 weight_to_token_ratio=0.196904 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381922250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282484736 dispatch_meta_bytes=352152 combine_payload_bytes=282484736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282836888 ingress_bytes=386293760 link_bytes=386293760 token_bytes=565321624 weight_movement_bytes=103809024 total_nvlink_bytes=669130648 weight_to_token_ratio=0.183628 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453592320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75834 payload_rows_remote=65452 dup_rows=22470 dup_rows_remote=18182 dedup_ratio=0.228577 router_tokens_per_expert=[724, 933, 940, 539, 920, 235, 321, 1410, 73, 996, 1412, 411, 568, 1034, 904, 1381, 956, 1107, 715, 272, 1313, 235, 1680, 1532, 739, 618, 784, 898, 215, 416, 217, 96, 401, 310, 441, 155, 956, 1314, 1033, 995, 2244, 1476, 917, 1245, 367, 1436, 44, 305, 969, 1165, 1018, 8, 127, 1607, 196, 1075, 888, 375, 933, 528, 831, 410, 374, 415] payload_rows_per_dst=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] dup_rows_per_dst=[4288, 3672, 2674, 4266, 1718, 1932, 2396, 1524] combine_rows_per_src=[10382, 9886, 10668, 11140, 9144, 6696, 9934, 7984] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76074 payload_rows_remote=66260 dup_rows=22230 dup_rows_remote=19562 dedup_ratio=0.226135 router_tokens_per_expert=[92, 1114, 368, 446, 24, 971, 1036, 219, 574, 284, 1201, 1451, 1113, 1052, 298, 1762, 900, 1372, 555, 670, 112, 240, 1233, 1159, 495, 852, 1043, 643, 422, 624, 1042, 184, 438, 958, 741, 401, 2196, 1579, 1683, 496, 866, 162, 378, 397, 1482, 1626, 162, 1592, 1098, 424, 476, 965, 57, 159, 283, 661, 995, 310, 129, 641, 124, 1623, 370, 2129] payload_rows_per_dst=[9144, 9390, 9814, 11434, 7414, 9276, 9282, 10320] dup_rows_per_dst=[2762, 2600, 2668, 3568, 1812, 4054, 2444, 2322] combine_rows_per_src=[9144, 9390, 9814, 11434, 7414, 9276, 9282, 10320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276070400 dispatch_meta_bytes=338256 combine_payload_bytes=276070400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276408656 ingress_bytes=379879424 link_bytes=379879424 token_bytes=552479056 weight_movement_bytes=103809024 total_nvlink_bytes=656288080 weight_to_token_ratio=0.187897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429455690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271843328 dispatch_meta_bytes=340560 combine_payload_bytes=271843328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272183888 ingress_bytes=306446336 link_bytes=306446336 token_bytes=544027216 weight_movement_bytes=34603008 total_nvlink_bytes=578630224 weight_to_token_ratio=0.063605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153132891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76984 payload_rows_remote=68966 dup_rows=21320 dup_rows_remote=19072 dedup_ratio=0.216878 router_tokens_per_expert=[286, 1012, 380, 442, 90, 966, 869, 241, 699, 324, 898, 1337, 1272, 1519, 880, 1914, 884, 1268, 266, 784, 73, 361, 1250, 968, 290, 825, 772, 429, 409, 643, 931, 322, 351, 1002, 699, 614, 1580, 1510, 2005, 512, 679, 199, 332, 269, 1593, 1540, 195, 1526, 1145, 689, 916, 757, 85, 308, 356, 850, 918, 313, 84, 690, 222, 1276, 536, 1797] payload_rows_per_dst=[9406, 10616, 9704, 9666, 8018, 8768, 10702, 10104] dup_rows_per_dst=[3176, 3084, 2004, 2004, 2248, 3898, 3338, 1568] combine_rows_per_src=[9406, 10616, 9704, 9666, 8018, 8768, 10702, 10104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=276070400 dispatch_meta_bytes=338256 combine_payload_bytes=276070400 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=276408656 ingress_bytes=379879424 link_bytes=379879424 token_bytes=552479056 weight_movement_bytes=103809024 total_nvlink_bytes=656288080 weight_to_token_ratio=0.187897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429455690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=263438336 dispatch_meta_bytes=330864 combine_payload_bytes=263438336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=263769200 ingress_bytes=367247360 link_bytes=367247360 token_bytes=527207536 weight_movement_bytes=103809024 total_nvlink_bytes=631016560 weight_to_token_ratio=0.196904 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381922250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268091392 dispatch_meta_bytes=334536 combine_payload_bytes=268091392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268425928 ingress_bytes=302694400 link_bytes=302694400 token_bytes=536517320 weight_movement_bytes=34603008 total_nvlink_bytes=571120328 weight_to_token_ratio=0.064496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139014658e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271843328 dispatch_meta_bytes=340560 combine_payload_bytes=271843328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272183888 ingress_bytes=306446336 link_bytes=306446336 token_bytes=544027216 weight_movement_bytes=34603008 total_nvlink_bytes=578630224 weight_to_token_ratio=0.063605 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153132891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273285120 dispatch_meta_bytes=336920 combine_payload_bytes=273285120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273622040 ingress_bytes=307888128 link_bytes=307888128 token_bytes=546907160 weight_movement_bytes=34603008 total_nvlink_bytes=581510168 weight_to_token_ratio=0.063270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158558239e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265175040 dispatch_meta_bytes=338008 combine_payload_bytes=265175040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265513048 ingress_bytes=299778048 link_bytes=299778048 token_bytes=530688088 weight_movement_bytes=34603008 total_nvlink_bytes=565291096 weight_to_token_ratio=0.065204 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=15616 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128040661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=271400960 dispatch_meta_bytes=343288 combine_payload_bytes=271400960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271744248 ingress_bytes=306003968 link_bytes=306003968 token_bytes=543145208 weight_movement_bytes=34603008 total_nvlink_bytes=577748216 weight_to_token_ratio=0.063709 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.151468296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282484736 dispatch_meta_bytes=352152 combine_payload_bytes=282484736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282836888 ingress_bytes=386293760 link_bytes=386293760 token_bytes=565321624 weight_movement_bytes=103809024 total_nvlink_bytes=669130648 weight_to_token_ratio=0.183628 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453592320e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=67714 dup_rows=22768 dup_rows_remote=20564 dedup_ratio=0.231608 router_tokens_per_expert=[433, 1045, 1190, 645, 473, 444, 959, 666, 1151, 2200, 170, 385, 849, 712, 774, 440, 1528, 67, 2201, 463, 724, 298, 19, 600, 1015, 655, 536, 480, 638, 739, 1657, 1146, 593, 703, 272, 301, 915, 531, 2177, 782, 518, 1816, 157, 456, 392, 876, 798, 2900, 246, 376, 441, 933, 740, 992, 332, 538, 270, 721, 1470, 530, 692, 739, 359, 284] payload_rows_per_dst=[9200, 7950, 8826, 10472, 9792, 7822, 9932, 11542] dup_rows_per_dst=[2510, 1012, 2974, 3260, 2756, 2204, 3664, 4388] combine_rows_per_src=[9200, 7950, 8826, 10472, 9792, 7822, 9932, 11542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75764 payload_rows_remote=64798 dup_rows=22540 dup_rows_remote=18478 dedup_ratio=0.229289 router_tokens_per_expert=[388, 758, 1364, 404, 565, 522, 1070, 709, 1607, 2746, 284, 392, 881, 707, 1053, 389, 1598, 51, 1989, 521, 826, 292, 13, 557, 1056, 859, 619, 464, 536, 929, 1587, 1007, 570, 709, 273, 247, 994, 664, 1537, 752, 577, 2444, 196, 470, 229, 640, 519, 2871, 116, 368, 401, 911, 578, 915, 369, 416, 266, 478, 1546, 599, 563, 739, 196, 256] payload_rows_per_dst=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] dup_rows_per_dst=[2306, 1468, 3088, 2676, 2382, 3052, 3506, 4062] combine_rows_per_src=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75842 payload_rows_remote=66062 dup_rows=22462 dup_rows_remote=19102 dedup_ratio=0.228495 router_tokens_per_expert=[485, 831, 1330, 403, 468, 389, 1073, 694, 1531, 2540, 229, 342, 814, 738, 1047, 365, 1421, 52, 2263, 475, 728, 401, 15, 665, 1055, 884, 674, 475, 501, 1023, 1433, 1000, 565, 653, 227, 239, 1246, 530, 1837, 761, 423, 2177, 195, 601, 386, 612, 645, 3171, 175, 262, 289, 998, 591, 936, 331, 448, 220, 465, 1310, 666, 770, 623, 231, 225] payload_rows_per_dst=[9016, 8756, 8948, 10812, 9524, 7854, 9780, 11152] dup_rows_per_dst=[2330, 1376, 3092, 3278, 2592, 2224, 3360, 4210] combine_rows_per_src=[9016, 8756, 8948, 10812, 9524, 7854, 9780, 11152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75530 payload_rows_remote=64428 dup_rows=22774 dup_rows_remote=19662 dedup_ratio=0.231669 router_tokens_per_expert=[313, 604, 1486, 351, 469, 463, 1116, 909, 1642, 2864, 250, 369, 812, 615, 1121, 362, 1628, 75, 2127, 452, 988, 443, 4, 615, 1045, 807, 625, 422, 553, 1124, 1514, 885, 566, 674, 176, 221, 1076, 633, 2052, 842, 327, 2228, 176, 473, 371, 586, 401, 2935, 41, 244, 252, 1096, 730, 896, 179, 266, 208, 338, 1612, 599, 748, 702, 192, 259] payload_rows_per_dst=[10942, 11102, 9122, 8942, 9742, 10530, 7268, 7882] dup_rows_per_dst=[3508, 3112, 3542, 1980, 2738, 4464, 1996, 1434] combine_rows_per_src=[10942, 11102, 9122, 8942, 9742, 10530, 7268, 7882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75260 payload_rows_remote=65808 dup_rows=23044 dup_rows_remote=20464 dedup_ratio=0.234416 router_tokens_per_expert=[413, 885, 1128, 415, 470, 367, 1026, 679, 1320, 2348, 145, 360, 830, 748, 1030, 445, 1491, 96, 2125, 485, 694, 352, 27, 634, 1034, 654, 682, 486, 497, 952, 1510, 1163, 528, 755, 241, 273, 889, 537, 2043, 750, 487, 2092, 188, 573, 322, 634, 660, 3129, 349, 449, 341, 935, 865, 999, 428, 601, 246, 436, 1461, 648, 699, 605, 272, 226] payload_rows_per_dst=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] dup_rows_per_dst=[2114, 1250, 3156, 3374, 2580, 2344, 4196, 4030] combine_rows_per_src=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75536 payload_rows_remote=67714 dup_rows=22768 dup_rows_remote=20564 dedup_ratio=0.231608 router_tokens_per_expert=[433, 1045, 1190, 645, 473, 444, 959, 666, 1151, 2200, 170, 385, 849, 712, 774, 440, 1528, 67, 2201, 463, 724, 298, 19, 600, 1015, 655, 536, 480, 638, 739, 1657, 1146, 593, 703, 272, 301, 915, 531, 2177, 782, 518, 1816, 157, 456, 392, 876, 798, 2900, 246, 376, 441, 933, 740, 992, 332, 538, 270, 721, 1470, 530, 692, 739, 359, 284] payload_rows_per_dst=[9200, 7950, 8826, 10472, 9792, 7822, 9932, 11542] dup_rows_per_dst=[2510, 1012, 2974, 3260, 2756, 2204, 3664, 4388] combine_rows_per_src=[9200, 7950, 8826, 10472, 9792, 7822, 9932, 11542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265412608 dispatch_meta_bytes=333104 combine_payload_bytes=265412608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265745712 ingress_bytes=300015616 link_bytes=300015616 token_bytes=531158320 weight_movement_bytes=34603008 total_nvlink_bytes=565761328 weight_to_token_ratio=0.065146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128934610e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75392 payload_rows_remote=66104 dup_rows=22912 dup_rows_remote=20656 dedup_ratio=0.233073 router_tokens_per_expert=[402, 735, 1226, 398, 448, 461, 1005, 629, 1386, 2650, 215, 371, 925, 672, 977, 411, 1796, 61, 2431, 482, 721, 389, 14, 614, 1063, 856, 702, 461, 519, 1000, 1432, 1020, 563, 715, 197, 238, 1096, 575, 1853, 689, 467, 2141, 161, 589, 363, 613, 510, 3100, 168, 342, 281, 875, 746, 973, 310, 483, 265, 514, 1506, 678, 673, 575, 224, 197] payload_rows_per_dst=[8464, 9916, 9582, 9288, 9288, 10912, 10038, 7904] dup_rows_per_dst=[2144, 2560, 3434, 2256, 2564, 4976, 3618, 1360] combine_rows_per_src=[8464, 9916, 9582, 9288, 9288, 10912, 10038, 7904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75694 payload_rows_remote=66424 dup_rows=22610 dup_rows_remote=19622 dedup_ratio=0.230001 router_tokens_per_expert=[90, 1140, 533, 403, 29, 1158, 866, 249, 603, 387, 991, 1293, 1576, 867, 236, 1290, 1004, 1449, 454, 729, 96, 151, 1053, 1112, 651, 1026, 1043, 773, 421, 562, 892, 208, 406, 1173, 905, 472, 2172, 1937, 1661, 433, 920, 151, 341, 351, 1396, 1774, 189, 1370, 998, 203, 657, 889, 59, 216, 292, 682, 1071, 441, 88, 453, 151, 1560, 216, 2190] payload_rows_per_dst=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] dup_rows_per_dst=[2988, 4062, 4034, 4010, 2360, 1416, 1458, 2282] combine_rows_per_src=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270589952 dispatch_meta_bytes=340656 combine_payload_bytes=270589952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270930608 ingress_bytes=305192960 link_bytes=305192960 token_bytes=541520560 weight_movement_bytes=34603008 total_nvlink_bytes=576123568 weight_to_token_ratio=0.063900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148416538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76214 payload_rows_remote=66936 dup_rows=22090 dup_rows_remote=18872 dedup_ratio=0.224711 router_tokens_per_expert=[228, 585, 1531, 423, 560, 509, 1135, 696, 1765, 2929, 218, 383, 767, 662, 1267, 468, 1918, 75, 2007, 460, 850, 356, 14, 568, 1009, 807, 548, 429, 578, 999, 1883, 1019, 561, 742, 239, 232, 1166, 424, 1806, 817, 366, 1938, 217, 398, 236, 619, 407, 2629, 61, 429, 261, 918, 819, 901, 263, 281, 362, 465, 1466, 528, 718, 782, 188, 267] payload_rows_per_dst=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] dup_rows_per_dst=[2884, 3164, 3218, 1876, 2306, 3694, 3404, 1544] combine_rows_per_src=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75842 payload_rows_remote=66062 dup_rows=22462 dup_rows_remote=19102 dedup_ratio=0.228495 router_tokens_per_expert=[485, 831, 1330, 403, 468, 389, 1073, 694, 1531, 2540, 229, 342, 814, 738, 1047, 365, 1421, 52, 2263, 475, 728, 401, 15, 665, 1055, 884, 674, 475, 501, 1023, 1433, 1000, 565, 653, 227, 239, 1246, 530, 1837, 761, 423, 2177, 195, 601, 386, 612, 645, 3171, 175, 262, 289, 998, 591, 936, 331, 448, 220, 465, 1310, 666, 770, 623, 231, 225] payload_rows_per_dst=[9016, 8756, 8948, 10812, 9524, 7854, 9780, 11152] dup_rows_per_dst=[2330, 1376, 3092, 3278, 2592, 2224, 3360, 4210] combine_rows_per_src=[9016, 8756, 8948, 10812, 9524, 7854, 9780, 11152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75764 payload_rows_remote=64798 dup_rows=22540 dup_rows_remote=18478 dedup_ratio=0.229289 router_tokens_per_expert=[388, 758, 1364, 404, 565, 522, 1070, 709, 1607, 2746, 284, 392, 881, 707, 1053, 389, 1598, 51, 1989, 521, 826, 292, 13, 557, 1056, 859, 619, 464, 536, 929, 1587, 1007, 570, 709, 273, 247, 994, 664, 1537, 752, 577, 2444, 196, 470, 229, 640, 519, 2871, 116, 368, 401, 911, 578, 915, 369, 416, 266, 478, 1546, 599, 563, 739, 196, 256] payload_rows_per_dst=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] dup_rows_per_dst=[2306, 1468, 3088, 2676, 2382, 3052, 3506, 4062] combine_rows_per_src=[9254, 9158, 8546, 10202, 9110, 8394, 10134, 10966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=336360 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264233448 ingress_bytes=333103104 link_bytes=333103104 token_bytes=528130536 weight_movement_bytes=69206016 total_nvlink_bytes=597336552 weight_to_token_ratio=0.131040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253440163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75260 payload_rows_remote=65808 dup_rows=23044 dup_rows_remote=20464 dedup_ratio=0.234416 router_tokens_per_expert=[413, 885, 1128, 415, 470, 367, 1026, 679, 1320, 2348, 145, 360, 830, 748, 1030, 445, 1491, 96, 2125, 485, 694, 352, 27, 634, 1034, 654, 682, 486, 497, 952, 1510, 1163, 528, 755, 241, 273, 889, 537, 2043, 750, 487, 2092, 188, 573, 322, 634, 660, 3129, 349, 449, 341, 935, 865, 999, 428, 601, 246, 436, 1461, 648, 699, 605, 272, 226] payload_rows_per_dst=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] dup_rows_per_dst=[2114, 1250, 3156, 3374, 2580, 2344, 4196, 4030] combine_rows_per_src=[8652, 8506, 8652, 10582, 9452, 8082, 10434, 10900] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269549568 dispatch_meta_bytes=345088 combine_payload_bytes=269549568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269894656 ingress_bytes=304152576 link_bytes=304152576 token_bytes=539444224 weight_movement_bytes=34603008 total_nvlink_bytes=574047232 weight_to_token_ratio=0.064146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144501657e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76214 payload_rows_remote=66936 dup_rows=22090 dup_rows_remote=18872 dedup_ratio=0.224711 router_tokens_per_expert=[228, 585, 1531, 423, 560, 509, 1135, 696, 1765, 2929, 218, 383, 767, 662, 1267, 468, 1918, 75, 2007, 460, 850, 356, 14, 568, 1009, 807, 548, 429, 578, 999, 1883, 1019, 561, 742, 239, 232, 1166, 424, 1806, 817, 366, 1938, 217, 398, 236, 619, 407, 2629, 61, 429, 261, 918, 819, 901, 263, 281, 362, 465, 1466, 528, 718, 782, 188, 267] payload_rows_per_dst=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] dup_rows_per_dst=[2884, 3164, 3218, 1876, 2306, 3694, 3404, 1544] combine_rows_per_src=[9608, 10504, 9278, 8902, 9668, 9926, 10320, 8008] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277356544 dispatch_meta_bytes=353112 combine_payload_bytes=277356544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277709656 ingress_bytes=381165568 link_bytes=381165568 token_bytes=555066200 weight_movement_bytes=103809024 total_nvlink_bytes=658875224 weight_to_token_ratio=0.187021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.434295346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=336360 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264233448 ingress_bytes=333103104 link_bytes=333103104 token_bytes=528130536 weight_movement_bytes=69206016 total_nvlink_bytes=597336552 weight_to_token_ratio=0.131040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.253440163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75694 payload_rows_remote=66424 dup_rows=22610 dup_rows_remote=19622 dedup_ratio=0.230001 router_tokens_per_expert=[90, 1140, 533, 403, 29, 1158, 866, 249, 603, 387, 991, 1293, 1576, 867, 236, 1290, 1004, 1449, 454, 729, 96, 151, 1053, 1112, 651, 1026, 1043, 773, 421, 562, 892, 208, 406, 1173, 905, 472, 2172, 1937, 1661, 433, 920, 151, 341, 351, 1396, 1774, 189, 1370, 998, 203, 657, 889, 59, 216, 292, 682, 1071, 441, 88, 453, 151, 1560, 216, 2190] payload_rows_per_dst=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] dup_rows_per_dst=[2988, 4062, 4034, 4010, 2360, 1416, 1458, 2282] combine_rows_per_src=[9270, 10424, 11610, 11486, 8292, 8020, 6534, 10058] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270761984 dispatch_meta_bytes=347040 combine_payload_bytes=270761984 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=271109024 ingress_bytes=409174016 link_bytes=409174016 token_bytes=541871008 weight_movement_bytes=138412032 total_nvlink_bytes=680283040 weight_to_token_ratio=0.255434 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.539688881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270589952 dispatch_meta_bytes=340656 combine_payload_bytes=270589952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270930608 ingress_bytes=305192960 link_bytes=305192960 token_bytes=541520560 weight_movement_bytes=34603008 total_nvlink_bytes=576123568 weight_to_token_ratio=0.063900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.148416538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265412608 dispatch_meta_bytes=333104 combine_payload_bytes=265412608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265745712 ingress_bytes=300015616 link_bytes=300015616 token_bytes=531158320 weight_movement_bytes=34603008 total_nvlink_bytes=565761328 weight_to_token_ratio=0.065146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128934610e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=344184 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272416888 ingress_bytes=306675712 link_bytes=306675712 token_bytes=544489592 weight_movement_bytes=34603008 total_nvlink_bytes=579092600 weight_to_token_ratio=0.063551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153996015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=63732 dup_rows=24306 dup_rows_remote=18812 dedup_ratio=0.247253 router_tokens_per_expert=[3, 1598, 1591, 720, 986, 1073, 717, 1011, 309, 713, 2223, 62, 164, 414, 360, 1228, 1532, 371, 218, 95, 360, 478, 268, 273, 691, 402, 1766, 586, 1805, 984, 809, 36, 474, 14, 32, 622, 1384, 587, 177, 1432, 76, 307, 331, 1031, 604, 0, 2257, 5, 175, 608, 232, 919, 457, 1411, 731, 1230, 865, 985, 131, 1177, 1056, 2330, 1927, 1739] payload_rows_per_dst=[7716, 8602, 9504, 8128, 9302, 10316, 10164, 10266] dup_rows_per_dst=[1304, 2344, 2346, 2420, 3338, 2516, 4544, 5494] combine_rows_per_src=[7716, 8602, 9504, 8128, 9302, 10316, 10164, 10266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=274169856 dispatch_meta_bytes=343232 combine_payload_bytes=274169856 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=274513088 ingress_bytes=308772864 link_bytes=308772864 token_bytes=548682944 weight_movement_bytes=34603008 total_nvlink_bytes=583285952 weight_to_token_ratio=0.063066 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161887429e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269549568 dispatch_meta_bytes=345088 combine_payload_bytes=269549568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269894656 ingress_bytes=304152576 link_bytes=304152576 token_bytes=539444224 weight_movement_bytes=34603008 total_nvlink_bytes=574047232 weight_to_token_ratio=0.064146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144501657e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=344184 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272416888 ingress_bytes=306675712 link_bytes=306675712 token_bytes=544489592 weight_movement_bytes=34603008 total_nvlink_bytes=579092600 weight_to_token_ratio=0.063551 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153996015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73562 payload_rows_remote=64548 dup_rows=24742 dup_rows_remote=21736 dedup_ratio=0.251689 router_tokens_per_expert=[3, 1883, 1557, 568, 1119, 1131, 821, 867, 235, 536, 1896, 93, 269, 323, 339, 1445, 1467, 397, 179, 148, 286, 281, 206, 356, 691, 418, 1905, 615, 1789, 918, 1110, 91, 349, 22, 84, 569, 1459, 462, 204, 1399, 172, 264, 352, 1020, 764, 6, 2378, 9, 361, 520, 270, 889, 535, 1243, 647, 1335, 878, 725, 164, 1236, 1138, 2378, 2001, 1377] payload_rows_per_dst=[8184, 8314, 9360, 8680, 9448, 10524, 9014, 10038] dup_rows_per_dst=[3528, 1958, 2036, 2816, 3414, 2984, 3006, 5000] combine_rows_per_src=[8184, 8314, 9360, 8680, 9448, 10524, 9014, 10038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=63472 dup_rows=24316 dup_rows_remote=20058 dedup_ratio=0.247355 router_tokens_per_expert=[0, 1602, 1436, 821, 1031, 1396, 666, 1095, 415, 504, 2193, 86, 185, 274, 280, 1318, 1610, 354, 112, 98, 254, 611, 264, 269, 572, 351, 2479, 419, 2399, 842, 859, 8, 297, 1, 20, 652, 1238, 307, 115, 1254, 97, 260, 108, 1017, 653, 0, 2438, 0, 105, 811, 303, 934, 423, 1267, 856, 1302, 679, 1062, 85, 1458, 1114, 1996, 2132, 1365] payload_rows_per_dst=[9082, 10516, 10380, 10772, 8320, 8580, 9022, 7316] dup_rows_per_dst=[3808, 4258, 3486, 4634, 2652, 1018, 2980, 1480] combine_rows_per_src=[9082, 10516, 10380, 10772, 8320, 8580, 9022, 7316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73014 payload_rows_remote=63030 dup_rows=25290 dup_rows_remote=21328 dedup_ratio=0.257263 router_tokens_per_expert=[10, 1862, 1492, 510, 1107, 1304, 944, 649, 362, 431, 1623, 131, 274, 323, 365, 1599, 1504, 429, 203, 169, 409, 318, 215, 326, 497, 413, 1503, 754, 1355, 1301, 1033, 115, 548, 81, 94, 393, 1526, 715, 267, 1487, 208, 392, 662, 1101, 821, 29, 1887, 18, 529, 381, 249, 1193, 530, 1052, 730, 1340, 861, 775, 203, 1087, 1040, 2329, 1738, 1356] payload_rows_per_dst=[8202, 8086, 9448, 8462, 9984, 10076, 9116, 9640] dup_rows_per_dst=[3830, 2130, 2356, 2770, 3962, 2870, 2892, 4480] combine_rows_per_src=[8202, 8086, 9448, 8462, 9984, 10076, 9116, 9640] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73182 payload_rows_remote=63410 dup_rows=25122 dup_rows_remote=22356 dedup_ratio=0.255554 router_tokens_per_expert=[13, 1941, 1445, 499, 996, 1485, 958, 833, 405, 321, 1698, 169, 361, 291, 432, 1707, 1328, 413, 202, 233, 432, 308, 227, 242, 549, 333, 1466, 702, 1272, 1366, 1157, 155, 480, 81, 143, 552, 1591, 913, 360, 1244, 135, 341, 834, 925, 962, 26, 1741, 33, 529, 363, 279, 1151, 511, 997, 613, 1204, 865, 750, 203, 950, 1006, 2274, 1723, 1434] payload_rows_per_dst=[8564, 8552, 9158, 8458, 10370, 9772, 8748, 9560] dup_rows_per_dst=[3894, 2216, 2160, 2998, 4240, 2766, 2546, 4302] combine_rows_per_src=[8564, 8552, 9158, 8458, 10370, 9772, 8748, 9560] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261046272 dispatch_meta_bytes=330176 combine_payload_bytes=261046272 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=261376448 ingress_bytes=434061312 link_bytes=434061312 token_bytes=522422720 weight_movement_bytes=173015040 total_nvlink_bytes=695437760 weight_to_token_ratio=0.331178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633337772e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73334 payload_rows_remote=64550 dup_rows=24970 dup_rows_remote=22172 dedup_ratio=0.254008 router_tokens_per_expert=[2, 1733, 1537, 655, 1034, 1209, 777, 823, 407, 568, 1914, 92, 244, 325, 342, 1577, 1628, 392, 142, 118, 362, 371, 206, 193, 478, 426, 1970, 572, 1644, 1152, 1112, 81, 372, 8, 50, 557, 1419, 562, 282, 1324, 124, 351, 290, 957, 717, 1, 2249, 0, 382, 487, 249, 1000, 520, 1219, 737, 1355, 698, 887, 116, 1244, 1207, 2377, 1874, 1451] payload_rows_per_dst=[8244, 9372, 9450, 8784, 9234, 10006, 8972, 9272] dup_rows_per_dst=[3830, 3094, 2128, 2798, 3380, 2660, 2926, 4154] combine_rows_per_src=[8244, 9372, 9450, 8784, 9234, 10006, 8972, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=63472 dup_rows=24316 dup_rows_remote=20058 dedup_ratio=0.247355 router_tokens_per_expert=[0, 1602, 1436, 821, 1031, 1396, 666, 1095, 415, 504, 2193, 86, 185, 274, 280, 1318, 1610, 354, 112, 98, 254, 611, 264, 269, 572, 351, 2479, 419, 2399, 842, 859, 8, 297, 1, 20, 652, 1238, 307, 115, 1254, 97, 260, 108, 1017, 653, 0, 2438, 0, 105, 811, 303, 934, 423, 1267, 856, 1302, 679, 1062, 85, 1458, 1114, 1996, 2132, 1365] payload_rows_per_dst=[9082, 10516, 10380, 10772, 8320, 8580, 9022, 7316] dup_rows_per_dst=[3808, 4258, 3486, 4634, 2652, 1018, 2980, 1480] combine_rows_per_src=[9082, 10516, 10380, 10772, 8320, 8580, 9022, 7316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=63732 dup_rows=24306 dup_rows_remote=18812 dedup_ratio=0.247253 router_tokens_per_expert=[3, 1598, 1591, 720, 986, 1073, 717, 1011, 309, 713, 2223, 62, 164, 414, 360, 1228, 1532, 371, 218, 95, 360, 478, 268, 273, 691, 402, 1766, 586, 1805, 984, 809, 36, 474, 14, 32, 622, 1384, 587, 177, 1432, 76, 307, 331, 1031, 604, 0, 2257, 5, 175, 608, 232, 919, 457, 1411, 731, 1230, 865, 985, 131, 1177, 1056, 2330, 1927, 1739] payload_rows_per_dst=[7716, 8602, 9504, 8128, 9302, 10316, 10164, 10266] dup_rows_per_dst=[1304, 2344, 2346, 2420, 3338, 2516, 4544, 5494] combine_rows_per_src=[7716, 8602, 9504, 8128, 9302, 10316, 10164, 10266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73562 payload_rows_remote=64548 dup_rows=24742 dup_rows_remote=21736 dedup_ratio=0.251689 router_tokens_per_expert=[3, 1883, 1557, 568, 1119, 1131, 821, 867, 235, 536, 1896, 93, 269, 323, 339, 1445, 1467, 397, 179, 148, 286, 281, 206, 356, 691, 418, 1905, 615, 1789, 918, 1110, 91, 349, 22, 84, 569, 1459, 462, 204, 1399, 172, 264, 352, 1020, 764, 6, 2378, 9, 361, 520, 270, 889, 535, 1243, 647, 1335, 878, 725, 164, 1236, 1138, 2378, 2001, 1377] payload_rows_per_dst=[8184, 8314, 9360, 8680, 9448, 10524, 9014, 10038] dup_rows_per_dst=[3528, 1958, 2036, 2816, 3414, 2984, 3006, 5000] combine_rows_per_src=[8184, 8314, 9360, 8680, 9448, 10524, 9014, 10038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75386 payload_rows_remote=64146 dup_rows=22918 dup_rows_remote=18786 dedup_ratio=0.233134 router_tokens_per_expert=[241, 436, 1536, 349, 614, 793, 1130, 921, 2317, 3059, 359, 335, 840, 716, 1352, 257, 1980, 54, 1954, 479, 931, 496, 3, 526, 923, 940, 632, 409, 519, 991, 1666, 788, 582, 615, 199, 242, 1093, 522, 1399, 731, 355, 2503, 211, 479, 215, 572, 377, 2723, 12, 253, 279, 1035, 484, 894, 167, 203, 224, 240, 1694, 578, 591, 762, 123, 249] payload_rows_per_dst=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] dup_rows_per_dst=[4132, 5412, 3582, 1780, 2936, 2662, 1122, 1292] combine_rows_per_src=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259727360 dispatch_meta_bytes=343064 combine_payload_bytes=259727360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260070424 ingress_bytes=294330368 link_bytes=294330368 token_bytes=519797784 weight_movement_bytes=34603008 total_nvlink_bytes=554400792 weight_to_token_ratio=0.066570 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107541479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264388608 dispatch_meta_bytes=345136 combine_payload_bytes=264388608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264733744 ingress_bytes=298991616 link_bytes=298991616 token_bytes=529122352 weight_movement_bytes=34603008 total_nvlink_bytes=563725360 weight_to_token_ratio=0.065397 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125081380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259981312 dispatch_meta_bytes=334120 combine_payload_bytes=259981312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260315432 ingress_bytes=294584320 link_bytes=294584320 token_bytes=520296744 weight_movement_bytes=34603008 total_nvlink_bytes=554899752 weight_to_token_ratio=0.066506 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108497080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258170880 dispatch_meta_bytes=337432 combine_payload_bytes=258170880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258508312 ingress_bytes=292773888 link_bytes=292773888 token_bytes=516679192 weight_movement_bytes=34603008 total_nvlink_bytes=551282200 weight_to_token_ratio=0.066972 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.101684570e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259727360 dispatch_meta_bytes=343064 combine_payload_bytes=259727360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260070424 ingress_bytes=294330368 link_bytes=294330368 token_bytes=519797784 weight_movement_bytes=34603008 total_nvlink_bytes=554400792 weight_to_token_ratio=0.066570 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.107541479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73664 payload_rows_remote=63988 dup_rows=24640 dup_rows_remote=21446 dedup_ratio=0.250651 router_tokens_per_expert=[27, 1561, 1393, 512, 1027, 2227, 691, 969, 484, 506, 2079, 76, 186, 433, 380, 1147, 1622, 325, 192, 160, 502, 393, 281, 241, 439, 338, 1963, 508, 2141, 1060, 975, 55, 378, 47, 28, 741, 1096, 548, 178, 1352, 127, 508, 507, 1032, 541, 27, 2163, 44, 194, 640, 239, 909, 444, 1372, 833, 1118, 622, 915, 135, 1225, 1111, 2016, 1747, 1422] payload_rows_per_dst=[9272, 10168, 9676, 8320, 9112, 10860, 8636, 7620] dup_rows_per_dst=[4420, 3908, 3194, 2356, 2746, 3320, 2862, 1834] combine_rows_per_src=[9272, 10168, 9676, 8320, 9112, 10860, 8636, 7620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73014 payload_rows_remote=63030 dup_rows=25290 dup_rows_remote=21328 dedup_ratio=0.257263 router_tokens_per_expert=[10, 1862, 1492, 510, 1107, 1304, 944, 649, 362, 431, 1623, 131, 274, 323, 365, 1599, 1504, 429, 203, 169, 409, 318, 215, 326, 497, 413, 1503, 754, 1355, 1301, 1033, 115, 548, 81, 94, 393, 1526, 715, 267, 1487, 208, 392, 662, 1101, 821, 29, 1887, 18, 529, 381, 249, 1193, 530, 1052, 730, 1340, 861, 775, 203, 1087, 1040, 2329, 1738, 1356] payload_rows_per_dst=[8202, 8086, 9448, 8462, 9984, 10076, 9116, 9640] dup_rows_per_dst=[3830, 2130, 2356, 2770, 3962, 2870, 2892, 4480] combine_rows_per_src=[8202, 8086, 9448, 8462, 9984, 10076, 9116, 9640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 2, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264396800 dispatch_meta_bytes=346888 combine_payload_bytes=264396800 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264743688 ingress_bytes=333602816 link_bytes=333602816 token_bytes=529140488 weight_movement_bytes=69206016 total_nvlink_bytes=598346504 weight_to_token_ratio=0.130789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255320539e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75386 payload_rows_remote=64146 dup_rows=22918 dup_rows_remote=18786 dedup_ratio=0.233134 router_tokens_per_expert=[241, 436, 1536, 349, 614, 793, 1130, 921, 2317, 3059, 359, 335, 840, 716, 1352, 257, 1980, 54, 1954, 479, 931, 496, 3, 526, 923, 940, 632, 409, 519, 991, 1666, 788, 582, 615, 199, 242, 1093, 522, 1399, 731, 355, 2503, 211, 479, 215, 572, 377, 2723, 12, 253, 279, 1035, 484, 894, 167, 203, 224, 240, 1694, 578, 591, 762, 123, 249] payload_rows_per_dst=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] dup_rows_per_dst=[4132, 5412, 3582, 1780, 2936, 2662, 1122, 1292] combine_rows_per_src=[11240, 13058, 9264, 8624, 10874, 9164, 5532, 7630] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=259981312 dispatch_meta_bytes=334120 combine_payload_bytes=259981312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260315432 ingress_bytes=294584320 link_bytes=294584320 token_bytes=520296744 weight_movement_bytes=34603008 total_nvlink_bytes=554899752 weight_to_token_ratio=0.066506 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.108497080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262742016 dispatch_meta_bytes=331728 combine_payload_bytes=262742016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263073744 ingress_bytes=297345024 link_bytes=297345024 token_bytes=525815760 weight_movement_bytes=34603008 total_nvlink_bytes=560418768 weight_to_token_ratio=0.065808 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118885387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261046272 dispatch_meta_bytes=330176 combine_payload_bytes=261046272 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=261376448 ingress_bytes=434061312 link_bytes=434061312 token_bytes=522422720 weight_movement_bytes=173015040 total_nvlink_bytes=695437760 weight_to_token_ratio=0.331178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.633337772e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264388608 dispatch_meta_bytes=345136 combine_payload_bytes=264388608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264733744 ingress_bytes=298991616 link_bytes=298991616 token_bytes=529122352 weight_movement_bytes=34603008 total_nvlink_bytes=563725360 weight_to_token_ratio=0.065397 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125081380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76516 payload_rows_remote=63948 dup_rows=21788 dup_rows_remote=17678 dedup_ratio=0.221639 router_tokens_per_expert=[788, 1543, 840, 228, 193, 154, 3016, 456, 20, 0, 912, 667, 1428, 1, 5, 794, 793, 846, 3417, 105, 513, 638, 299, 10, 135, 2281, 767, 133, 292, 235, 3437, 503, 1665, 572, 170, 1812, 511, 315, 1300, 1056, 117, 3, 124, 1497, 144, 635, 1596, 592, 845, 958, 1376, 1132, 177, 655, 4, 1545, 1075, 473, 486, 599, 277, 479, 866, 647] payload_rows_per_dst=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] dup_rows_per_dst=[1318, 2798, 410, 1050, 5526, 3412, 3164, 4110] combine_rows_per_src=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=262094848 dispatch_meta_bytes=341736 combine_payload_bytes=262094848 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262436584 ingress_bytes=331300864 link_bytes=331300864 token_bytes=524531432 weight_movement_bytes=69206016 total_nvlink_bytes=593737448 weight_to_token_ratio=0.131939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246658479e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=258170880 dispatch_meta_bytes=337432 combine_payload_bytes=258170880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258508312 ingress_bytes=292773888 link_bytes=292773888 token_bytes=516679192 weight_movement_bytes=34603008 total_nvlink_bytes=551282200 weight_to_token_ratio=0.066972 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.101684570e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77456 payload_rows_remote=67646 dup_rows=20848 dup_rows_remote=17070 dedup_ratio=0.212077 router_tokens_per_expert=[861, 1554, 1350, 253, 529, 358, 2956, 404, 39, 3, 900, 800, 1355, 0, 15, 1257, 1030, 876, 3085, 104, 611, 393, 279, 7, 283, 1716, 925, 143, 360, 280, 2701, 679, 1405, 434, 214, 1761, 697, 349, 1641, 930, 420, 0, 139, 1987, 122, 527, 1374, 464, 712, 1099, 869, 511, 194, 734, 0, 1308, 862, 464, 402, 603, 381, 635, 1166, 642] payload_rows_per_dst=[8620, 11068, 10322, 7734, 9220, 9810, 8868, 11814] dup_rows_per_dst=[1998, 3582, 2448, 1038, 2120, 3778, 1986, 3898] combine_rows_per_src=[8620, 11068, 10322, 7734, 9220, 9810, 8868, 11814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277078016 dispatch_meta_bytes=338864 combine_payload_bytes=277078016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277416880 ingress_bytes=311681024 link_bytes=311681024 token_bytes=554494896 weight_movement_bytes=34603008 total_nvlink_bytes=589097904 weight_to_token_ratio=0.062405 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172830601e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78018 payload_rows_remote=66910 dup_rows=20286 dup_rows_remote=16746 dedup_ratio=0.206360 router_tokens_per_expert=[689, 1638, 987, 219, 238, 278, 3055, 388, 23, 0, 802, 911, 1459, 0, 1, 1064, 881, 877, 3527, 103, 455, 586, 244, 2, 102, 2142, 854, 124, 300, 198, 3162, 442, 1402, 544, 250, 1557, 559, 314, 1486, 842, 220, 0, 123, 1722, 105, 664, 1613, 581, 978, 1236, 1245, 831, 133, 695, 0, 1306, 956, 451, 489, 590, 308, 554, 1010, 637] payload_rows_per_dst=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] dup_rows_per_dst=[1716, 2826, 2470, 3540, 1844, 3554, 2758, 1578] combine_rows_per_src=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=262742016 dispatch_meta_bytes=331728 combine_payload_bytes=262742016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263073744 ingress_bytes=297345024 link_bytes=297345024 token_bytes=525815760 weight_movement_bytes=34603008 total_nvlink_bytes=560418768 weight_to_token_ratio=0.065808 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118885387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76498 payload_rows_remote=69274 dup_rows=21806 dup_rows_remote=20590 dedup_ratio=0.221822 router_tokens_per_expert=[790, 1849, 754, 211, 111, 192, 3082, 504, 7, 0, 767, 670, 1749, 0, 0, 1027, 595, 1004, 3815, 66, 522, 1049, 346, 0, 110, 2477, 993, 103, 166, 123, 3147, 378, 1699, 420, 125, 1324, 514, 317, 1276, 830, 139, 0, 35, 1684, 88, 798, 1438, 499, 1083, 1057, 1356, 1141, 147, 566, 0, 1352, 1152, 402, 153, 547, 324, 532, 841, 706] payload_rows_per_dst=[11880, 7224, 11630, 11126, 9930, 7542, 9390, 7776] dup_rows_per_dst=[4458, 1216, 3164, 3868, 3080, 1820, 2662, 1538] combine_rows_per_src=[11880, 7224, 11630, 11126, 9930, 7542, 9390, 7776] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64960 dup_rows=24012 dup_rows_remote=18918 dedup_ratio=0.244263 router_tokens_per_expert=[0, 875, 1384, 1028, 865, 1416, 615, 1262, 476, 687, 2703, 69, 138, 349, 276, 1049, 1596, 350, 36, 33, 220, 709, 362, 211, 464, 311, 2305, 354, 2989, 707, 608, 0, 337, 1, 3, 992, 1087, 220, 69, 1369, 52, 271, 14, 973, 465, 0, 3243, 0, 36, 843, 236, 918, 474, 1715, 1030, 1186, 525, 986, 63, 1559, 1098, 1789, 1646, 1505] payload_rows_per_dst=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] dup_rows_per_dst=[5094, 4124, 3300, 4986, 1406, 736, 3232, 1134] combine_rows_per_src=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76516 payload_rows_remote=63948 dup_rows=21788 dup_rows_remote=17678 dedup_ratio=0.221639 router_tokens_per_expert=[788, 1543, 840, 228, 193, 154, 3016, 456, 20, 0, 912, 667, 1428, 1, 5, 794, 793, 846, 3417, 105, 513, 638, 299, 10, 135, 2281, 767, 133, 292, 235, 3437, 503, 1665, 572, 170, 1812, 511, 315, 1300, 1056, 117, 3, 124, 1497, 144, 635, 1596, 592, 845, 958, 1376, 1132, 177, 655, 4, 1545, 1075, 473, 486, 599, 277, 479, 866, 647] payload_rows_per_dst=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] dup_rows_per_dst=[1318, 2798, 410, 1050, 5526, 3412, 3164, 4110] combine_rows_per_src=[7086, 10888, 5998, 7890, 12238, 9628, 10220, 12568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77178 payload_rows_remote=67390 dup_rows=21126 dup_rows_remote=18514 dedup_ratio=0.214905 router_tokens_per_expert=[843, 1783, 1072, 204, 247, 312, 3302, 448, 62, 1, 975, 853, 1087, 0, 8, 1267, 796, 903, 2912, 64, 659, 543, 274, 0, 204, 2187, 940, 120, 291, 168, 3080, 413, 1272, 533, 211, 1455, 589, 328, 1421, 966, 369, 0, 178, 1742, 137, 648, 1306, 510, 784, 1155, 1339, 728, 165, 728, 0, 1301, 1077, 409, 413, 664, 317, 610, 1030, 749] payload_rows_per_dst=[8272, 11344, 8258, 9212, 8776, 9160, 9788, 12368] dup_rows_per_dst=[1546, 3766, 1320, 2158, 1864, 3530, 2612, 4330] combine_rows_per_src=[8272, 11344, 8258, 9212, 8776, 9160, 9788, 12368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76670 payload_rows_remote=65742 dup_rows=21634 dup_rows_remote=18846 dedup_ratio=0.220072 router_tokens_per_expert=[953, 2010, 824, 268, 208, 288, 2910, 522, 23, 20, 853, 651, 1812, 0, 3, 981, 837, 752, 3383, 46, 661, 792, 352, 35, 113, 2103, 1002, 247, 297, 244, 2484, 445, 1757, 445, 237, 2147, 465, 406, 1463, 723, 183, 55, 59, 1297, 172, 619, 1401, 581, 986, 990, 1463, 991, 294, 541, 38, 1584, 1004, 440, 363, 404, 249, 335, 868, 473] payload_rows_per_dst=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] dup_rows_per_dst=[4448, 992, 2788, 3252, 2230, 3478, 3394, 1052] combine_rows_per_src=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77248 payload_rows_remote=68132 dup_rows=21056 dup_rows_remote=19006 dedup_ratio=0.214193 router_tokens_per_expert=[1038, 1565, 1183, 225, 304, 373, 2717, 399, 86, 4, 964, 910, 1230, 0, 30, 1419, 906, 940, 2886, 162, 647, 377, 209, 14, 217, 1515, 759, 221, 347, 244, 2421, 460, 1266, 586, 315, 1869, 458, 455, 1611, 892, 473, 1, 184, 2001, 214, 621, 1341, 524, 882, 1057, 1263, 614, 223, 697, 1, 1563, 881, 473, 492, 807, 260, 666, 1140, 550] payload_rows_per_dst=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] dup_rows_per_dst=[1964, 3790, 2204, 1238, 2050, 4202, 2614, 2994] combine_rows_per_src=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274063360 dispatch_meta_bytes=334624 combine_payload_bytes=274063360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274397984 ingress_bytes=343269376 link_bytes=343269376 token_bytes=548461344 weight_movement_bytes=69206016 total_nvlink_bytes=617667360 weight_to_token_ratio=0.126182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291695026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283746304 dispatch_meta_bytes=359456 combine_payload_bytes=283746304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284105760 ingress_bytes=318349312 link_bytes=318349312 token_bytes=567852064 weight_movement_bytes=34603008 total_nvlink_bytes=602455072 weight_to_token_ratio=0.060937 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197922832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=335512 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266411672 ingress_bytes=369885184 link_bytes=369885184 token_bytes=532487832 weight_movement_bytes=103809024 total_nvlink_bytes=636296856 weight_to_token_ratio=0.194951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391848169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=326504 combine_payload_bytes=261931008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262257512 ingress_bytes=296534016 link_bytes=296534016 token_bytes=524188520 weight_movement_bytes=34603008 total_nvlink_bytes=558791528 weight_to_token_ratio=0.066013 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115833629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77456 payload_rows_remote=67646 dup_rows=20848 dup_rows_remote=17070 dedup_ratio=0.212077 router_tokens_per_expert=[861, 1554, 1350, 253, 529, 358, 2956, 404, 39, 3, 900, 800, 1355, 0, 15, 1257, 1030, 876, 3085, 104, 611, 393, 279, 7, 283, 1716, 925, 143, 360, 280, 2701, 679, 1405, 434, 214, 1761, 697, 349, 1641, 930, 420, 0, 139, 1987, 122, 527, 1374, 464, 712, 1099, 869, 511, 194, 734, 0, 1308, 862, 464, 402, 603, 381, 635, 1166, 642] payload_rows_per_dst=[8620, 11068, 10322, 7734, 9220, 9810, 8868, 11814] dup_rows_per_dst=[1998, 3582, 2448, 1038, 2120, 3778, 1986, 3898] combine_rows_per_src=[8620, 11068, 10322, 7734, 9220, 9810, 8868, 11814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77248 payload_rows_remote=68132 dup_rows=21056 dup_rows_remote=19006 dedup_ratio=0.214193 router_tokens_per_expert=[1038, 1565, 1183, 225, 304, 373, 2717, 399, 86, 4, 964, 910, 1230, 0, 30, 1419, 906, 940, 2886, 162, 647, 377, 209, 14, 217, 1515, 759, 221, 347, 244, 2421, 460, 1266, 586, 315, 1869, 458, 455, 1611, 892, 473, 1, 184, 2001, 214, 621, 1341, 524, 882, 1057, 1263, 614, 223, 697, 1, 1563, 881, 473, 492, 807, 260, 666, 1140, 550] payload_rows_per_dst=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] dup_rows_per_dst=[1964, 3790, 2204, 1238, 2050, 4202, 2614, 2994] combine_rows_per_src=[8210, 10930, 10078, 8040, 9116, 10254, 9986, 10634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78018 payload_rows_remote=66910 dup_rows=20286 dup_rows_remote=16746 dedup_ratio=0.206360 router_tokens_per_expert=[689, 1638, 987, 219, 238, 278, 3055, 388, 23, 0, 802, 911, 1459, 0, 1, 1064, 881, 877, 3527, 103, 455, 586, 244, 2, 102, 2142, 854, 124, 300, 198, 3162, 442, 1402, 544, 250, 1557, 559, 314, 1486, 842, 220, 0, 123, 1722, 105, 664, 1613, 581, 978, 1236, 1245, 831, 133, 695, 0, 1306, 956, 451, 489, 590, 308, 554, 1010, 637] payload_rows_per_dst=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] dup_rows_per_dst=[1716, 2826, 2470, 3540, 1844, 3554, 2758, 1578] combine_rows_per_src=[8438, 10524, 10880, 11108, 8950, 9616, 10090, 8412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=283746304 dispatch_meta_bytes=359456 combine_payload_bytes=283746304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284105760 ingress_bytes=318349312 link_bytes=318349312 token_bytes=567852064 weight_movement_bytes=34603008 total_nvlink_bytes=602455072 weight_to_token_ratio=0.060937 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197922832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=276029440 dispatch_meta_bytes=343616 combine_payload_bytes=276029440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=276373056 ingress_bytes=345235456 link_bytes=345235456 token_bytes=552402496 weight_movement_bytes=69206016 total_nvlink_bytes=621608512 weight_to_token_ratio=0.125282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299093227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269279232 dispatch_meta_bytes=338352 combine_payload_bytes=269279232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269617584 ingress_bytes=373088256 link_bytes=373088256 token_bytes=538896816 weight_movement_bytes=103809024 total_nvlink_bytes=642705840 weight_to_token_ratio=0.192632 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403901071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=277078016 dispatch_meta_bytes=338864 combine_payload_bytes=277078016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277416880 ingress_bytes=311681024 link_bytes=311681024 token_bytes=554494896 weight_movement_bytes=34603008 total_nvlink_bytes=589097904 weight_to_token_ratio=0.062405 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172830601e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=279068672 dispatch_meta_bytes=348552 combine_payload_bytes=279068672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=279417224 ingress_bytes=348274688 link_bytes=348274688 token_bytes=558485896 weight_movement_bytes=69206016 total_nvlink_bytes=627691912 weight_to_token_ratio=0.123917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.310529613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80748 payload_rows_remote=71764 dup_rows=17556 dup_rows_remote=15248 dedup_ratio=0.178589 router_tokens_per_expert=[712, 129, 1, 2727, 88, 9, 228, 72, 0, 466, 1983, 963, 1036, 574, 467, 1583, 0, 7, 694, 3, 237, 562, 475, 2676, 122, 63, 820, 3, 379, 547, 0, 13, 72, 1365, 348, 36, 0, 0, 0, 2381, 1544, 200, 210, 2282, 4842, 319, 1524, 2388, 339, 464, 0, 1107, 356, 873, 5854, 0, 4258, 0, 0, 23, 312, 0, 311, 105] payload_rows_per_dst=[7002, 10842, 8684, 12002, 6822, 8984, 14584, 11828] dup_rows_per_dst=[930, 3302, 1702, 1576, 1582, 2308, 3402, 2754] combine_rows_per_src=[7002, 10842, 8684, 12002, 6822, 8984, 14584, 11828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293945344 dispatch_meta_bytes=348048 combine_payload_bytes=293945344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=294293392 ingress_bytes=432357376 link_bytes=432357376 token_bytes=588238736 weight_movement_bytes=138412032 total_nvlink_bytes=726650768 weight_to_token_ratio=0.235299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.626925998e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80448 payload_rows_remote=67728 dup_rows=17856 dup_rows_remote=16496 dedup_ratio=0.181641 router_tokens_per_expert=[669, 116, 0, 3790, 63, 1, 236, 62, 0, 431, 1316, 814, 1011, 638, 511, 1269, 0, 0, 676, 0, 33, 559, 419, 2419, 121, 89, 727, 0, 290, 560, 0, 0, 62, 1533, 237, 10, 0, 0, 0, 2702, 1588, 189, 185, 2090, 5253, 124, 1531, 2791, 447, 173, 0, 1117, 294, 849, 6068, 0, 4316, 0, 0, 16, 414, 0, 289, 54] payload_rows_per_dst=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] dup_rows_per_dst=[966, 2670, 2812, 1360, 7400, 1960, 156, 532] combine_rows_per_src=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76216 payload_rows_remote=63888 dup_rows=22088 dup_rows_remote=17584 dedup_ratio=0.224691 router_tokens_per_expert=[659, 1697, 652, 213, 111, 205, 2921, 577, 1, 0, 996, 620, 2245, 0, 0, 969, 524, 828, 3699, 31, 405, 1386, 278, 0, 104, 2626, 1305, 132, 160, 50, 3484, 355, 1695, 395, 158, 1323, 376, 296, 1143, 911, 51, 0, 22, 1164, 82, 1051, 1444, 673, 1011, 1079, 1548, 1417, 144, 549, 0, 1381, 1055, 431, 103, 435, 242, 385, 744, 611] payload_rows_per_dst=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] dup_rows_per_dst=[4504, 1422, 3594, 4940, 2942, 1472, 2018, 1196] combine_rows_per_src=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76670 payload_rows_remote=65742 dup_rows=21634 dup_rows_remote=18846 dedup_ratio=0.220072 router_tokens_per_expert=[953, 2010, 824, 268, 208, 288, 2910, 522, 23, 20, 853, 651, 1812, 0, 3, 981, 837, 752, 3383, 46, 661, 792, 352, 35, 113, 2103, 1002, 247, 297, 244, 2484, 445, 1757, 445, 237, 2147, 465, 406, 1463, 723, 183, 55, 59, 1297, 172, 619, 1401, 581, 986, 990, 1463, 991, 294, 541, 38, 1584, 1004, 440, 363, 404, 249, 335, 868, 473] payload_rows_per_dst=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] dup_rows_per_dst=[4448, 992, 2788, 3252, 2230, 3478, 3394, 1052] combine_rows_per_src=[11518, 7694, 10928, 10618, 8766, 9546, 10380, 7220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64960 dup_rows=24012 dup_rows_remote=18918 dedup_ratio=0.244263 router_tokens_per_expert=[0, 875, 1384, 1028, 865, 1416, 615, 1262, 476, 687, 2703, 69, 138, 349, 276, 1049, 1596, 350, 36, 33, 220, 709, 362, 211, 464, 311, 2305, 354, 2989, 707, 608, 0, 337, 1, 3, 992, 1087, 220, 69, 1369, 52, 271, 14, 973, 465, 0, 3243, 0, 36, 843, 236, 918, 474, 1715, 1030, 1186, 525, 986, 63, 1559, 1098, 1789, 1646, 1505] payload_rows_per_dst=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] dup_rows_per_dst=[5094, 4124, 3300, 4986, 1406, 736, 3232, 1134] combine_rows_per_src=[9332, 10662, 10322, 10922, 7396, 9300, 9644, 6714] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274063360 dispatch_meta_bytes=334624 combine_payload_bytes=274063360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274397984 ingress_bytes=343269376 link_bytes=343269376 token_bytes=548461344 weight_movement_bytes=69206016 total_nvlink_bytes=617667360 weight_to_token_ratio=0.126182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.291695026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76054 payload_rows_remote=64446 dup_rows=22250 dup_rows_remote=19742 dedup_ratio=0.226339 router_tokens_per_expert=[670, 88, 2, 3779, 91, 0, 220, 63, 1, 434, 1338, 668, 1005, 654, 476, 967, 0, 1, 717, 1, 39, 636, 433, 1861, 129, 53, 835, 5, 232, 538, 6, 5, 63, 1545, 171, 27, 0, 0, 7, 3077, 1361, 164, 193, 2129, 5168, 194, 1456, 3093, 414, 183, 3, 1548, 309, 1111, 6057, 3, 4118, 1, 0, 14, 403, 2, 329, 62] payload_rows_per_dst=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] dup_rows_per_dst=[934, 2340, 972, 192, 1602, 8380, 5322, 2508] combine_rows_per_src=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=269279232 dispatch_meta_bytes=338352 combine_payload_bytes=269279232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=269617584 ingress_bytes=373088256 link_bytes=373088256 token_bytes=538896816 weight_movement_bytes=103809024 total_nvlink_bytes=642705840 weight_to_token_ratio=0.192632 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403901071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=335512 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266411672 ingress_bytes=369885184 link_bytes=369885184 token_bytes=532487832 weight_movement_bytes=103809024 total_nvlink_bytes=636296856 weight_to_token_ratio=0.194951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391848169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80444 payload_rows_remote=70224 dup_rows=17860 dup_rows_remote=13920 dedup_ratio=0.181681 router_tokens_per_expert=[655, 98, 0, 3724, 98, 0, 179, 82, 0, 355, 1249, 654, 989, 518, 421, 1537, 0, 0, 594, 0, 3, 484, 432, 2379, 106, 61, 484, 0, 127, 481, 0, 0, 83, 1635, 176, 3, 0, 0, 0, 3344, 1348, 171, 176, 1953, 5371, 215, 1594, 3541, 382, 149, 0, 1019, 208, 1054, 6192, 0, 4360, 0, 0, 1, 81, 0, 322, 64] payload_rows_per_dst=[13408, 10220, 11512, 12472, 8888, 8970, 5454, 9520] dup_rows_per_dst=[5934, 3940, 3354, 788, 1594, 1944, 170, 136] combine_rows_per_src=[13408, 10220, 11512, 12472, 8888, 8970, 5454, 9520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79324 payload_rows_remote=68426 dup_rows=18980 dup_rows_remote=15642 dedup_ratio=0.193075 router_tokens_per_expert=[814, 130, 16, 3268, 113, 32, 237, 98, 26, 593, 950, 770, 895, 345, 482, 939, 1, 15, 824, 32, 100, 799, 404, 1551, 135, 81, 464, 37, 162, 502, 7, 14, 105, 1464, 155, 36, 37, 73, 40, 3192, 1769, 199, 263, 2195, 5104, 203, 1401, 3392, 324, 572, 29, 1022, 463, 1124, 5857, 111, 4194, 77, 43, 52, 402, 4, 340, 74] payload_rows_per_dst=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] dup_rows_per_dst=[1050, 3962, 3338, 924, 5872, 2756, 612, 466] combine_rows_per_src=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82584 payload_rows_remote=67590 dup_rows=15720 dup_rows_remote=12840 dedup_ratio=0.159912 router_tokens_per_expert=[667, 105, 0, 3684, 78, 2, 239, 70, 0, 438, 1223, 833, 1052, 581, 475, 1259, 0, 0, 648, 4, 74, 520, 450, 2677, 94, 87, 643, 9, 314, 601, 0, 5, 70, 1395, 262, 23, 0, 0, 0, 2837, 1524, 203, 203, 2069, 5186, 180, 1613, 2721, 421, 244, 0, 953, 324, 879, 6116, 0, 4410, 0, 0, 7, 335, 0, 294, 51] payload_rows_per_dst=[8598, 9194, 7760, 12394, 7534, 10202, 14994, 11908] dup_rows_per_dst=[1092, 2528, 986, 1178, 1640, 2992, 2880, 2424] combine_rows_per_src=[8598, 9194, 7760, 12394, 7534, 10202, 14994, 11908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80748 payload_rows_remote=71764 dup_rows=17556 dup_rows_remote=15248 dedup_ratio=0.178589 router_tokens_per_expert=[712, 129, 1, 2727, 88, 9, 228, 72, 0, 466, 1983, 963, 1036, 574, 467, 1583, 0, 7, 694, 3, 237, 562, 475, 2676, 122, 63, 820, 3, 379, 547, 0, 13, 72, 1365, 348, 36, 0, 0, 0, 2381, 1544, 200, 210, 2282, 4842, 319, 1524, 2388, 339, 464, 0, 1107, 356, 873, 5854, 0, 4258, 0, 0, 23, 312, 0, 311, 105] payload_rows_per_dst=[7002, 10842, 8684, 12002, 6822, 8984, 14584, 11828] dup_rows_per_dst=[930, 3302, 1702, 1576, 1582, 2308, 3402, 2754] combine_rows_per_src=[7002, 10842, 8684, 12002, 6822, 8984, 14584, 11828] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80802 payload_rows_remote=72974 dup_rows=17502 dup_rows_remote=15404 dedup_ratio=0.178040 router_tokens_per_expert=[834, 156, 0, 2949, 81, 37, 259, 65, 1, 527, 1596, 1050, 1092, 526, 529, 1082, 1, 6, 734, 30, 258, 624, 464, 2794, 121, 73, 906, 109, 465, 615, 5, 9, 66, 1404, 308, 74, 1, 0, 6, 2561, 1604, 268, 257, 2120, 4726, 318, 1208, 2366, 413, 434, 4, 930, 354, 920, 5694, 0, 4301, 17, 0, 23, 412, 5, 296, 64] payload_rows_per_dst=[7600, 10194, 11142, 12044, 7828, 6328, 13680, 11986] dup_rows_per_dst=[1162, 2612, 3412, 2014, 2098, 982, 2732, 2490] combine_rows_per_src=[7600, 10194, 11142, 12044, 7828, 6328, 13680, 11986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76054 payload_rows_remote=64446 dup_rows=22250 dup_rows_remote=19742 dedup_ratio=0.226339 router_tokens_per_expert=[670, 88, 2, 3779, 91, 0, 220, 63, 1, 434, 1338, 668, 1005, 654, 476, 967, 0, 1, 717, 1, 39, 636, 433, 1861, 129, 53, 835, 5, 232, 538, 6, 5, 63, 1545, 171, 27, 0, 0, 7, 3077, 1361, 164, 193, 2129, 5168, 194, 1456, 3093, 414, 183, 3, 1548, 309, 1111, 6057, 3, 4118, 1, 0, 14, 403, 2, 329, 62] payload_rows_per_dst=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] dup_rows_per_dst=[934, 2340, 972, 192, 1602, 8380, 5322, 2508] combine_rows_per_src=[8892, 8746, 6404, 3414, 8178, 13646, 15166, 11608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277413888 dispatch_meta_bytes=336896 combine_payload_bytes=277413888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277750784 ingress_bytes=312016896 link_bytes=312016896 token_bytes=555164672 weight_movement_bytes=34603008 total_nvlink_bytes=589767680 weight_to_token_ratio=0.062329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174094460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261685248 dispatch_meta_bytes=325888 combine_payload_bytes=261685248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262011136 ingress_bytes=330891264 link_bytes=330891264 token_bytes=523696384 weight_movement_bytes=69206016 total_nvlink_bytes=592902400 weight_to_token_ratio=0.132149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245117188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263970816 dispatch_meta_bytes=336752 combine_payload_bytes=263970816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264307568 ingress_bytes=298573824 link_bytes=298573824 token_bytes=528278384 weight_movement_bytes=34603008 total_nvlink_bytes=562881392 weight_to_token_ratio=0.065501 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123509263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80802 payload_rows_remote=72974 dup_rows=17502 dup_rows_remote=15404 dedup_ratio=0.178040 router_tokens_per_expert=[834, 156, 0, 2949, 81, 37, 259, 65, 1, 527, 1596, 1050, 1092, 526, 529, 1082, 1, 6, 734, 30, 258, 624, 464, 2794, 121, 73, 906, 109, 465, 615, 5, 9, 66, 1404, 308, 74, 1, 0, 6, 2561, 1604, 268, 257, 2120, 4726, 318, 1208, 2366, 413, 434, 4, 930, 354, 920, 5694, 0, 4301, 17, 0, 23, 412, 5, 296, 64] payload_rows_per_dst=[7600, 10194, 11142, 12044, 7828, 6328, 13680, 11986] dup_rows_per_dst=[1162, 2612, 3412, 2014, 2098, 982, 2732, 2490] combine_rows_per_src=[7600, 10194, 11142, 12044, 7828, 6328, 13680, 11986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80448 payload_rows_remote=67728 dup_rows=17856 dup_rows_remote=16496 dedup_ratio=0.181641 router_tokens_per_expert=[669, 116, 0, 3790, 63, 1, 236, 62, 0, 431, 1316, 814, 1011, 638, 511, 1269, 0, 0, 676, 0, 33, 559, 419, 2419, 121, 89, 727, 0, 290, 560, 0, 0, 62, 1533, 237, 10, 0, 0, 0, 2702, 1588, 189, 185, 2090, 5253, 124, 1531, 2791, 447, 173, 0, 1117, 294, 849, 6068, 0, 4316, 0, 0, 16, 414, 0, 289, 54] payload_rows_per_dst=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] dup_rows_per_dst=[966, 2670, 2812, 1360, 7400, 1960, 156, 532] combine_rows_per_src=[8908, 9310, 10982, 12720, 13824, 8804, 5604, 10296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=287637504 dispatch_meta_bytes=336576 combine_payload_bytes=287637504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=287974080 ingress_bytes=322240512 link_bytes=322240512 token_bytes=575611584 weight_movement_bytes=34603008 total_nvlink_bytes=610214592 weight_to_token_ratio=0.060115 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.212565104e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81000 payload_rows_remote=72322 dup_rows=17304 dup_rows_remote=16082 dedup_ratio=0.176025 router_tokens_per_expert=[1780, 3666, 85, 348, 931, 242, 63, 190, 138, 326, 180, 327, 104, 329, 578, 636, 416, 502, 0, 6, 709, 84, 3019, 737, 1847, 3865, 697, 267, 245, 263, 101, 520, 0, 217, 2431, 0, 91, 1419, 894, 203, 724, 396, 0, 57, 3070, 5802, 703, 0, 5385, 0, 1710, 524, 412, 114, 370, 69, 5, 336, 0, 0, 0, 506, 110, 403] payload_rows_per_dst=[6140, 11138, 9376, 10866, 8872, 8678, 13492, 12438] dup_rows_per_dst=[1138, 1828, 1570, 3900, 2084, 1222, 3676, 1886] combine_rows_per_src=[6140, 11138, 9376, 10866, 8872, 8678, 13492, 12438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280272896 dispatch_meta_bytes=336272 combine_payload_bytes=280272896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280609168 ingress_bytes=314875904 link_bytes=314875904 token_bytes=560882064 weight_movement_bytes=34603008 total_nvlink_bytes=595485072 weight_to_token_ratio=0.061694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184852677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=276848640 dispatch_meta_bytes=321720 combine_payload_bytes=276848640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=277170360 ingress_bytes=415260672 link_bytes=415260672 token_bytes=554019000 weight_movement_bytes=138412032 total_nvlink_bytes=692431032 weight_to_token_ratio=0.249833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.562592478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=293945344 dispatch_meta_bytes=348048 combine_payload_bytes=293945344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=294293392 ingress_bytes=432357376 link_bytes=432357376 token_bytes=588238736 weight_movement_bytes=138412032 total_nvlink_bytes=726650768 weight_to_token_ratio=0.235299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.626925998e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79324 payload_rows_remote=68426 dup_rows=18980 dup_rows_remote=15642 dedup_ratio=0.193075 router_tokens_per_expert=[814, 130, 16, 3268, 113, 32, 237, 98, 26, 593, 950, 770, 895, 345, 482, 939, 1, 15, 824, 32, 100, 799, 404, 1551, 135, 81, 464, 37, 162, 502, 7, 14, 105, 1464, 155, 36, 37, 73, 40, 3192, 1769, 199, 263, 2195, 5104, 203, 1401, 3392, 324, 572, 29, 1022, 463, 1124, 5857, 111, 4194, 77, 43, 52, 402, 4, 340, 74] payload_rows_per_dst=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] dup_rows_per_dst=[1050, 3962, 3338, 924, 5872, 2756, 612, 466] combine_rows_per_src=[8366, 10196, 10898, 12088, 11888, 9304, 6678, 9906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76216 payload_rows_remote=63888 dup_rows=22088 dup_rows_remote=17584 dedup_ratio=0.224691 router_tokens_per_expert=[659, 1697, 652, 213, 111, 205, 2921, 577, 1, 0, 996, 620, 2245, 0, 0, 969, 524, 828, 3699, 31, 405, 1386, 278, 0, 104, 2626, 1305, 132, 160, 50, 3484, 355, 1695, 395, 158, 1323, 376, 296, 1143, 911, 51, 0, 22, 1164, 82, 1051, 1444, 673, 1011, 1079, 1548, 1417, 144, 549, 0, 1381, 1055, 431, 103, 435, 242, 385, 744, 611] payload_rows_per_dst=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] dup_rows_per_dst=[4504, 1422, 3594, 4940, 2942, 1472, 2018, 1196] combine_rows_per_src=[12328, 8240, 11812, 11492, 9652, 7502, 8374, 6816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=298901504 dispatch_meta_bytes=353512 combine_payload_bytes=298901504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299255016 ingress_bytes=333504512 link_bytes=333504512 token_bytes=598156520 weight_movement_bytes=34603008 total_nvlink_bytes=632759528 weight_to_token_ratio=0.057849 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.254950629e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=263970816 dispatch_meta_bytes=336752 combine_payload_bytes=263970816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264307568 ingress_bytes=298573824 link_bytes=298573824 token_bytes=528278384 weight_movement_bytes=34603008 total_nvlink_bytes=562881392 weight_to_token_ratio=0.065501 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123509263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=296230912 dispatch_meta_bytes=353616 combine_payload_bytes=296230912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296584528 ingress_bytes=365436928 link_bytes=365436928 token_bytes=592815440 weight_movement_bytes=69206016 total_nvlink_bytes=662021456 weight_to_token_ratio=0.116741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375109740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80128 payload_rows_remote=68796 dup_rows=18176 dup_rows_remote=13912 dedup_ratio=0.184896 router_tokens_per_expert=[2000, 5245, 0, 169, 676, 101, 54, 75, 0, 317, 124, 253, 83, 302, 529, 761, 357, 820, 0, 1, 506, 99, 2424, 693, 2465, 3927, 1102, 324, 142, 0, 64, 462, 0, 248, 1666, 0, 63, 1622, 757, 129, 703, 223, 0, 88, 2836, 6243, 294, 0, 5619, 0, 1221, 608, 491, 87, 342, 79, 21, 455, 0, 0, 0, 832, 118, 332] payload_rows_per_dst=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] dup_rows_per_dst=[2074, 1980, 1642, 4264, 1292, 904, 3352, 2668] combine_rows_per_src=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80488 payload_rows_remote=65664 dup_rows=17816 dup_rows_remote=9178 dedup_ratio=0.181234 router_tokens_per_expert=[660, 81, 0, 4339, 89, 0, 179, 69, 0, 407, 711, 517, 923, 527, 409, 630, 0, 0, 652, 0, 0, 599, 433, 1563, 130, 95, 316, 1, 124, 452, 0, 0, 69, 1585, 137, 4, 0, 0, 0, 3982, 1515, 43, 180, 2032, 5451, 184, 1418, 3926, 376, 98, 0, 1349, 280, 1332, 6314, 0, 4489, 0, 0, 0, 0, 0, 421, 61] payload_rows_per_dst=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] dup_rows_per_dst=[8638, 1564, 2984, 454, 1608, 2008, 316, 244] combine_rows_per_src=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277413888 dispatch_meta_bytes=336896 combine_payload_bytes=277413888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277750784 ingress_bytes=312016896 link_bytes=312016896 token_bytes=555164672 weight_movement_bytes=34603008 total_nvlink_bytes=589767680 weight_to_token_ratio=0.062329 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174094460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=280272896 dispatch_meta_bytes=336272 combine_payload_bytes=280272896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280609168 ingress_bytes=314875904 link_bytes=314875904 token_bytes=560882064 weight_movement_bytes=34603008 total_nvlink_bytes=595485072 weight_to_token_ratio=0.061694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.184852677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261685248 dispatch_meta_bytes=325888 combine_payload_bytes=261685248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=262011136 ingress_bytes=330891264 link_bytes=330891264 token_bytes=523696384 weight_movement_bytes=69206016 total_nvlink_bytes=592902400 weight_to_token_ratio=0.132149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.245117188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80284 payload_rows_remote=66686 dup_rows=18020 dup_rows_remote=15228 dedup_ratio=0.183309 router_tokens_per_expert=[2236, 5720, 20, 209, 702, 62, 52, 47, 0, 246, 152, 239, 131, 315, 564, 820, 402, 902, 9, 4, 472, 79, 2325, 820, 2395, 3887, 1102, 286, 35, 11, 138, 371, 5, 273, 1118, 4, 88, 1474, 1033, 113, 700, 175, 1, 65, 2415, 6432, 166, 4, 5802, 4, 1266, 526, 489, 92, 339, 52, 31, 413, 2, 1, 0, 844, 137, 335] payload_rows_per_dst=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] dup_rows_per_dst=[710, 1888, 1636, 1210, 5784, 438, 3562, 2792] combine_rows_per_src=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76784 payload_rows_remote=66776 dup_rows=21520 dup_rows_remote=20078 dedup_ratio=0.218913 router_tokens_per_expert=[2319, 6171, 0, 160, 565, 51, 64, 20, 0, 162, 65, 250, 109, 312, 488, 716, 408, 1027, 0, 0, 452, 88, 1630, 636, 3670, 3623, 1202, 384, 0, 0, 78, 493, 0, 248, 1026, 0, 98, 2229, 731, 140, 695, 129, 0, 60, 2730, 6279, 144, 0, 5085, 0, 786, 709, 446, 102, 224, 55, 9, 379, 0, 0, 0, 1259, 143, 303] payload_rows_per_dst=[13708, 10008, 13172, 9708, 7920, 13842, 4256, 4170] dup_rows_per_dst=[4992, 1442, 5480, 1946, 1024, 6232, 388, 16] combine_rows_per_src=[13708, 10008, 13172, 9708, 7920, 13842, 4256, 4170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77890 payload_rows_remote=67594 dup_rows=20414 dup_rows_remote=16806 dedup_ratio=0.207662 router_tokens_per_expert=[2054, 5096, 187, 258, 959, 43, 96, 96, 1, 489, 227, 432, 109, 381, 535, 564, 428, 1585, 138, 110, 464, 88, 1454, 714, 2939, 3196, 1325, 201, 297, 59, 152, 520, 56, 377, 924, 16, 79, 2234, 1220, 132, 636, 119, 56, 69, 2776, 5538, 311, 92, 4544, 158, 1164, 535, 447, 109, 296, 47, 135, 360, 100, 33, 8, 734, 296, 354] payload_rows_per_dst=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] dup_rows_per_dst=[4712, 1846, 3608, 2236, 1578, 3748, 448, 2238] combine_rows_per_src=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79208 payload_rows_remote=66036 dup_rows=19096 dup_rows_remote=15760 dedup_ratio=0.194255 router_tokens_per_expert=[2094, 4933, 26, 208, 694, 112, 51, 43, 6, 276, 124, 226, 98, 314, 529, 753, 389, 725, 0, 1, 484, 79, 2732, 640, 2581, 3854, 769, 340, 167, 0, 2, 503, 0, 228, 1819, 0, 78, 1829, 834, 169, 758, 254, 0, 84, 2959, 5874, 449, 0, 5276, 0, 1325, 615, 494, 100, 369, 75, 7, 431, 0, 0, 0, 896, 134, 342] payload_rows_per_dst=[5732, 10510, 8520, 7218, 13208, 8030, 13172, 12818] dup_rows_per_dst=[724, 1850, 1580, 1506, 6572, 978, 3336, 2550] combine_rows_per_src=[5732, 10510, 8520, 7218, 13208, 8030, 13172, 12818] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81000 payload_rows_remote=72322 dup_rows=17304 dup_rows_remote=16082 dedup_ratio=0.176025 router_tokens_per_expert=[1780, 3666, 85, 348, 931, 242, 63, 190, 138, 326, 180, 327, 104, 329, 578, 636, 416, 502, 0, 6, 709, 84, 3019, 737, 1847, 3865, 697, 267, 245, 263, 101, 520, 0, 217, 2431, 0, 91, 1419, 894, 203, 724, 396, 0, 57, 3070, 5802, 703, 0, 5385, 0, 1710, 524, 412, 114, 370, 69, 5, 336, 0, 0, 0, 506, 110, 403] payload_rows_per_dst=[6140, 11138, 9376, 10866, 8872, 8678, 13492, 12438] dup_rows_per_dst=[1138, 1828, 1570, 3900, 2084, 1222, 3676, 1886] combine_rows_per_src=[6140, 11138, 9376, 10866, 8872, 8678, 13492, 12438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273145856 dispatch_meta_bytes=327656 combine_payload_bytes=273145856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273473512 ingress_bytes=307748864 link_bytes=307748864 token_bytes=546619368 weight_movement_bytes=34603008 total_nvlink_bytes=581222376 weight_to_token_ratio=0.063304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158034199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80128 payload_rows_remote=68796 dup_rows=18176 dup_rows_remote=13912 dedup_ratio=0.184896 router_tokens_per_expert=[2000, 5245, 0, 169, 676, 101, 54, 75, 0, 317, 124, 253, 83, 302, 529, 761, 357, 820, 0, 1, 506, 99, 2424, 693, 2465, 3927, 1102, 324, 142, 0, 64, 462, 0, 248, 1666, 0, 63, 1622, 757, 129, 703, 223, 0, 88, 2836, 6243, 294, 0, 5619, 0, 1221, 608, 491, 87, 342, 79, 21, 455, 0, 0, 0, 832, 118, 332] payload_rows_per_dst=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] dup_rows_per_dst=[2074, 1980, 1642, 4264, 1292, 904, 3352, 2668] combine_rows_per_src=[8088, 10612, 8158, 11332, 7678, 7788, 13138, 13334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273514496 dispatch_meta_bytes=347416 combine_payload_bytes=273514496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=273861912 ingress_bytes=342720512 link_bytes=342720512 token_bytes=547376408 weight_movement_bytes=69206016 total_nvlink_bytes=616582424 weight_to_token_ratio=0.126432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289629695e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270483456 dispatch_meta_bytes=327184 combine_payload_bytes=270483456 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270810640 ingress_bytes=374292480 link_bytes=374292480 token_bytes=541294096 weight_movement_bytes=103809024 total_nvlink_bytes=645103120 weight_to_token_ratio=0.191779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408432469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80466 payload_rows_remote=71630 dup_rows=17838 dup_rows_remote=15928 dedup_ratio=0.181458 router_tokens_per_expert=[1770, 3270, 80, 248, 1033, 130, 54, 191, 202, 440, 179, 365, 108, 304, 566, 715, 384, 793, 5, 16, 727, 84, 2828, 713, 2185, 3791, 792, 286, 243, 353, 39, 528, 0, 220, 2420, 0, 81, 1514, 923, 215, 745, 254, 0, 87, 3216, 5521, 784, 1, 5047, 2, 1725, 447, 489, 127, 357, 71, 27, 432, 0, 0, 0, 540, 107, 378] payload_rows_per_dst=[5948, 11212, 9250, 11222, 8836, 8758, 13148, 12092] dup_rows_per_dst=[1064, 2128, 1850, 4170, 1910, 1416, 3382, 1918] combine_rows_per_src=[5948, 11212, 9250, 11222, 8836, 8758, 13148, 12092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=296230912 dispatch_meta_bytes=353616 combine_payload_bytes=296230912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=296584528 ingress_bytes=365436928 link_bytes=365436928 token_bytes=592815440 weight_movement_bytes=69206016 total_nvlink_bytes=662021456 weight_to_token_ratio=0.116741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375109740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77890 payload_rows_remote=67594 dup_rows=20414 dup_rows_remote=16806 dedup_ratio=0.207662 router_tokens_per_expert=[2054, 5096, 187, 258, 959, 43, 96, 96, 1, 489, 227, 432, 109, 381, 535, 564, 428, 1585, 138, 110, 464, 88, 1454, 714, 2939, 3196, 1325, 201, 297, 59, 152, 520, 56, 377, 924, 16, 79, 2234, 1220, 132, 636, 119, 56, 69, 2776, 5538, 311, 92, 4544, 158, 1164, 535, 447, 109, 296, 47, 135, 360, 100, 33, 8, 734, 296, 354] payload_rows_per_dst=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] dup_rows_per_dst=[4712, 1846, 3608, 2236, 1578, 3748, 448, 2238] combine_rows_per_src=[12866, 10022, 10296, 8750, 8498, 11368, 5064, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=281788416 dispatch_meta_bytes=330832 combine_payload_bytes=281788416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282119248 ingress_bytes=385597440 link_bytes=385597440 token_bytes=563907664 weight_movement_bytes=103809024 total_nvlink_bytes=667716688 weight_to_token_ratio=0.184089 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450972124e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80488 payload_rows_remote=65664 dup_rows=17816 dup_rows_remote=9178 dedup_ratio=0.181234 router_tokens_per_expert=[660, 81, 0, 4339, 89, 0, 179, 69, 0, 407, 711, 517, 923, 527, 409, 630, 0, 0, 652, 0, 0, 599, 433, 1563, 130, 95, 316, 1, 124, 452, 0, 0, 69, 1585, 137, 4, 0, 0, 0, 3982, 1515, 43, 180, 2032, 5451, 184, 1418, 3926, 376, 98, 0, 1349, 280, 1332, 6314, 0, 4489, 0, 0, 0, 0, 0, 421, 61] payload_rows_per_dst=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] dup_rows_per_dst=[8638, 1564, 2984, 454, 1608, 2008, 316, 244] combine_rows_per_src=[14824, 6684, 11362, 12684, 9946, 8736, 6554, 9698] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=293396480 dispatch_meta_bytes=350232 combine_payload_bytes=293396480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293746712 ingress_bytes=327999488 link_bytes=327999488 token_bytes=587143192 weight_movement_bytes=34603008 total_nvlink_bytes=621746200 weight_to_token_ratio=0.058935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234235667e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=337600 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277202624 ingress_bytes=311468032 link_bytes=311468032 token_bytes=554067648 weight_movement_bytes=34603008 total_nvlink_bytes=588670656 weight_to_token_ratio=0.062453 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172029129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78268 payload_rows_remote=66798 dup_rows=20036 dup_rows_remote=14866 dedup_ratio=0.203817 router_tokens_per_expert=[565, 532, 4077, 2628, 1616, 7, 5899, 4769, 5, 178, 4, 470, 10, 1500, 3, 151, 307, 347, 2212, 113, 928, 56, 406, 124, 47, 92, 1190, 0, 0, 140, 8, 553, 5838, 907, 2, 89, 232, 1458, 679, 4, 402, 5, 68, 9, 0, 7, 17, 454, 2670, 5, 7, 344, 1, 1200, 119, 6, 1806, 15, 46, 2068, 1625, 122, 10, 0] payload_rows_per_dst=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] dup_rows_per_dst=[618, 2586, 1156, 1584, 2004, 640, 6278, 5170] combine_rows_per_src=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75104 payload_rows_remote=65574 dup_rows=23200 dup_rows_remote=20806 dedup_ratio=0.236003 router_tokens_per_expert=[573, 389, 3674, 2819, 1452, 0, 6228, 4949, 0, 195, 0, 440, 0, 1511, 0, 142, 298, 333, 2460, 115, 936, 65, 382, 135, 62, 97, 1056, 0, 0, 162, 0, 565, 5854, 852, 0, 98, 270, 2164, 671, 0, 410, 0, 17, 0, 0, 0, 0, 312, 2363, 0, 4, 183, 0, 1111, 140, 0, 1250, 0, 16, 2265, 2024, 110, 0, 0] payload_rows_per_dst=[11720, 9530, 15274, 7096, 6032, 9436, 6974, 9042] dup_rows_per_dst=[8336, 2394, 5882, 1106, 2078, 488, 628, 2288] combine_rows_per_src=[11720, 9530, 15274, 7096, 6032, 9436, 6974, 9042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78774 payload_rows_remote=64538 dup_rows=19530 dup_rows_remote=13892 dedup_ratio=0.198669 router_tokens_per_expert=[574, 548, 4246, 3179, 1168, 0, 6044, 5277, 0, 156, 0, 420, 0, 1617, 0, 118, 336, 374, 2050, 108, 986, 27, 417, 144, 44, 78, 1220, 0, 1, 129, 0, 586, 5865, 945, 0, 78, 186, 1549, 664, 0, 392, 1, 69, 0, 0, 0, 7, 378, 2446, 0, 4, 463, 1, 1045, 113, 0, 1934, 0, 59, 1546, 1462, 98, 0, 0] payload_rows_per_dst=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] dup_rows_per_dst=[504, 2702, 1252, 1712, 2326, 616, 5638, 4780] combine_rows_per_src=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268959744 dispatch_meta_bytes=299368 combine_payload_bytes=268959744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269259112 ingress_bytes=303562752 link_bytes=303562752 token_bytes=538218856 weight_movement_bytes=34603008 total_nvlink_bytes=572821864 weight_to_token_ratio=0.064292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.142282197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80284 payload_rows_remote=66686 dup_rows=18020 dup_rows_remote=15228 dedup_ratio=0.183309 router_tokens_per_expert=[2236, 5720, 20, 209, 702, 62, 52, 47, 0, 246, 152, 239, 131, 315, 564, 820, 402, 902, 9, 4, 472, 79, 2325, 820, 2395, 3887, 1102, 286, 35, 11, 138, 371, 5, 273, 1118, 4, 88, 1474, 1033, 113, 700, 175, 1, 65, 2415, 6432, 166, 4, 5802, 4, 1266, 526, 489, 92, 339, 52, 31, 413, 2, 1, 0, 844, 137, 335] payload_rows_per_dst=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] dup_rows_per_dst=[710, 1888, 1636, 1210, 5784, 438, 3562, 2792] combine_rows_per_src=[5946, 10820, 8390, 7466, 13872, 6614, 13578, 13598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273514496 dispatch_meta_bytes=347416 combine_payload_bytes=273514496 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=273861912 ingress_bytes=342720512 link_bytes=342720512 token_bytes=547376408 weight_movement_bytes=69206016 total_nvlink_bytes=616582424 weight_to_token_ratio=0.126432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.289629695e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79392 payload_rows_remote=65386 dup_rows=18912 dup_rows_remote=18088 dedup_ratio=0.192383 router_tokens_per_expert=[601, 608, 3621, 3421, 1019, 0, 6197, 5921, 0, 176, 0, 450, 0, 1548, 0, 141, 325, 375, 1974, 105, 838, 62, 330, 145, 49, 88, 1621, 0, 0, 150, 0, 283, 5816, 830, 0, 88, 281, 1023, 688, 0, 417, 0, 135, 0, 34, 0, 277, 355, 2604, 0, 16, 469, 1, 1330, 67, 0, 1604, 0, 97, 1567, 1296, 109, 0, 0] payload_rows_per_dst=[3906, 9668, 7234, 13572, 10624, 14006, 12772, 7610] dup_rows_per_dst=[550, 2204, 1074, 2652, 4748, 824, 5124, 1736] combine_rows_per_src=[3906, 9668, 7234, 13572, 10624, 14006, 12772, 7610] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=276865024 dispatch_meta_bytes=337600 combine_payload_bytes=276865024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277202624 ingress_bytes=311468032 link_bytes=311468032 token_bytes=554067648 weight_movement_bytes=34603008 total_nvlink_bytes=588670656 weight_to_token_ratio=0.062453 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172029129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270483456 dispatch_meta_bytes=327184 combine_payload_bytes=270483456 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=270810640 ingress_bytes=374292480 link_bytes=374292480 token_bytes=541294096 weight_movement_bytes=103809024 total_nvlink_bytes=645103120 weight_to_token_ratio=0.191779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=27008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.408432469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=293396480 dispatch_meta_bytes=350232 combine_payload_bytes=293396480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293746712 ingress_bytes=327999488 link_bytes=327999488 token_bytes=587143192 weight_movement_bytes=34603008 total_nvlink_bytes=621746200 weight_to_token_ratio=0.058935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234235667e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=273145856 dispatch_meta_bytes=327656 combine_payload_bytes=273145856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273473512 ingress_bytes=307748864 link_bytes=307748864 token_bytes=546619368 weight_movement_bytes=34603008 total_nvlink_bytes=581222376 weight_to_token_ratio=0.063304 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158034199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267821056 dispatch_meta_bytes=333896 combine_payload_bytes=267821056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268154952 ingress_bytes=302424064 link_bytes=302424064 token_bytes=535976008 weight_movement_bytes=34603008 total_nvlink_bytes=570579016 weight_to_token_ratio=0.064561 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137997406e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82428 payload_rows_remote=69500 dup_rows=15876 dup_rows_remote=14198 dedup_ratio=0.161499 router_tokens_per_expert=[567, 465, 3995, 3032, 1380, 0, 6004, 5289, 0, 124, 0, 393, 0, 1607, 0, 121, 333, 356, 2173, 104, 950, 44, 446, 145, 50, 61, 1238, 0, 0, 117, 0, 548, 5862, 870, 0, 63, 217, 1675, 678, 0, 365, 0, 47, 0, 0, 0, 1, 334, 2622, 0, 6, 360, 1, 1131, 113, 0, 1857, 0, 50, 1834, 1436, 88, 0, 0] payload_rows_per_dst=[9218, 9954, 7806, 12928, 13468, 12926, 7606, 8522] dup_rows_per_dst=[1670, 2526, 1296, 1678, 5262, 576, 860, 2008] combine_rows_per_src=[9218, 9954, 7806, 12928, 13468, 12926, 7606, 8522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76096 payload_rows_remote=61062 dup_rows=22208 dup_rows_remote=12780 dedup_ratio=0.225911 router_tokens_per_expert=[2961, 6779, 1, 103, 552, 1, 61, 3, 0, 146, 61, 203, 89, 321, 459, 550, 403, 1366, 0, 0, 268, 75, 1131, 611, 4214, 3644, 1297, 306, 0, 0, 61, 414, 0, 300, 706, 1, 91, 2475, 947, 68, 756, 50, 0, 95, 1757, 6459, 27, 0, 4804, 0, 670, 680, 459, 79, 202, 66, 8, 291, 0, 0, 0, 1483, 243, 355] payload_rows_per_dst=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] dup_rows_per_dst=[9428, 1216, 3620, 2034, 1066, 4474, 312, 58] combine_rows_per_src=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273604608 dispatch_meta_bytes=326656 combine_payload_bytes=273604608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273931264 ingress_bytes=308207616 link_bytes=308207616 token_bytes=547535872 weight_movement_bytes=34603008 total_nvlink_bytes=582138880 weight_to_token_ratio=0.063198 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159760446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=345520 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268936624 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537527728 weight_movement_bytes=34603008 total_nvlink_bytes=572130736 weight_to_token_ratio=0.064374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264347648 dispatch_meta_bytes=313720 combine_payload_bytes=264347648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264661368 ingress_bytes=298950656 link_bytes=298950656 token_bytes=529009016 weight_movement_bytes=34603008 total_nvlink_bytes=563612024 weight_to_token_ratio=0.065411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124927251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81396 payload_rows_remote=71124 dup_rows=16908 dup_rows_remote=14522 dedup_ratio=0.171997 router_tokens_per_expert=[539, 497, 4026, 2858, 822, 105, 5859, 3895, 62, 167, 90, 424, 192, 1750, 21, 270, 303, 292, 1936, 87, 684, 84, 440, 166, 62, 115, 1026, 25, 15, 133, 185, 628, 5606, 701, 51, 130, 479, 2113, 661, 40, 425, 39, 495, 87, 11, 157, 359, 438, 2208, 16, 38, 146, 140, 1088, 155, 155, 1168, 46, 87, 1764, 2422, 63, 90, 16] payload_rows_per_dst=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] dup_rows_per_dst=[1256, 3164, 2386, 1566, 3990, 1668, 692, 2186] combine_rows_per_src=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=68784 dup_rows=16192 dup_rows_remote=11416 dedup_ratio=0.164714 router_tokens_per_expert=[557, 629, 3607, 3174, 1204, 1, 6154, 5599, 7, 162, 1, 432, 0, 1609, 1, 179, 312, 363, 1985, 119, 969, 61, 438, 140, 52, 88, 1515, 1, 4, 132, 1, 301, 5723, 847, 0, 82, 222, 1236, 652, 0, 386, 6, 316, 2, 21, 8, 355, 438, 2191, 0, 10, 481, 21, 1314, 69, 0, 1929, 34, 72, 1477, 1342, 104, 17, 0] payload_rows_per_dst=[8924, 9588, 7574, 13124, 13328, 14258, 7296, 8020] dup_rows_per_dst=[1626, 2408, 1200, 2262, 4776, 1114, 876, 1930] combine_rows_per_src=[8924, 9588, 7574, 13124, 13328, 14258, 7296, 8020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75104 payload_rows_remote=65574 dup_rows=23200 dup_rows_remote=20806 dedup_ratio=0.236003 router_tokens_per_expert=[573, 389, 3674, 2819, 1452, 0, 6228, 4949, 0, 195, 0, 440, 0, 1511, 0, 142, 298, 333, 2460, 115, 936, 65, 382, 135, 62, 97, 1056, 0, 0, 162, 0, 565, 5854, 852, 0, 98, 270, 2164, 671, 0, 410, 0, 17, 0, 0, 0, 0, 312, 2363, 0, 4, 183, 0, 1111, 140, 0, 1250, 0, 16, 2265, 2024, 110, 0, 0] payload_rows_per_dst=[11720, 9530, 15274, 7096, 6032, 9436, 6974, 9042] dup_rows_per_dst=[8336, 2394, 5882, 1106, 2078, 488, 628, 2288] combine_rows_per_src=[11720, 9530, 15274, 7096, 6032, 9436, 6974, 9042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78774 payload_rows_remote=64538 dup_rows=19530 dup_rows_remote=13892 dedup_ratio=0.198669 router_tokens_per_expert=[574, 548, 4246, 3179, 1168, 0, 6044, 5277, 0, 156, 0, 420, 0, 1617, 0, 118, 336, 374, 2050, 108, 986, 27, 417, 144, 44, 78, 1220, 0, 1, 129, 0, 586, 5865, 945, 0, 78, 186, 1549, 664, 0, 392, 1, 69, 0, 0, 0, 7, 378, 2446, 0, 4, 463, 1, 1045, 113, 0, 1934, 0, 59, 1546, 1462, 98, 0, 0] payload_rows_per_dst=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] dup_rows_per_dst=[504, 2702, 1252, 1712, 2326, 616, 5638, 4780] combine_rows_per_src=[4076, 10412, 7632, 12958, 5008, 13166, 14236, 11286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267821056 dispatch_meta_bytes=333896 combine_payload_bytes=267821056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268154952 ingress_bytes=302424064 link_bytes=302424064 token_bytes=535976008 weight_movement_bytes=34603008 total_nvlink_bytes=570579016 weight_to_token_ratio=0.064561 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40448 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137997406e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81396 payload_rows_remote=71124 dup_rows=16908 dup_rows_remote=14522 dedup_ratio=0.171997 router_tokens_per_expert=[539, 497, 4026, 2858, 822, 105, 5859, 3895, 62, 167, 90, 424, 192, 1750, 21, 270, 303, 292, 1936, 87, 684, 84, 440, 166, 62, 115, 1026, 25, 15, 133, 185, 628, 5606, 701, 51, 130, 479, 2113, 661, 40, 425, 39, 495, 87, 11, 157, 359, 438, 2208, 16, 38, 146, 140, 1088, 155, 155, 1168, 46, 87, 1764, 2422, 63, 90, 16] payload_rows_per_dst=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] dup_rows_per_dst=[1256, 3164, 2386, 1566, 3990, 1668, 692, 2186] combine_rows_per_src=[8386, 10840, 10272, 10602, 10898, 14072, 7200, 9126] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284672000 dispatch_meta_bytes=334792 combine_payload_bytes=284672000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285006792 ingress_bytes=319275008 link_bytes=319275008 token_bytes=569678792 weight_movement_bytes=34603008 total_nvlink_bytes=604281800 weight_to_token_ratio=0.060741 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201406151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78268 payload_rows_remote=66798 dup_rows=20036 dup_rows_remote=14866 dedup_ratio=0.203817 router_tokens_per_expert=[565, 532, 4077, 2628, 1616, 7, 5899, 4769, 5, 178, 4, 470, 10, 1500, 3, 151, 307, 347, 2212, 113, 928, 56, 406, 124, 47, 92, 1190, 0, 0, 140, 8, 553, 5838, 907, 2, 89, 232, 1458, 679, 4, 402, 5, 68, 9, 0, 7, 17, 454, 2670, 5, 7, 344, 1, 1200, 119, 6, 1806, 15, 46, 2068, 1625, 122, 10, 0] payload_rows_per_dst=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] dup_rows_per_dst=[618, 2586, 1156, 1584, 2004, 640, 6278, 5170] combine_rows_per_src=[4822, 10210, 7830, 12014, 4738, 13082, 14102, 11470] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81184 payload_rows_remote=68468 dup_rows=17120 dup_rows_remote=12170 dedup_ratio=0.174154 router_tokens_per_expert=[0, 7033, 1533, 0, 77, 66, 625, 5188, 0, 0, 0, 0, 0, 552, 2579, 168, 251, 111, 104, 377, 72, 0, 254, 829, 1233, 257, 0, 291, 257, 66, 270, 0, 26, 7296, 124, 0, 1262, 0, 1194, 0, 638, 0, 4, 0, 2747, 256, 0, 0, 85, 0, 117, 686, 134, 508, 89, 0, 3, 325, 5416, 306, 257, 2964, 0, 2522] payload_rows_per_dst=[4528, 6598, 12950, 3836, 15134, 12716, 15020, 10402] dup_rows_per_dst=[74, 0, 1878, 912, 4670, 4950, 2284, 2352] combine_rows_per_src=[4528, 6598, 12950, 3836, 15134, 12716, 15020, 10402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=250109952 dispatch_meta_bytes=295368 combine_payload_bytes=250109952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250405320 ingress_bytes=353918976 link_bytes=353918976 token_bytes=500515272 weight_movement_bytes=103809024 total_nvlink_bytes=604324296 weight_to_token_ratio=0.207404 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331768614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76096 payload_rows_remote=61062 dup_rows=22208 dup_rows_remote=12780 dedup_ratio=0.225911 router_tokens_per_expert=[2961, 6779, 1, 103, 552, 1, 61, 3, 0, 146, 61, 203, 89, 321, 459, 550, 403, 1366, 0, 0, 268, 75, 1131, 611, 4214, 3644, 1297, 306, 0, 0, 61, 414, 0, 300, 706, 1, 91, 2475, 947, 68, 756, 50, 0, 95, 1757, 6459, 27, 0, 4804, 0, 670, 680, 459, 79, 202, 66, 8, 291, 0, 0, 0, 1483, 243, 355] payload_rows_per_dst=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] dup_rows_per_dst=[9428, 1216, 3620, 2034, 1066, 4474, 312, 58] combine_rows_per_src=[15034, 9804, 10156, 10476, 8110, 13814, 4000, 4702] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=291323904 dispatch_meta_bytes=342584 combine_payload_bytes=291323904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291666488 ingress_bytes=325926912 link_bytes=325926912 token_bytes=582990392 weight_movement_bytes=34603008 total_nvlink_bytes=617593400 weight_to_token_ratio=0.059354 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226436731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=281739264 dispatch_meta_bytes=320800 combine_payload_bytes=281739264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=282060064 ingress_bytes=385548288 link_bytes=385548288 token_bytes=563799328 weight_movement_bytes=103809024 total_nvlink_bytes=667608352 weight_to_token_ratio=0.184124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.450787169e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=268591104 dispatch_meta_bytes=345520 combine_payload_bytes=268591104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268936624 ingress_bytes=303194112 link_bytes=303194112 token_bytes=537527728 weight_movement_bytes=34603008 total_nvlink_bytes=572130736 weight_to_token_ratio=0.064374 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140895034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264347648 dispatch_meta_bytes=313720 combine_payload_bytes=264347648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264661368 ingress_bytes=298950656 link_bytes=298950656 token_bytes=529009016 weight_movement_bytes=34603008 total_nvlink_bytes=563612024 weight_to_token_ratio=0.065411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124927251e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=280444928 dispatch_meta_bytes=322552 combine_payload_bytes=280444928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280767480 ingress_bytes=315047936 link_bytes=315047936 token_bytes=561212408 weight_movement_bytes=34603008 total_nvlink_bytes=595815416 weight_to_token_ratio=0.061658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185500020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=273604608 dispatch_meta_bytes=326656 combine_payload_bytes=273604608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273931264 ingress_bytes=308207616 link_bytes=308207616 token_bytes=547535872 weight_movement_bytes=34603008 total_nvlink_bytes=582138880 weight_to_token_ratio=0.063198 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159760446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71662 payload_rows_remote=56288 dup_rows=26642 dup_rows_remote=11052 dedup_ratio=0.271016 router_tokens_per_expert=[2, 7347, 1816, 7, 54, 57, 605, 4556, 1, 23, 4, 2, 2, 728, 3206, 154, 50, 114, 135, 307, 64, 2, 179, 809, 842, 276, 6, 385, 251, 59, 293, 0, 86, 7226, 90, 2, 1994, 13, 1235, 0, 886, 3, 24, 4, 2778, 252, 3, 13, 79, 13, 200, 607, 32, 506, 98, 11, 17, 420, 5258, 492, 280, 2670, 22, 1502] payload_rows_per_dst=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] dup_rows_per_dst=[160, 34, 356, 1012, 2656, 4660, 2174, 15590] combine_rows_per_src=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79040 payload_rows_remote=63054 dup_rows=19264 dup_rows_remote=12894 dedup_ratio=0.195964 router_tokens_per_expert=[0, 7423, 1497, 0, 0, 64, 616, 4314, 0, 0, 0, 0, 0, 766, 2822, 167, 0, 164, 110, 313, 81, 0, 147, 857, 1526, 231, 0, 316, 229, 64, 285, 0, 117, 7270, 99, 0, 2819, 0, 930, 0, 999, 0, 0, 0, 2614, 231, 0, 0, 98, 0, 140, 428, 17, 393, 96, 0, 0, 382, 5096, 1063, 295, 2580, 0, 1493] payload_rows_per_dst=[12734, 15986, 12312, 15276, 7298, 7336, 2092, 6006] dup_rows_per_dst=[5408, 6370, 1224, 4566, 632, 352, 252, 460] combine_rows_per_src=[12734, 15986, 12312, 15276, 7298, 7336, 2092, 6006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80264 payload_rows_remote=64786 dup_rows=18040 dup_rows_remote=16178 dedup_ratio=0.183512 router_tokens_per_expert=[0, 7283, 1820, 0, 56, 56, 646, 4819, 0, 5, 0, 0, 0, 786, 2756, 142, 39, 85, 98, 323, 70, 0, 230, 844, 1338, 259, 0, 347, 244, 51, 247, 1, 108, 7246, 95, 0, 1665, 0, 947, 0, 829, 0, 0, 0, 2766, 244, 0, 0, 78, 0, 110, 550, 67, 504, 78, 0, 1, 417, 5533, 462, 287, 2587, 0, 2033] payload_rows_per_dst=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] dup_rows_per_dst=[90, 46, 966, 908, 5088, 4820, 1862, 4260] combine_rows_per_src=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=250109952 dispatch_meta_bytes=295368 combine_payload_bytes=250109952 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=250405320 ingress_bytes=353918976 link_bytes=353918976 token_bytes=500515272 weight_movement_bytes=103809024 total_nvlink_bytes=604324296 weight_to_token_ratio=0.207404 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.331768614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81184 payload_rows_remote=68468 dup_rows=17120 dup_rows_remote=12170 dedup_ratio=0.174154 router_tokens_per_expert=[0, 7033, 1533, 0, 77, 66, 625, 5188, 0, 0, 0, 0, 0, 552, 2579, 168, 251, 111, 104, 377, 72, 0, 254, 829, 1233, 257, 0, 291, 257, 66, 270, 0, 26, 7296, 124, 0, 1262, 0, 1194, 0, 638, 0, 4, 0, 2747, 256, 0, 0, 85, 0, 117, 686, 134, 508, 89, 0, 3, 325, 5416, 306, 257, 2964, 0, 2522] payload_rows_per_dst=[4528, 6598, 12950, 3836, 15134, 12716, 15020, 10402] dup_rows_per_dst=[74, 0, 1878, 912, 4670, 4950, 2284, 2352] combine_rows_per_src=[4528, 6598, 12950, 3836, 15134, 12716, 15020, 10402] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83966 payload_rows_remote=68654 dup_rows=14338 dup_rows_remote=10518 dedup_ratio=0.145854 router_tokens_per_expert=[0, 7387, 1710, 0, 22, 54, 599, 4896, 0, 0, 0, 0, 0, 768, 2719, 128, 36, 84, 89, 300, 62, 0, 176, 898, 1046, 249, 0, 442, 240, 55, 210, 0, 97, 7324, 68, 0, 1971, 0, 989, 0, 950, 0, 0, 0, 2762, 242, 0, 0, 63, 0, 111, 500, 36, 500, 61, 0, 0, 446, 5510, 574, 235, 2698, 0, 1845] payload_rows_per_dst=[13114, 7188, 13108, 15312, 5780, 8030, 15612, 5822] dup_rows_per_dst=[6016, 42, 1202, 3820, 470, 706, 1704, 378] combine_rows_per_src=[13114, 7188, 13108, 15312, 5780, 8030, 15612, 5822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230555648 dispatch_meta_bytes=269360 combine_payload_bytes=230555648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=230825008 ingress_bytes=334364672 link_bytes=334364672 token_bytes=461380656 weight_movement_bytes=103809024 total_nvlink_bytes=565189680 weight_to_token_ratio=0.224996 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258187342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258269184 dispatch_meta_bytes=303792 combine_payload_bytes=258269184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258572976 ingress_bytes=292872192 link_bytes=292872192 token_bytes=516842160 weight_movement_bytes=34603008 total_nvlink_bytes=551445168 weight_to_token_ratio=0.066951 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102054480e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=323856 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265687312 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531050768 weight_movement_bytes=34603008 total_nvlink_bytes=565653776 weight_to_token_ratio=0.065160 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66108 payload_rows_remote=51448 dup_rows=32196 dup_rows_remote=14164 dedup_ratio=0.327515 router_tokens_per_expert=[593, 365, 3934, 2538, 1391, 0, 5985, 4105, 0, 178, 0, 467, 0, 1723, 0, 126, 355, 342, 2033, 101, 896, 42, 376, 150, 59, 89, 792, 0, 0, 151, 0, 634, 5923, 909, 0, 90, 501, 2294, 718, 0, 410, 0, 13, 0, 0, 0, 0, 293, 2454, 0, 0, 199, 1, 1058, 299, 0, 1277, 0, 24, 2689, 2463, 112, 0, 0] payload_rows_per_dst=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] dup_rows_per_dst=[18032, 1872, 5178, 494, 2196, 504, 850, 3070] combine_rows_per_src=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77116 payload_rows_remote=64980 dup_rows=21188 dup_rows_remote=19852 dedup_ratio=0.215535 router_tokens_per_expert=[47, 6874, 3359, 92, 9, 60, 684, 2640, 6, 363, 202, 32, 13, 992, 3089, 142, 120, 142, 117, 307, 99, 141, 233, 704, 1402, 290, 87, 397, 262, 93, 264, 2, 332, 6461, 87, 53, 1974, 128, 541, 1, 312, 2, 290, 27, 2144, 300, 29, 197, 88, 187, 130, 458, 44, 558, 145, 275, 46, 326, 4873, 1105, 353, 2503, 152, 1767] payload_rows_per_dst=[12342, 14024, 12136, 14026, 5848, 6086, 5746, 6908] dup_rows_per_dst=[6446, 6446, 1336, 4490, 384, 516, 980, 590] combine_rows_per_src=[12342, 14024, 12136, 14026, 5848, 6086, 5746, 6908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281206784 dispatch_meta_bytes=316688 combine_payload_bytes=281206784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281523472 ingress_bytes=315809792 link_bytes=315809792 token_bytes=562730256 weight_movement_bytes=34603008 total_nvlink_bytes=597333264 weight_to_token_ratio=0.061491 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188366823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=68850 dup_rows=16192 dup_rows_remote=12348 dedup_ratio=0.164714 router_tokens_per_expert=[0, 7017, 1875, 13, 75, 75, 550, 4510, 0, 22, 0, 3, 0, 716, 2756, 143, 166, 96, 109, 309, 62, 1, 262, 892, 1229, 249, 7, 426, 242, 75, 253, 0, 93, 7102, 123, 2, 1348, 8, 1041, 0, 688, 1, 48, 2, 2666, 243, 0, 0, 73, 1, 107, 717, 149, 536, 85, 0, 19, 409, 5394, 368, 272, 2603, 15, 2906] payload_rows_per_dst=[5520, 7222, 12990, 5644, 13262, 12164, 15046, 10264] dup_rows_per_dst=[264, 58, 1592, 1646, 3844, 4152, 2324, 2312] combine_rows_per_src=[5520, 7222, 12990, 5644, 13262, 12164, 15046, 10264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79040 payload_rows_remote=63054 dup_rows=19264 dup_rows_remote=12894 dedup_ratio=0.195964 router_tokens_per_expert=[0, 7423, 1497, 0, 0, 64, 616, 4314, 0, 0, 0, 0, 0, 766, 2822, 167, 0, 164, 110, 313, 81, 0, 147, 857, 1526, 231, 0, 316, 229, 64, 285, 0, 117, 7270, 99, 0, 2819, 0, 930, 0, 999, 0, 0, 0, 2614, 231, 0, 0, 98, 0, 140, 428, 17, 393, 96, 0, 0, 382, 5096, 1063, 295, 2580, 0, 1493] payload_rows_per_dst=[12734, 15986, 12312, 15276, 7298, 7336, 2092, 6006] dup_rows_per_dst=[5408, 6370, 1224, 4566, 632, 352, 252, 460] combine_rows_per_src=[12734, 15986, 12312, 15276, 7298, 7336, 2092, 6006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80264 payload_rows_remote=64786 dup_rows=18040 dup_rows_remote=16178 dedup_ratio=0.183512 router_tokens_per_expert=[0, 7283, 1820, 0, 56, 56, 646, 4819, 0, 5, 0, 0, 0, 786, 2756, 142, 39, 85, 98, 323, 70, 0, 230, 844, 1338, 259, 0, 347, 244, 51, 247, 1, 108, 7246, 95, 0, 1665, 0, 947, 0, 829, 0, 0, 0, 2766, 244, 0, 0, 78, 0, 110, 550, 67, 504, 78, 0, 1, 417, 5533, 462, 287, 2587, 0, 2033] payload_rows_per_dst=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] dup_rows_per_dst=[90, 46, 966, 908, 5088, 4820, 1862, 4260] combine_rows_per_src=[5066, 7332, 9598, 4066, 15034, 12496, 15478, 11194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84314 payload_rows_remote=75696 dup_rows=13990 dup_rows_remote=12720 dedup_ratio=0.142314 router_tokens_per_expert=[0, 0, 0, 197, 0, 0, 0, 0, 395, 78, 1400, 0, 702, 345, 0, 179, 57, 0, 1, 0, 0, 72, 0, 0, 0, 4391, 0, 275, 222, 1099, 0, 217, 1843, 2740, 0, 0, 0, 1223, 0, 1124, 1472, 1949, 5246, 0, 484, 528, 879, 0, 0, 190, 0, 3981, 236, 0, 57, 0, 736, 0, 4175, 5651, 5200, 1632, 0, 176] payload_rows_per_dst=[8692, 7338, 11560, 11270, 9826, 8618, 14188, 12822] dup_rows_per_dst=[52, 1804, 2, 1138, 1826, 1270, 5232, 2666] combine_rows_per_src=[8692, 7338, 11560, 11270, 9826, 8618, 14188, 12822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=280444928 dispatch_meta_bytes=322552 combine_payload_bytes=280444928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280767480 ingress_bytes=315047936 link_bytes=315047936 token_bytes=561212408 weight_movement_bytes=34603008 total_nvlink_bytes=595815416 weight_to_token_ratio=0.061658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.185500020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=281206784 dispatch_meta_bytes=316688 combine_payload_bytes=281206784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281523472 ingress_bytes=315809792 link_bytes=315809792 token_bytes=562730256 weight_movement_bytes=34603008 total_nvlink_bytes=597333264 weight_to_token_ratio=0.061491 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.188366823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71662 payload_rows_remote=56288 dup_rows=26642 dup_rows_remote=11052 dedup_ratio=0.271016 router_tokens_per_expert=[2, 7347, 1816, 7, 54, 57, 605, 4556, 1, 23, 4, 2, 2, 728, 3206, 154, 50, 114, 135, 307, 64, 2, 179, 809, 842, 276, 6, 385, 251, 59, 293, 0, 86, 7226, 90, 2, 1994, 13, 1235, 0, 886, 3, 24, 4, 2778, 252, 3, 13, 79, 13, 200, 607, 32, 506, 98, 11, 17, 420, 5258, 492, 280, 2670, 22, 1502] payload_rows_per_dst=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] dup_rows_per_dst=[160, 34, 356, 1012, 2656, 4660, 2174, 15590] combine_rows_per_src=[4922, 8206, 2964, 3212, 8994, 12378, 15612, 15374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=210731008 dispatch_meta_bytes=262448 combine_payload_bytes=210731008 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=210993456 ingress_bytes=383746048 link_bytes=383746048 token_bytes=421724464 weight_movement_bytes=173015040 total_nvlink_bytes=594739504 weight_to_token_ratio=0.410256 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444005485e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66108 payload_rows_remote=51448 dup_rows=32196 dup_rows_remote=14164 dedup_ratio=0.327515 router_tokens_per_expert=[593, 365, 3934, 2538, 1391, 0, 5985, 4105, 0, 178, 0, 467, 0, 1723, 0, 126, 355, 342, 2033, 101, 896, 42, 376, 150, 59, 89, 792, 0, 0, 151, 0, 634, 5923, 909, 0, 90, 501, 2294, 718, 0, 410, 0, 13, 0, 0, 0, 0, 293, 2454, 0, 0, 199, 1, 1058, 299, 0, 1277, 0, 24, 2689, 2463, 112, 0, 0] payload_rows_per_dst=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] dup_rows_per_dst=[18032, 1872, 5178, 494, 2196, 504, 850, 3070] combine_rows_per_src=[14660, 8246, 15258, 2956, 6828, 928, 7172, 10060] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=339328 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266497408 ingress_bytes=300761088 link_bytes=300761088 token_bytes=532655488 weight_movement_bytes=34603008 total_nvlink_bytes=567258496 weight_to_token_ratio=0.064963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131739761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=282009600 dispatch_meta_bytes=324792 combine_payload_bytes=282009600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=282334392 ingress_bytes=351215616 link_bytes=351215616 token_bytes=564343992 weight_movement_bytes=69206016 total_nvlink_bytes=633550008 weight_to_token_ratio=0.122631 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.321596088e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=258269184 dispatch_meta_bytes=303792 combine_payload_bytes=258269184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258572976 ingress_bytes=292872192 link_bytes=292872192 token_bytes=516842160 weight_movement_bytes=34603008 total_nvlink_bytes=551445168 weight_to_token_ratio=0.066951 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.102054480e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=323856 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265687312 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531050768 weight_movement_bytes=34603008 total_nvlink_bytes=565653776 weight_to_token_ratio=0.065160 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37632 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=310050816 dispatch_meta_bytes=353664 combine_payload_bytes=310050816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=310404480 ingress_bytes=448462848 link_bytes=448462848 token_bytes=620455296 weight_movement_bytes=138412032 total_nvlink_bytes=758867328 weight_to_token_ratio=0.223081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.687529593e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=230555648 dispatch_meta_bytes=269360 combine_payload_bytes=230555648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=230825008 ingress_bytes=334364672 link_bytes=334364672 token_bytes=461380656 weight_movement_bytes=103809024 total_nvlink_bytes=565189680 weight_to_token_ratio=0.224996 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258187342e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=67302 dup_rows=16192 dup_rows_remote=8686 dedup_ratio=0.164714 router_tokens_per_expert=[4, 2, 0, 223, 3, 2, 0, 12, 378, 92, 1659, 8, 594, 339, 5, 92, 51, 1, 2, 0, 13, 63, 0, 23, 3, 4645, 4, 338, 247, 1475, 17, 144, 1131, 3890, 0, 7, 0, 1010, 2, 1123, 2550, 1448, 4563, 8, 588, 654, 918, 4, 1, 130, 13, 2417, 33, 13, 62, 2, 1047, 12, 5338, 5523, 4905, 1152, 0, 169] payload_rows_per_dst=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] dup_rows_per_dst=[68, 2428, 18, 48, 964, 2718, 2442, 7506] combine_rows_per_src=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80398 payload_rows_remote=70278 dup_rows=17906 dup_rows_remote=14824 dedup_ratio=0.182149 router_tokens_per_expert=[0, 0, 0, 242, 0, 0, 0, 0, 376, 148, 1735, 0, 625, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 0, 5277, 0, 320, 240, 1543, 0, 110, 865, 3990, 0, 0, 0, 1101, 0, 1110, 3137, 1090, 4313, 0, 559, 1045, 1197, 0, 0, 111, 0, 2035, 0, 0, 60, 0, 911, 0, 5287, 5914, 4074, 975, 0, 195] payload_rows_per_dst=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] dup_rows_per_dst=[3812, 3082, 2194, 2056, 2360, 3614, 0, 788] combine_rows_per_src=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85568 payload_rows_remote=72470 dup_rows=12736 dup_rows_remote=9340 dedup_ratio=0.129557 router_tokens_per_expert=[0, 0, 0, 258, 0, 0, 0, 0, 366, 117, 2043, 0, 601, 322, 0, 92, 84, 0, 1, 0, 0, 70, 0, 0, 0, 4686, 0, 254, 254, 1273, 1, 0, 1535, 2868, 0, 0, 0, 1112, 0, 1076, 2253, 1467, 4799, 0, 618, 659, 750, 0, 0, 123, 0, 3117, 124, 0, 84, 0, 939, 0, 5115, 5633, 4946, 1357, 0, 155] payload_rows_per_dst=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] dup_rows_per_dst=[84, 2710, 2, 980, 1382, 1894, 3396, 2288] combine_rows_per_src=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=210731008 dispatch_meta_bytes=262448 combine_payload_bytes=210731008 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=210993456 ingress_bytes=383746048 link_bytes=383746048 token_bytes=421724464 weight_movement_bytes=173015040 total_nvlink_bytes=594739504 weight_to_token_ratio=0.410256 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.444005485e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275668992 dispatch_meta_bytes=303952 combine_payload_bytes=275668992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275972944 ingress_bytes=448684032 link_bytes=448684032 token_bytes=551641936 weight_movement_bytes=173015040 total_nvlink_bytes=724656976 weight_to_token_ratio=0.313636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688361890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85986 payload_rows_remote=74348 dup_rows=12318 dup_rows_remote=10982 dedup_ratio=0.125305 router_tokens_per_expert=[0, 0, 0, 226, 0, 0, 0, 0, 358, 138, 1861, 0, 642, 303, 0, 61, 88, 0, 0, 0, 0, 62, 0, 0, 0, 4702, 0, 252, 240, 1286, 1, 6, 1124, 3519, 0, 0, 0, 1190, 0, 1126, 2170, 1127, 4919, 0, 631, 633, 754, 0, 0, 96, 0, 2990, 93, 0, 88, 0, 986, 0, 5172, 5804, 5113, 1268, 0, 123] payload_rows_per_dst=[10714, 8540, 11908, 11638, 11466, 6036, 12902, 12782] dup_rows_per_dst=[82, 2526, 0, 1336, 2452, 370, 3354, 2198] combine_rows_per_src=[10714, 8540, 11908, 11638, 11466, 6036, 12902, 12782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=341320 combine_payload_bytes=304529408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=304870728 ingress_bytes=373735424 link_bytes=373735424 token_bytes=609400136 weight_movement_bytes=69206016 total_nvlink_bytes=678606152 weight_to_token_ratio=0.113564 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406336312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287858688 dispatch_meta_bytes=340408 combine_payload_bytes=287858688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=288199096 ingress_bytes=357064704 link_bytes=357064704 token_bytes=576057784 weight_movement_bytes=69206016 total_nvlink_bytes=645263800 weight_to_token_ratio=0.120137 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343605735e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84314 payload_rows_remote=75696 dup_rows=13990 dup_rows_remote=12720 dedup_ratio=0.142314 router_tokens_per_expert=[0, 0, 0, 197, 0, 0, 0, 0, 395, 78, 1400, 0, 702, 345, 0, 179, 57, 0, 1, 0, 0, 72, 0, 0, 0, 4391, 0, 275, 222, 1099, 0, 217, 1843, 2740, 0, 0, 0, 1223, 0, 1124, 1472, 1949, 5246, 0, 484, 528, 879, 0, 0, 190, 0, 3981, 236, 0, 57, 0, 736, 0, 4175, 5651, 5200, 1632, 0, 176] payload_rows_per_dst=[8692, 7338, 11560, 11270, 9826, 8618, 14188, 12822] dup_rows_per_dst=[52, 1804, 2, 1138, 1826, 1270, 5232, 2666] combine_rows_per_src=[8692, 7338, 11560, 11270, 9826, 8618, 14188, 12822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=327240 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297164360 ingress_bytes=331440128 link_bytes=331440128 token_bytes=594001480 weight_movement_bytes=34603008 total_nvlink_bytes=628604488 weight_to_token_ratio=0.058254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247182519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80198 payload_rows_remote=68548 dup_rows=18106 dup_rows_remote=15384 dedup_ratio=0.184184 router_tokens_per_expert=[40, 17, 2, 308, 170, 17, 27, 305, 422, 123, 1913, 161, 699, 395, 130, 86, 38, 45, 0, 11, 155, 118, 0, 197, 24, 4494, 26, 645, 338, 1569, 115, 369, 627, 3368, 0, 75, 0, 1277, 60, 1283, 3825, 1881, 3695, 180, 531, 879, 1624, 65, 3, 117, 267, 1563, 100, 179, 81, 43, 1321, 139, 4279, 4516, 3361, 712, 4, 138] payload_rows_per_dst=[10012, 11778, 11650, 12540, 10990, 12800, 4254, 6174] dup_rows_per_dst=[318, 3730, 2722, 2620, 2390, 4910, 452, 964] combine_rows_per_src=[10012, 11778, 11650, 12540, 10990, 12800, 4254, 6174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78406 payload_rows_remote=64692 dup_rows=19898 dup_rows_remote=11992 dedup_ratio=0.202413 router_tokens_per_expert=[0, 7451, 1549, 0, 13, 61, 571, 4005, 0, 1, 0, 0, 0, 974, 3245, 151, 0, 123, 123, 321, 67, 0, 163, 828, 1331, 253, 0, 333, 227, 65, 301, 0, 145, 7279, 90, 0, 3508, 0, 862, 0, 729, 0, 0, 0, 2490, 252, 0, 0, 90, 0, 112, 424, 3, 366, 64, 0, 0, 376, 5221, 1335, 286, 2165, 0, 1199] payload_rows_per_dst=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] dup_rows_per_dst=[7906, 4978, 1194, 4296, 654, 230, 186, 454] combine_rows_per_src=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84744 payload_rows_remote=74058 dup_rows=13560 dup_rows_remote=11010 dedup_ratio=0.137939 router_tokens_per_expert=[5, 0, 1, 279, 0, 4, 1, 4, 371, 65, 1302, 1, 700, 322, 2, 199, 69, 0, 17, 0, 4, 63, 0, 8, 4, 4612, 0, 274, 264, 1050, 14, 310, 1652, 2622, 0, 0, 0, 1208, 4, 1132, 1937, 2055, 5472, 3, 615, 461, 638, 4, 0, 202, 0, 3441, 256, 0, 78, 0, 1027, 1, 4636, 5094, 4828, 1685, 0, 156] payload_rows_per_dst=[9790, 7854, 10490, 11864, 10686, 7078, 14386, 12596] dup_rows_per_dst=[70, 1944, 20, 1192, 2550, 474, 4512, 2798] combine_rows_per_src=[9790, 7854, 10490, 11864, 10686, 7078, 14386, 12596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88184 payload_rows_remote=77970 dup_rows=10120 dup_rows_remote=9656 dedup_ratio=0.102946 router_tokens_per_expert=[0, 62, 263, 73, 30, 3686, 0, 489, 0, 0, 17, 0, 88, 0, 0, 0, 1403, 0, 0, 0, 0, 1248, 0, 1074, 0, 110, 2, 765, 0, 0, 0, 62, 5477, 17, 88, 698, 0, 0, 160, 1255, 306, 0, 0, 825, 72, 0, 0, 8, 307, 0, 4893, 107, 5554, 62, 6058, 0, 18, 3962, 4128, 715, 3047, 1827, 0, 196] payload_rows_per_dst=[9130, 12302, 6730, 12386, 12552, 10214, 10384, 14486] dup_rows_per_dst=[76, 24, 720, 600, 2838, 464, 354, 5044] combine_rows_per_src=[9130, 12302, 6730, 12386, 12552, 10214, 10384, 14486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85568 payload_rows_remote=72470 dup_rows=12736 dup_rows_remote=9340 dedup_ratio=0.129557 router_tokens_per_expert=[0, 0, 0, 258, 0, 0, 0, 0, 366, 117, 2043, 0, 601, 322, 0, 92, 84, 0, 1, 0, 0, 70, 0, 0, 0, 4686, 0, 254, 254, 1273, 1, 0, 1535, 2868, 0, 0, 0, 1112, 0, 1076, 2253, 1467, 4799, 0, 618, 659, 750, 0, 0, 123, 0, 3117, 124, 0, 84, 0, 939, 0, 5115, 5633, 4946, 1357, 0, 155] payload_rows_per_dst=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] dup_rows_per_dst=[84, 2710, 2, 980, 1382, 1894, 3396, 2288] combine_rows_per_src=[10662, 8878, 11574, 9668, 8352, 10830, 13098, 12506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80398 payload_rows_remote=70278 dup_rows=17906 dup_rows_remote=14824 dedup_ratio=0.182149 router_tokens_per_expert=[0, 0, 0, 242, 0, 0, 0, 0, 376, 148, 1735, 0, 625, 328, 0, 97, 60, 0, 0, 0, 0, 82, 0, 0, 0, 5277, 0, 320, 240, 1543, 0, 110, 865, 3990, 0, 0, 0, 1101, 0, 1110, 3137, 1090, 4313, 0, 559, 1045, 1197, 0, 0, 111, 0, 2035, 0, 0, 60, 0, 911, 0, 5287, 5914, 4074, 975, 0, 195] payload_rows_per_dst=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] dup_rows_per_dst=[3812, 3082, 2194, 2056, 2360, 3614, 0, 788] combine_rows_per_src=[12608, 10120, 12704, 12924, 11772, 12484, 4412, 3374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=304529408 dispatch_meta_bytes=341320 combine_payload_bytes=304529408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=304870728 ingress_bytes=373735424 link_bytes=373735424 token_bytes=609400136 weight_movement_bytes=69206016 total_nvlink_bytes=678606152 weight_to_token_ratio=0.113564 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406336312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=310050816 dispatch_meta_bytes=353664 combine_payload_bytes=310050816 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=310404480 ingress_bytes=448462848 link_bytes=448462848 token_bytes=620455296 weight_movement_bytes=138412032 total_nvlink_bytes=758867328 weight_to_token_ratio=0.223081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.687529593e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78406 payload_rows_remote=64692 dup_rows=19898 dup_rows_remote=11992 dedup_ratio=0.202413 router_tokens_per_expert=[0, 7451, 1549, 0, 13, 61, 571, 4005, 0, 1, 0, 0, 0, 974, 3245, 151, 0, 123, 123, 321, 67, 0, 163, 828, 1331, 253, 0, 333, 227, 65, 301, 0, 145, 7279, 90, 0, 3508, 0, 862, 0, 729, 0, 0, 0, 2490, 252, 0, 0, 90, 0, 112, 424, 3, 366, 64, 0, 0, 376, 5221, 1335, 286, 2165, 0, 1199] payload_rows_per_dst=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] dup_rows_per_dst=[7906, 4978, 1194, 4296, 654, 230, 186, 454] combine_rows_per_src=[13714, 13774, 12498, 15282, 8556, 6712, 1932, 5938] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82112 payload_rows_remote=67302 dup_rows=16192 dup_rows_remote=8686 dedup_ratio=0.164714 router_tokens_per_expert=[4, 2, 0, 223, 3, 2, 0, 12, 378, 92, 1659, 8, 594, 339, 5, 92, 51, 1, 2, 0, 13, 63, 0, 23, 3, 4645, 4, 338, 247, 1475, 17, 144, 1131, 3890, 0, 7, 0, 1010, 2, 1123, 2550, 1448, 4563, 8, 588, 654, 918, 4, 1, 130, 13, 2417, 33, 13, 62, 2, 1047, 12, 5338, 5523, 4905, 1152, 0, 169] payload_rows_per_dst=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] dup_rows_per_dst=[68, 2428, 18, 48, 964, 2718, 2442, 7506] combine_rows_per_src=[11100, 9006, 11334, 4408, 7126, 12302, 12026, 14810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=280772608 dispatch_meta_bytes=335728 combine_payload_bytes=280772608 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=281108336 ingress_bytes=349978624 link_bytes=349978624 token_bytes=561880944 weight_movement_bytes=69206016 total_nvlink_bytes=631086960 weight_to_token_ratio=0.123168 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.316941387e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=306736 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265285168 ingress_bytes=403390464 link_bytes=403390464 token_bytes=530263600 weight_movement_bytes=138412032 total_nvlink_bytes=668675632 weight_to_token_ratio=0.261025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517925840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=303341568 dispatch_meta_bytes=340272 combine_payload_bytes=303341568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=303681840 ingress_bytes=372547584 link_bytes=372547584 token_bytes=607023408 weight_movement_bytes=69206016 total_nvlink_bytes=676229424 weight_to_token_ratio=0.114009 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401866566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=319365120 dispatch_meta_bytes=350504 combine_payload_bytes=319365120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=319715624 ingress_bytes=388571136 link_bytes=388571136 token_bytes=639080744 weight_movement_bytes=69206016 total_nvlink_bytes=708286760 weight_to_token_ratio=0.108290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462161902e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=296837120 dispatch_meta_bytes=327240 combine_payload_bytes=296837120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297164360 ingress_bytes=331440128 link_bytes=331440128 token_bytes=594001480 weight_movement_bytes=34603008 total_nvlink_bytes=628604488 weight_to_token_ratio=0.058254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247182519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=287858688 dispatch_meta_bytes=340408 combine_payload_bytes=287858688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=288199096 ingress_bytes=357064704 link_bytes=357064704 token_bytes=576057784 weight_movement_bytes=69206016 total_nvlink_bytes=645263800 weight_to_token_ratio=0.120137 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.343605735e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275668992 dispatch_meta_bytes=303952 combine_payload_bytes=275668992 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=275972944 ingress_bytes=448684032 link_bytes=448684032 token_bytes=551641936 weight_movement_bytes=173015040 total_nvlink_bytes=724656976 weight_to_token_ratio=0.313636 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.688361890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82130 payload_rows_remote=67714 dup_rows=16174 dup_rows_remote=9786 dedup_ratio=0.164530 router_tokens_per_expert=[3, 56, 261, 88, 21, 3013, 30, 462, 2, 6, 33, 2, 90, 16, 0, 9, 1320, 0, 1, 0, 1, 1322, 2, 1113, 6, 146, 64, 1342, 0, 9, 0, 56, 5684, 28, 90, 602, 0, 17, 151, 1055, 202, 15, 28, 1079, 63, 6, 16, 7, 360, 8, 4773, 87, 5634, 51, 6071, 3, 31, 3433, 4548, 717, 2300, 2337, 3, 279] payload_rows_per_dst=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] dup_rows_per_dst=[86, 90, 758, 56, 2506, 664, 5626, 6388] combine_rows_per_src=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83208 payload_rows_remote=71148 dup_rows=15096 dup_rows_remote=15032 dedup_ratio=0.153564 router_tokens_per_expert=[0, 63, 231, 82, 17, 2912, 0, 437, 0, 0, 66, 0, 98, 0, 0, 0, 1792, 0, 0, 0, 0, 1586, 0, 1115, 0, 80, 23, 2018, 0, 0, 0, 63, 5562, 4, 98, 538, 0, 0, 179, 883, 242, 0, 0, 1281, 82, 0, 0, 0, 379, 0, 4416, 97, 6069, 63, 5898, 0, 12, 3604, 3864, 711, 2099, 2183, 0, 305] payload_rows_per_dst=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] dup_rows_per_dst=[2560, 64, 5596, 2816, 1568, 1726, 354, 412] combine_rows_per_src=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83122 payload_rows_remote=69432 dup_rows=15182 dup_rows_remote=9462 dedup_ratio=0.154439 router_tokens_per_expert=[0, 51, 244, 77, 63, 3223, 0, 473, 0, 0, 38, 0, 78, 0, 0, 0, 1530, 0, 0, 0, 0, 1426, 0, 1097, 0, 98, 16, 1053, 0, 0, 0, 51, 5421, 2, 78, 631, 0, 0, 148, 1262, 211, 0, 0, 1013, 70, 0, 0, 1, 363, 0, 4093, 79, 5947, 51, 6020, 0, 65, 3679, 4486, 663, 3082, 2029, 0, 240] payload_rows_per_dst=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] dup_rows_per_dst=[86, 60, 746, 206, 2754, 560, 5720, 5050] combine_rows_per_src=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=306736 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265285168 ingress_bytes=403390464 link_bytes=403390464 token_bytes=530263600 weight_movement_bytes=138412032 total_nvlink_bytes=668675632 weight_to_token_ratio=0.261025 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517925840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277356544 dispatch_meta_bytes=310000 combine_payload_bytes=277356544 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=277666544 ingress_bytes=450371584 link_bytes=450371584 token_bytes=555023088 weight_movement_bytes=173015040 total_nvlink_bytes=728038128 weight_to_token_ratio=0.311726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.694712013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88090 payload_rows_remote=74712 dup_rows=10214 dup_rows_remote=8972 dedup_ratio=0.103902 router_tokens_per_expert=[0, 54, 243, 64, 30, 3225, 0, 442, 0, 0, 54, 0, 63, 0, 0, 0, 1636, 0, 0, 0, 0, 1581, 0, 1155, 0, 83, 12, 1189, 0, 0, 0, 54, 5740, 1, 63, 570, 0, 0, 123, 1211, 180, 0, 0, 979, 62, 0, 0, 1, 401, 0, 4311, 66, 5972, 54, 5927, 0, 58, 3648, 4012, 613, 2707, 2383, 0, 185] payload_rows_per_dst=[12452, 12028, 7710, 13378, 12812, 10116, 9288, 10306] dup_rows_per_dst=[2960, 60, 1034, 1242, 2604, 352, 376, 1586] combine_rows_per_src=[12452, 12028, 7710, 13378, 12812, 10116, 9288, 10306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291422208 dispatch_meta_bytes=344720 combine_payload_bytes=291422208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291766928 ingress_bytes=326025216 link_bytes=326025216 token_bytes=583189136 weight_movement_bytes=34603008 total_nvlink_bytes=617792144 weight_to_token_ratio=0.059334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226806641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284393472 dispatch_meta_bytes=315576 combine_payload_bytes=284393472 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=284709048 ingress_bytes=388202496 link_bytes=388202496 token_bytes=569102520 weight_movement_bytes=103809024 total_nvlink_bytes=672911544 weight_to_token_ratio=0.182408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460774740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88184 payload_rows_remote=77970 dup_rows=10120 dup_rows_remote=9656 dedup_ratio=0.102946 router_tokens_per_expert=[0, 62, 263, 73, 30, 3686, 0, 489, 0, 0, 17, 0, 88, 0, 0, 0, 1403, 0, 0, 0, 0, 1248, 0, 1074, 0, 110, 2, 765, 0, 0, 0, 62, 5477, 17, 88, 698, 0, 0, 160, 1255, 306, 0, 0, 825, 72, 0, 0, 8, 307, 0, 4893, 107, 5554, 62, 6058, 0, 18, 3962, 4128, 715, 3047, 1827, 0, 196] payload_rows_per_dst=[9130, 12302, 6730, 12386, 12552, 10214, 10384, 14486] dup_rows_per_dst=[76, 24, 720, 600, 2838, 464, 354, 5044] combine_rows_per_src=[9130, 12302, 6730, 12386, 12552, 10214, 10384, 14486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84118 payload_rows_remote=70550 dup_rows=14186 dup_rows_remote=9224 dedup_ratio=0.144307 router_tokens_per_expert=[44, 87, 305, 344, 57, 1939, 218, 542, 9, 33, 60, 87, 127, 345, 11, 51, 1406, 0, 14, 1, 49, 1215, 114, 1447, 47, 186, 453, 1580, 55, 41, 8, 118, 5220, 170, 84, 474, 2, 226, 157, 619, 212, 228, 446, 1044, 89, 23, 330, 47, 387, 14, 4202, 71, 5514, 49, 5744, 57, 40, 3191, 4080, 818, 1659, 2623, 76, 263] payload_rows_per_dst=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] dup_rows_per_dst=[2194, 452, 4962, 2954, 724, 1888, 352, 660] combine_rows_per_src=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=306020352 dispatch_meta_bytes=334736 combine_payload_bytes=306020352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=306355088 ingress_bytes=340623360 link_bytes=340623360 token_bytes=612375440 weight_movement_bytes=34603008 total_nvlink_bytes=646978448 weight_to_token_ratio=0.056506 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.281738281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87290 payload_rows_remote=74180 dup_rows=11014 dup_rows_remote=7924 dedup_ratio=0.112040 router_tokens_per_expert=[3, 52, 247, 71, 74, 3582, 0, 485, 0, 0, 37, 1, 82, 1, 0, 0, 1601, 0, 0, 9, 1, 1385, 0, 1120, 0, 135, 41, 847, 3, 3, 0, 64, 5711, 5, 84, 675, 5, 1, 142, 1477, 265, 1, 1, 905, 62, 3, 1, 9, 390, 29, 4468, 93, 5396, 52, 5968, 0, 47, 3923, 4009, 674, 2939, 1753, 0, 220] payload_rows_per_dst=[12694, 12124, 7438, 12370, 13110, 10070, 9620, 9864] dup_rows_per_dst=[3624, 54, 794, 608, 3090, 442, 444, 1958] combine_rows_per_src=[12694, 12124, 7438, 12370, 13110, 10070, 9620, 9864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=288972800 dispatch_meta_bytes=319096 combine_payload_bytes=288972800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289291896 ingress_bytes=323575808 link_bytes=323575808 token_bytes=578264696 weight_movement_bytes=34603008 total_nvlink_bytes=612867704 weight_to_token_ratio=0.059839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217589716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79860 payload_rows_remote=65802 dup_rows=18444 dup_rows_remote=12262 dedup_ratio=0.187622 router_tokens_per_expert=[1, 0, 0, 187, 0, 1, 0, 0, 382, 143, 2053, 0, 616, 341, 0, 89, 95, 0, 0, 0, 0, 69, 0, 0, 0, 5045, 0, 379, 241, 1603, 0, 43, 585, 4396, 0, 0, 0, 1028, 0, 1072, 3731, 928, 3242, 0, 556, 1335, 1320, 0, 0, 125, 0, 1605, 0, 0, 95, 0, 1067, 0, 5998, 5899, 3933, 772, 0, 177] payload_rows_per_dst=[14058, 11538, 12126, 12520, 12238, 10456, 3650, 3274] dup_rows_per_dst=[6182, 5028, 0, 2102, 1924, 2450, 0, 758] combine_rows_per_src=[14058, 11538, 12126, 12520, 12238, 10456, 3650, 3274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84118 payload_rows_remote=70550 dup_rows=14186 dup_rows_remote=9224 dedup_ratio=0.144307 router_tokens_per_expert=[44, 87, 305, 344, 57, 1939, 218, 542, 9, 33, 60, 87, 127, 345, 11, 51, 1406, 0, 14, 1, 49, 1215, 114, 1447, 47, 186, 453, 1580, 55, 41, 8, 118, 5220, 170, 84, 474, 2, 226, 157, 619, 212, 228, 446, 1044, 89, 23, 330, 47, 387, 14, 4202, 71, 5514, 49, 5744, 57, 40, 3191, 4080, 818, 1659, 2623, 76, 263] payload_rows_per_dst=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] dup_rows_per_dst=[2194, 452, 4962, 2954, 724, 1888, 352, 660] combine_rows_per_src=[11260, 12482, 13568, 13050, 5804, 11110, 9208, 7636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85162 payload_rows_remote=72756 dup_rows=13142 dup_rows_remote=8542 dedup_ratio=0.133687 router_tokens_per_expert=[1, 0, 62, 76, 62, 0, 0, 0, 0, 2806, 0, 236, 88, 738, 0, 1083, 0, 6919, 0, 358, 0, 0, 57, 0, 7155, 0, 267, 264, 0, 647, 88, 4687, 7566, 5415, 0, 418, 0, 0, 0, 0, 194, 57, 2859, 0, 0, 1599, 257, 0, 151, 0, 0, 1429, 0, 0, 0, 0, 0, 3459, 97, 0, 0, 0, 0, 57] payload_rows_per_dst=[14588, 8630, 14554, 4480, 836, 12406, 15852, 13816] dup_rows_per_dst=[124, 1272, 114, 352, 0, 4600, 2440, 4240] combine_rows_per_src=[14588, 8630, 14554, 4480, 836, 12406, 15852, 13816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83122 payload_rows_remote=69432 dup_rows=15182 dup_rows_remote=9462 dedup_ratio=0.154439 router_tokens_per_expert=[0, 51, 244, 77, 63, 3223, 0, 473, 0, 0, 38, 0, 78, 0, 0, 0, 1530, 0, 0, 0, 0, 1426, 0, 1097, 0, 98, 16, 1053, 0, 0, 0, 51, 5421, 2, 78, 631, 0, 0, 148, 1262, 211, 0, 0, 1013, 70, 0, 0, 1, 363, 0, 4093, 79, 5947, 51, 6020, 0, 65, 3679, 4486, 663, 3082, 2029, 0, 240] payload_rows_per_dst=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] dup_rows_per_dst=[86, 60, 746, 206, 2754, 560, 5720, 5050] combine_rows_per_src=[8176, 12212, 7360, 3886, 12330, 11002, 13690, 14466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83208 payload_rows_remote=71148 dup_rows=15096 dup_rows_remote=15032 dedup_ratio=0.153564 router_tokens_per_expert=[0, 63, 231, 82, 17, 2912, 0, 437, 0, 0, 66, 0, 98, 0, 0, 0, 1792, 0, 0, 0, 0, 1586, 0, 1115, 0, 80, 23, 2018, 0, 0, 0, 63, 5562, 4, 98, 538, 0, 0, 179, 883, 242, 0, 0, 1281, 82, 0, 0, 0, 379, 0, 4416, 97, 6069, 63, 5898, 0, 12, 3604, 3864, 711, 2099, 2183, 0, 305] payload_rows_per_dst=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] dup_rows_per_dst=[2560, 64, 5596, 2816, 1568, 1726, 354, 412] combine_rows_per_src=[12132, 12060, 14514, 13690, 6034, 11172, 9556, 4050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82130 payload_rows_remote=67714 dup_rows=16174 dup_rows_remote=9786 dedup_ratio=0.164530 router_tokens_per_expert=[3, 56, 261, 88, 21, 3013, 30, 462, 2, 6, 33, 2, 90, 16, 0, 9, 1320, 0, 1, 0, 1, 1322, 2, 1113, 6, 146, 64, 1342, 0, 9, 0, 56, 5684, 28, 90, 602, 0, 17, 151, 1055, 202, 15, 28, 1079, 63, 6, 16, 7, 360, 8, 4773, 87, 5634, 51, 6071, 3, 31, 3433, 4548, 717, 2300, 2337, 3, 279] payload_rows_per_dst=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] dup_rows_per_dst=[86, 90, 758, 56, 2506, 664, 5626, 6388] combine_rows_per_src=[7782, 12342, 6760, 3190, 12748, 11264, 13628, 14416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=319365120 dispatch_meta_bytes=350504 combine_payload_bytes=319365120 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=319715624 ingress_bytes=388571136 link_bytes=388571136 token_bytes=639080744 weight_movement_bytes=69206016 total_nvlink_bytes=708286760 weight_to_token_ratio=0.108290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=33280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.462161902e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82382 payload_rows_remote=68332 dup_rows=15922 dup_rows_remote=11350 dedup_ratio=0.161967 router_tokens_per_expert=[16, 2, 51, 56, 52, 1, 1, 0, 10, 3107, 0, 223, 91, 209, 25, 1390, 0, 6845, 15, 151, 0, 0, 58, 2, 7055, 11, 353, 251, 4, 758, 89, 5093, 7293, 5506, 12, 407, 9, 4, 0, 1, 493, 53, 1891, 21, 1, 1633, 256, 26, 59, 2, 0, 1745, 3, 1, 5, 7, 7, 3589, 130, 1, 9, 16, 1, 52] payload_rows_per_dst=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] dup_rows_per_dst=[102, 1316, 110, 1764, 12, 4988, 3058, 4572] combine_rows_per_src=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=303841280 dispatch_meta_bytes=328416 combine_payload_bytes=303841280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=304169696 ingress_bytes=373047296 link_bytes=373047296 token_bytes=608010976 weight_movement_bytes=69206016 total_nvlink_bytes=677216992 weight_to_token_ratio=0.113824 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403746942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80044 payload_rows_remote=67968 dup_rows=18260 dup_rows_remote=13670 dedup_ratio=0.185750 router_tokens_per_expert=[2, 0, 63, 0, 63, 0, 0, 0, 0, 2781, 0, 215, 98, 116, 0, 1456, 0, 6918, 0, 120, 0, 0, 60, 0, 7182, 0, 439, 231, 0, 1221, 97, 4891, 7384, 5686, 0, 347, 0, 0, 0, 0, 214, 58, 1463, 0, 0, 1692, 231, 0, 64, 0, 0, 2227, 0, 0, 0, 0, 0, 3664, 109, 0, 0, 0, 0, 60] payload_rows_per_dst=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] dup_rows_per_dst=[126, 4590, 58, 1844, 11000, 618, 24, 0] combine_rows_per_src=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=269524992 dispatch_meta_bytes=312256 combine_payload_bytes=269524992 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269837248 ingress_bytes=338731008 link_bytes=338731008 token_bytes=539362240 weight_movement_bytes=69206016 total_nvlink_bytes=608568256 weight_to_token_ratio=0.128311 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274617513e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=288972800 dispatch_meta_bytes=319096 combine_payload_bytes=288972800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289291896 ingress_bytes=323575808 link_bytes=323575808 token_bytes=578264696 weight_movement_bytes=34603008 total_nvlink_bytes=612867704 weight_to_token_ratio=0.059839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217589716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=298008576 dispatch_meta_bytes=325192 combine_payload_bytes=298008576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298333768 ingress_bytes=332611584 link_bytes=332611584 token_bytes=596342344 weight_movement_bytes=34603008 total_nvlink_bytes=630945352 weight_to_token_ratio=0.058025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251590613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284393472 dispatch_meta_bytes=315576 combine_payload_bytes=284393472 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=284709048 ingress_bytes=388202496 link_bytes=388202496 token_bytes=569102520 weight_movement_bytes=103809024 total_nvlink_bytes=672911544 weight_to_token_ratio=0.182408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.460774740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291422208 dispatch_meta_bytes=344720 combine_payload_bytes=291422208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291766928 ingress_bytes=326025216 link_bytes=326025216 token_bytes=583189136 weight_movement_bytes=34603008 total_nvlink_bytes=617792144 weight_to_token_ratio=0.059334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=47488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226806641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=277356544 dispatch_meta_bytes=310000 combine_payload_bytes=277356544 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=277666544 ingress_bytes=450371584 link_bytes=450371584 token_bytes=555023088 weight_movement_bytes=173015040 total_nvlink_bytes=728038128 weight_to_token_ratio=0.311726 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.694712013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84160 payload_rows_remote=68598 dup_rows=14144 dup_rows_remote=11356 dedup_ratio=0.143880 router_tokens_per_expert=[0, 0, 51, 60, 51, 0, 0, 0, 0, 3080, 0, 192, 78, 271, 0, 1507, 0, 6925, 0, 309, 0, 0, 84, 0, 7158, 0, 280, 249, 0, 697, 78, 5112, 7488, 5299, 0, 386, 0, 0, 0, 0, 238, 84, 2101, 0, 0, 1425, 244, 0, 55, 0, 0, 1632, 0, 0, 0, 0, 0, 3843, 91, 0, 0, 0, 0, 84] payload_rows_per_dst=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] dup_rows_per_dst=[102, 1396, 168, 316, 0, 4746, 2788, 4628] combine_rows_per_src=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=318728 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280206600 ingress_bytes=314490880 link_bytes=314490880 token_bytes=560094472 weight_movement_bytes=34603008 total_nvlink_bytes=594697480 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183403863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88134 payload_rows_remote=76256 dup_rows=10170 dup_rows_remote=8740 dedup_ratio=0.103455 router_tokens_per_expert=[0, 0, 54, 5, 54, 0, 0, 0, 0, 3060, 0, 154, 63, 254, 0, 1476, 2, 6974, 0, 100, 0, 0, 88, 0, 7226, 0, 328, 242, 0, 989, 61, 5034, 7548, 5710, 0, 366, 0, 0, 0, 0, 124, 85, 2145, 0, 0, 1376, 242, 0, 52, 0, 0, 1635, 0, 0, 0, 0, 0, 3528, 89, 0, 0, 0, 0, 88] payload_rows_per_dst=[14570, 8740, 14148, 11878, 13564, 7316, 10508, 7410] dup_rows_per_dst=[108, 1274, 180, 1430, 5018, 628, 1532, 0] combine_rows_per_src=[14570, 8740, 14148, 11878, 13564, 7316, 10508, 7410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83958 payload_rows_remote=69726 dup_rows=14346 dup_rows_remote=14040 dedup_ratio=0.145935 router_tokens_per_expert=[127, 13, 65, 87, 48, 45, 13, 0, 30, 2734, 0, 141, 138, 705, 72, 780, 3, 6422, 169, 398, 8, 0, 159, 110, 6216, 256, 810, 267, 102, 1515, 81, 5230, 6400, 3772, 222, 526, 134, 109, 0, 96, 455, 68, 1505, 275, 15, 1962, 344, 402, 323, 8, 22, 1999, 20, 2, 50, 30, 172, 2794, 83, 21, 203, 305, 21, 70] payload_rows_per_dst=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] dup_rows_per_dst=[180, 852, 306, 3228, 7554, 1516, 158, 552] combine_rows_per_src=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=278396928 dispatch_meta_bytes=326552 combine_payload_bytes=278396928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278723480 ingress_bytes=312999936 link_bytes=312999936 token_bytes=557120408 weight_movement_bytes=34603008 total_nvlink_bytes=591723416 weight_to_token_ratio=0.062110 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177793561e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=319816 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281297224 ingress_bytes=315580416 link_bytes=315580416 token_bytes=562274632 weight_movement_bytes=34603008 total_nvlink_bytes=596877640 weight_to_token_ratio=0.061541 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187503699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85162 payload_rows_remote=72756 dup_rows=13142 dup_rows_remote=8542 dedup_ratio=0.133687 router_tokens_per_expert=[1, 0, 62, 76, 62, 0, 0, 0, 0, 2806, 0, 236, 88, 738, 0, 1083, 0, 6919, 0, 358, 0, 0, 57, 0, 7155, 0, 267, 264, 0, 647, 88, 4687, 7566, 5415, 0, 418, 0, 0, 0, 0, 194, 57, 2859, 0, 0, 1599, 257, 0, 151, 0, 0, 1429, 0, 0, 0, 0, 0, 3459, 97, 0, 0, 0, 0, 57] payload_rows_per_dst=[14588, 8630, 14554, 4480, 836, 12406, 15852, 13816] dup_rows_per_dst=[124, 1272, 114, 352, 0, 4600, 2440, 4240] combine_rows_per_src=[14588, 8630, 14554, 4480, 836, 12406, 15852, 13816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=312344576 dispatch_meta_bytes=339984 combine_payload_bytes=312344576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=312684560 ingress_bytes=416153600 link_bytes=416153600 token_bytes=625029136 weight_movement_bytes=103809024 total_nvlink_bytes=728838160 weight_to_token_ratio=0.166087 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.565952494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285597696 dispatch_meta_bytes=335064 combine_payload_bytes=285597696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285932760 ingress_bytes=354803712 link_bytes=354803712 token_bytes=571530456 weight_movement_bytes=69206016 total_nvlink_bytes=640736472 weight_to_token_ratio=0.121089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335097804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89590 payload_rows_remote=83756 dup_rows=8714 dup_rows_remote=8680 dedup_ratio=0.088643 router_tokens_per_expert=[0, 5, 56, 80, 63, 1, 3, 0, 1, 2149, 0, 213, 80, 681, 15, 1572, 3, 6834, 2, 495, 0, 3, 63, 0, 6748, 0, 327, 249, 1, 597, 80, 4794, 7442, 5468, 0, 468, 0, 1, 1, 0, 476, 63, 3025, 0, 0, 1653, 242, 0, 54, 0, 3, 1339, 0, 0, 0, 1, 0, 3706, 28, 5, 0, 0, 0, 62] payload_rows_per_dst=[13786, 7998, 14660, 10778, 5834, 10044, 15490, 11000] dup_rows_per_dst=[126, 1424, 140, 1318, 34, 874, 2188, 2610] combine_rows_per_src=[13786, 7998, 14660, 10778, 5834, 10044, 15490, 11000] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80628 payload_rows_remote=69538 dup_rows=17676 dup_rows_remote=15674 dedup_ratio=0.179810 router_tokens_per_expert=[0, 61, 252, 119, 9, 2890, 0, 437, 0, 0, 74, 0, 90, 0, 0, 0, 1740, 0, 0, 0, 0, 1624, 0, 1260, 0, 0, 20, 2788, 0, 0, 0, 61, 5183, 13, 90, 420, 0, 0, 158, 761, 218, 0, 0, 1546, 69, 0, 0, 0, 475, 0, 4227, 89, 6067, 61, 5811, 0, 6, 2778, 4104, 683, 1973, 2693, 0, 302] payload_rows_per_dst=[11090, 11896, 14142, 13806, 5540, 13182, 9344, 1628] dup_rows_per_dst=[2002, 54, 5472, 4066, 1290, 4078, 360, 354] combine_rows_per_src=[11090, 11896, 14142, 13806, 5540, 13182, 9344, 1628] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89100 payload_rows_remote=78424 dup_rows=9204 dup_rows_remote=9202 dedup_ratio=0.093628 router_tokens_per_expert=[0, 0, 104, 2011, 0, 0, 819, 0, 0, 4642, 829, 0, 0, 0, 652, 0, 186, 334, 0, 2386, 0, 57, 0, 0, 6185, 71, 0, 88, 0, 0, 0, 0, 148, 389, 0, 0, 1674, 993, 0, 88, 0, 0, 5338, 0, 0, 1, 0, 0, 567, 0, 88, 0, 6371, 0, 0, 80, 6837, 0, 0, 0, 6872, 554, 788, 0] payload_rows_per_dst=[5718, 10870, 5630, 12688, 15736, 10676, 13094, 14688] dup_rows_per_dst=[150, 1376, 296, 0, 4592, 2, 1118, 1670] combine_rows_per_src=[5718, 10870, 5630, 12688, 15736, 10676, 13094, 14688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83958 payload_rows_remote=69726 dup_rows=14346 dup_rows_remote=14040 dedup_ratio=0.145935 router_tokens_per_expert=[127, 13, 65, 87, 48, 45, 13, 0, 30, 2734, 0, 141, 138, 705, 72, 780, 3, 6422, 169, 398, 8, 0, 159, 110, 6216, 256, 810, 267, 102, 1515, 81, 5230, 6400, 3772, 222, 526, 134, 109, 0, 96, 455, 68, 1505, 275, 15, 1962, 344, 402, 323, 8, 22, 1999, 20, 2, 50, 30, 172, 2794, 83, 21, 203, 305, 21, 70] payload_rows_per_dst=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] dup_rows_per_dst=[180, 852, 306, 3228, 7554, 1516, 158, 552] combine_rows_per_src=[13048, 8348, 14232, 13294, 14964, 8536, 4750, 6786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84160 payload_rows_remote=68598 dup_rows=14144 dup_rows_remote=11356 dedup_ratio=0.143880 router_tokens_per_expert=[0, 0, 51, 60, 51, 0, 0, 0, 0, 3080, 0, 192, 78, 271, 0, 1507, 0, 6925, 0, 309, 0, 0, 84, 0, 7158, 0, 280, 249, 0, 697, 78, 5112, 7488, 5299, 0, 386, 0, 0, 0, 0, 238, 84, 2101, 0, 0, 1425, 244, 0, 55, 0, 0, 1632, 0, 0, 0, 0, 0, 3843, 91, 0, 0, 0, 0, 84] payload_rows_per_dst=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] dup_rows_per_dst=[102, 1396, 168, 316, 0, 4746, 2788, 4628] combine_rows_per_src=[12074, 8860, 14468, 4756, 772, 13662, 15562, 14006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=298008576 dispatch_meta_bytes=325192 combine_payload_bytes=298008576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298333768 ingress_bytes=332611584 link_bytes=332611584 token_bytes=596342344 weight_movement_bytes=34603008 total_nvlink_bytes=630945352 weight_to_token_ratio=0.058025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.251590613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80044 payload_rows_remote=67968 dup_rows=18260 dup_rows_remote=13670 dedup_ratio=0.185750 router_tokens_per_expert=[2, 0, 63, 0, 63, 0, 0, 0, 0, 2781, 0, 215, 98, 116, 0, 1456, 0, 6918, 0, 120, 0, 0, 60, 0, 7182, 0, 439, 231, 0, 1221, 97, 4891, 7384, 5686, 0, 347, 0, 0, 0, 0, 214, 58, 1463, 0, 0, 1692, 231, 0, 64, 0, 0, 2227, 0, 0, 0, 0, 0, 3664, 109, 0, 0, 0, 0, 60] payload_rows_per_dst=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] dup_rows_per_dst=[126, 4590, 58, 1844, 11000, 618, 24, 0] combine_rows_per_src=[14494, 12076, 6804, 11914, 15834, 6698, 4558, 7666] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82382 payload_rows_remote=68332 dup_rows=15922 dup_rows_remote=11350 dedup_ratio=0.161967 router_tokens_per_expert=[16, 2, 51, 56, 52, 1, 1, 0, 10, 3107, 0, 223, 91, 209, 25, 1390, 0, 6845, 15, 151, 0, 0, 58, 2, 7055, 11, 353, 251, 4, 758, 89, 5093, 7293, 5506, 12, 407, 9, 4, 0, 1, 493, 53, 1891, 21, 1, 1633, 256, 26, 59, 2, 0, 1745, 3, 1, 5, 7, 7, 3589, 130, 1, 9, 16, 1, 52] payload_rows_per_dst=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] dup_rows_per_dst=[102, 1316, 110, 1764, 12, 4988, 3058, 4572] combine_rows_per_src=[256, 8794, 14032, 15278, 854, 13946, 15172, 14050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77532 payload_rows_remote=61398 dup_rows=20772 dup_rows_remote=5796 dedup_ratio=0.211304 router_tokens_per_expert=[1, 1, 130, 1280, 1, 10, 1024, 0, 0, 2965, 926, 5, 20, 28, 724, 1, 140, 410, 17, 2021, 6, 54, 1, 2, 5892, 54, 0, 169, 7, 1, 0, 1, 595, 388, 16, 13, 1706, 1322, 6, 89, 52, 10, 5776, 0, 3, 3, 11, 27, 580, 7, 88, 0, 6891, 36, 18, 69, 6894, 0, 8, 1, 7158, 532, 962, 0] payload_rows_per_dst=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] dup_rows_per_dst=[146, 1342, 260, 4, 1068, 130, 2846, 14976] combine_rows_per_src=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=343064576 dispatch_meta_bytes=369744 combine_payload_bytes=343064576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=343434320 ingress_bytes=481476608 link_bytes=481476608 token_bytes=686498896 weight_movement_bytes=138412032 total_nvlink_bytes=824910928 weight_to_token_ratio=0.201620 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=23168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.811757714e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81296 payload_rows_remote=69866 dup_rows=17008 dup_rows_remote=13790 dedup_ratio=0.173014 router_tokens_per_expert=[0, 0, 110, 1197, 0, 1, 1230, 0, 0, 2643, 819, 0, 0, 0, 744, 0, 141, 474, 0, 1662, 0, 60, 0, 0, 5923, 5, 0, 249, 0, 0, 0, 0, 519, 335, 0, 0, 1964, 1429, 0, 97, 27, 0, 5801, 0, 0, 0, 0, 0, 588, 0, 97, 0, 7110, 3, 77, 29, 6951, 0, 0, 0, 7162, 512, 1193, 0] payload_rows_per_dst=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] dup_rows_per_dst=[3524, 3218, 4196, 0, 1268, 4570, 228, 4] combine_rows_per_src=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=284827648 dispatch_meta_bytes=340848 combine_payload_bytes=284827648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285168496 ingress_bytes=319430656 link_bytes=319430656 token_bytes=569996144 weight_movement_bytes=34603008 total_nvlink_bytes=604599152 weight_to_token_ratio=0.060707 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.201991842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=321224704 dispatch_meta_bytes=350504 combine_payload_bytes=321224704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321575208 ingress_bytes=355827712 link_bytes=355827712 token_bytes=642799912 weight_movement_bytes=34603008 total_nvlink_bytes=677402920 weight_to_token_ratio=0.053832 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338951034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84234 payload_rows_remote=70348 dup_rows=14070 dup_rows_remote=12974 dedup_ratio=0.143127 router_tokens_per_expert=[0, 0, 98, 1711, 0, 0, 1090, 0, 0, 3321, 814, 0, 0, 0, 726, 0, 179, 442, 0, 1758, 6, 83, 0, 0, 6042, 46, 0, 109, 0, 0, 0, 0, 569, 396, 0, 0, 1888, 1104, 0, 78, 2, 0, 5700, 0, 0, 1, 2, 0, 562, 0, 78, 0, 6773, 0, 14, 64, 6796, 0, 0, 0, 7080, 584, 1036, 0] payload_rows_per_dst=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] dup_rows_per_dst=[226, 1334, 284, 0, 4454, 8, 1096, 6668] combine_rows_per_src=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87020 payload_rows_remote=74716 dup_rows=11284 dup_rows_remote=11284 dedup_ratio=0.114787 router_tokens_per_expert=[0, 0, 89, 1655, 0, 0, 970, 0, 0, 3497, 826, 0, 0, 0, 753, 0, 174, 359, 0, 1778, 3, 88, 0, 0, 6019, 45, 0, 88, 0, 0, 0, 0, 564, 309, 0, 0, 1817, 1218, 0, 61, 1, 0, 5727, 0, 0, 0, 0, 0, 545, 0, 61, 0, 6928, 1, 12, 53, 6905, 0, 0, 0, 7176, 544, 886, 0] payload_rows_per_dst=[15702, 8846, 14876, 12304, 6854, 11454, 14128, 2856] dup_rows_per_dst=[3536, 1306, 4280, 0, 1084, 2, 1072, 4] combine_rows_per_src=[15702, 8846, 14876, 12304, 6854, 11454, 14128, 2856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285597696 dispatch_meta_bytes=335064 combine_payload_bytes=285597696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=285932760 ingress_bytes=354803712 link_bytes=354803712 token_bytes=571530456 weight_movement_bytes=69206016 total_nvlink_bytes=640736472 weight_to_token_ratio=0.121089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335097804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=280977408 dispatch_meta_bytes=319816 combine_payload_bytes=280977408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281297224 ingress_bytes=315580416 link_bytes=315580416 token_bytes=562274632 weight_movement_bytes=34603008 total_nvlink_bytes=596877640 weight_to_token_ratio=0.061541 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=34176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.187503699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=278396928 dispatch_meta_bytes=326552 combine_payload_bytes=278396928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278723480 ingress_bytes=312999936 link_bytes=312999936 token_bytes=557120408 weight_movement_bytes=34603008 total_nvlink_bytes=591723416 weight_to_token_ratio=0.062110 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.177793561e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=279887872 dispatch_meta_bytes=318728 combine_payload_bytes=279887872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=280206600 ingress_bytes=314490880 link_bytes=314490880 token_bytes=560094472 weight_movement_bytes=34603008 total_nvlink_bytes=594697480 weight_to_token_ratio=0.061781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.183403863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=251486208 dispatch_meta_bytes=268776 combine_payload_bytes=251486208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=251754984 ingress_bytes=389898240 link_bytes=389898240 token_bytes=503241192 weight_movement_bytes=138412032 total_nvlink_bytes=641653224 weight_to_token_ratio=0.275041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.467155688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286171136 dispatch_meta_bytes=334624 combine_payload_bytes=286171136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286505760 ingress_bytes=320774144 link_bytes=320774144 token_bytes=572676896 weight_movement_bytes=34603008 total_nvlink_bytes=607279904 weight_to_token_ratio=0.060423 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207047279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84716 payload_rows_remote=71262 dup_rows=13588 dup_rows_remote=9638 dedup_ratio=0.138224 router_tokens_per_expert=[44, 21, 122, 786, 43, 165, 1601, 0, 0, 1683, 1451, 15, 353, 477, 597, 21, 132, 420, 282, 1776, 82, 39, 8, 29, 5723, 37, 0, 324, 167, 15, 37, 74, 500, 354, 244, 165, 1408, 984, 84, 79, 160, 166, 5584, 0, 53, 108, 98, 496, 638, 238, 75, 1, 6784, 339, 107, 86, 5980, 6, 112, 57, 5934, 578, 1208, 2] payload_rows_per_dst=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] dup_rows_per_dst=[3832, 1856, 3950, 98, 1014, 890, 1714, 234] combine_rows_per_src=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288145408 dispatch_meta_bytes=333288 combine_payload_bytes=288145408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288478696 ingress_bytes=391954432 link_bytes=391954432 token_bytes=576624104 weight_movement_bytes=103809024 total_nvlink_bytes=680433128 weight_to_token_ratio=0.180029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474892973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89100 payload_rows_remote=78424 dup_rows=9204 dup_rows_remote=9202 dedup_ratio=0.093628 router_tokens_per_expert=[0, 0, 104, 2011, 0, 0, 819, 0, 0, 4642, 829, 0, 0, 0, 652, 0, 186, 334, 0, 2386, 0, 57, 0, 0, 6185, 71, 0, 88, 0, 0, 0, 0, 148, 389, 0, 0, 1674, 993, 0, 88, 0, 0, 5338, 0, 0, 1, 0, 0, 567, 0, 88, 0, 6371, 0, 0, 80, 6837, 0, 0, 0, 6872, 554, 788, 0] payload_rows_per_dst=[5718, 10870, 5630, 12688, 15736, 10676, 13094, 14688] dup_rows_per_dst=[150, 1376, 296, 0, 4592, 2, 1118, 1670] combine_rows_per_src=[5718, 10870, 5630, 12688, 15736, 10676, 13094, 14688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=306036736 dispatch_meta_bytes=344000 combine_payload_bytes=306036736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=306380736 ingress_bytes=375242752 link_bytes=375242752 token_bytes=612417472 weight_movement_bytes=69206016 total_nvlink_bytes=681623488 weight_to_token_ratio=0.113005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.412008266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90334 payload_rows_remote=79600 dup_rows=7970 dup_rows_remote=7350 dedup_ratio=0.081075 router_tokens_per_expert=[3112, 0, 0, 225, 736, 2429, 0, 0, 1520, 72, 3224, 0, 0, 0, 0, 0, 0, 88, 135, 0, 0, 88, 0, 0, 74, 0, 0, 0, 0, 88, 0, 2, 0, 0, 7158, 0, 0, 0, 1619, 727, 0, 6981, 656, 7567, 0, 0, 4937, 84, 0, 0, 0, 88, 101, 0, 0, 5678, 316, 0, 0, 368, 600, 164, 257, 58] payload_rows_per_dst=[8190, 8164, 14282, 15138, 7040, 10734, 11734, 15052] dup_rows_per_dst=[1118, 1468, 302, 324, 1348, 620, 0, 2790] combine_rows_per_src=[8190, 8164, 14282, 15138, 7040, 10734, 11734, 15052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=291889152 dispatch_meta_bytes=323600 combine_payload_bytes=291889152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292212752 ingress_bytes=326492160 link_bytes=326492160 token_bytes=584101904 weight_movement_bytes=34603008 total_nvlink_bytes=618704912 weight_to_token_ratio=0.059241 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228563713e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87800 payload_rows_remote=73568 dup_rows=10504 dup_rows_remote=5800 dedup_ratio=0.106852 router_tokens_per_expert=[0, 0, 108, 2064, 0, 0, 911, 0, 0, 4251, 811, 0, 0, 0, 704, 2, 253, 306, 0, 2201, 48, 62, 0, 0, 6206, 75, 0, 82, 0, 0, 0, 0, 526, 353, 1, 0, 1630, 1152, 1, 80, 13, 0, 5178, 0, 0, 0, 1, 0, 580, 0, 80, 0, 6531, 2, 2, 83, 6542, 0, 0, 0, 6878, 585, 850, 0] payload_rows_per_dst=[6416, 10228, 6900, 12752, 14232, 10374, 13398, 13500] dup_rows_per_dst=[464, 1308, 1146, 122, 4704, 10, 1158, 1592] combine_rows_per_src=[6416, 10228, 6900, 12752, 14232, 10374, 13398, 13500] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79168 payload_rows_remote=64674 dup_rows=19136 dup_rows_remote=19010 dedup_ratio=0.194661 router_tokens_per_expert=[3, 0, 61, 2, 61, 0, 0, 0, 0, 2544, 0, 212, 87, 87, 0, 1533, 10, 6904, 0, 102, 0, 0, 95, 0, 7183, 0, 631, 252, 0, 1049, 89, 5217, 7042, 6080, 0, 449, 0, 0, 0, 0, 394, 85, 1244, 0, 0, 1888, 252, 0, 40, 0, 0, 2072, 0, 0, 0, 0, 0, 3264, 125, 0, 0, 0, 0, 95] payload_rows_per_dst=[14494, 15464, 4254, 12756, 13906, 7106, 4220, 6968] dup_rows_per_dst=[126, 7270, 76, 2918, 8122, 620, 4, 0] combine_rows_per_src=[14494, 15464, 4254, 12756, 13906, 7106, 4220, 6968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84716 payload_rows_remote=71262 dup_rows=13588 dup_rows_remote=9638 dedup_ratio=0.138224 router_tokens_per_expert=[44, 21, 122, 786, 43, 165, 1601, 0, 0, 1683, 1451, 15, 353, 477, 597, 21, 132, 420, 282, 1776, 82, 39, 8, 29, 5723, 37, 0, 324, 167, 15, 37, 74, 500, 354, 244, 165, 1408, 984, 84, 79, 160, 166, 5584, 0, 53, 108, 98, 496, 638, 238, 75, 1, 6784, 339, 107, 86, 5980, 6, 112, 57, 5934, 578, 1208, 2] payload_rows_per_dst=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] dup_rows_per_dst=[3832, 1856, 3950, 98, 1014, 890, 1714, 234] combine_rows_per_src=[13692, 7338, 13454, 12656, 6622, 12440, 14822, 3692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84234 payload_rows_remote=70348 dup_rows=14070 dup_rows_remote=12974 dedup_ratio=0.143127 router_tokens_per_expert=[0, 0, 98, 1711, 0, 0, 1090, 0, 0, 3321, 814, 0, 0, 0, 726, 0, 179, 442, 0, 1758, 6, 83, 0, 0, 6042, 46, 0, 109, 0, 0, 0, 0, 569, 396, 0, 0, 1888, 1104, 0, 78, 2, 0, 5700, 0, 0, 1, 2, 0, 562, 0, 78, 0, 6773, 0, 14, 64, 6796, 0, 0, 0, 7080, 584, 1036, 0] payload_rows_per_dst=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] dup_rows_per_dst=[226, 1334, 284, 0, 4454, 8, 1096, 6668] combine_rows_per_src=[5572, 8388, 4652, 12394, 12608, 11402, 13886, 15332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=321224704 dispatch_meta_bytes=350504 combine_payload_bytes=321224704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=321575208 ingress_bytes=355827712 link_bytes=355827712 token_bytes=642799912 weight_movement_bytes=34603008 total_nvlink_bytes=677402920 weight_to_token_ratio=0.053832 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.338951034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81296 payload_rows_remote=69866 dup_rows=17008 dup_rows_remote=13790 dedup_ratio=0.173014 router_tokens_per_expert=[0, 0, 110, 1197, 0, 1, 1230, 0, 0, 2643, 819, 0, 0, 0, 744, 0, 141, 474, 0, 1662, 0, 60, 0, 0, 5923, 5, 0, 249, 0, 0, 0, 0, 519, 335, 0, 0, 1964, 1429, 0, 97, 27, 0, 5801, 0, 0, 0, 0, 0, 588, 0, 97, 0, 7110, 3, 77, 29, 6951, 0, 0, 0, 7162, 512, 1193, 0] payload_rows_per_dst=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] dup_rows_per_dst=[3524, 3218, 4196, 0, 1268, 4570, 228, 4] combine_rows_per_src=[15454, 11430, 14802, 12354, 7420, 12712, 3718, 3406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77532 payload_rows_remote=61398 dup_rows=20772 dup_rows_remote=5796 dedup_ratio=0.211304 router_tokens_per_expert=[1, 1, 130, 1280, 1, 10, 1024, 0, 0, 2965, 926, 5, 20, 28, 724, 1, 140, 410, 17, 2021, 6, 54, 1, 2, 5892, 54, 0, 169, 7, 1, 0, 1, 595, 388, 16, 13, 1706, 1322, 6, 89, 52, 10, 5776, 0, 3, 3, 11, 27, 580, 7, 88, 0, 6891, 36, 18, 69, 6894, 0, 8, 1, 7158, 532, 962, 0] payload_rows_per_dst=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] dup_rows_per_dst=[146, 1342, 260, 4, 1068, 130, 2846, 14976] combine_rows_per_src=[4748, 7996, 5042, 10416, 7202, 11634, 14360, 16134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=326041600 dispatch_meta_bytes=347800 combine_payload_bytes=326041600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326389400 ingress_bytes=464453632 link_bytes=464453632 token_bytes=652431000 weight_movement_bytes=138412032 total_nvlink_bytes=790843032 weight_to_token_ratio=0.212148 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747701626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60722 payload_rows_remote=46096 dup_rows=37582 dup_rows_remote=34354 dedup_ratio=0.382304 router_tokens_per_expert=[2837, 1, 32, 93, 810, 3882, 5, 4, 1396, 64, 1894, 0, 3, 3, 1, 0, 0, 87, 145, 28, 0, 88, 8, 4, 95, 1, 4, 9, 7, 88, 1, 57, 14, 0, 6649, 0, 0, 6, 1587, 776, 4, 6821, 1196, 7496, 6, 20, 4656, 88, 3, 16, 0, 87, 45, 4, 26, 5727, 675, 26, 1, 365, 598, 182, 256, 175] payload_rows_per_dst=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] dup_rows_per_dst=[192, 850, 30, 2, 1462, 28360, 3458, 3228] combine_rows_per_src=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86682 payload_rows_remote=73886 dup_rows=11622 dup_rows_remote=9688 dedup_ratio=0.118225 router_tokens_per_expert=[3312, 0, 0, 104, 762, 4347, 0, 0, 1410, 82, 1605, 0, 0, 0, 0, 0, 0, 98, 146, 0, 0, 97, 0, 0, 132, 0, 0, 0, 0, 98, 0, 80, 0, 0, 6764, 0, 0, 0, 1559, 588, 0, 6953, 1080, 7601, 0, 0, 4268, 54, 0, 0, 0, 98, 25, 0, 0, 5808, 363, 0, 0, 334, 593, 170, 231, 390] payload_rows_per_dst=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] dup_rows_per_dst=[3620, 1934, 322, 460, 3396, 0, 0, 1890] combine_rows_per_src=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=301334528 dispatch_meta_bytes=317472 combine_payload_bytes=301334528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301652000 ingress_bytes=335937536 link_bytes=335937536 token_bytes=602986528 weight_movement_bytes=34603008 total_nvlink_bytes=637589536 weight_to_token_ratio=0.057386 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86920 payload_rows_remote=71618 dup_rows=11384 dup_rows_remote=11022 dedup_ratio=0.115804 router_tokens_per_expert=[3009, 0, 0, 113, 840, 3487, 0, 0, 1499, 62, 2367, 0, 0, 0, 0, 0, 0, 63, 116, 0, 0, 61, 0, 0, 118, 0, 0, 0, 0, 63, 0, 22, 0, 0, 6907, 0, 0, 0, 1477, 744, 0, 6924, 724, 7629, 0, 0, 4789, 58, 0, 0, 0, 63, 75, 0, 0, 5873, 543, 0, 0, 356, 604, 177, 242, 147] payload_rows_per_dst=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] dup_rows_per_dst=[2410, 3594, 230, 362, 2462, 0, 0, 2326] combine_rows_per_src=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78990 payload_rows_remote=67126 dup_rows=19314 dup_rows_remote=18974 dedup_ratio=0.196472 router_tokens_per_expert=[3432, 0, 0, 149, 802, 3434, 0, 0, 1428, 69, 2015, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 110, 0, 0, 0, 1, 78, 0, 18, 0, 0, 6962, 0, 0, 0, 1706, 513, 0, 6929, 847, 7596, 0, 0, 4800, 73, 0, 0, 0, 78, 90, 0, 0, 5683, 589, 0, 0, 372, 644, 186, 244, 27] payload_rows_per_dst=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] dup_rows_per_dst=[1166, 1010, 184, 108, 1280, 12224, 340, 3002] combine_rows_per_src=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=334736 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239440 ingress_bytes=299507712 link_bytes=299507712 token_bytes=530144144 weight_movement_bytes=34603008 total_nvlink_bytes=564747152 weight_to_token_ratio=0.065271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=47744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127023408e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=291889152 dispatch_meta_bytes=323600 combine_payload_bytes=291889152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292212752 ingress_bytes=326492160 link_bytes=326492160 token_bytes=584101904 weight_movement_bytes=34603008 total_nvlink_bytes=618704912 weight_to_token_ratio=0.059241 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228563713e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86920 payload_rows_remote=71618 dup_rows=11384 dup_rows_remote=11022 dedup_ratio=0.115804 router_tokens_per_expert=[3009, 0, 0, 113, 840, 3487, 0, 0, 1499, 62, 2367, 0, 0, 0, 0, 0, 0, 63, 116, 0, 0, 61, 0, 0, 118, 0, 0, 0, 0, 63, 0, 22, 0, 0, 6907, 0, 0, 0, 1477, 744, 0, 6924, 724, 7629, 0, 0, 4789, 58, 0, 0, 0, 63, 75, 0, 0, 5873, 543, 0, 0, 356, 604, 177, 242, 147] payload_rows_per_dst=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] dup_rows_per_dst=[2410, 3594, 230, 362, 2462, 0, 0, 2326] combine_rows_per_src=[12488, 13840, 14098, 15302, 9720, 1564, 12022, 7886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=188809216 dispatch_meta_bytes=321800 combine_payload_bytes=188809216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=189131016 ingress_bytes=223412224 link_bytes=223412224 token_bytes=377940232 weight_movement_bytes=34603008 total_nvlink_bytes=412543240 weight_to_token_ratio=0.091557 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.406822128e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=302637056 dispatch_meta_bytes=334296 combine_payload_bytes=302637056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302971352 ingress_bytes=337240064 link_bytes=337240064 token_bytes=605608408 weight_movement_bytes=34603008 total_nvlink_bytes=640211416 weight_to_token_ratio=0.057138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269007211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=274948096 dispatch_meta_bytes=344400 combine_payload_bytes=274948096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275292496 ingress_bytes=309551104 link_bytes=309551104 token_bytes=550240592 weight_movement_bytes=34603008 total_nvlink_bytes=584843600 weight_to_token_ratio=0.062887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164815883e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84920 payload_rows_remote=70986 dup_rows=13384 dup_rows_remote=12468 dedup_ratio=0.136149 router_tokens_per_expert=[1921, 12, 564, 75, 699, 5533, 175, 64, 1643, 176, 980, 0, 32, 48, 20, 22, 1, 66, 491, 378, 81, 78, 197, 54, 154, 11, 32, 175, 57, 84, 25, 52, 209, 1, 5055, 0, 5, 109, 2077, 544, 1, 6079, 1183, 6711, 61, 299, 4245, 86, 56, 218, 31, 69, 10, 94, 254, 4040, 1015, 519, 15, 581, 606, 104, 303, 672] payload_rows_per_dst=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] dup_rows_per_dst=[3430, 1970, 916, 450, 3234, 24, 368, 2992] combine_rows_per_src=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90334 payload_rows_remote=79600 dup_rows=7970 dup_rows_remote=7350 dedup_ratio=0.081075 router_tokens_per_expert=[3112, 0, 0, 225, 736, 2429, 0, 0, 1520, 72, 3224, 0, 0, 0, 0, 0, 0, 88, 135, 0, 0, 88, 0, 0, 74, 0, 0, 0, 0, 88, 0, 2, 0, 0, 7158, 0, 0, 0, 1619, 727, 0, 6981, 656, 7567, 0, 0, 4937, 84, 0, 0, 0, 88, 101, 0, 0, 5678, 316, 0, 0, 368, 600, 164, 257, 58] payload_rows_per_dst=[8190, 8164, 14282, 15138, 7040, 10734, 11734, 15052] dup_rows_per_dst=[1118, 1468, 302, 324, 1348, 620, 0, 2790] combine_rows_per_src=[8190, 8164, 14282, 15138, 7040, 10734, 11734, 15052] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=288145408 dispatch_meta_bytes=333288 combine_payload_bytes=288145408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=288478696 ingress_bytes=391954432 link_bytes=391954432 token_bytes=576624104 weight_movement_bytes=103809024 total_nvlink_bytes=680433128 weight_to_token_ratio=0.180029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.474892973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=286171136 dispatch_meta_bytes=334624 combine_payload_bytes=286171136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286505760 ingress_bytes=320774144 link_bytes=320774144 token_bytes=572676896 weight_movement_bytes=34603008 total_nvlink_bytes=607279904 weight_to_token_ratio=0.060423 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.207047279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=251486208 dispatch_meta_bytes=268776 combine_payload_bytes=251486208 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=251754984 ingress_bytes=389898240 link_bytes=389898240 token_bytes=503241192 weight_movement_bytes=138412032 total_nvlink_bytes=641653224 weight_to_token_ratio=0.275041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=26752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.467155688e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=293347328 dispatch_meta_bytes=330560 combine_payload_bytes=293347328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293677888 ingress_bytes=327950336 link_bytes=327950336 token_bytes=587025216 weight_movement_bytes=34603008 total_nvlink_bytes=621628224 weight_to_token_ratio=0.058946 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.234050712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87778 payload_rows_remote=81178 dup_rows=10526 dup_rows_remote=10386 dedup_ratio=0.107076 router_tokens_per_expert=[4901, 507, 0, 0, 0, 0, 1500, 13, 0, 3071, 0, 4522, 0, 0, 1231, 0, 0, 176, 2989, 0, 0, 333, 0, 0, 0, 0, 2285, 582, 1489, 0, 307, 304, 0, 0, 0, 0, 0, 0, 0, 5562, 0, 0, 402, 134, 2617, 217, 7331, 0, 6906, 70, 0, 152, 0, 88, 0, 0, 72, 0, 0, 0, 439, 88, 0, 864] payload_rows_per_dst=[10360, 9204, 12874, 8664, 11124, 6600, 13838, 15114] dup_rows_per_dst=[1850, 1032, 3166, 1270, 0, 140, 594, 2474] combine_rows_per_src=[10360, 9204, 12874, 8664, 11124, 6600, 13838, 15114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=290758656 dispatch_meta_bytes=333816 combine_payload_bytes=290758656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291092472 ingress_bytes=325361664 link_bytes=325361664 token_bytes=581851128 weight_movement_bytes=34603008 total_nvlink_bytes=616454136 weight_to_token_ratio=0.059471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224309748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=326041600 dispatch_meta_bytes=347800 combine_payload_bytes=326041600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=326389400 ingress_bytes=464453632 link_bytes=464453632 token_bytes=652431000 weight_movement_bytes=138412032 total_nvlink_bytes=790843032 weight_to_token_ratio=0.212148 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.747701626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83378 payload_rows_remote=68156 dup_rows=14926 dup_rows_remote=11024 dedup_ratio=0.151835 router_tokens_per_expert=[0, 0, 124, 878, 0, 0, 1522, 0, 0, 2074, 857, 0, 0, 0, 721, 0, 185, 499, 0, 1327, 0, 95, 0, 0, 5923, 0, 0, 335, 0, 0, 0, 0, 597, 364, 0, 0, 1836, 1837, 0, 89, 68, 0, 5932, 0, 0, 0, 0, 0, 579, 0, 89, 0, 7082, 38, 69, 58, 7038, 0, 0, 0, 7162, 525, 1249, 0] payload_rows_per_dst=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] dup_rows_per_dst=[3902, 5684, 3724, 0, 1288, 134, 192, 2] combine_rows_per_src=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88828 payload_rows_remote=83564 dup_rows=9476 dup_rows_remote=8418 dedup_ratio=0.096395 router_tokens_per_expert=[2611, 2, 0, 147, 885, 2223, 0, 1, 1796, 61, 2839, 0, 0, 0, 0, 0, 0, 80, 114, 0, 0, 80, 0, 0, 68, 0, 0, 0, 5, 80, 0, 7, 0, 0, 7196, 0, 0, 2, 2342, 817, 0, 6873, 637, 7597, 3, 0, 4837, 81, 0, 0, 0, 80, 109, 0, 29, 5692, 448, 0, 0, 355, 622, 189, 242, 2] payload_rows_per_dst=[10376, 10988, 14030, 15218, 5264, 5950, 11818, 15184] dup_rows_per_dst=[1362, 3270, 264, 296, 1058, 300, 2, 2924] combine_rows_per_src=[10376, 10988, 14030, 15218, 5264, 5950, 11818, 15184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83378 payload_rows_remote=68156 dup_rows=14926 dup_rows_remote=11024 dedup_ratio=0.151835 router_tokens_per_expert=[0, 0, 124, 878, 0, 0, 1522, 0, 0, 2074, 857, 0, 0, 0, 721, 0, 185, 499, 0, 1327, 0, 95, 0, 0, 5923, 0, 0, 335, 0, 0, 0, 0, 597, 364, 0, 0, 1836, 1837, 0, 89, 68, 0, 5932, 0, 0, 0, 0, 0, 579, 0, 89, 0, 7082, 38, 69, 58, 7038, 0, 0, 0, 7162, 525, 1249, 0] payload_rows_per_dst=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] dup_rows_per_dst=[3902, 5684, 3724, 0, 1288, 134, 192, 2] combine_rows_per_src=[15222, 15784, 14812, 12516, 8158, 11866, 1474, 3546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78990 payload_rows_remote=67126 dup_rows=19314 dup_rows_remote=18974 dedup_ratio=0.196472 router_tokens_per_expert=[3432, 0, 0, 149, 802, 3434, 0, 0, 1428, 69, 2015, 0, 0, 0, 0, 0, 0, 78, 121, 0, 0, 78, 0, 0, 110, 0, 0, 0, 1, 78, 0, 18, 0, 0, 6962, 0, 0, 0, 1706, 513, 0, 6929, 847, 7596, 0, 0, 4800, 73, 0, 0, 0, 78, 90, 0, 0, 5683, 589, 0, 0, 372, 644, 186, 244, 27] payload_rows_per_dst=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] dup_rows_per_dst=[1166, 1010, 184, 108, 1280, 12224, 340, 3002] combine_rows_per_src=[7098, 6014, 10272, 4618, 10026, 14052, 11864, 15046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84920 payload_rows_remote=70986 dup_rows=13384 dup_rows_remote=12468 dedup_ratio=0.136149 router_tokens_per_expert=[1921, 12, 564, 75, 699, 5533, 175, 64, 1643, 176, 980, 0, 32, 48, 20, 22, 1, 66, 491, 378, 81, 78, 197, 54, 154, 11, 32, 175, 57, 84, 25, 52, 209, 1, 5055, 0, 5, 109, 2077, 544, 1, 6079, 1183, 6711, 61, 299, 4245, 86, 56, 218, 31, 69, 10, 94, 254, 4040, 1015, 519, 15, 581, 606, 104, 303, 672] payload_rows_per_dst=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] dup_rows_per_dst=[3430, 1970, 916, 450, 3234, 24, 368, 2992] combine_rows_per_src=[14656, 12362, 13934, 14152, 12766, 3236, 9176, 4638] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=332505088 dispatch_meta_bytes=366256 combine_payload_bytes=332505088 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332871344 ingress_bytes=436314112 link_bytes=436314112 token_bytes=665376432 weight_movement_bytes=103809024 total_nvlink_bytes=769185456 weight_to_token_ratio=0.156015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641814877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86682 payload_rows_remote=73886 dup_rows=11622 dup_rows_remote=9688 dedup_ratio=0.118225 router_tokens_per_expert=[3312, 0, 0, 104, 762, 4347, 0, 0, 1410, 82, 1605, 0, 0, 0, 0, 0, 0, 98, 146, 0, 0, 97, 0, 0, 132, 0, 0, 0, 0, 98, 0, 80, 0, 0, 6764, 0, 0, 0, 1559, 588, 0, 6953, 1080, 7601, 0, 0, 4268, 54, 0, 0, 0, 98, 25, 0, 0, 5808, 363, 0, 0, 334, 593, 170, 231, 390] payload_rows_per_dst=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] dup_rows_per_dst=[3620, 1934, 322, 460, 3396, 0, 0, 1890] combine_rows_per_src=[13430, 12796, 14266, 15362, 14426, 2268, 11862, 2272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65340 dup_rows=17956 dup_rows_remote=15076 dedup_ratio=0.182658 router_tokens_per_expert=[5290, 953, 2, 1, 4, 30, 1572, 80, 27, 2787, 16, 4397, 3, 0, 979, 0, 10, 175, 2267, 2, 10, 361, 3, 10, 0, 2, 3243, 650, 1553, 14, 303, 429, 32, 15, 8, 0, 24, 0, 0, 4462, 3, 24, 428, 114, 2730, 169, 7247, 9, 6697, 50, 0, 147, 0, 87, 19, 17, 63, 0, 0, 31, 604, 89, 0, 910] payload_rows_per_dst=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] dup_rows_per_dst=[1058, 4100, 2472, 2472, 76, 24, 4874, 2880] combine_rows_per_src=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67268 dup_rows=19806 dup_rows_remote=17650 dedup_ratio=0.201477 router_tokens_per_expert=[4912, 822, 0, 0, 0, 0, 1906, 97, 0, 2420, 0, 4136, 0, 0, 710, 0, 0, 195, 1959, 0, 0, 368, 0, 0, 0, 0, 3517, 613, 1715, 0, 317, 746, 0, 0, 0, 0, 0, 0, 0, 4597, 0, 0, 577, 143, 3142, 133, 7293, 0, 6671, 6, 0, 162, 0, 97, 0, 0, 82, 0, 0, 0, 732, 98, 0, 986] payload_rows_per_dst=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] dup_rows_per_dst=[3232, 2156, 282, 2642, 3948, 6852, 272, 422] combine_rows_per_src=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84180 payload_rows_remote=70162 dup_rows=14124 dup_rows_remote=12430 dedup_ratio=0.143677 router_tokens_per_expert=[4930, 471, 0, 0, 0, 0, 1801, 68, 0, 2958, 0, 4373, 0, 0, 1146, 0, 0, 156, 2675, 0, 0, 401, 0, 0, 0, 0, 2568, 661, 1824, 0, 287, 406, 0, 0, 0, 0, 0, 0, 0, 5079, 0, 0, 417, 120, 2840, 174, 7339, 0, 6790, 62, 0, 173, 0, 78, 0, 0, 70, 0, 0, 0, 309, 78, 0, 898] payload_rows_per_dst=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] dup_rows_per_dst=[750, 4702, 2838, 1646, 0, 200, 1694, 2294] combine_rows_per_src=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279166976 dispatch_meta_bytes=316720 combine_payload_bytes=279166976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279483696 ingress_bytes=313769984 link_bytes=313769984 token_bytes=558650672 weight_movement_bytes=34603008 total_nvlink_bytes=593253680 weight_to_token_ratio=0.061940 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180691189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=342278144 dispatch_meta_bytes=367928 combine_payload_bytes=342278144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=342646072 ingress_bytes=411484160 link_bytes=411484160 token_bytes=684924216 weight_movement_bytes=69206016 total_nvlink_bytes=754130232 weight_to_token_ratio=0.101042 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.548381767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87656 payload_rows_remote=77400 dup_rows=10648 dup_rows_remote=8762 dedup_ratio=0.108317 router_tokens_per_expert=[5082, 491, 0, 0, 0, 0, 1781, 60, 0, 3165, 0, 4544, 0, 0, 1111, 0, 0, 124, 2175, 0, 0, 356, 0, 0, 0, 0, 2979, 688, 1699, 0, 272, 433, 0, 0, 0, 0, 0, 0, 0, 4923, 0, 0, 426, 113, 2704, 142, 7373, 0, 6791, 40, 0, 158, 0, 61, 0, 0, 62, 0, 0, 0, 471, 62, 0, 866] payload_rows_per_dst=[12126, 8384, 12076, 10256, 9846, 7930, 13714, 13324] dup_rows_per_dst=[2702, 168, 2322, 1886, 0, 1038, 386, 2146] combine_rows_per_src=[12126, 8384, 12076, 10256, 9846, 7930, 13714, 13324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=60722 payload_rows_remote=46096 dup_rows=37582 dup_rows_remote=34354 dedup_ratio=0.382304 router_tokens_per_expert=[2837, 1, 32, 93, 810, 3882, 5, 4, 1396, 64, 1894, 0, 3, 3, 1, 0, 0, 87, 145, 28, 0, 88, 8, 4, 95, 1, 4, 9, 7, 88, 1, 57, 14, 0, 6649, 0, 0, 6, 1587, 776, 4, 6821, 1196, 7496, 6, 20, 4656, 88, 3, 16, 0, 87, 45, 4, 26, 5727, 675, 26, 1, 365, 598, 182, 256, 175] payload_rows_per_dst=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] dup_rows_per_dst=[192, 850, 30, 2, 1462, 28360, 3458, 3228] combine_rows_per_src=[1698, 5872, 690, 522, 7160, 16122, 14032, 14626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=290758656 dispatch_meta_bytes=333816 combine_payload_bytes=290758656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291092472 ingress_bytes=325361664 link_bytes=325361664 token_bytes=581851128 weight_movement_bytes=34603008 total_nvlink_bytes=616454136 weight_to_token_ratio=0.059471 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224309748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=279166976 dispatch_meta_bytes=316720 combine_payload_bytes=279166976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279483696 ingress_bytes=313769984 link_bytes=313769984 token_bytes=558650672 weight_movement_bytes=34603008 total_nvlink_bytes=593253680 weight_to_token_ratio=0.061940 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27264 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.180691189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=321664 combine_payload_bytes=267632640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267954304 ingress_bytes=302235648 link_bytes=302235648 token_bytes=535586944 weight_movement_bytes=34603008 total_nvlink_bytes=570189952 weight_to_token_ratio=0.064608 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137288411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=275529728 dispatch_meta_bytes=339672 combine_payload_bytes=275529728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275869400 ingress_bytes=344735744 link_bytes=344735744 token_bytes=551399128 weight_movement_bytes=69206016 total_nvlink_bytes=620605144 weight_to_token_ratio=0.125510 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297212851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=287383552 dispatch_meta_bytes=330368 combine_payload_bytes=287383552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287713920 ingress_bytes=356589568 link_bytes=356589568 token_bytes=575097472 weight_movement_bytes=69206016 total_nvlink_bytes=644303488 weight_to_token_ratio=0.120338 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341817837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=274948096 dispatch_meta_bytes=344400 combine_payload_bytes=274948096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275292496 ingress_bytes=309551104 link_bytes=309551104 token_bytes=550240592 weight_movement_bytes=34603008 total_nvlink_bytes=584843600 weight_to_token_ratio=0.062887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164815883e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=302637056 dispatch_meta_bytes=334296 combine_payload_bytes=302637056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302971352 ingress_bytes=337240064 link_bytes=337240064 token_bytes=605608408 weight_movement_bytes=34603008 total_nvlink_bytes=640211416 weight_to_token_ratio=0.057138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269007211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=317030400 dispatch_meta_bytes=344648 combine_payload_bytes=317030400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317375048 ingress_bytes=351633408 link_bytes=351633408 token_bytes=634405448 weight_movement_bytes=34603008 total_nvlink_bytes=669008456 weight_to_token_ratio=0.054544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323168205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=188809216 dispatch_meta_bytes=321800 combine_payload_bytes=188809216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=189131016 ingress_bytes=223412224 link_bytes=223412224 token_bytes=377940232 weight_movement_bytes=34603008 total_nvlink_bytes=412543240 weight_to_token_ratio=0.091557 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.406822128e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90778 payload_rows_remote=75788 dup_rows=7526 dup_rows_remote=3032 dedup_ratio=0.076558 router_tokens_per_expert=[0, 0, 639, 0, 0, 0, 2507, 0, 0, 0, 159, 319, 7246, 0, 0, 4838, 0, 0, 0, 0, 0, 5576, 0, 0, 0, 0, 0, 7962, 0, 0, 3764, 0, 0, 0, 150, 0, 0, 4598, 0, 0, 0, 0, 2450, 0, 0, 0, 0, 46, 0, 0, 336, 0, 0, 858, 0, 88, 0, 6746, 0, 0, 795, 75, 0, 0] payload_rows_per_dst=[5968, 9954, 11152, 7888, 9496, 14990, 16208, 15122] dup_rows_per_dst=[324, 92, 0, 226, 0, 4494, 2280, 110] combine_rows_per_src=[5968, 9954, 11152, 7888, 9496, 14990, 16208, 15122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87670 payload_rows_remote=76906 dup_rows=10634 dup_rows_remote=10634 dedup_ratio=0.108175 router_tokens_per_expert=[4297, 435, 0, 0, 0, 0, 1564, 48, 0, 4170, 0, 4092, 3, 0, 1216, 0, 0, 160, 3281, 0, 0, 385, 7, 0, 0, 0, 2112, 684, 1773, 0, 281, 382, 0, 0, 0, 0, 0, 0, 0, 5382, 0, 0, 368, 111, 2215, 285, 7329, 0, 6533, 59, 0, 154, 0, 83, 23, 192, 62, 0, 0, 0, 537, 82, 0, 847] payload_rows_per_dst=[10478, 10218, 12518, 9144, 10764, 5896, 13594, 15058] dup_rows_per_dst=[2210, 560, 3332, 1320, 0, 62, 494, 2656] combine_rows_per_src=[10478, 10218, 12518, 9144, 10764, 5896, 13594, 15058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=310427648 dispatch_meta_bytes=315280 combine_payload_bytes=310427648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310742928 ingress_bytes=345030656 link_bytes=345030656 token_bytes=621170576 weight_movement_bytes=34603008 total_nvlink_bytes=655773584 weight_to_token_ratio=0.055706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298322581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79432 payload_rows_remote=69302 dup_rows=18872 dup_rows_remote=17230 dedup_ratio=0.191976 router_tokens_per_expert=[4409, 1620, 19, 4, 115, 101, 1625, 93, 89, 2229, 234, 3177, 68, 29, 458, 1, 105, 180, 1673, 42, 134, 393, 44, 138, 15, 69, 2172, 571, 2212, 187, 262, 1058, 567, 262, 69, 0, 464, 17, 7, 3461, 12, 443, 1417, 133, 3396, 180, 6267, 202, 5704, 89, 33, 168, 0, 76, 7, 306, 81, 23, 2, 541, 856, 96, 23, 724] payload_rows_per_dst=[12006, 6002, 10130, 10536, 8508, 15430, 12378, 4442] dup_rows_per_dst=[3966, 214, 1642, 2556, 1186, 8670, 388, 250] combine_rows_per_src=[12006, 6002, 10130, 10536, 8508, 15430, 12378, 4442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87120 payload_rows_remote=73408 dup_rows=11184 dup_rows_remote=7826 dedup_ratio=0.113770 router_tokens_per_expert=[2805, 0, 0, 90, 755, 4885, 0, 0, 1339, 69, 1470, 0, 0, 0, 0, 0, 0, 89, 130, 0, 0, 89, 0, 0, 246, 0, 0, 0, 12, 90, 0, 57, 0, 0, 6725, 0, 0, 0, 1380, 726, 0, 6934, 1425, 7590, 0, 0, 4208, 0, 0, 0, 0, 90, 1, 0, 0, 5723, 380, 0, 0, 344, 593, 219, 252, 436] payload_rows_per_dst=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] dup_rows_per_dst=[3358, 1654, 300, 672, 3278, 0, 2, 1920] combine_rows_per_src=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84180 payload_rows_remote=70162 dup_rows=14124 dup_rows_remote=12430 dedup_ratio=0.143677 router_tokens_per_expert=[4930, 471, 0, 0, 0, 0, 1801, 68, 0, 2958, 0, 4373, 0, 0, 1146, 0, 0, 156, 2675, 0, 0, 401, 0, 0, 0, 0, 2568, 661, 1824, 0, 287, 406, 0, 0, 0, 0, 0, 0, 0, 5079, 0, 0, 417, 120, 2840, 174, 7339, 0, 6790, 62, 0, 173, 0, 78, 0, 0, 70, 0, 0, 0, 309, 78, 0, 898] payload_rows_per_dst=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] dup_rows_per_dst=[750, 4702, 2838, 1646, 0, 200, 1694, 2294] combine_rows_per_src=[3930, 13366, 12372, 9846, 10158, 5396, 14018, 15094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=315006976 dispatch_meta_bytes=350160 combine_payload_bytes=315006976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315357136 ingress_bytes=349609984 link_bytes=349609984 token_bytes=630364112 weight_movement_bytes=34603008 total_nvlink_bytes=664967120 weight_to_token_ratio=0.054894 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315554224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87778 payload_rows_remote=81178 dup_rows=10526 dup_rows_remote=10386 dedup_ratio=0.107076 router_tokens_per_expert=[4901, 507, 0, 0, 0, 0, 1500, 13, 0, 3071, 0, 4522, 0, 0, 1231, 0, 0, 176, 2989, 0, 0, 333, 0, 0, 0, 0, 2285, 582, 1489, 0, 307, 304, 0, 0, 0, 0, 0, 0, 0, 5562, 0, 0, 402, 134, 2617, 217, 7331, 0, 6906, 70, 0, 152, 0, 88, 0, 0, 72, 0, 0, 0, 439, 88, 0, 864] payload_rows_per_dst=[10360, 9204, 12874, 8664, 11124, 6600, 13838, 15114] dup_rows_per_dst=[1850, 1032, 3166, 1270, 0, 140, 594, 2474] combine_rows_per_src=[10360, 9204, 12874, 8664, 11124, 6600, 13838, 15114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=317030400 dispatch_meta_bytes=344648 combine_payload_bytes=317030400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=317375048 ingress_bytes=351633408 link_bytes=351633408 token_bytes=634405448 weight_movement_bytes=34603008 total_nvlink_bytes=669008456 weight_to_token_ratio=0.054544 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=22784 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.323168205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=85802 dup_rows=9670 dup_rows_remote=9670 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 954, 0, 0, 0, 2227, 0, 0, 0, 118, 272, 7329, 0, 0, 3990, 0, 0, 0, 0, 0, 4491, 0, 0, 0, 0, 0, 7889, 0, 0, 4608, 0, 0, 0, 115, 0, 0, 6535, 0, 0, 0, 0, 1741, 0, 0, 0, 0, 47, 0, 0, 307, 0, 0, 955, 0, 63, 0, 6413, 0, 0, 863, 235, 0, 0] payload_rows_per_dst=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] dup_rows_per_dst=[3574, 1432, 0, 0, 0, 2516, 1836, 312] combine_rows_per_src=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78498 payload_rows_remote=67268 dup_rows=19806 dup_rows_remote=17650 dedup_ratio=0.201477 router_tokens_per_expert=[4912, 822, 0, 0, 0, 0, 1906, 97, 0, 2420, 0, 4136, 0, 0, 710, 0, 0, 195, 1959, 0, 0, 368, 0, 0, 0, 0, 3517, 613, 1715, 0, 317, 746, 0, 0, 0, 0, 0, 0, 0, 4597, 0, 0, 577, 143, 3142, 133, 7293, 0, 6671, 6, 0, 162, 0, 97, 0, 0, 82, 0, 0, 0, 732, 98, 0, 986] payload_rows_per_dst=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] dup_rows_per_dst=[3232, 2156, 282, 2642, 3948, 6852, 272, 422] combine_rows_per_src=[12242, 11230, 5908, 11174, 11974, 15724, 6872, 3374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80348 payload_rows_remote=65340 dup_rows=17956 dup_rows_remote=15076 dedup_ratio=0.182658 router_tokens_per_expert=[5290, 953, 2, 1, 4, 30, 1572, 80, 27, 2787, 16, 4397, 3, 0, 979, 0, 10, 175, 2267, 2, 10, 361, 3, 10, 0, 2, 3243, 650, 1553, 14, 303, 429, 32, 15, 8, 0, 24, 0, 0, 4462, 3, 24, 428, 114, 2730, 169, 7247, 9, 6697, 50, 0, 147, 0, 87, 19, 17, 63, 0, 0, 31, 604, 89, 0, 910] payload_rows_per_dst=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] dup_rows_per_dst=[1058, 4100, 2472, 2472, 76, 24, 4874, 2880] combine_rows_per_src=[4226, 13242, 11998, 10778, 9006, 1470, 14620, 15008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=283860992 dispatch_meta_bytes=346128 combine_payload_bytes=283860992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=284207120 ingress_bytes=318464000 link_bytes=318464000 token_bytes=568068112 weight_movement_bytes=34603008 total_nvlink_bytes=602671120 weight_to_token_ratio=0.060913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.198354394e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=287383552 dispatch_meta_bytes=330368 combine_payload_bytes=287383552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=287713920 ingress_bytes=356589568 link_bytes=356589568 token_bytes=575097472 weight_movement_bytes=69206016 total_nvlink_bytes=644303488 weight_to_token_ratio=0.120338 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.341817837e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87120 payload_rows_remote=73408 dup_rows=11184 dup_rows_remote=7826 dedup_ratio=0.113770 router_tokens_per_expert=[2805, 0, 0, 90, 755, 4885, 0, 0, 1339, 69, 1470, 0, 0, 0, 0, 0, 0, 89, 130, 0, 0, 89, 0, 0, 246, 0, 0, 0, 12, 90, 0, 57, 0, 0, 6725, 0, 0, 0, 1380, 726, 0, 6934, 1425, 7590, 0, 0, 4208, 0, 0, 0, 0, 90, 1, 0, 0, 5723, 380, 0, 0, 344, 593, 219, 252, 436] payload_rows_per_dst=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] dup_rows_per_dst=[3358, 1654, 300, 672, 3278, 0, 2, 1920] combine_rows_per_src=[13712, 12518, 14184, 15318, 14384, 2850, 11626, 2528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=332505088 dispatch_meta_bytes=366256 combine_payload_bytes=332505088 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=332871344 ingress_bytes=436314112 link_bytes=436314112 token_bytes=665376432 weight_movement_bytes=103809024 total_nvlink_bytes=769185456 weight_to_token_ratio=0.156015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.641814877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87670 payload_rows_remote=76906 dup_rows=10634 dup_rows_remote=10634 dedup_ratio=0.108175 router_tokens_per_expert=[4297, 435, 0, 0, 0, 0, 1564, 48, 0, 4170, 0, 4092, 3, 0, 1216, 0, 0, 160, 3281, 0, 0, 385, 7, 0, 0, 0, 2112, 684, 1773, 0, 281, 382, 0, 0, 0, 0, 0, 0, 0, 5382, 0, 0, 368, 111, 2215, 285, 7329, 0, 6533, 59, 0, 154, 0, 83, 23, 192, 62, 0, 0, 0, 537, 82, 0, 847] payload_rows_per_dst=[10478, 10218, 12518, 9144, 10764, 5896, 13594, 15058] dup_rows_per_dst=[2210, 560, 3332, 1320, 0, 62, 494, 2656] combine_rows_per_src=[10478, 10218, 12518, 9144, 10764, 5896, 13594, 15058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83944 payload_rows_remote=68702 dup_rows=14360 dup_rows_remote=8700 dedup_ratio=0.146077 router_tokens_per_expert=[0, 31, 1433, 7, 0, 1, 1780, 0, 0, 23, 138, 276, 7200, 0, 34, 4244, 5, 0, 1, 2, 4, 4262, 5, 19, 0, 2, 9, 7759, 7, 35, 4616, 2, 6, 1, 138, 2, 7, 6657, 12, 1, 3, 0, 1450, 0, 36, 38, 22, 36, 2, 18, 300, 5, 7, 901, 23, 87, 4, 6247, 0, 13, 880, 326, 24, 11] payload_rows_per_dst=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] dup_rows_per_dst=[356, 0, 2, 3712, 14, 2542, 2074, 5660] combine_rows_per_src=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80088 payload_rows_remote=64456 dup_rows=18216 dup_rows_remote=10680 dedup_ratio=0.185303 router_tokens_per_expert=[0, 0, 1281, 0, 0, 0, 2310, 0, 0, 0, 160, 249, 7167, 0, 0, 4008, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 7859, 0, 0, 5015, 0, 0, 0, 160, 0, 0, 7094, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 22, 0, 0, 270, 0, 0, 1055, 0, 98, 0, 6344, 0, 0, 827, 347, 0, 0] payload_rows_per_dst=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] dup_rows_per_dst=[4004, 7536, 5626, 0, 0, 0, 730, 320] combine_rows_per_src=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=300679168 dispatch_meta_bytes=324936 combine_payload_bytes=300679168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=301004104 ingress_bytes=404488192 link_bytes=404488192 token_bytes=601683272 weight_movement_bytes=103809024 total_nvlink_bytes=705492296 weight_to_token_ratio=0.172531 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522056503e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=351444992 dispatch_meta_bytes=381888 combine_payload_bytes=351444992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=351826880 ingress_bytes=455254016 link_bytes=455254016 token_bytes=703271872 weight_movement_bytes=103809024 total_nvlink_bytes=807080896 weight_to_token_ratio=0.147609 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.713084211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=275529728 dispatch_meta_bytes=339672 combine_payload_bytes=275529728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=275869400 ingress_bytes=344735744 link_bytes=344735744 token_bytes=551399128 weight_movement_bytes=69206016 total_nvlink_bytes=620605144 weight_to_token_ratio=0.125510 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297212851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267632640 dispatch_meta_bytes=321664 combine_payload_bytes=267632640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267954304 ingress_bytes=302235648 link_bytes=302235648 token_bytes=535586944 weight_movement_bytes=34603008 total_nvlink_bytes=570189952 weight_to_token_ratio=0.064608 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137288411e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=76500 dup_rows=6712 dup_rows_remote=6712 dedup_ratio=0.068278 router_tokens_per_expert=[0, 0, 0, 0, 0, 88, 433, 0, 0, 71, 105, 0, 0, 62, 1228, 0, 4285, 0, 91, 7206, 7489, 0, 0, 0, 0, 492, 428, 0, 7608, 0, 62, 0, 2570, 0, 1, 0, 256, 5842, 7103, 88, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1640, 0, 633, 0, 0, 0, 0, 0, 0, 0, 224, 415, 0, 310, 365] payload_rows_per_dst=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] dup_rows_per_dst=[170, 1516, 0, 1038, 514, 0, 1892, 1582] combine_rows_per_src=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91414 payload_rows_remote=81936 dup_rows=6890 dup_rows_remote=6848 dedup_ratio=0.070089 router_tokens_per_expert=[0, 9, 629, 7, 0, 0, 3109, 0, 4, 0, 207, 300, 7249, 0, 0, 4270, 0, 0, 0, 0, 0, 5466, 1, 0, 0, 0, 0, 7894, 2, 0, 4075, 0, 21, 0, 132, 0, 0, 4606, 1, 0, 0, 0, 1958, 0, 0, 0, 0, 112, 0, 0, 312, 3, 0, 928, 0, 80, 1, 6759, 0, 9, 944, 64, 0, 0] payload_rows_per_dst=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] dup_rows_per_dst=[580, 128, 0, 0, 42, 3632, 2210, 298] combine_rows_per_src=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=313344000 dispatch_meta_bytes=332848 combine_payload_bytes=313344000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313676848 ingress_bytes=347947008 link_bytes=347947008 token_bytes=627020848 weight_movement_bytes=34603008 total_nvlink_bytes=661623856 weight_to_token_ratio=0.055186 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309296579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83276 payload_rows_remote=74038 dup_rows=15028 dup_rows_remote=13636 dedup_ratio=0.152873 router_tokens_per_expert=[30, 142, 2457, 67, 13, 4, 2216, 0, 0, 6, 114, 169, 6248, 19, 535, 3113, 98, 0, 2, 44, 27, 2369, 143, 402, 24, 7, 32, 6842, 44, 636, 4195, 36, 55, 18, 117, 4, 20, 6669, 218, 90, 25, 0, 613, 1, 220, 697, 302, 144, 4, 128, 241, 103, 21, 621, 512, 77, 58, 4698, 0, 106, 676, 2205, 385, 90] payload_rows_per_dst=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] dup_rows_per_dst=[5090, 5722, 1392, 102, 286, 806, 474, 1156] combine_rows_per_src=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=335609856 dispatch_meta_bytes=355136 combine_payload_bytes=335609856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=335964992 ingress_bytes=404815872 link_bytes=404815872 token_bytes=671574848 weight_movement_bytes=69206016 total_nvlink_bytes=740780864 weight_to_token_ratio=0.103050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523289536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=72406 dup_rows=9670 dup_rows_remote=7602 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 860, 0, 0, 0, 2273, 0, 0, 0, 140, 276, 7251, 0, 0, 3933, 0, 0, 0, 0, 0, 4915, 0, 0, 0, 0, 0, 7871, 0, 0, 4365, 0, 0, 0, 129, 0, 0, 6274, 0, 0, 0, 0, 1998, 0, 0, 0, 0, 35, 0, 0, 307, 0, 0, 892, 0, 78, 0, 6499, 0, 0, 891, 165, 0, 0] payload_rows_per_dst=[5894, 832, 9830, 13102, 12806, 15026, 16228, 14916] dup_rows_per_dst=[372, 0, 0, 3494, 0, 3542, 2068, 194] combine_rows_per_src=[5894, 832, 9830, 13102, 12806, 15026, 16228, 14916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=61332 dup_rows=24072 dup_rows_remote=19804 dedup_ratio=0.244873 router_tokens_per_expert=[5020, 1166, 0, 0, 0, 0, 2120, 87, 0, 2422, 0, 3976, 0, 0, 465, 0, 0, 179, 1635, 0, 0, 373, 0, 0, 0, 0, 3883, 628, 1416, 0, 273, 857, 0, 0, 0, 0, 0, 0, 0, 4190, 0, 0, 633, 99, 3486, 103, 7306, 0, 6667, 0, 0, 195, 0, 89, 0, 0, 69, 0, 0, 0, 707, 72, 0, 1036] payload_rows_per_dst=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] dup_rows_per_dst=[4268, 2170, 0, 2724, 7098, 7330, 178, 304] combine_rows_per_src=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92186 payload_rows_remote=76078 dup_rows=6118 dup_rows_remote=5738 dedup_ratio=0.062236 router_tokens_per_expert=[2, 0, 0, 0, 0, 61, 231, 0, 0, 29, 136, 0, 0, 54, 1340, 0, 4234, 0, 98, 7461, 7549, 0, 0, 0, 0, 174, 497, 0, 7519, 0, 54, 0, 3536, 0, 3, 0, 242, 5211, 6995, 63, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1579, 0, 570, 0, 0, 0, 0, 0, 0, 0, 258, 507, 0, 305, 356] payload_rows_per_dst=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] dup_rows_per_dst=[52, 1292, 0, 380, 490, 0, 1910, 1994] combine_rows_per_src=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80088 payload_rows_remote=64456 dup_rows=18216 dup_rows_remote=10680 dedup_ratio=0.185303 router_tokens_per_expert=[0, 0, 1281, 0, 0, 0, 2310, 0, 0, 0, 160, 249, 7167, 0, 0, 4008, 0, 0, 0, 0, 0, 3630, 0, 0, 0, 0, 0, 7859, 0, 0, 5015, 0, 0, 0, 160, 0, 0, 7094, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 22, 0, 0, 270, 0, 0, 1055, 0, 98, 0, 6344, 0, 0, 827, 347, 0, 0] payload_rows_per_dst=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] dup_rows_per_dst=[4004, 7536, 5626, 0, 0, 0, 730, 320] combine_rows_per_src=[13208, 15632, 14322, 15718, 14508, 2556, 2116, 2028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83944 payload_rows_remote=68702 dup_rows=14360 dup_rows_remote=8700 dedup_ratio=0.146077 router_tokens_per_expert=[0, 31, 1433, 7, 0, 1, 1780, 0, 0, 23, 138, 276, 7200, 0, 34, 4244, 5, 0, 1, 2, 4, 4262, 5, 19, 0, 2, 9, 7759, 7, 35, 4616, 2, 6, 1, 138, 2, 7, 6657, 12, 1, 3, 0, 1450, 0, 36, 38, 22, 36, 2, 18, 300, 5, 7, 901, 23, 87, 4, 6247, 0, 13, 880, 326, 24, 11] payload_rows_per_dst=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] dup_rows_per_dst=[356, 0, 2, 3712, 14, 2542, 2074, 5660] combine_rows_per_src=[6148, 942, 8594, 12604, 9256, 15028, 16130, 15242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=303259648 dispatch_meta_bytes=350696 combine_payload_bytes=303259648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303610344 ingress_bytes=337862656 link_bytes=337862656 token_bytes=606869992 weight_movement_bytes=34603008 total_nvlink_bytes=641473000 weight_to_token_ratio=0.057019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271349974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=320032 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296895008 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593469984 weight_movement_bytes=34603008 total_nvlink_bytes=628072992 weight_to_token_ratio=0.058306 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=315006976 dispatch_meta_bytes=350160 combine_payload_bytes=315006976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=315357136 ingress_bytes=349609984 link_bytes=349609984 token_bytes=630364112 weight_movement_bytes=34603008 total_nvlink_bytes=664967120 weight_to_token_ratio=0.054894 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=37760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.315554224e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88634 payload_rows_remote=85802 dup_rows=9670 dup_rows_remote=9670 dedup_ratio=0.098368 router_tokens_per_expert=[0, 0, 954, 0, 0, 0, 2227, 0, 0, 0, 118, 272, 7329, 0, 0, 3990, 0, 0, 0, 0, 0, 4491, 0, 0, 0, 0, 0, 7889, 0, 0, 4608, 0, 0, 0, 115, 0, 0, 6535, 0, 0, 0, 0, 1741, 0, 0, 0, 0, 47, 0, 0, 307, 0, 0, 955, 0, 63, 0, 6413, 0, 0, 863, 235, 0, 0] payload_rows_per_dst=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] dup_rows_per_dst=[3574, 1432, 0, 0, 0, 2516, 1836, 312] combine_rows_per_src=[12004, 10204, 8982, 2832, 13300, 12842, 13760, 14710] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=251215872 dispatch_meta_bytes=324544 combine_payload_bytes=251215872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251540416 ingress_bytes=355024896 link_bytes=355024896 token_bytes=502756288 weight_movement_bytes=103809024 total_nvlink_bytes=606565312 weight_to_token_ratio=0.206480 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311615488 dispatch_meta_bytes=327264 combine_payload_bytes=311615488 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=311942752 ingress_bytes=380821504 link_bytes=380821504 token_bytes=623558240 weight_movement_bytes=69206016 total_nvlink_bytes=692764256 weight_to_token_ratio=0.110986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433000661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90330 payload_rows_remote=74946 dup_rows=7974 dup_rows_remote=7214 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 88, 0, 88, 883, 0, 0, 0, 3169, 3031, 7588, 742, 0, 88, 7586, 0, 0, 7084, 104, 62, 0, 0, 0, 0, 192, 0, 61, 0, 0, 6218, 0, 382, 0, 0, 0, 0, 102, 0, 0, 0, 0, 320, 3213, 0, 634, 144, 1204, 88, 0, 0, 0, 5133, 1, 460] payload_rows_per_dst=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] dup_rows_per_dst=[774, 1024, 740, 1878, 128, 760, 1188, 1482] combine_rows_per_src=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=281403392 dispatch_meta_bytes=309608 combine_payload_bytes=281403392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=281713000 ingress_bytes=350609408 link_bytes=350609408 token_bytes=563116392 weight_movement_bytes=69206016 total_nvlink_bytes=632322408 weight_to_token_ratio=0.122898 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35200 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.319314976e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83276 payload_rows_remote=74038 dup_rows=15028 dup_rows_remote=13636 dedup_ratio=0.152873 router_tokens_per_expert=[30, 142, 2457, 67, 13, 4, 2216, 0, 0, 6, 114, 169, 6248, 19, 535, 3113, 98, 0, 2, 44, 27, 2369, 143, 402, 24, 7, 32, 6842, 44, 636, 4195, 36, 55, 18, 117, 4, 20, 6669, 218, 90, 25, 0, 613, 1, 220, 697, 302, 144, 4, 128, 241, 103, 21, 621, 512, 77, 58, 4698, 0, 106, 676, 2205, 385, 90] payload_rows_per_dst=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] dup_rows_per_dst=[5090, 5722, 1392, 102, 286, 806, 474, 1156] combine_rows_per_src=[13158, 14686, 9238, 15140, 14096, 3198, 2940, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90778 payload_rows_remote=75788 dup_rows=7526 dup_rows_remote=3032 dedup_ratio=0.076558 router_tokens_per_expert=[0, 0, 639, 0, 0, 0, 2507, 0, 0, 0, 159, 319, 7246, 0, 0, 4838, 0, 0, 0, 0, 0, 5576, 0, 0, 0, 0, 0, 7962, 0, 0, 3764, 0, 0, 0, 150, 0, 0, 4598, 0, 0, 0, 0, 2450, 0, 0, 0, 0, 46, 0, 0, 336, 0, 0, 858, 0, 88, 0, 6746, 0, 0, 795, 75, 0, 0] payload_rows_per_dst=[5968, 9954, 11152, 7888, 9496, 14990, 16208, 15122] dup_rows_per_dst=[324, 92, 0, 226, 0, 4494, 2280, 110] combine_rows_per_src=[5968, 9954, 11152, 7888, 9496, 14990, 16208, 15122] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=300544 combine_payload_bytes=264011776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264312320 ingress_bytes=367820800 link_bytes=367820800 token_bytes=528324096 weight_movement_bytes=103809024 total_nvlink_bytes=632133120 weight_to_token_ratio=0.196487 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.384080058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92004 payload_rows_remote=86546 dup_rows=6300 dup_rows_remote=5812 dedup_ratio=0.064087 router_tokens_per_expert=[1, 0, 0, 0, 0, 81, 441, 0, 0, 62, 115, 0, 0, 53, 1266, 0, 4528, 1, 83, 7219, 7513, 0, 0, 0, 0, 374, 417, 0, 7551, 0, 52, 0, 2636, 0, 1, 0, 242, 5679, 7048, 94, 0, 0, 0, 0, 0, 2, 0, 73, 0, 1605, 0, 605, 0, 0, 0, 0, 0, 0, 0, 231, 488, 0, 337, 354] payload_rows_per_dst=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] dup_rows_per_dst=[164, 1352, 0, 628, 488, 20, 1856, 1792] combine_rows_per_src=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306978816 dispatch_meta_bytes=328640 combine_payload_bytes=306978816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307307456 ingress_bytes=341581824 link_bytes=341581824 token_bytes=614286272 weight_movement_bytes=34603008 total_nvlink_bytes=648889280 weight_to_token_ratio=0.056330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285344904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88148 payload_rows_remote=73530 dup_rows=10156 dup_rows_remote=10050 dedup_ratio=0.103312 router_tokens_per_expert=[66, 0, 524, 0, 663, 70, 385, 49, 25, 61, 198, 96, 0, 107, 1194, 6, 4323, 409, 162, 6326, 5948, 1, 367, 97, 0, 628, 1513, 0, 6301, 206, 388, 8, 3054, 152, 328, 89, 248, 2303, 5454, 79, 1, 127, 6, 485, 13, 15, 3, 37, 32, 801, 66, 1865, 0, 520, 0, 1, 218, 26, 408, 281, 565, 608, 856, 390] payload_rows_per_dst=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] dup_rows_per_dst=[966, 554, 106, 3220, 826, 124, 2692, 1668] combine_rows_per_src=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=354492416 dispatch_meta_bytes=369432 combine_payload_bytes=354492416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354861848 ingress_bytes=458301440 link_bytes=458301440 token_bytes=709354264 weight_movement_bytes=103809024 total_nvlink_bytes=813163288 weight_to_token_ratio=0.146343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724551422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90274 payload_rows_remote=78840 dup_rows=8030 dup_rows_remote=6194 dedup_ratio=0.081685 router_tokens_per_expert=[0, 0, 0, 0, 0, 78, 283, 0, 0, 46, 154, 0, 0, 51, 1364, 0, 4586, 0, 120, 7350, 7492, 0, 0, 0, 0, 199, 484, 0, 7504, 0, 51, 0, 3058, 0, 0, 0, 244, 5381, 7026, 78, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1431, 0, 618, 0, 0, 0, 0, 0, 0, 0, 273, 495, 0, 330, 372] payload_rows_per_dst=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] dup_rows_per_dst=[72, 1444, 0, 400, 2144, 0, 1836, 2134] combine_rows_per_src=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81386 payload_rows_remote=66796 dup_rows=16918 dup_rows_remote=11182 dedup_ratio=0.172099 router_tokens_per_expert=[0, 0, 1509, 0, 0, 0, 2500, 0, 0, 0, 150, 261, 7118, 0, 0, 3326, 1, 0, 0, 0, 0, 2960, 2, 0, 0, 0, 0, 7851, 0, 0, 6154, 0, 0, 0, 150, 0, 0, 7361, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 39, 0, 0, 258, 0, 0, 1165, 0, 90, 0, 6153, 0, 0, 558, 522, 0, 0] payload_rows_per_dst=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] dup_rows_per_dst=[5736, 6220, 4122, 0, 0, 0, 696, 144] combine_rows_per_src=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91260 payload_rows_remote=85198 dup_rows=7044 dup_rows_remote=7010 dedup_ratio=0.071655 router_tokens_per_expert=[0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 2, 63, 0, 63, 888, 0, 0, 0, 2548, 2905, 7575, 709, 0, 60, 7658, 0, 0, 7184, 89, 54, 0, 0, 0, 0, 152, 0, 39, 0, 0, 6307, 0, 583, 0, 0, 0, 0, 61, 0, 0, 0, 0, 212, 3489, 0, 725, 174, 1441, 61, 0, 0, 0, 4833, 6, 635] payload_rows_per_dst=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] dup_rows_per_dst=[1270, 1030, 692, 34, 84, 912, 1146, 1876] combine_rows_per_src=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=320032 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296895008 ingress_bytes=331177984 link_bytes=331177984 token_bytes=593469984 weight_movement_bytes=34603008 total_nvlink_bytes=628072992 weight_to_token_ratio=0.058306 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246196092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=351444992 dispatch_meta_bytes=381888 combine_payload_bytes=351444992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=351826880 ingress_bytes=455254016 link_bytes=455254016 token_bytes=703271872 weight_movement_bytes=103809024 total_nvlink_bytes=807080896 weight_to_token_ratio=0.147609 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=24064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.713084211e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=303259648 dispatch_meta_bytes=350696 combine_payload_bytes=303259648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=303610344 ingress_bytes=337862656 link_bytes=337862656 token_bytes=606869992 weight_movement_bytes=34603008 total_nvlink_bytes=641473000 weight_to_token_ratio=0.057019 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=33792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.271349974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=61332 dup_rows=24072 dup_rows_remote=19804 dedup_ratio=0.244873 router_tokens_per_expert=[5020, 1166, 0, 0, 0, 0, 2120, 87, 0, 2422, 0, 3976, 0, 0, 465, 0, 0, 179, 1635, 0, 0, 373, 0, 0, 0, 0, 3883, 628, 1416, 0, 273, 857, 0, 0, 0, 0, 0, 0, 0, 4190, 0, 0, 633, 99, 3486, 103, 7306, 0, 6667, 0, 0, 195, 0, 89, 0, 0, 69, 0, 0, 0, 707, 72, 0, 1036] payload_rows_per_dst=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] dup_rows_per_dst=[4268, 2170, 0, 2724, 7098, 7330, 178, 304] combine_rows_per_src=[12900, 11556, 4374, 11390, 14616, 15542, 390, 3464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=310427648 dispatch_meta_bytes=315280 combine_payload_bytes=310427648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310742928 ingress_bytes=345030656 link_bytes=345030656 token_bytes=621170576 weight_movement_bytes=34603008 total_nvlink_bytes=655773584 weight_to_token_ratio=0.055706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=34944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298322581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91414 payload_rows_remote=81936 dup_rows=6890 dup_rows_remote=6848 dedup_ratio=0.070089 router_tokens_per_expert=[0, 9, 629, 7, 0, 0, 3109, 0, 4, 0, 207, 300, 7249, 0, 0, 4270, 0, 0, 0, 0, 0, 5466, 1, 0, 0, 0, 0, 7894, 2, 0, 4075, 0, 21, 0, 132, 0, 0, 4606, 1, 0, 0, 0, 1958, 0, 0, 0, 0, 112, 0, 0, 312, 3, 0, 928, 0, 80, 1, 6759, 0, 9, 944, 64, 0, 0] payload_rows_per_dst=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] dup_rows_per_dst=[580, 128, 0, 0, 42, 3632, 2210, 298] combine_rows_per_src=[7298, 9434, 10934, 7784, 9478, 15006, 16224, 15256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=69090 payload_rows_remote=53672 dup_rows=29214 dup_rows_remote=27268 dedup_ratio=0.297180 router_tokens_per_expert=[0, 0, 29, 0, 38, 86, 316, 3, 1, 24, 166, 12, 0, 55, 1510, 1, 4011, 39, 176, 7169, 7386, 0, 16, 10, 0, 188, 549, 0, 7416, 11, 63, 0, 3737, 6, 23, 3, 250, 5123, 6909, 87, 3, 10, 1, 23, 0, 5, 0, 52, 24, 1343, 12, 726, 0, 27, 0, 0, 8, 1, 22, 225, 503, 36, 366, 352] payload_rows_per_dst=[874, 9860, 474, 1586, 15544, 14948, 10386, 15418] dup_rows_per_dst=[70, 1050, 8, 36, 24192, 12, 1900, 1946] combine_rows_per_src=[874, 9860, 474, 1586, 15544, 14948, 10386, 15418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82042 payload_rows_remote=71438 dup_rows=16262 dup_rows_remote=14854 dedup_ratio=0.165426 router_tokens_per_expert=[13, 0, 0, 0, 0, 86, 1, 0, 0, 0, 212, 0, 0, 63, 1681, 0, 4448, 0, 195, 7554, 7478, 0, 0, 0, 0, 168, 570, 0, 7461, 0, 64, 0, 4183, 0, 4, 0, 228, 4050, 7012, 98, 0, 0, 0, 0, 0, 1, 0, 60, 0, 1477, 0, 629, 0, 0, 0, 0, 0, 0, 0, 250, 479, 0, 353, 334] payload_rows_per_dst=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] dup_rows_per_dst=[0, 1408, 12138, 220, 464, 0, 2032, 0] combine_rows_per_src=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=301178880 dispatch_meta_bytes=334320 combine_payload_bytes=301178880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301513200 ingress_bytes=439590912 link_bytes=439590912 token_bytes=602692080 weight_movement_bytes=138412032 total_nvlink_bytes=741104112 weight_to_token_ratio=0.229656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654145212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322928640 dispatch_meta_bytes=340136 combine_payload_bytes=322928640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323268776 ingress_bytes=357531648 link_bytes=357531648 token_bytes=646197416 weight_movement_bytes=34603008 total_nvlink_bytes=680800424 weight_to_token_ratio=0.053549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345362808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=251215872 dispatch_meta_bytes=324544 combine_payload_bytes=251215872 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=251540416 ingress_bytes=355024896 link_bytes=355024896 token_bytes=502756288 weight_movement_bytes=103809024 total_nvlink_bytes=606565312 weight_to_token_ratio=0.206480 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.335930102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82042 payload_rows_remote=71438 dup_rows=16262 dup_rows_remote=14854 dedup_ratio=0.165426 router_tokens_per_expert=[13, 0, 0, 0, 0, 86, 1, 0, 0, 0, 212, 0, 0, 63, 1681, 0, 4448, 0, 195, 7554, 7478, 0, 0, 0, 0, 168, 570, 0, 7461, 0, 64, 0, 4183, 0, 4, 0, 228, 4050, 7012, 98, 0, 0, 0, 0, 0, 1, 0, 60, 0, 1477, 0, 629, 0, 0, 0, 0, 0, 0, 0, 250, 479, 0, 353, 334] payload_rows_per_dst=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] dup_rows_per_dst=[0, 1408, 12138, 220, 464, 0, 2032, 0] combine_rows_per_src=[14224, 10604, 15840, 9714, 8562, 9190, 11076, 2832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90274 payload_rows_remote=78840 dup_rows=8030 dup_rows_remote=6194 dedup_ratio=0.081685 router_tokens_per_expert=[0, 0, 0, 0, 0, 78, 283, 0, 0, 46, 154, 0, 0, 51, 1364, 0, 4586, 0, 120, 7350, 7492, 0, 0, 0, 0, 199, 484, 0, 7504, 0, 51, 0, 3058, 0, 0, 0, 244, 5381, 7026, 78, 0, 0, 0, 0, 0, 0, 0, 84, 0, 1431, 0, 618, 0, 0, 0, 0, 0, 0, 0, 273, 495, 0, 330, 372] payload_rows_per_dst=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] dup_rows_per_dst=[72, 1444, 0, 400, 2144, 0, 1836, 2134] combine_rows_per_src=[9638, 12548, 240, 15986, 9770, 15152, 11434, 15506] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92186 payload_rows_remote=76078 dup_rows=6118 dup_rows_remote=5738 dedup_ratio=0.062236 router_tokens_per_expert=[2, 0, 0, 0, 0, 61, 231, 0, 0, 29, 136, 0, 0, 54, 1340, 0, 4234, 0, 98, 7461, 7549, 0, 0, 0, 0, 174, 497, 0, 7519, 0, 54, 0, 3536, 0, 3, 0, 242, 5211, 6995, 63, 0, 0, 0, 0, 0, 0, 0, 88, 0, 1579, 0, 570, 0, 0, 0, 0, 0, 0, 0, 258, 507, 0, 305, 356] payload_rows_per_dst=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] dup_rows_per_dst=[52, 1292, 0, 380, 490, 0, 1910, 1994] combine_rows_per_src=[14526, 12248, 1398, 16108, 7198, 15274, 10856, 14578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=335609856 dispatch_meta_bytes=355136 combine_payload_bytes=335609856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=335964992 ingress_bytes=404815872 link_bytes=404815872 token_bytes=671574848 weight_movement_bytes=69206016 total_nvlink_bytes=740780864 weight_to_token_ratio=0.103050 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523289536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=273596416 dispatch_meta_bytes=311912 combine_payload_bytes=273596416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273908328 ingress_bytes=308199424 link_bytes=308199424 token_bytes=547504744 weight_movement_bytes=34603008 total_nvlink_bytes=582107752 weight_to_token_ratio=0.063201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159729620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89750 payload_rows_remote=74218 dup_rows=8554 dup_rows_remote=5768 dedup_ratio=0.087016 router_tokens_per_expert=[0, 3967, 961, 1158, 7576, 7578, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 1274, 1529, 88, 0, 0, 0, 0, 0, 204, 0, 0, 6659, 10, 140, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 1459, 2, 0, 0, 0, 72, 6996, 212, 5820, 0, 0, 88, 0, 0, 0] payload_rows_per_dst=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] dup_rows_per_dst=[354, 514, 1168, 332, 334, 2786, 2642, 424] combine_rows_per_src=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=348971008 dispatch_meta_bytes=368832 combine_payload_bytes=348971008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=349339840 ingress_bytes=452780032 link_bytes=452780032 token_bytes=698310848 weight_movement_bytes=103809024 total_nvlink_bytes=802119872 weight_to_token_ratio=0.148657 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.703774809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=219840512 dispatch_meta_bytes=323760 combine_payload_bytes=219840512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=220164272 ingress_bytes=254443520 link_bytes=254443520 token_bytes=440004784 weight_movement_bytes=34603008 total_nvlink_bytes=474607792 weight_to_token_ratio=0.078642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.574504814e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88148 payload_rows_remote=73530 dup_rows=10156 dup_rows_remote=10050 dedup_ratio=0.103312 router_tokens_per_expert=[66, 0, 524, 0, 663, 70, 385, 49, 25, 61, 198, 96, 0, 107, 1194, 6, 4323, 409, 162, 6326, 5948, 1, 367, 97, 0, 628, 1513, 0, 6301, 206, 388, 8, 3054, 152, 328, 89, 248, 2303, 5454, 79, 1, 127, 6, 485, 13, 15, 3, 37, 32, 801, 66, 1865, 0, 520, 0, 1, 218, 26, 408, 281, 565, 608, 856, 390] payload_rows_per_dst=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] dup_rows_per_dst=[966, 554, 106, 3220, 826, 124, 2692, 1668] combine_rows_per_src=[13456, 7426, 14618, 14868, 7074, 13146, 12524, 5036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=292610048 dispatch_meta_bytes=345168 combine_payload_bytes=292610048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292955216 ingress_bytes=327213056 link_bytes=327213056 token_bytes=585565264 weight_movement_bytes=34603008 total_nvlink_bytes=620168272 weight_to_token_ratio=0.059093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=35584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.231276387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=78564 dup_rows=7098 dup_rows_remote=6994 dedup_ratio=0.072205 router_tokens_per_expert=[0, 0, 0, 627, 0, 0, 0, 0, 0, 1, 1, 91, 0, 84, 786, 0, 0, 0, 2919, 3011, 7615, 685, 0, 80, 7615, 0, 0, 7190, 116, 52, 1, 0, 0, 0, 188, 0, 52, 0, 0, 6133, 0, 493, 0, 0, 0, 0, 86, 0, 0, 0, 2, 271, 2839, 0, 560, 130, 1469, 80, 1, 0, 0, 5390, 25, 559] payload_rows_per_dst=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] dup_rows_per_dst=[1236, 942, 722, 62, 104, 896, 1050, 2086] combine_rows_per_src=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=303996928 dispatch_meta_bytes=319944 combine_payload_bytes=303996928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304316872 ingress_bytes=338599936 link_bytes=338599936 token_bytes=608313800 weight_movement_bytes=34603008 total_nvlink_bytes=642916808 weight_to_token_ratio=0.056883 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274124300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=321798144 dispatch_meta_bytes=342232 combine_payload_bytes=321798144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=322140376 ingress_bytes=391004160 link_bytes=391004160 token_bytes=643938520 weight_movement_bytes=69206016 total_nvlink_bytes=713144536 weight_to_token_ratio=0.107473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471317176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85282 payload_rows_remote=79226 dup_rows=13022 dup_rows_remote=12700 dedup_ratio=0.132467 router_tokens_per_expert=[0, 0, 468, 1194, 121, 278, 0, 1, 29, 19, 0, 73, 266, 52, 1052, 358, 9, 43, 801, 1674, 5788, 608, 81, 79, 6318, 260, 5, 4962, 76, 94, 23, 4, 0, 477, 514, 645, 46, 13, 0, 5286, 371, 920, 40, 610, 27, 0, 110, 2, 127, 102, 152, 290, 5888, 115, 2714, 228, 1790, 150, 237, 3, 619, 1718, 214, 1008] payload_rows_per_dst=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] dup_rows_per_dst=[1906, 1318, 322, 52, 974, 1556, 5628, 1266] combine_rows_per_src=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73532 payload_rows_remote=65834 dup_rows=24772 dup_rows_remote=23672 dedup_ratio=0.251994 router_tokens_per_expert=[0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 839, 0, 0, 0, 2588, 2701, 7618, 723, 0, 78, 7613, 0, 0, 7151, 98, 51, 0, 0, 0, 0, 186, 0, 50, 0, 0, 6288, 0, 615, 0, 0, 0, 0, 80, 0, 0, 0, 0, 235, 3282, 0, 718, 164, 1468, 78, 0, 0, 0, 5168, 2, 617] payload_rows_per_dst=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] dup_rows_per_dst=[0, 156, 708, 19588, 116, 1136, 1100, 1968] combine_rows_per_src=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59008 payload_rows_remote=44972 dup_rows=39296 dup_rows_remote=39296 dedup_ratio=0.399740 router_tokens_per_expert=[28, 0, 0, 0, 0, 31, 29, 0, 0, 0, 239, 0, 0, 61, 1798, 0, 4333, 0, 179, 7546, 7452, 0, 0, 0, 0, 129, 719, 0, 7463, 0, 61, 0, 4782, 0, 18, 0, 242, 3354, 6930, 90, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1474, 0, 652, 0, 0, 0, 0, 0, 0, 0, 288, 470, 0, 345, 344] payload_rows_per_dst=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] dup_rows_per_dst=[0, 1170, 36748, 0, 520, 0, 846, 12] combine_rows_per_src=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89886 payload_rows_remote=82600 dup_rows=8418 dup_rows_remote=8128 dedup_ratio=0.085632 router_tokens_per_expert=[0, 3229, 780, 1240, 7754, 7583, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 1381, 2346, 61, 0, 0, 0, 0, 0, 144, 0, 0, 6487, 10, 120, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 1415, 8, 0, 0, 0, 62, 6447, 203, 6022, 0, 0, 61, 0, 0, 0] payload_rows_per_dst=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] dup_rows_per_dst=[248, 482, 1740, 290, 312, 2484, 2456, 406] combine_rows_per_src=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=322928640 dispatch_meta_bytes=340136 combine_payload_bytes=322928640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323268776 ingress_bytes=357531648 link_bytes=357531648 token_bytes=646197416 weight_movement_bytes=34603008 total_nvlink_bytes=680800424 weight_to_token_ratio=0.053549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345362808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91592 payload_rows_remote=76500 dup_rows=6712 dup_rows_remote=6712 dedup_ratio=0.068278 router_tokens_per_expert=[0, 0, 0, 0, 0, 88, 433, 0, 0, 71, 105, 0, 0, 62, 1228, 0, 4285, 0, 91, 7206, 7489, 0, 0, 0, 0, 492, 428, 0, 7608, 0, 62, 0, 2570, 0, 1, 0, 256, 5842, 7103, 88, 0, 0, 0, 0, 0, 0, 0, 57, 0, 1640, 0, 633, 0, 0, 0, 0, 0, 0, 0, 224, 415, 0, 310, 365] payload_rows_per_dst=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] dup_rows_per_dst=[170, 1516, 0, 1038, 514, 0, 1892, 1582] combine_rows_per_src=[15078, 13100, 182, 16142, 5316, 15092, 11224, 15458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311615488 dispatch_meta_bytes=327264 combine_payload_bytes=311615488 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=311942752 ingress_bytes=380821504 link_bytes=380821504 token_bytes=623558240 weight_movement_bytes=69206016 total_nvlink_bytes=692764256 weight_to_token_ratio=0.110986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.030000 gemm_active_groups=7 gemm_peak_group_rows=40704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433000661e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=301178880 dispatch_meta_bytes=334320 combine_payload_bytes=301178880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301513200 ingress_bytes=439590912 link_bytes=439590912 token_bytes=602692080 weight_movement_bytes=138412032 total_nvlink_bytes=741104112 weight_to_token_ratio=0.229656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.654145212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81386 payload_rows_remote=66796 dup_rows=16918 dup_rows_remote=11182 dedup_ratio=0.172099 router_tokens_per_expert=[0, 0, 1509, 0, 0, 0, 2500, 0, 0, 0, 150, 261, 7118, 0, 0, 3326, 1, 0, 0, 0, 0, 2960, 2, 0, 0, 0, 0, 7851, 0, 0, 6154, 0, 0, 0, 150, 0, 0, 7361, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 39, 0, 0, 258, 0, 0, 1165, 0, 90, 0, 6153, 0, 0, 558, 522, 0, 0] payload_rows_per_dst=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] dup_rows_per_dst=[5736, 6220, 4122, 0, 0, 0, 696, 144] combine_rows_per_src=[14590, 15490, 14110, 15702, 15022, 2126, 2330, 2016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=313344000 dispatch_meta_bytes=332848 combine_payload_bytes=313344000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313676848 ingress_bytes=347947008 link_bytes=347947008 token_bytes=627020848 weight_movement_bytes=34603008 total_nvlink_bytes=661623856 weight_to_token_ratio=0.055186 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=48000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309296579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72696 payload_rows_remote=60822 dup_rows=25608 dup_rows_remote=23100 dedup_ratio=0.260498 router_tokens_per_expert=[0, 0, 29, 653, 8, 19, 0, 0, 1, 1, 3, 87, 14, 78, 844, 22, 1, 1, 2399, 2830, 7363, 700, 3, 88, 7524, 18, 1, 6985, 134, 53, 1, 6, 0, 29, 253, 30, 51, 0, 0, 6249, 25, 771, 5, 35, 0, 0, 89, 0, 5, 4, 26, 255, 3962, 3, 735, 242, 1542, 94, 10, 0, 39, 4179, 40, 613] payload_rows_per_dst=[1400, 1926, 5744, 15612, 11744, 15344, 9052, 11874] dup_rows_per_dst=[18, 174, 640, 19492, 132, 1232, 1412, 2508] combine_rows_per_src=[1400, 1926, 5744, 15612, 11744, 15344, 9052, 11874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92004 payload_rows_remote=86546 dup_rows=6300 dup_rows_remote=5812 dedup_ratio=0.064087 router_tokens_per_expert=[1, 0, 0, 0, 0, 81, 441, 0, 0, 62, 115, 0, 0, 53, 1266, 0, 4528, 1, 83, 7219, 7513, 0, 0, 0, 0, 374, 417, 0, 7551, 0, 52, 0, 2636, 0, 1, 0, 242, 5679, 7048, 94, 0, 0, 0, 0, 0, 2, 0, 73, 0, 1605, 0, 605, 0, 0, 0, 0, 0, 0, 0, 231, 488, 0, 337, 354] payload_rows_per_dst=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] dup_rows_per_dst=[164, 1352, 0, 628, 488, 20, 1856, 1792] combine_rows_per_src=[14978, 12998, 168, 16160, 5458, 15156, 11620, 15466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=273596416 dispatch_meta_bytes=311912 combine_payload_bytes=273596416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273908328 ingress_bytes=308199424 link_bytes=308199424 token_bytes=547504744 weight_movement_bytes=34603008 total_nvlink_bytes=582107752 weight_to_token_ratio=0.063201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.159729620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73532 payload_rows_remote=65834 dup_rows=24772 dup_rows_remote=23672 dedup_ratio=0.251994 router_tokens_per_expert=[0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 0, 78, 0, 78, 839, 0, 0, 0, 2588, 2701, 7618, 723, 0, 78, 7613, 0, 0, 7151, 98, 51, 0, 0, 0, 0, 186, 0, 50, 0, 0, 6288, 0, 615, 0, 0, 0, 0, 80, 0, 0, 0, 0, 235, 3282, 0, 718, 164, 1468, 78, 0, 0, 0, 5168, 2, 617] payload_rows_per_dst=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] dup_rows_per_dst=[0, 156, 708, 19588, 116, 1136, 1100, 1968] combine_rows_per_src=[1170, 1834, 6070, 15640, 12932, 15490, 7698, 12698] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81990 payload_rows_remote=66744 dup_rows=16314 dup_rows_remote=15286 dedup_ratio=0.165955 router_tokens_per_expert=[0, 0, 0, 534, 0, 0, 0, 0, 0, 0, 0, 98, 0, 98, 812, 0, 0, 0, 2165, 2459, 7456, 757, 0, 98, 7574, 0, 0, 7129, 110, 64, 0, 0, 0, 0, 238, 0, 42, 0, 0, 6315, 0, 756, 0, 0, 0, 0, 97, 0, 0, 0, 0, 241, 3854, 0, 979, 250, 1488, 97, 0, 0, 0, 4725, 25, 691] payload_rows_per_dst=[15192, 15246, 15984, 924, 13084, 1706, 11790, 8064] dup_rows_per_dst=[1024, 1028, 9310, 0, 106, 0, 3696, 1150] combine_rows_per_src=[15192, 15246, 15984, 924, 13084, 1706, 11790, 8064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91260 payload_rows_remote=85198 dup_rows=7044 dup_rows_remote=7010 dedup_ratio=0.071655 router_tokens_per_expert=[0, 0, 0, 636, 0, 0, 0, 0, 0, 0, 2, 63, 0, 63, 888, 0, 0, 0, 2548, 2905, 7575, 709, 0, 60, 7658, 0, 0, 7184, 89, 54, 0, 0, 0, 0, 152, 0, 39, 0, 0, 6307, 0, 583, 0, 0, 0, 0, 61, 0, 0, 0, 0, 212, 3489, 0, 725, 174, 1441, 61, 0, 0, 0, 4833, 6, 635] payload_rows_per_dst=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] dup_rows_per_dst=[1270, 1030, 692, 34, 84, 912, 1146, 1876] combine_rows_per_src=[15318, 15370, 5942, 6062, 12912, 15526, 8054, 12076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=249126912 dispatch_meta_bytes=335688 combine_payload_bytes=249126912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=249462600 ingress_bytes=318332928 link_bytes=318332928 token_bytes=498589512 weight_movement_bytes=69206016 total_nvlink_bytes=567795528 weight_to_token_ratio=0.138804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197861180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=354492416 dispatch_meta_bytes=369432 combine_payload_bytes=354492416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=354861848 ingress_bytes=458301440 link_bytes=458301440 token_bytes=709354264 weight_movement_bytes=103809024 total_nvlink_bytes=813163288 weight_to_token_ratio=0.146343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.724551422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85282 payload_rows_remote=79226 dup_rows=13022 dup_rows_remote=12700 dedup_ratio=0.132467 router_tokens_per_expert=[0, 0, 468, 1194, 121, 278, 0, 1, 29, 19, 0, 73, 266, 52, 1052, 358, 9, 43, 801, 1674, 5788, 608, 81, 79, 6318, 260, 5, 4962, 76, 94, 23, 4, 0, 477, 514, 645, 46, 13, 0, 5286, 371, 920, 40, 610, 27, 0, 110, 2, 127, 102, 152, 290, 5888, 115, 2714, 228, 1790, 150, 237, 3, 619, 1718, 214, 1008] payload_rows_per_dst=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] dup_rows_per_dst=[1906, 1318, 322, 52, 974, 1556, 5628, 1266] combine_rows_per_src=[14854, 12304, 6056, 4220, 12988, 11044, 13604, 10212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=269656064 dispatch_meta_bytes=358024 combine_payload_bytes=269656064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270014088 ingress_bytes=304259072 link_bytes=304259072 token_bytes=539670152 weight_movement_bytes=34603008 total_nvlink_bytes=574273160 weight_to_token_ratio=0.064119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.144902393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273383424 dispatch_meta_bytes=328120 combine_payload_bytes=273383424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273711544 ingress_bytes=307986432 link_bytes=307986432 token_bytes=547094968 weight_movement_bytes=34603008 total_nvlink_bytes=581697976 weight_to_token_ratio=0.063249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158928149e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90330 payload_rows_remote=74946 dup_rows=7974 dup_rows_remote=7214 dedup_ratio=0.081116 router_tokens_per_expert=[0, 0, 0, 487, 0, 0, 0, 0, 0, 0, 0, 88, 0, 88, 883, 0, 0, 0, 3169, 3031, 7588, 742, 0, 88, 7586, 0, 0, 7084, 104, 62, 0, 0, 0, 0, 192, 0, 61, 0, 0, 6218, 0, 382, 0, 0, 0, 0, 102, 0, 0, 0, 0, 320, 3213, 0, 634, 144, 1204, 88, 0, 0, 0, 5133, 1, 460] payload_rows_per_dst=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] dup_rows_per_dst=[774, 1024, 740, 1878, 128, 760, 1188, 1482] combine_rows_per_src=[12616, 14462, 7258, 8072, 12814, 15384, 7434, 12290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=348971008 dispatch_meta_bytes=368832 combine_payload_bytes=348971008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=349339840 ingress_bytes=452780032 link_bytes=452780032 token_bytes=698310848 weight_movement_bytes=103809024 total_nvlink_bytes=802119872 weight_to_token_ratio=0.148657 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16640 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.703774809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=324509696 dispatch_meta_bytes=367704 combine_payload_bytes=324509696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=324877400 ingress_bytes=428318720 link_bytes=428318720 token_bytes=649387096 weight_movement_bytes=103809024 total_nvlink_bytes=753196120 weight_to_token_ratio=0.159857 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.611728861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90050 payload_rows_remote=73958 dup_rows=8254 dup_rows_remote=7818 dedup_ratio=0.083964 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 3148, 479, 0, 164, 0, 0, 0, 267, 5720, 0, 0, 0, 5361, 180, 1485, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 5298, 0, 0, 0, 8046, 0, 0, 218, 0, 0, 0, 0, 2440, 0, 7979, 0, 0, 0, 0, 0, 3872, 160, 456, 0, 88] payload_rows_per_dst=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] dup_rows_per_dst=[0, 1320, 1532, 0, 4062, 436, 0, 904] combine_rows_per_src=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=59008 payload_rows_remote=44972 dup_rows=39296 dup_rows_remote=39296 dedup_ratio=0.399740 router_tokens_per_expert=[28, 0, 0, 0, 0, 31, 29, 0, 0, 0, 239, 0, 0, 61, 1798, 0, 4333, 0, 179, 7546, 7452, 0, 0, 0, 0, 129, 719, 0, 7463, 0, 61, 0, 4782, 0, 18, 0, 242, 3354, 6930, 90, 0, 0, 0, 0, 0, 0, 0, 95, 0, 1474, 0, 652, 0, 0, 0, 0, 0, 0, 0, 288, 470, 0, 345, 344] payload_rows_per_dst=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] dup_rows_per_dst=[0, 1170, 36748, 0, 520, 0, 846, 12] combine_rows_per_src=[14036, 9734, 16384, 1818, 9744, 190, 4220, 2882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=306978816 dispatch_meta_bytes=328640 combine_payload_bytes=306978816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307307456 ingress_bytes=341581824 link_bytes=341581824 token_bytes=614286272 weight_movement_bytes=34603008 total_nvlink_bytes=648889280 weight_to_token_ratio=0.056330 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285344904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89284 payload_rows_remote=77606 dup_rows=9020 dup_rows_remote=8312 dedup_ratio=0.091756 router_tokens_per_expert=[0, 2815, 900, 1113, 7418, 7484, 9, 0, 1, 1, 2, 250, 7, 3, 15, 12, 38, 5, 2, 4, 28, 2406, 0, 0, 0, 0, 1301, 3099, 110, 0, 23, 2, 1, 5, 350, 1, 0, 6257, 59, 123, 3, 0, 1363, 0, 11, 13, 37, 32, 11, 1604, 66, 0, 5, 39, 63, 6069, 188, 5663, 5, 27, 87, 3, 3, 16] payload_rows_per_dst=[3708, 14906, 9062, 8632, 12860, 15398, 13040, 11678] dup_rows_per_dst=[336, 512, 1534, 438, 330, 2488, 2674, 708] combine_rows_per_src=[3708, 14906, 9062, 8632, 12860, 15398, 13040, 11678] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=184205312 dispatch_meta_bytes=337072 combine_payload_bytes=184205312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=184542384 ingress_bytes=218808320 link_bytes=218808320 token_bytes=368747696 weight_movement_bytes=34603008 total_nvlink_bytes=403350704 weight_to_token_ratio=0.093839 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.233580926e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89306 payload_rows_remote=75606 dup_rows=8998 dup_rows_remote=8736 dedup_ratio=0.091532 router_tokens_per_expert=[0, 3731, 927, 1168, 7448, 7547, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1657, 1, 0, 0, 0, 1311, 1793, 80, 2, 0, 0, 0, 0, 246, 1, 0, 6614, 0, 120, 0, 62, 1719, 0, 0, 1, 0, 0, 0, 1681, 0, 0, 0, 0, 62, 6680, 197, 5740, 0, 0, 80, 0, 1, 0] payload_rows_per_dst=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] dup_rows_per_dst=[316, 480, 1368, 320, 262, 3188, 2676, 388] combine_rows_per_src=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=249126912 dispatch_meta_bytes=335688 combine_payload_bytes=249126912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=249462600 ingress_bytes=318332928 link_bytes=318332928 token_bytes=498589512 weight_movement_bytes=69206016 total_nvlink_bytes=567795528 weight_to_token_ratio=0.138804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.197861180e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=273383424 dispatch_meta_bytes=328120 combine_payload_bytes=273383424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273711544 ingress_bytes=307986432 link_bytes=307986432 token_bytes=547094968 weight_movement_bytes=34603008 total_nvlink_bytes=581697976 weight_to_token_ratio=0.063249 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.158928149e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=362912 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338692512 ingress_bytes=372932608 link_bytes=372932608 token_bytes=677022112 weight_movement_bytes=34603008 total_nvlink_bytes=711625120 weight_to_token_ratio=0.051111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403315380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=302931968 dispatch_meta_bytes=327104 combine_payload_bytes=302931968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303259072 ingress_bytes=406740992 link_bytes=406740992 token_bytes=606191040 weight_movement_bytes=103809024 total_nvlink_bytes=710000064 weight_to_token_ratio=0.171248 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530533608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309682176 dispatch_meta_bytes=337368 combine_payload_bytes=309682176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310019544 ingress_bytes=378888192 link_bytes=378888192 token_bytes=619701720 weight_movement_bytes=69206016 total_nvlink_bytes=688907736 weight_to_token_ratio=0.111676 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425725763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91206 payload_rows_remote=78564 dup_rows=7098 dup_rows_remote=6994 dedup_ratio=0.072205 router_tokens_per_expert=[0, 0, 0, 627, 0, 0, 0, 0, 0, 1, 1, 91, 0, 84, 786, 0, 0, 0, 2919, 3011, 7615, 685, 0, 80, 7615, 0, 0, 7190, 116, 52, 1, 0, 0, 0, 188, 0, 52, 0, 0, 6133, 0, 493, 0, 0, 0, 0, 86, 0, 0, 0, 2, 271, 2839, 0, 560, 130, 1469, 80, 1, 0, 0, 5390, 25, 559] payload_rows_per_dst=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] dup_rows_per_dst=[1236, 942, 722, 62, 104, 896, 1050, 2086] combine_rows_per_src=[15248, 15364, 6646, 6298, 12642, 15492, 6554, 12962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89796 payload_rows_remote=76582 dup_rows=8508 dup_rows_remote=5908 dedup_ratio=0.086548 router_tokens_per_expert=[0, 3213, 848, 1317, 7686, 7562, 0, 0, 0, 0, 0, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2240, 0, 0, 0, 0, 1334, 2237, 78, 0, 0, 0, 0, 0, 135, 0, 0, 6399, 13, 133, 0, 0, 1399, 0, 0, 0, 0, 0, 0, 1467, 17, 0, 0, 0, 70, 6353, 213, 6116, 0, 0, 78, 0, 0, 0] payload_rows_per_dst=[4010, 15372, 9424, 7000, 13002, 15386, 13214, 12388] dup_rows_per_dst=[320, 488, 1482, 298, 358, 2536, 2600, 426] combine_rows_per_src=[4010, 15372, 9424, 7000, 13002, 15386, 13214, 12388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75208 payload_rows_remote=70540 dup_rows=23096 dup_rows_remote=22964 dedup_ratio=0.234945 router_tokens_per_expert=[0, 2798, 795, 1186, 7761, 7537, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 0, 0, 0, 1225, 3783, 97, 0, 0, 0, 0, 0, 131, 0, 0, 6133, 19, 149, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 1449, 124, 0, 0, 0, 82, 5681, 219, 6294, 0, 0, 97, 0, 0, 0] payload_rows_per_dst=[16078, 4668, 8618, 15092, 12464, 2444, 3062, 12782] dup_rows_per_dst=[14140, 132, 1258, 6480, 400, 0, 248, 438] combine_rows_per_src=[16078, 4668, 8618, 15092, 12464, 2444, 3062, 12782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317874176 dispatch_meta_bytes=343672 combine_payload_bytes=317874176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318217848 ingress_bytes=387080192 link_bytes=387080192 token_bytes=636092024 weight_movement_bytes=69206016 total_nvlink_bytes=705298040 weight_to_token_ratio=0.108799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.456551600e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89886 payload_rows_remote=82600 dup_rows=8418 dup_rows_remote=8128 dedup_ratio=0.085632 router_tokens_per_expert=[0, 3229, 780, 1240, 7754, 7583, 0, 0, 0, 0, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2181, 0, 0, 0, 0, 1381, 2346, 61, 0, 0, 0, 0, 0, 144, 0, 0, 6487, 10, 120, 0, 0, 1377, 0, 0, 0, 0, 0, 0, 1415, 8, 0, 0, 0, 62, 6447, 203, 6022, 0, 0, 61, 0, 0, 0] payload_rows_per_dst=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] dup_rows_per_dst=[248, 482, 1740, 290, 312, 2484, 2456, 406] combine_rows_per_src=[3792, 15508, 9832, 7286, 13210, 14684, 13408, 12166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80974 payload_rows_remote=65654 dup_rows=17330 dup_rows_remote=9322 dedup_ratio=0.176290 router_tokens_per_expert=[1, 1860, 651, 1000, 6302, 6300, 102, 0, 32, 15, 94, 245, 134, 49, 243, 230, 669, 173, 40, 9, 465, 2846, 7, 0, 3, 0, 1125, 4493, 71, 1, 361, 23, 13, 40, 624, 25, 4, 4969, 461, 117, 129, 7, 654, 7, 397, 117, 135, 597, 217, 1241, 447, 0, 70, 712, 64, 3664, 146, 5917, 50, 441, 71, 33, 36, 203] payload_rows_per_dst=[4422, 13850, 15320, 13930, 11690, 3564, 4952, 13246] dup_rows_per_dst=[496, 838, 8008, 4960, 816, 522, 1142, 548] combine_rows_per_src=[4422, 13850, 15320, 13930, 11690, 3564, 4952, 13246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=60150 dup_rows=22936 dup_rows_remote=21876 dedup_ratio=0.233317 router_tokens_per_expert=[0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 1, 90, 0, 89, 792, 0, 0, 0, 2352, 2501, 7344, 616, 0, 84, 7607, 0, 0, 7091, 124, 61, 0, 0, 0, 0, 245, 0, 34, 0, 0, 6331, 0, 824, 0, 0, 0, 0, 89, 0, 0, 0, 0, 212, 3836, 0, 1195, 236, 2006, 89, 0, 0, 0, 3970, 67, 734] payload_rows_per_dst=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] dup_rows_per_dst=[1060, 982, 15686, 0, 50, 0, 4676, 482] combine_rows_per_src=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=321798144 dispatch_meta_bytes=342232 combine_payload_bytes=321798144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=322140376 ingress_bytes=391004160 link_bytes=391004160 token_bytes=643938520 weight_movement_bytes=69206016 total_nvlink_bytes=713144536 weight_to_token_ratio=0.107473 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=45696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.471317176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=288931840 dispatch_meta_bytes=374016 combine_payload_bytes=288931840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289305856 ingress_bytes=323534848 link_bytes=323534848 token_bytes=578237696 weight_movement_bytes=34603008 total_nvlink_bytes=612840704 weight_to_token_ratio=0.059842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217435586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=313679872 dispatch_meta_bytes=329960 combine_payload_bytes=313679872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314009832 ingress_bytes=382885888 link_bytes=382885888 token_bytes=627689704 weight_movement_bytes=69206016 total_nvlink_bytes=696895720 weight_to_token_ratio=0.110255 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440768772e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89750 payload_rows_remote=74218 dup_rows=8554 dup_rows_remote=5768 dedup_ratio=0.087016 router_tokens_per_expert=[0, 3967, 961, 1158, 7576, 7578, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 0, 0, 0, 1274, 1529, 88, 0, 0, 0, 0, 0, 204, 0, 0, 6659, 10, 140, 0, 0, 1581, 0, 0, 0, 0, 0, 0, 1459, 2, 0, 0, 0, 72, 6996, 212, 5820, 0, 0, 88, 0, 0, 0] payload_rows_per_dst=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] dup_rows_per_dst=[354, 514, 1168, 332, 334, 2786, 2642, 424] combine_rows_per_src=[3884, 15152, 9808, 5450, 13692, 15532, 14416, 11816] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89284 payload_rows_remote=77606 dup_rows=9020 dup_rows_remote=8312 dedup_ratio=0.091756 router_tokens_per_expert=[0, 2815, 900, 1113, 7418, 7484, 9, 0, 1, 1, 2, 250, 7, 3, 15, 12, 38, 5, 2, 4, 28, 2406, 0, 0, 0, 0, 1301, 3099, 110, 0, 23, 2, 1, 5, 350, 1, 0, 6257, 59, 123, 3, 0, 1363, 0, 11, 13, 37, 32, 11, 1604, 66, 0, 5, 39, 63, 6069, 188, 5663, 5, 27, 87, 3, 3, 16] payload_rows_per_dst=[3708, 14906, 9062, 8632, 12860, 15398, 13040, 11678] dup_rows_per_dst=[336, 512, 1534, 438, 330, 2488, 2674, 708] combine_rows_per_src=[3708, 14906, 9062, 8632, 12860, 15398, 13040, 11678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91850 payload_rows_remote=76138 dup_rows=6454 dup_rows_remote=6454 dedup_ratio=0.065653 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 2896, 536, 0, 250, 0, 0, 0, 223, 6097, 0, 0, 0, 4525, 248, 733, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 5409, 0, 0, 0, 7992, 0, 0, 213, 0, 0, 0, 0, 3543, 0, 7856, 0, 0, 0, 0, 0, 4245, 123, 419, 0, 63] payload_rows_per_dst=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] dup_rows_per_dst=[2916, 1686, 590, 0, 0, 424, 0, 838] combine_rows_per_src=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75208 payload_rows_remote=70540 dup_rows=23096 dup_rows_remote=22964 dedup_ratio=0.234945 router_tokens_per_expert=[0, 2798, 795, 1186, 7761, 7537, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2140, 0, 0, 0, 0, 1225, 3783, 97, 0, 0, 0, 0, 0, 131, 0, 0, 6133, 19, 149, 0, 0, 1222, 0, 0, 0, 0, 0, 0, 1449, 124, 0, 0, 0, 82, 5681, 219, 6294, 0, 0, 97, 0, 0, 0] payload_rows_per_dst=[16078, 4668, 8618, 15092, 12464, 2444, 3062, 12782] dup_rows_per_dst=[14140, 132, 1258, 6480, 400, 0, 248, 438] combine_rows_per_src=[16078, 4668, 8618, 15092, 12464, 2444, 3062, 12782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=338329600 dispatch_meta_bytes=362912 combine_payload_bytes=338329600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338692512 ingress_bytes=372932608 link_bytes=372932608 token_bytes=677022112 weight_movement_bytes=34603008 total_nvlink_bytes=711625120 weight_to_token_ratio=0.051111 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.403315380e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75368 payload_rows_remote=60150 dup_rows=22936 dup_rows_remote=21876 dedup_ratio=0.233317 router_tokens_per_expert=[0, 0, 0, 532, 0, 0, 0, 0, 0, 0, 1, 90, 0, 89, 792, 0, 0, 0, 2352, 2501, 7344, 616, 0, 84, 7607, 0, 0, 7091, 124, 61, 0, 0, 0, 0, 245, 0, 34, 0, 0, 6331, 0, 824, 0, 0, 0, 0, 89, 0, 0, 0, 0, 212, 3836, 0, 1195, 236, 2006, 89, 0, 0, 0, 3970, 67, 734] payload_rows_per_dst=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] dup_rows_per_dst=[1060, 982, 15686, 0, 50, 0, 4676, 482] combine_rows_per_src=[15218, 15144, 15784, 370, 7494, 1826, 14222, 5310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268918784 dispatch_meta_bytes=299904 combine_payload_bytes=268918784 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269218688 ingress_bytes=338124800 link_bytes=338124800 token_bytes=538137472 weight_movement_bytes=69206016 total_nvlink_bytes=607343488 weight_to_token_ratio=0.128603 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.272336401e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92824 payload_rows_remote=86012 dup_rows=5480 dup_rows_remote=4328 dedup_ratio=0.055745 router_tokens_per_expert=[0, 0, 874, 0, 0, 2, 0, 0, 0, 243, 0, 0, 7475, 0, 1684, 0, 0, 369, 62, 0, 0, 0, 225, 0, 588, 0, 0, 0, 0, 57, 0, 6586, 7575, 0, 192, 0, 0, 0, 175, 0, 388, 2472, 0, 0, 0, 7134, 7462, 57, 0, 414, 0, 0, 257, 0, 88, 0, 0, 0, 4773, 0, 0, 0, 0, 0] payload_rows_per_dst=[13466, 3368, 15334, 13300, 15358, 6812, 15640, 9546] dup_rows_per_dst=[886, 486, 440, 1162, 526, 1152, 828, 0] combine_rows_per_src=[13466, 3368, 15334, 13300, 15358, 6812, 15640, 9546] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=313679872 dispatch_meta_bytes=329960 combine_payload_bytes=313679872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=314009832 ingress_bytes=382885888 link_bytes=382885888 token_bytes=627689704 weight_movement_bytes=69206016 total_nvlink_bytes=696895720 weight_to_token_ratio=0.110255 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=32896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.440768772e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=303996928 dispatch_meta_bytes=319944 combine_payload_bytes=303996928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=304316872 ingress_bytes=338599936 link_bytes=338599936 token_bytes=608313800 weight_movement_bytes=34603008 total_nvlink_bytes=642916808 weight_to_token_ratio=0.056883 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=37504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.274124300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64804 payload_rows_remote=49592 dup_rows=33500 dup_rows_remote=26056 dedup_ratio=0.340780 router_tokens_per_expert=[177, 36, 1, 6, 1, 5, 2, 2972, 547, 42, 318, 1, 4, 0, 256, 6064, 0, 12, 1, 4250, 270, 665, 1, 672, 0, 1, 0, 0, 0, 1, 0, 9, 2510, 32, 5, 0, 0, 0, 0, 5377, 0, 39, 6, 7660, 29, 35, 183, 2, 1, 3, 0, 3509, 10, 7678, 0, 26, 0, 10, 1, 5052, 150, 432, 1, 87] payload_rows_per_dst=[6176, 2148, 11630, 22, 12844, 634, 16138, 15212] dup_rows_per_dst=[224, 188, 882, 0, 3004, 122, 21636, 7444] combine_rows_per_src=[6176, 2148, 11630, 22, 12844, 634, 16138, 15212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=246374400 dispatch_meta_bytes=328104 combine_payload_bytes=246374400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246702504 ingress_bytes=280977408 link_bytes=280977408 token_bytes=493076904 weight_movement_bytes=34603008 total_nvlink_bytes=527679912 weight_to_token_ratio=0.070178 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.057295366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90124 payload_rows_remote=79414 dup_rows=8180 dup_rows_remote=6156 dedup_ratio=0.083211 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 2945, 567, 0, 121, 0, 0, 0, 267, 5712, 1, 0, 0, 5513, 285, 1890, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 8076, 0, 0, 266, 1, 0, 0, 0, 1746, 0, 7901, 0, 0, 0, 0, 0, 3969, 142, 435, 0, 82] payload_rows_per_dst=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] dup_rows_per_dst=[1268, 1436, 2068, 0, 2024, 532, 0, 852] combine_rows_per_src=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=317874176 dispatch_meta_bytes=343672 combine_payload_bytes=317874176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318217848 ingress_bytes=387080192 link_bytes=387080192 token_bytes=636092024 weight_movement_bytes=69206016 total_nvlink_bytes=705298040 weight_to_token_ratio=0.108799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.456551600e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=311861248 dispatch_meta_bytes=330368 combine_payload_bytes=311861248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312191616 ingress_bytes=346464256 link_bytes=346464256 token_bytes=624052864 weight_movement_bytes=34603008 total_nvlink_bytes=658655872 weight_to_token_ratio=0.055449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303717103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=288931840 dispatch_meta_bytes=374016 combine_payload_bytes=288931840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289305856 ingress_bytes=323534848 link_bytes=323534848 token_bytes=578237696 weight_movement_bytes=34603008 total_nvlink_bytes=612840704 weight_to_token_ratio=0.059842 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.217435586e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=352305152 dispatch_meta_bytes=361360 combine_payload_bytes=352305152 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352666512 ingress_bytes=490717184 link_bytes=490717184 token_bytes=704971664 weight_movement_bytes=138412032 total_nvlink_bytes=843383696 weight_to_token_ratio=0.196337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846529257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325279744 dispatch_meta_bytes=342280 combine_payload_bytes=325279744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325622024 ingress_bytes=394485760 link_bytes=394485760 token_bytes=650901768 weight_movement_bytes=69206016 total_nvlink_bytes=720107784 weight_to_token_ratio=0.106323 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484418156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86956 payload_rows_remote=81032 dup_rows=11348 dup_rows_remote=10862 dedup_ratio=0.115438 router_tokens_per_expert=[1926, 675, 15, 23, 7, 10, 65, 1479, 426, 719, 2378, 51, 68, 4, 249, 5037, 4, 177, 40, 1543, 269, 350, 44, 778, 0, 13, 0, 14, 44, 77, 17, 99, 4111, 593, 108, 0, 9, 0, 0, 3952, 1, 710, 5, 5228, 340, 627, 181, 26, 12, 8, 2, 2749, 228, 6163, 0, 387, 7, 211, 5, 6266, 153, 385, 16, 68] payload_rows_per_dst=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] dup_rows_per_dst=[3200, 3846, 486, 58, 462, 1972, 438, 886] combine_rows_per_src=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89306 payload_rows_remote=75606 dup_rows=8998 dup_rows_remote=8736 dedup_ratio=0.091532 router_tokens_per_expert=[0, 3731, 927, 1168, 7448, 7547, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 41, 0, 1657, 1, 0, 0, 0, 1311, 1793, 80, 2, 0, 0, 0, 0, 246, 1, 0, 6614, 0, 120, 0, 62, 1719, 0, 0, 1, 0, 0, 0, 1681, 0, 0, 0, 0, 62, 6680, 197, 5740, 0, 0, 80, 0, 1, 0] payload_rows_per_dst=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] dup_rows_per_dst=[316, 480, 1368, 320, 262, 3188, 2676, 388] combine_rows_per_src=[3874, 14900, 9492, 6052, 13700, 15470, 14170, 11648] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91732 payload_rows_remote=79338 dup_rows=6572 dup_rows_remote=5016 dedup_ratio=0.066854 router_tokens_per_expert=[30, 0, 0, 0, 0, 0, 0, 3025, 495, 0, 317, 0, 0, 0, 202, 5961, 0, 0, 0, 3670, 275, 452, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 5760, 0, 0, 0, 7821, 0, 0, 201, 0, 0, 0, 0, 4111, 0, 7750, 0, 0, 0, 0, 0, 4868, 179, 432, 0, 98] payload_rows_per_dst=[14198, 12394, 9830, 15500, 5640, 15658, 8222, 10290] dup_rows_per_dst=[3432, 1556, 334, 0, 0, 386, 0, 864] combine_rows_per_src=[14198, 12394, 9830, 15500, 5640, 15658, 8222, 10290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=203128832 dispatch_meta_bytes=302592 combine_payload_bytes=203128832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203431424 ingress_bytes=237731840 link_bytes=237731840 token_bytes=406560256 weight_movement_bytes=34603008 total_nvlink_bytes=441163264 weight_to_token_ratio=0.085112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.945657749e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77874 payload_rows_remote=65476 dup_rows=20430 dup_rows_remote=11174 dedup_ratio=0.207825 router_tokens_per_expert=[3, 0, 0, 0, 0, 0, 0, 2974, 561, 0, 216, 0, 0, 0, 269, 6029, 0, 0, 0, 4334, 256, 769, 0, 581, 0, 0, 0, 0, 0, 0, 0, 0, 3755, 0, 0, 0, 0, 0, 0, 5398, 0, 0, 0, 8060, 0, 0, 205, 0, 0, 0, 0, 3392, 0, 7855, 0, 0, 0, 0, 0, 3819, 148, 450, 0, 78] payload_rows_per_dst=[5948, 8146, 12602, 6914, 13556, 10218, 12398, 8092] dup_rows_per_dst=[6, 1002, 4280, 0, 4750, 238, 9256, 898] combine_rows_per_src=[5948, 8146, 12602, 6914, 13556, 10218, 12398, 8092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63000 payload_rows_remote=46738 dup_rows=35304 dup_rows_remote=9240 dedup_ratio=0.359131 router_tokens_per_expert=[0, 2225, 804, 1150, 7789, 7510, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 0, 1202, 4315, 88, 0, 0, 0, 0, 0, 98, 0, 0, 6064, 53, 132, 0, 0, 900, 0, 0, 0, 11, 0, 0, 1620, 133, 0, 0, 0, 69, 5267, 245, 6267, 0, 0, 89, 0, 0, 0] payload_rows_per_dst=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] dup_rows_per_dst=[26064, 0, 1218, 6576, 124, 22, 928, 372] combine_rows_per_src=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91850 payload_rows_remote=76138 dup_rows=6454 dup_rows_remote=6454 dedup_ratio=0.065653 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 2896, 536, 0, 250, 0, 0, 0, 223, 6097, 0, 0, 0, 4525, 248, 733, 0, 599, 0, 0, 0, 0, 0, 0, 0, 0, 3180, 0, 0, 0, 0, 0, 0, 5409, 0, 0, 0, 7992, 0, 0, 213, 0, 0, 0, 0, 3543, 0, 7856, 0, 0, 0, 0, 0, 4245, 123, 419, 0, 63] payload_rows_per_dst=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] dup_rows_per_dst=[2916, 1686, 590, 0, 0, 424, 0, 838] combine_rows_per_src=[13698, 12526, 11620, 15712, 6360, 15986, 7086, 8862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86956 payload_rows_remote=81032 dup_rows=11348 dup_rows_remote=10862 dedup_ratio=0.115438 router_tokens_per_expert=[1926, 675, 15, 23, 7, 10, 65, 1479, 426, 719, 2378, 51, 68, 4, 249, 5037, 4, 177, 40, 1543, 269, 350, 44, 778, 0, 13, 0, 14, 44, 77, 17, 99, 4111, 593, 108, 0, 9, 0, 0, 3952, 1, 710, 5, 5228, 340, 627, 181, 26, 12, 8, 2, 2749, 228, 6163, 0, 387, 7, 211, 5, 6266, 153, 385, 16, 68] payload_rows_per_dst=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] dup_rows_per_dst=[3200, 3846, 486, 58, 462, 1972, 438, 886] combine_rows_per_src=[13104, 14018, 5924, 12796, 9180, 12264, 6334, 13336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=64804 payload_rows_remote=49592 dup_rows=33500 dup_rows_remote=26056 dedup_ratio=0.340780 router_tokens_per_expert=[177, 36, 1, 6, 1, 5, 2, 2972, 547, 42, 318, 1, 4, 0, 256, 6064, 0, 12, 1, 4250, 270, 665, 1, 672, 0, 1, 0, 0, 0, 1, 0, 9, 2510, 32, 5, 0, 0, 0, 0, 5377, 0, 39, 6, 7660, 29, 35, 183, 2, 1, 3, 0, 3509, 10, 7678, 0, 26, 0, 10, 1, 5052, 150, 432, 1, 87] payload_rows_per_dst=[6176, 2148, 11630, 22, 12844, 634, 16138, 15212] dup_rows_per_dst=[224, 188, 882, 0, 3004, 122, 21636, 7444] combine_rows_per_src=[6176, 2148, 11630, 22, 12844, 634, 16138, 15212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93490 payload_rows_remote=80298 dup_rows=4814 dup_rows_remote=3742 dedup_ratio=0.048971 router_tokens_per_expert=[0, 0, 900, 0, 0, 0, 0, 0, 0, 111, 0, 0, 7620, 0, 1598, 0, 0, 356, 54, 0, 0, 0, 249, 0, 608, 0, 0, 0, 0, 0, 0, 6524, 7612, 0, 143, 0, 0, 0, 164, 0, 376, 1964, 0, 0, 0, 7265, 7626, 87, 0, 596, 0, 0, 242, 0, 63, 0, 0, 0, 4994, 0, 0, 0, 0, 0] payload_rows_per_dst=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] dup_rows_per_dst=[1050, 222, 498, 1072, 452, 328, 1192, 0] combine_rows_per_src=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91732 payload_rows_remote=79338 dup_rows=6572 dup_rows_remote=5016 dedup_ratio=0.066854 router_tokens_per_expert=[30, 0, 0, 0, 0, 0, 0, 3025, 495, 0, 317, 0, 0, 0, 202, 5961, 0, 0, 0, 3670, 275, 452, 0, 685, 0, 0, 0, 0, 0, 0, 0, 0, 2820, 0, 0, 0, 0, 0, 0, 5760, 0, 0, 0, 7821, 0, 0, 201, 0, 0, 0, 0, 4111, 0, 7750, 0, 0, 0, 0, 0, 4868, 179, 432, 0, 98] payload_rows_per_dst=[14198, 12394, 9830, 15500, 5640, 15658, 8222, 10290] dup_rows_per_dst=[3432, 1556, 334, 0, 0, 386, 0, 864] combine_rows_per_src=[14198, 12394, 9830, 15500, 5640, 15658, 8222, 10290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=331907072 dispatch_meta_bytes=367576 combine_payload_bytes=331907072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332274648 ingress_bytes=366510080 link_bytes=366510080 token_bytes=664181720 weight_movement_bytes=34603008 total_nvlink_bytes=698784728 weight_to_token_ratio=0.052099 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379147925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89642 payload_rows_remote=87456 dup_rows=8662 dup_rows_remote=8462 dedup_ratio=0.088114 router_tokens_per_expert=[0, 216, 0, 0, 0, 0, 439, 0, 88, 7421, 2, 0, 0, 88, 0, 148, 57, 0, 0, 62, 0, 182, 0, 0, 0, 0, 0, 0, 501, 7842, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 88, 0, 0, 1003, 12, 0, 90, 0, 257, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 581, 7856, 0, 0, 0, 594] payload_rows_per_dst=[12966, 14986, 10674, 14896, 5750, 2186, 12296, 15888] dup_rows_per_dst=[1068, 508, 402, 802, 2994, 200, 514, 2174] combine_rows_per_src=[12966, 14986, 10674, 14896, 5750, 2186, 12296, 15888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=309682176 dispatch_meta_bytes=337368 combine_payload_bytes=309682176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=310019544 ingress_bytes=378888192 link_bytes=378888192 token_bytes=619701720 weight_movement_bytes=69206016 total_nvlink_bytes=688907736 weight_to_token_ratio=0.111676 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.425725763e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324968448 dispatch_meta_bytes=337416 combine_payload_bytes=324968448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325305864 ingress_bytes=428777472 link_bytes=428777472 token_bytes=650274312 weight_movement_bytes=103809024 total_nvlink_bytes=754083336 weight_to_token_ratio=0.159639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613455108e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90050 payload_rows_remote=73958 dup_rows=8254 dup_rows_remote=7818 dedup_ratio=0.083964 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 3148, 479, 0, 164, 0, 0, 0, 267, 5720, 0, 0, 0, 5361, 180, 1485, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 5298, 0, 0, 0, 8046, 0, 0, 218, 0, 0, 0, 0, 2440, 0, 7979, 0, 0, 0, 0, 0, 3872, 160, 456, 0, 88] payload_rows_per_dst=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] dup_rows_per_dst=[0, 1320, 1532, 0, 4062, 436, 0, 904] combine_rows_per_src=[6298, 11940, 13288, 15958, 13346, 16092, 4880, 8248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268189696 dispatch_meta_bytes=306600 combine_payload_bytes=268189696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268496296 ingress_bytes=337395712 link_bytes=337395712 token_bytes=536685992 weight_movement_bytes=69206016 total_nvlink_bytes=605892008 weight_to_token_ratio=0.128951 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.269592902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=311861248 dispatch_meta_bytes=330368 combine_payload_bytes=311861248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312191616 ingress_bytes=346464256 link_bytes=346464256 token_bytes=624052864 weight_movement_bytes=34603008 total_nvlink_bytes=658655872 weight_to_token_ratio=0.055449 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=48896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303717103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=331907072 dispatch_meta_bytes=367576 combine_payload_bytes=331907072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=332274648 ingress_bytes=366510080 link_bytes=366510080 token_bytes=664181720 weight_movement_bytes=34603008 total_nvlink_bytes=698784728 weight_to_token_ratio=0.052099 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=32384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.379147925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=63000 payload_rows_remote=46738 dup_rows=35304 dup_rows_remote=9240 dedup_ratio=0.359131 router_tokens_per_expert=[0, 2225, 804, 1150, 7789, 7510, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2870, 0, 0, 0, 0, 1202, 4315, 88, 0, 0, 0, 0, 0, 98, 0, 0, 6064, 53, 132, 0, 0, 900, 0, 0, 0, 11, 0, 0, 1620, 133, 0, 0, 0, 69, 5267, 245, 6267, 0, 0, 89, 0, 0, 0] payload_rows_per_dst=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] dup_rows_per_dst=[26064, 0, 1218, 6576, 124, 22, 928, 372] combine_rows_per_src=[16262, 502, 8972, 15168, 4750, 1800, 6676, 8870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=302931968 dispatch_meta_bytes=327104 combine_payload_bytes=302931968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=303259072 ingress_bytes=406740992 link_bytes=406740992 token_bytes=606191040 weight_movement_bytes=103809024 total_nvlink_bytes=710000064 weight_to_token_ratio=0.171248 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.530533608e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71642 payload_rows_remote=55818 dup_rows=26662 dup_rows_remote=19218 dedup_ratio=0.271220 router_tokens_per_expert=[4, 2, 872, 6, 5, 0, 2, 0, 40, 121, 1, 0, 7471, 1, 2101, 0, 19, 358, 52, 1, 0, 0, 238, 6, 693, 5, 4, 0, 0, 6, 0, 6528, 7306, 1, 218, 0, 42, 0, 164, 0, 632, 1429, 4, 11, 43, 7102, 7465, 48, 44, 604, 0, 6, 304, 0, 87, 0, 0, 44, 5023, 32, 3, 1, 1, 2] payload_rows_per_dst=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] dup_rows_per_dst=[14, 246, 114, 8, 488, 17092, 1256, 7444] combine_rows_per_src=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93608 payload_rows_remote=78182 dup_rows=4696 dup_rows_remote=4210 dedup_ratio=0.047770 router_tokens_per_expert=[0, 0, 887, 0, 0, 6, 0, 0, 0, 238, 0, 0, 7500, 0, 2213, 0, 0, 355, 52, 0, 0, 0, 269, 0, 534, 0, 0, 0, 0, 52, 0, 6570, 7605, 0, 192, 0, 0, 0, 158, 1, 355, 2103, 0, 0, 0, 7097, 7495, 61, 0, 488, 0, 0, 242, 0, 80, 0, 0, 0, 4598, 0, 0, 0, 1, 0] payload_rows_per_dst=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] dup_rows_per_dst=[942, 474, 532, 1068, 486, 218, 976, 0] combine_rows_per_src=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=203128832 dispatch_meta_bytes=302592 combine_payload_bytes=203128832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=203431424 ingress_bytes=237731840 link_bytes=237731840 token_bytes=406560256 weight_movement_bytes=34603008 total_nvlink_bytes=441163264 weight_to_token_ratio=0.085112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.945657749e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328900608 dispatch_meta_bytes=336160 combine_payload_bytes=328900608 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329236768 ingress_bytes=398106624 link_bytes=398106624 token_bytes=658137376 weight_movement_bytes=69206016 total_nvlink_bytes=727343392 weight_to_token_ratio=0.105154 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498043176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=324968448 dispatch_meta_bytes=337416 combine_payload_bytes=324968448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325305864 ingress_bytes=428777472 link_bytes=428777472 token_bytes=650274312 weight_movement_bytes=103809024 total_nvlink_bytes=754083336 weight_to_token_ratio=0.159639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=38400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.613455108e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=191438848 dispatch_meta_bytes=223912 combine_payload_bytes=191438848 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=191662760 ingress_bytes=364453888 link_bytes=364453888 token_bytes=383101608 weight_movement_bytes=173015040 total_nvlink_bytes=556116648 weight_to_token_ratio=0.451617 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.371410640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=358219776 dispatch_meta_bytes=383672 combine_payload_bytes=358219776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=358603448 ingress_bytes=392822784 link_bytes=392822784 token_bytes=716823224 weight_movement_bytes=34603008 total_nvlink_bytes=751426232 weight_to_token_ratio=0.048273 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478160511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=320233472 dispatch_meta_bytes=329568 combine_payload_bytes=320233472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320563040 ingress_bytes=389439488 link_bytes=389439488 token_bytes=640796512 weight_movement_bytes=69206016 total_nvlink_bytes=710002528 weight_to_token_ratio=0.108000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465429441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13867317520 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6937598224 ingress_bytes=8556060672 link_bytes=8556060672 total_nvlink_bytes=15493658896 weight_to_token_ratio=0.117279 prefetch_share_of_ingress=0.088974 dedup_ratio=0.143243 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.399815978e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87586 payload_rows_remote=74490 dup_rows=10718 dup_rows_remote=10042 dedup_ratio=0.109029 router_tokens_per_expert=[39, 60, 1394, 114, 19, 7, 52, 24, 602, 134, 34, 0, 5123, 5, 2237, 25, 140, 363, 49, 2, 0, 8, 190, 46, 2442, 108, 104, 4, 19, 39, 8, 4419, 4329, 11, 118, 0, 675, 0, 327, 0, 1475, 2320, 58, 240, 747, 6267, 6088, 38, 741, 661, 4, 175, 890, 0, 68, 4, 3, 758, 4585, 659, 35, 26, 19, 21] payload_rows_per_dst=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] dup_rows_per_dst=[2222, 1714, 676, 2160, 476, 1088, 1066, 1316] combine_rows_per_src=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90124 payload_rows_remote=79414 dup_rows=8180 dup_rows_remote=6156 dedup_ratio=0.083211 router_tokens_per_expert=[0, 0, 0, 0, 0, 10, 0, 2945, 567, 0, 121, 0, 0, 0, 267, 5712, 1, 0, 0, 5513, 285, 1890, 0, 428, 0, 0, 0, 0, 0, 0, 0, 0, 3501, 0, 0, 0, 0, 0, 0, 5294, 0, 0, 0, 8076, 0, 0, 266, 1, 0, 0, 0, 1746, 0, 7901, 0, 0, 0, 0, 0, 3969, 142, 435, 0, 82] payload_rows_per_dst=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] dup_rows_per_dst=[1268, 1436, 2068, 0, 2024, 532, 0, 852] combine_rows_per_src=[9498, 11898, 14166, 15802, 10710, 16154, 3492, 8404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=228630528 dispatch_meta_bytes=300144 combine_payload_bytes=228630528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228930672 ingress_bytes=263233536 link_bytes=263233536 token_bytes=457561200 weight_movement_bytes=34603008 total_nvlink_bytes=492164208 weight_to_token_ratio=0.075625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.905266039e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77374 payload_rows_remote=61118 dup_rows=20930 dup_rows_remote=3714 dedup_ratio=0.212911 router_tokens_per_expert=[0, 0, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 1729, 0, 0, 334, 64, 0, 0, 0, 251, 0, 748, 0, 0, 0, 0, 0, 0, 6572, 7544, 0, 200, 0, 0, 0, 183, 0, 517, 1739, 0, 0, 0, 7187, 7597, 47, 0, 638, 0, 0, 231, 0, 98, 0, 0, 0, 5037, 0, 0, 0, 0, 0] payload_rows_per_dst=[15202, 16256, 15990, 13358, 3950, 1128, 1416, 10074] dup_rows_per_dst=[962, 17216, 502, 1282, 450, 0, 518, 0] combine_rows_per_src=[15202, 16256, 15990, 13358, 3950, 1128, 1416, 10074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74636 payload_rows_remote=58812 dup_rows=23668 dup_rows_remote=22678 dedup_ratio=0.240763 router_tokens_per_expert=[0, 0, 909, 0, 0, 0, 0, 0, 0, 152, 0, 0, 7590, 0, 1641, 0, 0, 372, 51, 0, 0, 0, 262, 0, 724, 0, 0, 0, 0, 13, 0, 6510, 7585, 0, 183, 0, 0, 0, 183, 0, 447, 1880, 0, 0, 0, 7202, 7578, 76, 0, 495, 0, 0, 244, 0, 78, 0, 0, 0, 4977, 0, 0, 0, 0, 0] payload_rows_per_dst=[1818, 3282, 1268, 9084, 15364, 16384, 15824, 11612] dup_rows_per_dst=[0, 304, 102, 850, 538, 18974, 990, 1910] combine_rows_per_src=[1818, 3282, 1268, 9084, 15364, 16384, 15824, 11612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=71642 payload_rows_remote=55818 dup_rows=26662 dup_rows_remote=19218 dedup_ratio=0.271220 router_tokens_per_expert=[4, 2, 872, 6, 5, 0, 2, 0, 40, 121, 1, 0, 7471, 1, 2101, 0, 19, 358, 52, 1, 0, 0, 238, 6, 693, 5, 4, 0, 0, 6, 0, 6528, 7306, 1, 218, 0, 42, 0, 164, 0, 632, 1429, 4, 11, 43, 7102, 7465, 48, 44, 604, 0, 6, 304, 0, 87, 0, 0, 44, 5023, 32, 3, 1, 1, 2] payload_rows_per_dst=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] dup_rows_per_dst=[14, 246, 114, 8, 488, 17092, 1256, 7444] combine_rows_per_src=[1768, 4282, 1234, 1408, 14974, 16376, 15776, 15824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93490 payload_rows_remote=80298 dup_rows=4814 dup_rows_remote=3742 dedup_ratio=0.048971 router_tokens_per_expert=[0, 0, 900, 0, 0, 0, 0, 0, 0, 111, 0, 0, 7620, 0, 1598, 0, 0, 356, 54, 0, 0, 0, 249, 0, 608, 0, 0, 0, 0, 0, 0, 6524, 7612, 0, 143, 0, 0, 0, 164, 0, 376, 1964, 0, 0, 0, 7265, 7626, 87, 0, 596, 0, 0, 242, 0, 63, 0, 0, 0, 4994, 0, 0, 0, 0, 0] payload_rows_per_dst=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] dup_rows_per_dst=[1050, 222, 498, 1072, 452, 328, 1192, 0] combine_rows_per_src=[15280, 3196, 16072, 13192, 15386, 4526, 15850, 9988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87586 payload_rows_remote=74490 dup_rows=10718 dup_rows_remote=10042 dedup_ratio=0.109029 router_tokens_per_expert=[39, 60, 1394, 114, 19, 7, 52, 24, 602, 134, 34, 0, 5123, 5, 2237, 25, 140, 363, 49, 2, 0, 8, 190, 46, 2442, 108, 104, 4, 19, 39, 8, 4419, 4329, 11, 118, 0, 675, 0, 327, 0, 1475, 2320, 58, 240, 747, 6267, 6088, 38, 741, 661, 4, 175, 890, 0, 68, 4, 3, 758, 4585, 659, 35, 26, 19, 21] payload_rows_per_dst=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] dup_rows_per_dst=[2222, 1714, 676, 2160, 476, 1088, 1066, 1316] combine_rows_per_src=[13730, 9376, 13096, 12126, 10956, 8156, 9250, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92314 payload_rows_remote=76588 dup_rows=5990 dup_rows_remote=4960 dedup_ratio=0.060933 router_tokens_per_expert=[0, 230, 0, 0, 0, 0, 326, 0, 61, 7430, 7, 0, 0, 61, 0, 108, 85, 0, 0, 54, 0, 198, 0, 0, 0, 0, 0, 0, 597, 7781, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 8131, 63, 0, 0, 1206, 15, 0, 67, 0, 243, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 568, 7882, 0, 0, 0, 626] payload_rows_per_dst=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] dup_rows_per_dst=[1012, 350, 674, 1030, 0, 2442, 482, 0] combine_rows_per_src=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=325279744 dispatch_meta_bytes=342280 combine_payload_bytes=325279744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=325622024 ingress_bytes=394485760 link_bytes=394485760 token_bytes=650901768 weight_movement_bytes=69206016 total_nvlink_bytes=720107784 weight_to_token_ratio=0.106323 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.035000 gemm_active_groups=7 gemm_peak_group_rows=25216 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484418156e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92824 payload_rows_remote=86012 dup_rows=5480 dup_rows_remote=4328 dedup_ratio=0.055745 router_tokens_per_expert=[0, 0, 874, 0, 0, 2, 0, 0, 0, 243, 0, 0, 7475, 0, 1684, 0, 0, 369, 62, 0, 0, 0, 225, 0, 588, 0, 0, 0, 0, 57, 0, 6586, 7575, 0, 192, 0, 0, 0, 175, 0, 388, 2472, 0, 0, 0, 7134, 7462, 57, 0, 414, 0, 0, 257, 0, 88, 0, 0, 0, 4773, 0, 0, 0, 0, 0] payload_rows_per_dst=[13466, 3368, 15334, 13300, 15358, 6812, 15640, 9546] dup_rows_per_dst=[886, 486, 440, 1162, 526, 1152, 828, 0] combine_rows_per_src=[13466, 3368, 15334, 13300, 15358, 6812, 15640, 9546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=250339328 dispatch_meta_bytes=259328 combine_payload_bytes=250339328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250598656 ingress_bytes=319545344 link_bytes=319545344 token_bytes=500937984 weight_movement_bytes=69206016 total_nvlink_bytes=570144000 weight_to_token_ratio=0.138153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202423404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=240893952 dispatch_meta_bytes=325960 combine_payload_bytes=240893952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241219912 ingress_bytes=275496960 link_bytes=275496960 token_bytes=482113864 weight_movement_bytes=34603008 total_nvlink_bytes=516716872 weight_to_token_ratio=0.071774 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.036672881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=328900608 dispatch_meta_bytes=336160 combine_payload_bytes=328900608 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=329236768 ingress_bytes=398106624 link_bytes=398106624 token_bytes=658137376 weight_movement_bytes=69206016 total_nvlink_bytes=727343392 weight_to_token_ratio=0.105154 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=41472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.498043176e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=305111040 dispatch_meta_bytes=338128 combine_payload_bytes=305111040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=305449168 ingress_bytes=339714048 link_bytes=339714048 token_bytes=610560208 weight_movement_bytes=34603008 total_nvlink_bytes=645163216 weight_to_token_ratio=0.056674 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.278316613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77374 payload_rows_remote=61118 dup_rows=20930 dup_rows_remote=3714 dedup_ratio=0.212911 router_tokens_per_expert=[0, 0, 895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7541, 0, 1729, 0, 0, 334, 64, 0, 0, 0, 251, 0, 748, 0, 0, 0, 0, 0, 0, 6572, 7544, 0, 200, 0, 0, 0, 183, 0, 517, 1739, 0, 0, 0, 7187, 7597, 47, 0, 638, 0, 0, 231, 0, 98, 0, 0, 0, 5037, 0, 0, 0, 0, 0] payload_rows_per_dst=[15202, 16256, 15990, 13358, 3950, 1128, 1416, 10074] dup_rows_per_dst=[962, 17216, 502, 1282, 450, 0, 518, 0] combine_rows_per_src=[15202, 16256, 15990, 13358, 3950, 1128, 1416, 10074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90432 payload_rows_remote=75836 dup_rows=7872 dup_rows_remote=4130 dedup_ratio=0.080078 router_tokens_per_expert=[187, 0, 0, 0, 0, 0, 0, 2920, 467, 0, 312, 0, 0, 0, 93, 6228, 0, 0, 0, 3093, 268, 468, 0, 707, 0, 0, 0, 0, 0, 0, 0, 0, 1874, 0, 0, 0, 0, 0, 0, 6062, 0, 0, 0, 7679, 0, 0, 224, 0, 0, 0, 0, 4212, 0, 7771, 0, 0, 0, 0, 0, 5905, 158, 434, 0, 90] payload_rows_per_dst=[14596, 12866, 8768, 15542, 10898, 7032, 8424, 12306] dup_rows_per_dst=[3742, 1334, 304, 0, 1468, 156, 0, 868] combine_rows_per_src=[14596, 12866, 8768, 15542, 10898, 7032, 8424, 12306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=240893952 dispatch_meta_bytes=325960 combine_payload_bytes=240893952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=241219912 ingress_bytes=275496960 link_bytes=275496960 token_bytes=482113864 weight_movement_bytes=34603008 total_nvlink_bytes=516716872 weight_to_token_ratio=0.071774 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.036672881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92180 payload_rows_remote=92180 dup_rows=6124 dup_rows_remote=6124 dedup_ratio=0.062297 router_tokens_per_expert=[0, 257, 0, 0, 0, 0, 477, 18, 80, 7419, 1, 0, 0, 80, 2, 131, 62, 0, 0, 55, 0, 207, 0, 0, 0, 0, 0, 0, 579, 7772, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 80, 0, 0, 1072, 11, 0, 80, 0, 242, 0, 0, 0, 0, 6342, 0, 0, 0, 0, 590, 7827, 0, 4, 0, 287] payload_rows_per_dst=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] dup_rows_per_dst=[1398, 462, 646, 996, 0, 2032, 484, 106] combine_rows_per_src=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=352305152 dispatch_meta_bytes=361360 combine_payload_bytes=352305152 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=352666512 ingress_bytes=490717184 link_bytes=490717184 token_bytes=704971664 weight_movement_bytes=138412032 total_nvlink_bytes=843383696 weight_to_token_ratio=0.196337 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.846529257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51272 payload_rows_remote=35008 dup_rows=47032 dup_rows_remote=30092 dedup_ratio=0.478434 router_tokens_per_expert=[0, 229, 29, 42, 43, 24, 332, 86, 87, 7404, 10, 0, 0, 114, 5, 75, 51, 15, 0, 67, 38, 192, 28, 0, 0, 14, 0, 0, 598, 7598, 23, 0, 2, 43, 0, 4, 7, 7176, 2, 8052, 87, 0, 0, 964, 93, 3, 118, 6, 297, 0, 1, 0, 0, 5995, 0, 0, 0, 0, 715, 7741, 4, 15, 2, 721] payload_rows_per_dst=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] dup_rows_per_dst=[536, 202, 56, 12, 24700, 360, 4226, 16940] combine_rows_per_src=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=228630528 dispatch_meta_bytes=300144 combine_payload_bytes=228630528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=228930672 ingress_bytes=263233536 link_bytes=263233536 token_bytes=457561200 weight_movement_bytes=34603008 total_nvlink_bytes=492164208 weight_to_token_ratio=0.075625 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=39296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.905266039e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313704448 dispatch_meta_bytes=326192 combine_payload_bytes=313704448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=314030640 ingress_bytes=417513472 link_bytes=417513472 token_bytes=627735088 weight_movement_bytes=103809024 total_nvlink_bytes=731544112 weight_to_token_ratio=0.165371 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.571069583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=250339328 dispatch_meta_bytes=259328 combine_payload_bytes=250339328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=250598656 ingress_bytes=319545344 link_bytes=319545344 token_bytes=500937984 weight_movement_bytes=69206016 total_nvlink_bytes=570144000 weight_to_token_ratio=0.138153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202423404e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=310624256 dispatch_meta_bytes=319864 combine_payload_bytes=310624256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310944120 ingress_bytes=345227264 link_bytes=345227264 token_bytes=621568376 weight_movement_bytes=34603008 total_nvlink_bytes=656171384 weight_to_token_ratio=0.055670 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299062401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=377569280 dispatch_meta_bytes=393216 combine_payload_bytes=377569280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377962496 ingress_bytes=481378304 link_bytes=481378304 token_bytes=755531776 weight_movement_bytes=103809024 total_nvlink_bytes=859340800 weight_to_token_ratio=0.137399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.811387804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13799475920 prefetch_bytes=726663168 reduce_bytes=795869184 weight_movement_bytes=1522532352 egress_bytes=6903663312 ingress_bytes=8418344960 link_bytes=8418344960 total_nvlink_bytes=15322008272 weight_to_token_ratio=0.110333 prefetch_share_of_ingress=0.086319 dedup_ratio=0.139510 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=23 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027609 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.377284972e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85690 payload_rows_remote=69604 dup_rows=12614 dup_rows_remote=10506 dedup_ratio=0.128316 router_tokens_per_expert=[1, 124, 504, 728, 787, 394, 303, 458, 69, 6390, 180, 0, 0, 360, 62, 72, 46, 136, 0, 221, 674, 246, 483, 0, 1, 139, 6, 20, 678, 5902, 548, 7, 64, 772, 0, 77, 138, 5540, 51, 7291, 68, 5, 25, 2039, 260, 87, 847, 27, 956, 13, 23, 7, 10, 1772, 0, 0, 5, 0, 744, 6021, 18, 31, 33, 2689] payload_rows_per_dst=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] dup_rows_per_dst=[4038, 870, 2108, 762, 144, 4534, 124, 34] combine_rows_per_src=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=143392768 dispatch_meta_bytes=260400 combine_payload_bytes=143392768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=143653168 ingress_bytes=212598784 link_bytes=212598784 token_bytes=287045936 weight_movement_bytes=69206016 total_nvlink_bytes=356251952 weight_to_token_ratio=0.241097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.999921086e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93608 payload_rows_remote=78182 dup_rows=4696 dup_rows_remote=4210 dedup_ratio=0.047770 router_tokens_per_expert=[0, 0, 887, 0, 0, 6, 0, 0, 0, 238, 0, 0, 7500, 0, 2213, 0, 0, 355, 52, 0, 0, 0, 269, 0, 534, 0, 0, 0, 0, 52, 0, 6570, 7605, 0, 192, 0, 0, 0, 158, 1, 355, 2103, 0, 0, 0, 7097, 7495, 61, 0, 488, 0, 0, 242, 0, 80, 0, 0, 0, 4598, 0, 0, 0, 1, 0] payload_rows_per_dst=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] dup_rows_per_dst=[942, 474, 532, 1068, 486, 218, 976, 0] combine_rows_per_src=[15038, 4428, 15810, 13244, 15426, 4820, 15644, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92574 payload_rows_remote=77416 dup_rows=5730 dup_rows_remote=5262 dedup_ratio=0.058289 router_tokens_per_expert=[0, 189, 0, 0, 0, 0, 319, 4, 97, 7497, 27, 0, 0, 97, 0, 95, 60, 0, 0, 71, 0, 222, 0, 0, 0, 0, 0, 0, 574, 7786, 0, 0, 0, 0, 0, 0, 0, 7338, 0, 8095, 98, 0, 0, 1027, 83, 0, 116, 0, 233, 0, 0, 0, 0, 5729, 0, 0, 0, 0, 627, 7820, 0, 0, 0, 948] payload_rows_per_dst=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] dup_rows_per_dst=[946, 468, 706, 928, 0, 2220, 462, 0] combine_rows_per_src=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=54138 payload_rows_remote=42330 dup_rows=44166 dup_rows_remote=43688 dedup_ratio=0.449280 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 354, 31, 78, 7450, 0, 0, 0, 78, 0, 112, 84, 0, 0, 51, 0, 204, 0, 0, 0, 0, 0, 0, 585, 7739, 0, 0, 0, 0, 0, 0, 0, 7319, 0, 8114, 78, 0, 0, 1167, 35, 0, 79, 0, 244, 0, 0, 0, 0, 5899, 0, 0, 0, 0, 660, 7833, 0, 0, 0, 714] payload_rows_per_dst=[870, 4522, 678, 1170, 16384, 2492, 11808, 16214] dup_rows_per_dst=[388, 278, 0, 0, 29960, 226, 478, 12836] combine_rows_per_src=[870, 4522, 678, 1170, 16384, 2492, 11808, 16214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14090289064 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=7049125800 ingress_bytes=8632901632 link_bytes=8632901632 total_nvlink_bytes=15682027432 weight_to_token_ratio=0.112967 prefetch_share_of_ingress=0.076157 dedup_ratio=0.139498 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=27178 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.412387558e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92314 payload_rows_remote=76588 dup_rows=5990 dup_rows_remote=4960 dedup_ratio=0.060933 router_tokens_per_expert=[0, 230, 0, 0, 0, 0, 326, 0, 61, 7430, 7, 0, 0, 61, 0, 108, 85, 0, 0, 54, 0, 198, 0, 0, 0, 0, 0, 0, 597, 7781, 0, 0, 0, 0, 0, 0, 0, 7416, 0, 8131, 63, 0, 0, 1206, 15, 0, 67, 0, 243, 0, 0, 0, 0, 5997, 0, 0, 0, 0, 568, 7882, 0, 0, 0, 626] payload_rows_per_dst=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] dup_rows_per_dst=[1012, 350, 674, 1030, 0, 2442, 482, 0] combine_rows_per_src=[14932, 14984, 16262, 15726, 0, 16024, 11998, 2388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85690 payload_rows_remote=69604 dup_rows=12614 dup_rows_remote=10506 dedup_ratio=0.128316 router_tokens_per_expert=[1, 124, 504, 728, 787, 394, 303, 458, 69, 6390, 180, 0, 0, 360, 62, 72, 46, 136, 0, 221, 674, 246, 483, 0, 1, 139, 6, 20, 678, 5902, 548, 7, 64, 772, 0, 77, 138, 5540, 51, 7291, 68, 5, 25, 2039, 260, 87, 847, 27, 956, 13, 23, 7, 10, 1772, 0, 0, 5, 0, 744, 6021, 18, 31, 33, 2689] payload_rows_per_dst=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] dup_rows_per_dst=[4038, 870, 2108, 762, 144, 4534, 124, 34] combine_rows_per_src=[13640, 13396, 16086, 13840, 2060, 14224, 5438, 7006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51272 payload_rows_remote=35008 dup_rows=47032 dup_rows_remote=30092 dedup_ratio=0.478434 router_tokens_per_expert=[0, 229, 29, 42, 43, 24, 332, 86, 87, 7404, 10, 0, 0, 114, 5, 75, 51, 15, 0, 67, 38, 192, 28, 0, 0, 14, 0, 0, 598, 7598, 23, 0, 2, 43, 0, 4, 7, 7176, 2, 8052, 87, 0, 0, 964, 93, 3, 118, 6, 297, 0, 1, 0, 0, 5995, 0, 0, 0, 0, 715, 7741, 4, 15, 2, 721] payload_rows_per_dst=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] dup_rows_per_dst=[536, 202, 56, 12, 24700, 360, 4226, 16940] combine_rows_per_src=[1034, 380, 726, 1258, 16312, 2182, 13116, 16264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=285097984 dispatch_meta_bytes=320440 combine_payload_bytes=285097984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285418424 ingress_bytes=319700992 link_bytes=319700992 token_bytes=570516408 weight_movement_bytes=34603008 total_nvlink_bytes=605119416 weight_to_token_ratio=0.060652 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203009095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11626757688 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=5817024056 ingress_bytes=7366868992 link_bytes=7366868992 total_nvlink_bytes=13183893048 weight_to_token_ratio=0.133927 prefetch_share_of_ingress=0.098639 dedup_ratio=0.236226 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205258041e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=320233472 dispatch_meta_bytes=329568 combine_payload_bytes=320233472 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=320563040 ingress_bytes=389439488 link_bytes=389439488 token_bytes=640796512 weight_movement_bytes=69206016 total_nvlink_bytes=710002528 weight_to_token_ratio=0.108000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.465429441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89642 payload_rows_remote=87456 dup_rows=8662 dup_rows_remote=8462 dedup_ratio=0.088114 router_tokens_per_expert=[0, 216, 0, 0, 0, 0, 439, 0, 88, 7421, 2, 0, 0, 88, 0, 148, 57, 0, 0, 62, 0, 182, 0, 0, 0, 0, 0, 0, 501, 7842, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 88, 0, 0, 1003, 12, 0, 90, 0, 257, 0, 0, 0, 0, 6148, 0, 0, 0, 0, 581, 7856, 0, 0, 0, 594] payload_rows_per_dst=[12966, 14986, 10674, 14896, 5750, 2186, 12296, 15888] dup_rows_per_dst=[1068, 508, 402, 802, 2994, 200, 514, 2174] combine_rows_per_src=[12966, 14986, 10674, 14896, 5750, 2186, 12296, 15888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=317095936 dispatch_meta_bytes=330712 combine_payload_bytes=317095936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317426648 ingress_bytes=386301952 link_bytes=386301952 token_bytes=634522584 weight_movement_bytes=69206016 total_nvlink_bytes=703728600 weight_to_token_ratio=0.109068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453623146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=173383680 dispatch_meta_bytes=344072 combine_payload_bytes=173383680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=173727752 ingress_bytes=207986688 link_bytes=207986688 token_bytes=347111432 weight_movement_bytes=34603008 total_nvlink_bytes=381714440 weight_to_token_ratio=0.099688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.826371626e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92574 payload_rows_remote=77416 dup_rows=5730 dup_rows_remote=5262 dedup_ratio=0.058289 router_tokens_per_expert=[0, 189, 0, 0, 0, 0, 319, 4, 97, 7497, 27, 0, 0, 97, 0, 95, 60, 0, 0, 71, 0, 222, 0, 0, 0, 0, 0, 0, 574, 7786, 0, 0, 0, 0, 0, 0, 0, 7338, 0, 8095, 98, 0, 0, 1027, 83, 0, 116, 0, 233, 0, 0, 0, 0, 5729, 0, 0, 0, 0, 627, 7820, 0, 0, 0, 948] payload_rows_per_dst=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] dup_rows_per_dst=[946, 468, 706, 928, 0, 2220, 462, 0] combine_rows_per_src=[14754, 15158, 16190, 15792, 0, 16068, 11462, 3150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=313704448 dispatch_meta_bytes=326192 combine_payload_bytes=313704448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=314030640 ingress_bytes=417513472 link_bytes=417513472 token_bytes=627735088 weight_movement_bytes=103809024 total_nvlink_bytes=731544112 weight_to_token_ratio=0.165371 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.571069583e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76996 payload_rows_remote=61952 dup_rows=21308 dup_rows_remote=20426 dedup_ratio=0.216756 router_tokens_per_expert=[0, 0, 791, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7606, 0, 1909, 0, 0, 344, 61, 0, 0, 0, 242, 0, 979, 0, 0, 0, 0, 0, 0, 6516, 7212, 0, 149, 0, 0, 0, 198, 0, 566, 1390, 0, 0, 0, 7172, 7671, 59, 0, 697, 0, 0, 252, 0, 90, 0, 0, 0, 5248, 0, 0, 0, 0, 0] payload_rows_per_dst=[15044, 16262, 16150, 13332, 3052, 1250, 1410, 10496] dup_rows_per_dst=[882, 17192, 486, 1658, 422, 0, 668, 0] combine_rows_per_src=[15044, 16262, 16150, 13332, 3052, 1250, 1410, 10496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13151731176 prefetch_bytes=830472192 reduce_bytes=484442112 weight_movement_bytes=1314914304 egress_bytes=6579756520 ingress_bytes=7886888960 link_bytes=7886888960 total_nvlink_bytes=14466645480 weight_to_token_ratio=0.099980 prefetch_share_of_ingress=0.105298 dedup_ratio=0.175322 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=14 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027391 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290336009e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=143392768 dispatch_meta_bytes=260400 combine_payload_bytes=143392768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=143653168 ingress_bytes=212598784 link_bytes=212598784 token_bytes=287045936 weight_movement_bytes=69206016 total_nvlink_bytes=356251952 weight_to_token_ratio=0.241097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=23936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.999921086e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=285097984 dispatch_meta_bytes=320440 combine_payload_bytes=285097984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285418424 ingress_bytes=319700992 link_bytes=319700992 token_bytes=570516408 weight_movement_bytes=34603008 total_nvlink_bytes=605119416 weight_to_token_ratio=0.060652 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.203009095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=358219776 dispatch_meta_bytes=383672 combine_payload_bytes=358219776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=358603448 ingress_bytes=392822784 link_bytes=392822784 token_bytes=716823224 weight_movement_bytes=34603008 total_nvlink_bytes=751426232 weight_to_token_ratio=0.048273 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.478160511e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12874962448 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6441317904 ingress_bytes=7713955840 link_bytes=7713955840 total_nvlink_bytes=14155273744 weight_to_token_ratio=0.099442 prefetch_share_of_ingress=0.094201 dedup_ratio=0.188405 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262043252e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12606409080 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6307023224 ingress_bytes=7579697152 link_bytes=7579697152 total_nvlink_bytes=13886720376 weight_to_token_ratio=0.101560 prefetch_share_of_ingress=0.091304 dedup_ratio=0.192807 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=27813 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240077833e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=317095936 dispatch_meta_bytes=330712 combine_payload_bytes=317095936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=317426648 ingress_bytes=386301952 link_bytes=386301952 token_bytes=634522584 weight_movement_bytes=69206016 total_nvlink_bytes=703728600 weight_to_token_ratio=0.109068 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.453623146e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13799475920 prefetch_bytes=726663168 reduce_bytes=795869184 weight_movement_bytes=1522532352 egress_bytes=6903663312 ingress_bytes=8418344960 link_bytes=8418344960 total_nvlink_bytes=15322008272 weight_to_token_ratio=0.110333 prefetch_share_of_ingress=0.086319 dedup_ratio=0.139510 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=23 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027609 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.377284972e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13151731176 prefetch_bytes=830472192 reduce_bytes=484442112 weight_movement_bytes=1314914304 egress_bytes=6579756520 ingress_bytes=7886888960 link_bytes=7886888960 total_nvlink_bytes=14466645480 weight_to_token_ratio=0.099980 prefetch_share_of_ingress=0.105298 dedup_ratio=0.175322 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=14 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027391 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.290336009e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=253755392 dispatch_meta_bytes=329512 combine_payload_bytes=253755392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=254084904 ingress_bytes=288358400 link_bytes=288358400 token_bytes=507840296 weight_movement_bytes=34603008 total_nvlink_bytes=542443304 weight_to_token_ratio=0.068138 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.085069444e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11626757688 prefetch_bytes=726663168 reduce_bytes=830472192 weight_movement_bytes=1557135360 egress_bytes=5817024056 ingress_bytes=7366868992 link_bytes=7366868992 total_nvlink_bytes=13183893048 weight_to_token_ratio=0.133927 prefetch_share_of_ingress=0.098639 dedup_ratio=0.236226 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=24 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205258041e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12874962448 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6441317904 ingress_bytes=7713955840 link_bytes=7713955840 total_nvlink_bytes=14155273744 weight_to_token_ratio=0.099442 prefetch_share_of_ingress=0.094201 dedup_ratio=0.188405 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262043252e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13867317520 prefetch_bytes=761266176 reduce_bytes=865075200 weight_movement_bytes=1626341376 egress_bytes=6937598224 ingress_bytes=8556060672 link_bytes=8556060672 total_nvlink_bytes=15493658896 weight_to_token_ratio=0.117279 prefetch_share_of_ingress=0.088974 dedup_ratio=0.143243 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=25 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.399815978e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92180 payload_rows_remote=92180 dup_rows=6124 dup_rows_remote=6124 dedup_ratio=0.062297 router_tokens_per_expert=[0, 257, 0, 0, 0, 0, 477, 18, 80, 7419, 1, 0, 0, 80, 2, 131, 62, 0, 0, 55, 0, 207, 0, 0, 0, 0, 0, 0, 579, 7772, 0, 0, 0, 0, 0, 0, 0, 7373, 0, 8104, 80, 0, 0, 1072, 11, 0, 80, 0, 242, 0, 0, 0, 0, 6342, 0, 0, 0, 0, 590, 7827, 0, 4, 0, 287] payload_rows_per_dst=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] dup_rows_per_dst=[1398, 462, 646, 996, 0, 2032, 484, 106] combine_rows_per_src=[14852, 14964, 16210, 15706, 0, 15234, 12684, 2530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=377569280 dispatch_meta_bytes=393216 combine_payload_bytes=377569280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=377962496 ingress_bytes=481378304 link_bytes=481378304 token_bytes=755531776 weight_movement_bytes=103809024 total_nvlink_bytes=859340800 weight_to_token_ratio=0.137399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19072 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.811387804e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92058 payload_rows_remote=77100 dup_rows=6246 dup_rows_remote=5334 dedup_ratio=0.063538 router_tokens_per_expert=[0, 175, 0, 0, 0, 0, 307, 2, 89, 7495, 20, 0, 0, 89, 0, 69, 86, 0, 0, 70, 0, 184, 0, 0, 0, 0, 0, 0, 571, 7748, 0, 0, 0, 0, 0, 0, 0, 7451, 0, 8103, 90, 0, 0, 810, 88, 0, 118, 0, 253, 0, 0, 0, 0, 6113, 0, 0, 0, 0, 494, 7834, 0, 0, 0, 893] payload_rows_per_dst=[14958, 15232, 16206, 14576, 0, 16082, 12230, 2774] dup_rows_per_dst=[912, 1544, 680, 810, 0, 1798, 502, 0] combine_rows_per_src=[14958, 15232, 16206, 14576, 0, 16082, 12230, 2774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=14090289064 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=7049125800 ingress_bytes=8632901632 link_bytes=8632901632 total_nvlink_bytes=15682027432 weight_to_token_ratio=0.112967 prefetch_share_of_ingress=0.076157 dedup_ratio=0.139498 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=27178 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.412387558e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=315801600 dispatch_meta_bytes=329736 combine_payload_bytes=315801600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=316131336 ingress_bytes=350404608 link_bytes=350404608 token_bytes=631932936 weight_movement_bytes=34603008 total_nvlink_bytes=666535944 weight_to_token_ratio=0.054757 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.318544330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12045631536 prefetch_bytes=761266176 reduce_bytes=830472192 weight_movement_bytes=1591738368 egress_bytes=6026469424 ingress_bytes=7610900480 link_bytes=7610900480 total_nvlink_bytes=13637369904 weight_to_token_ratio=0.132142 prefetch_share_of_ingress=0.100023 dedup_ratio=0.214449 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=24 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=27686 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245182860e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none - [2026-08-26 16:55:29] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 800.4 | throughput per GPU (TFLOP/s/GPU): 159.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583418E+00 | loss scale: 1.0 | grad norm: 24.333 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75802 payload_rows_remote=67122 dup_rows=22502 dup_rows_remote=20016 dedup_ratio=0.228902 router_tokens_per_expert=[1138, 374, 845, 1103, 629, 748, 807, 938, 1386, 362, 952, 368, 320, 1394, 872, 963, 850, 1006, 782, 525, 765, 737, 459, 1330, 518, 682, 828, 368, 808, 114, 887, 794, 1067, 855, 1260, 905, 352, 265, 462, 899, 595, 637, 503, 759, 1072, 554, 772, 691, 743, 629, 906, 531, 995, 1182, 833, 693, 1158, 293, 1120, 640, 210, 778, 1096, 1045] payload_rows_per_dst=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] dup_rows_per_dst=[2676, 3740, 2562, 2434, 2976, 2486, 3076, 2552] combine_rows_per_src=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] + [2026-08-26 18:07:36] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 804.9 | throughput per GPU (TFLOP/s/GPU): 158.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.583418E+00 | loss scale: 1.0 | grad norm: 24.333 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75918 payload_rows_remote=67146 dup_rows=22386 dup_rows_remote=19600 dedup_ratio=0.227722 router_tokens_per_expert=[1183, 436, 795, 1142, 712, 897, 924, 763, 1305, 479, 947, 368, 314, 1248, 1004, 1115, 818, 960, 845, 695, 587, 722, 401, 1223, 520, 656, 778, 392, 854, 115, 722, 582, 1049, 665, 1215, 822, 432, 243, 523, 830, 623, 575, 518, 751, 1097, 708, 797, 657, 821, 633, 782, 444, 957, 1352, 867, 807, 1154, 299, 1340, 634, 273, 856, 1069, 857] payload_rows_per_dst=[8914, 9384, 9874, 9420, 8772, 8862, 10178, 10514] dup_rows_per_dst=[3122, 3478, 2628, 2184, 2786, 2590, 3148, 2450] combine_rows_per_src=[8914, 9384, 9874, 9420, 8772, 8862, 10178, 10514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74650 payload_rows_remote=64048 dup_rows=23654 dup_rows_remote=21036 dedup_ratio=0.240621 router_tokens_per_expert=[1193, 615, 971, 1169, 656, 895, 814, 781, 1371, 372, 1031, 382, 322, 1271, 802, 1102, 746, 1040, 813, 563, 702, 744, 414, 1181, 453, 730, 758, 383, 778, 124, 753, 569, 1058, 888, 1211, 997, 371, 207, 512, 758, 574, 570, 574, 682, 1100, 649, 733, 594, 681, 564, 812, 598, 1032, 1330, 785, 764, 1194, 293, 1380, 624, 236, 843, 1091, 949] payload_rows_per_dst=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] dup_rows_per_dst=[2302, 3910, 1914, 3574, 3860, 2290, 3186, 2618] combine_rows_per_src=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76640 payload_rows_remote=67380 dup_rows=21664 dup_rows_remote=19370 dedup_ratio=0.220378 router_tokens_per_expert=[1208, 462, 892, 1137, 709, 825, 906, 791, 1371, 336, 1000, 377, 305, 1183, 879, 1044, 779, 1105, 794, 652, 610, 716, 392, 1222, 469, 719, 698, 414, 805, 140, 776, 548, 1016, 774, 1256, 921, 380, 195, 545, 844, 583, 594, 616, 643, 1165, 786, 768, 505, 760, 582, 869, 475, 1097, 1380, 811, 732, 1120, 337, 1371, 590, 223, 837, 1157, 956] payload_rows_per_dst=[10272, 8244, 9992, 9260, 8940, 9652, 9598, 10682] dup_rows_per_dst=[3914, 2004, 2548, 2294, 2922, 2820, 2662, 2500] combine_rows_per_src=[10272, 8244, 9992, 9260, 8940, 9652, 9598, 10682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75802 payload_rows_remote=67122 dup_rows=22502 dup_rows_remote=20016 dedup_ratio=0.228902 router_tokens_per_expert=[1138, 374, 845, 1103, 629, 748, 807, 938, 1386, 362, 952, 368, 320, 1394, 872, 963, 850, 1006, 782, 525, 765, 737, 459, 1330, 518, 682, 828, 368, 808, 114, 887, 794, 1067, 855, 1260, 905, 352, 265, 462, 899, 595, 637, 503, 759, 1072, 554, 772, 691, 743, 629, 906, 531, 995, 1182, 833, 693, 1158, 293, 1120, 640, 210, 778, 1096, 1045] payload_rows_per_dst=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] dup_rows_per_dst=[2676, 3740, 2562, 2434, 2976, 2486, 3076, 2552] combine_rows_per_src=[8212, 9494, 10346, 9840, 9154, 8680, 9948, 10128] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77102 payload_rows_remote=67044 dup_rows=21202 dup_rows_remote=18612 dedup_ratio=0.215678 router_tokens_per_expert=[1158, 459, 838, 1184, 612, 803, 762, 928, 1294, 284, 990, 309, 380, 1166, 887, 1100, 731, 997, 783, 551, 765, 689, 413, 1395, 562, 694, 822, 301, 768, 92, 954, 788, 1065, 1010, 1177, 853, 346, 180, 573, 973, 624, 618, 572, 781, 1149, 657, 813, 637, 663, 588, 822, 495, 961, 1214, 842, 733, 1236, 276, 1168, 668, 166, 802, 1040, 991] payload_rows_per_dst=[10136, 8308, 10058, 9918, 9194, 10706, 8584, 10198] dup_rows_per_dst=[3624, 1924, 2590, 2360, 3160, 3424, 1624, 2496] combine_rows_per_src=[10136, 8308, 10058, 9918, 9194, 10706, 8584, 10198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75448 payload_rows_remote=65358 dup_rows=22856 dup_rows_remote=19786 dedup_ratio=0.232503 router_tokens_per_expert=[1124, 465, 864, 1202, 687, 847, 830, 794, 1406, 290, 978, 340, 327, 1048, 824, 1064, 804, 971, 741, 613, 664, 739, 372, 1171, 627, 771, 735, 363, 769, 101, 873, 693, 1041, 805, 1196, 881, 460, 245, 502, 804, 675, 694, 582, 781, 1112, 668, 743, 526, 769, 658, 808, 438, 1071, 1275, 821, 740, 1186, 323, 1296, 786, 186, 876, 1128, 979] payload_rows_per_dst=[7840, 9146, 9788, 9956, 9026, 8870, 10090, 10732] dup_rows_per_dst=[2638, 3408, 2362, 3056, 2842, 2692, 3070, 2788] combine_rows_per_src=[7840, 9146, 9788, 9956, 9026, 8870, 10090, 10732] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76404 payload_rows_remote=67448 dup_rows=21900 dup_rows_remote=19136 dedup_ratio=0.222778 router_tokens_per_expert=[1161, 380, 894, 1044, 633, 1043, 969, 621, 1275, 773, 995, 395, 235, 1418, 843, 908, 730, 1041, 757, 668, 398, 613, 433, 1256, 435, 664, 929, 395, 920, 159, 761, 579, 1114, 781, 1314, 1006, 370, 210, 622, 842, 572, 517, 653, 572, 1075, 703, 745, 462, 898, 707, 734, 655, 1035, 1258, 711, 726, 1156, 344, 1352, 616, 257, 884, 1114, 822] payload_rows_per_dst=[9906, 8956, 9284, 9570, 9272, 10162, 9070, 10184] dup_rows_per_dst=[3812, 2764, 2508, 2436, 3246, 2952, 1862, 2320] combine_rows_per_src=[9906, 8956, 9284, 9570, 9272, 10162, 9070, 10184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75918 payload_rows_remote=67146 dup_rows=22386 dup_rows_remote=19600 dedup_ratio=0.227722 router_tokens_per_expert=[1183, 436, 795, 1142, 712, 897, 924, 763, 1305, 479, 947, 368, 314, 1248, 1004, 1115, 818, 960, 845, 695, 587, 722, 401, 1223, 520, 656, 778, 392, 854, 115, 722, 582, 1049, 665, 1215, 822, 432, 243, 523, 830, 623, 575, 518, 751, 1097, 708, 797, 657, 821, 633, 782, 444, 957, 1352, 867, 807, 1154, 299, 1340, 634, 273, 856, 1069, 857] payload_rows_per_dst=[8914, 9384, 9874, 9420, 8772, 8862, 10178, 10514] dup_rows_per_dst=[3122, 3478, 2628, 2184, 2786, 2590, 3148, 2450] combine_rows_per_src=[8914, 9384, 9874, 9420, 8772, 8862, 10178, 10514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274931712 dispatch_meta_bytes=348552 combine_payload_bytes=274931712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275280264 ingress_bytes=309534720 link_bytes=309534720 token_bytes=550211976 weight_movement_bytes=34603008 total_nvlink_bytes=584814984 weight_to_token_ratio=0.062890 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164754232e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75448 payload_rows_remote=65358 dup_rows=22856 dup_rows_remote=19786 dedup_ratio=0.232503 router_tokens_per_expert=[1124, 465, 864, 1202, 687, 847, 830, 794, 1406, 290, 978, 340, 327, 1048, 824, 1064, 804, 971, 741, 613, 664, 739, 372, 1171, 627, 771, 735, 363, 769, 101, 873, 693, 1041, 805, 1196, 881, 460, 245, 502, 804, 675, 694, 582, 781, 1112, 668, 743, 526, 769, 658, 808, 438, 1071, 1275, 821, 740, 1186, 323, 1296, 786, 186, 876, 1128, 979] payload_rows_per_dst=[7840, 9146, 9788, 9956, 9026, 8870, 10090, 10732] dup_rows_per_dst=[2638, 3408, 2362, 3056, 2842, 2692, 3070, 2788] combine_rows_per_src=[7840, 9146, 9788, 9956, 9026, 8870, 10090, 10732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275030016 dispatch_meta_bytes=346984 combine_payload_bytes=275030016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275377000 ingress_bytes=309633024 link_bytes=309633024 token_bytes=550407016 weight_movement_bytes=34603008 total_nvlink_bytes=585010024 weight_to_token_ratio=0.062868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165124142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262340608 dispatch_meta_bytes=340336 combine_payload_bytes=262340608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262680944 ingress_bytes=366149632 link_bytes=366149632 token_bytes=525021552 weight_movement_bytes=103809024 total_nvlink_bytes=628830576 weight_to_token_ratio=0.197723 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377791588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74650 payload_rows_remote=64048 dup_rows=23654 dup_rows_remote=21036 dedup_ratio=0.240621 router_tokens_per_expert=[1193, 615, 971, 1169, 656, 895, 814, 781, 1371, 372, 1031, 382, 322, 1271, 802, 1102, 746, 1040, 813, 563, 702, 744, 414, 1181, 453, 730, 758, 383, 778, 124, 753, 569, 1058, 888, 1211, 997, 371, 207, 512, 758, 574, 570, 574, 682, 1100, 649, 733, 594, 681, 564, 812, 598, 1032, 1330, 785, 764, 1194, 293, 1380, 624, 236, 843, 1091, 949] payload_rows_per_dst=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] dup_rows_per_dst=[2302, 3910, 1914, 3574, 3860, 2290, 3186, 2618] combine_rows_per_src=[7162, 9396, 8892, 10246, 9744, 8662, 9946, 10602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275988480 dispatch_meta_bytes=347000 combine_payload_bytes=275988480 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=276335480 ingress_bytes=345194496 link_bytes=345194496 token_bytes=552323960 weight_movement_bytes=69206016 total_nvlink_bytes=621529976 weight_to_token_ratio=0.125300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=9344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.298939098e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274931712 dispatch_meta_bytes=348552 combine_payload_bytes=274931712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275280264 ingress_bytes=309534720 link_bytes=309534720 token_bytes=550211976 weight_movement_bytes=34603008 total_nvlink_bytes=584814984 weight_to_token_ratio=0.062890 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.164754232e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=274612224 dispatch_meta_bytes=342624 combine_payload_bytes=274612224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=274954848 ingress_bytes=343818240 link_bytes=343818240 token_bytes=549567072 weight_movement_bytes=69206016 total_nvlink_bytes=618773088 weight_to_token_ratio=0.125928 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.293760357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276267008 dispatch_meta_bytes=346336 combine_payload_bytes=276267008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276613344 ingress_bytes=345473024 link_bytes=345473024 token_bytes=552880352 weight_movement_bytes=69206016 total_nvlink_bytes=622086368 weight_to_token_ratio=0.125174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299987176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=275030016 dispatch_meta_bytes=346984 combine_payload_bytes=275030016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=275377000 ingress_bytes=309633024 link_bytes=309633024 token_bytes=550407016 weight_movement_bytes=34603008 total_nvlink_bytes=585010024 weight_to_token_ratio=0.062868 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=9984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.165124142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=340576 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268046944 ingress_bytes=336912384 link_bytes=336912384 token_bytes=535753312 weight_movement_bytes=69206016 total_nvlink_bytes=604959328 weight_to_token_ratio=0.129175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267774177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=262340608 dispatch_meta_bytes=340336 combine_payload_bytes=262340608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262680944 ingress_bytes=366149632 link_bytes=366149632 token_bytes=525021552 weight_movement_bytes=103809024 total_nvlink_bytes=628830576 weight_to_token_ratio=0.197723 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.377791588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=276267008 dispatch_meta_bytes=346336 combine_payload_bytes=276267008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276613344 ingress_bytes=345473024 link_bytes=345473024 token_bytes=552880352 weight_movement_bytes=69206016 total_nvlink_bytes=622086368 weight_to_token_ratio=0.125174 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.299987176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74802 payload_rows_remote=65362 dup_rows=23502 dup_rows_remote=21526 dedup_ratio=0.239075 router_tokens_per_expert=[709, 957, 918, 458, 847, 287, 398, 1162, 70, 936, 1373, 377, 529, 868, 984, 1299, 989, 1166, 776, 481, 1488, 258, 1680, 1382, 701, 764, 839, 905, 253, 451, 168, 56, 430, 336, 450, 229, 927, 1302, 1013, 1021, 2057, 1542, 950, 1241, 449, 1281, 26, 274, 1045, 1087, 1001, 10, 242, 1489, 204, 1112, 763, 387, 1011, 551, 1026, 369, 348, 450] payload_rows_per_dst=[9100, 9710, 10250, 8712, 9440, 8076, 10002, 9512] dup_rows_per_dst=[2372, 3162, 2830, 2922, 1976, 3450, 2378, 4412] combine_rows_per_src=[9100, 9710, 10250, 8712, 9440, 8076, 10002, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65890 dup_rows=23626 dup_rows_remote=20616 dedup_ratio=0.240336 router_tokens_per_expert=[746, 932, 899, 580, 772, 292, 444, 1172, 90, 897, 1268, 393, 594, 816, 913, 1308, 991, 1234, 792, 374, 1430, 268, 1752, 1242, 723, 707, 881, 917, 202, 382, 209, 126, 452, 296, 475, 227, 939, 1367, 1021, 1174, 1988, 1489, 1032, 1351, 401, 1136, 32, 258, 958, 1139, 988, 32, 181, 1602, 195, 1062, 818, 458, 915, 501, 1145, 371, 344, 459] payload_rows_per_dst=[9146, 9402, 9870, 8788, 9722, 8904, 9880, 8966] dup_rows_per_dst=[2528, 3156, 2792, 3010, 2180, 3872, 2434, 3654] combine_rows_per_src=[9146, 9402, 9870, 8788, 9722, 8904, 9880, 8966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73658 payload_rows_remote=63854 dup_rows=24646 dup_rows_remote=20074 dedup_ratio=0.250712 router_tokens_per_expert=[651, 940, 778, 395, 982, 284, 293, 1080, 67, 967, 1521, 361, 634, 802, 765, 1465, 821, 1262, 775, 346, 1524, 296, 1784, 1274, 773, 672, 843, 950, 100, 300, 208, 95, 374, 282, 441, 187, 1115, 1180, 1062, 1068, 2401, 1604, 913, 1209, 394, 1336, 37, 388, 1104, 992, 942, 8, 158, 1769, 222, 1171, 781, 349, 868, 541, 1133, 349, 325, 441] payload_rows_per_dst=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] dup_rows_per_dst=[2298, 1692, 1350, 4136, 3558, 4504, 2536, 4572] combine_rows_per_src=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74916 payload_rows_remote=66296 dup_rows=23388 dup_rows_remote=19526 dedup_ratio=0.237915 router_tokens_per_expert=[654, 983, 919, 601, 774, 353, 488, 918, 86, 898, 1304, 314, 670, 1000, 864, 1337, 912, 1186, 833, 214, 1512, 238, 1858, 1151, 678, 716, 728, 834, 164, 287, 198, 79, 462, 296, 520, 224, 931, 1545, 1117, 1140, 2000, 1578, 1196, 1377, 510, 1252, 30, 298, 776, 1208, 780, 15, 230, 1616, 208, 1217, 764, 384, 855, 528, 1229, 321, 338, 456] payload_rows_per_dst=[9094, 9778, 9516, 8476, 10254, 8620, 9946, 9232] dup_rows_per_dst=[2286, 3168, 2442, 2742, 2216, 3862, 2154, 4518] combine_rows_per_src=[9094, 9778, 9516, 8476, 10254, 8620, 9946, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75490 payload_rows_remote=65472 dup_rows=22814 dup_rows_remote=19058 dedup_ratio=0.232076 router_tokens_per_expert=[1121, 404, 856, 1138, 614, 1067, 898, 652, 1258, 791, 1191, 319, 230, 1416, 802, 894, 841, 1122, 995, 664, 326, 648, 478, 1080, 444, 672, 821, 454, 885, 120, 779, 582, 1150, 786, 1294, 996, 414, 234, 565, 766, 502, 543, 696, 521, 1117, 683, 750, 493, 749, 622, 709, 529, 1133, 1423, 680, 639, 1122, 344, 1431, 616, 231, 863, 1178, 811] payload_rows_per_dst=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] dup_rows_per_dst=[3756, 3626, 3182, 2348, 3280, 2964, 2730, 928] combine_rows_per_src=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76706 payload_rows_remote=66990 dup_rows=21598 dup_rows_remote=19026 dedup_ratio=0.219706 router_tokens_per_expert=[673, 893, 960, 503, 763, 298, 473, 1037, 80, 964, 1198, 286, 648, 955, 949, 1336, 984, 1249, 824, 287, 1529, 172, 1824, 1099, 659, 677, 701, 749, 127, 293, 212, 78, 391, 274, 543, 220, 1037, 1909, 963, 1081, 2248, 1447, 1342, 1416, 440, 1070, 28, 245, 825, 1125, 770, 24, 164, 1729, 245, 1152, 755, 340, 880, 675, 1267, 298, 317, 452] payload_rows_per_dst=[9744, 9936, 9716, 8148, 10300, 10888, 9940, 8034] dup_rows_per_dst=[3056, 2896, 2572, 2492, 2536, 3984, 2128, 1934] combine_rows_per_src=[9744, 9936, 9716, 8148, 10300, 10888, 9940, 8034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=64666 dup_rows=23914 dup_rows_remote=21786 dedup_ratio=0.243266 router_tokens_per_expert=[587, 889, 891, 540, 837, 310, 438, 1072, 75, 1188, 1451, 360, 645, 1005, 938, 1329, 922, 1244, 833, 315, 1494, 194, 1724, 1086, 609, 650, 835, 851, 134, 340, 206, 93, 391, 226, 549, 306, 891, 1529, 1034, 1060, 2281, 1529, 1094, 1559, 433, 1006, 31, 255, 851, 1092, 884, 18, 161, 1539, 226, 1155, 800, 420, 893, 521, 1256, 307, 307, 463] payload_rows_per_dst=[8794, 9300, 7970, 10056, 10550, 8094, 9724, 9902] dup_rows_per_dst=[2334, 2956, 1218, 3816, 3148, 3720, 2128, 4594] combine_rows_per_src=[8794, 9300, 7970, 10056, 10550, 8094, 9724, 9902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76450 payload_rows_remote=66920 dup_rows=21854 dup_rows_remote=18698 dedup_ratio=0.222310 router_tokens_per_expert=[741, 1074, 926, 383, 1038, 289, 375, 994, 63, 785, 1609, 382, 527, 844, 746, 1387, 702, 1286, 784, 431, 1316, 369, 1865, 1555, 919, 677, 922, 1044, 124, 447, 226, 118, 507, 360, 345, 153, 899, 1084, 1157, 1145, 2050, 1584, 745, 1122, 496, 1482, 39, 329, 1186, 844, 1230, 17, 193, 1454, 176, 1201, 808, 346, 725, 481, 1024, 355, 344, 323] payload_rows_per_dst=[10856, 9530, 10566, 9274, 9396, 9160, 10144, 7524] dup_rows_per_dst=[3952, 3156, 3456, 3410, 1904, 2230, 2458, 1288] combine_rows_per_src=[10856, 9530, 10566, 9274, 9396, 9160, 10144, 7524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65890 dup_rows=23626 dup_rows_remote=20616 dedup_ratio=0.240336 router_tokens_per_expert=[746, 932, 899, 580, 772, 292, 444, 1172, 90, 897, 1268, 393, 594, 816, 913, 1308, 991, 1234, 792, 374, 1430, 268, 1752, 1242, 723, 707, 881, 917, 202, 382, 209, 126, 452, 296, 475, 227, 939, 1367, 1021, 1174, 1988, 1489, 1032, 1351, 401, 1136, 32, 258, 958, 1139, 988, 32, 181, 1602, 195, 1062, 818, 458, 915, 501, 1145, 371, 344, 459] payload_rows_per_dst=[9146, 9402, 9870, 8788, 9722, 8904, 9880, 8966] dup_rows_per_dst=[2528, 3156, 2792, 3010, 2180, 3872, 2434, 3654] combine_rows_per_src=[9146, 9402, 9870, 8788, 9722, 8904, 9880, 8966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=347552 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070304 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535793056 weight_movement_bytes=34603008 total_nvlink_bytes=570396064 weight_to_token_ratio=0.064583 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75490 payload_rows_remote=65472 dup_rows=22814 dup_rows_remote=19058 dedup_ratio=0.232076 router_tokens_per_expert=[1121, 404, 856, 1138, 614, 1067, 898, 652, 1258, 791, 1191, 319, 230, 1416, 802, 894, 841, 1122, 995, 664, 326, 648, 478, 1080, 444, 672, 821, 454, 885, 120, 779, 582, 1150, 786, 1294, 996, 414, 234, 565, 766, 502, 543, 696, 521, 1117, 683, 750, 493, 749, 622, 709, 529, 1133, 1423, 680, 639, 1122, 344, 1431, 616, 231, 863, 1178, 811] payload_rows_per_dst=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] dup_rows_per_dst=[3756, 3626, 3182, 2348, 3280, 2964, 2730, 928] combine_rows_per_src=[10018, 10522, 9776, 9408, 9130, 10492, 9532, 6612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269885440 dispatch_meta_bytes=346024 combine_payload_bytes=269885440 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270231464 ingress_bytes=339091456 link_bytes=339091456 token_bytes=540116904 weight_movement_bytes=69206016 total_nvlink_bytes=609322920 weight_to_token_ratio=0.128132 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275973850e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=261545984 dispatch_meta_bytes=335712 combine_payload_bytes=261545984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261881696 ingress_bytes=296148992 link_bytes=296148992 token_bytes=523427680 weight_movement_bytes=34603008 total_nvlink_bytes=558030688 weight_to_token_ratio=0.066108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114384815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=271548416 dispatch_meta_bytes=343288 combine_payload_bytes=271548416 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=271891704 ingress_bytes=409960448 link_bytes=409960448 token_bytes=543440120 weight_movement_bytes=138412032 total_nvlink_bytes=681852152 weight_to_token_ratio=0.254696 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.542648161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74802 payload_rows_remote=65362 dup_rows=23502 dup_rows_remote=21526 dedup_ratio=0.239075 router_tokens_per_expert=[709, 957, 918, 458, 847, 287, 398, 1162, 70, 936, 1373, 377, 529, 868, 984, 1299, 989, 1166, 776, 481, 1488, 258, 1680, 1382, 701, 764, 839, 905, 253, 451, 168, 56, 430, 336, 450, 229, 927, 1302, 1013, 1021, 2057, 1542, 950, 1241, 449, 1281, 26, 274, 1045, 1087, 1001, 10, 242, 1489, 204, 1112, 763, 387, 1011, 551, 1026, 369, 348, 450] payload_rows_per_dst=[9100, 9710, 10250, 8712, 9440, 8076, 10002, 9512] dup_rows_per_dst=[2372, 3162, 2830, 2922, 1976, 3450, 2378, 4412] combine_rows_per_src=[9100, 9710, 10250, 8712, 9440, 8076, 10002, 9512] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268173312 dispatch_meta_bytes=338120 combine_payload_bytes=268173312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268511432 ingress_bytes=371982336 link_bytes=371982336 token_bytes=536684744 weight_movement_bytes=103809024 total_nvlink_bytes=640493768 weight_to_token_ratio=0.193426 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399739583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=64666 dup_rows=23914 dup_rows_remote=21786 dedup_ratio=0.243266 router_tokens_per_expert=[587, 889, 891, 540, 837, 310, 438, 1072, 75, 1188, 1451, 360, 645, 1005, 938, 1329, 922, 1244, 833, 315, 1494, 194, 1724, 1086, 609, 650, 835, 851, 134, 340, 206, 93, 391, 226, 549, 306, 891, 1529, 1034, 1060, 2281, 1529, 1094, 1559, 433, 1006, 31, 255, 851, 1092, 884, 18, 161, 1539, 226, 1155, 800, 420, 893, 521, 1256, 307, 307, 463] payload_rows_per_dst=[8794, 9300, 7970, 10056, 10550, 8094, 9724, 9902] dup_rows_per_dst=[2334, 2956, 1218, 3816, 3148, 3720, 2128, 4594] combine_rows_per_src=[8794, 9300, 7970, 10056, 10550, 8094, 9724, 9902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73658 payload_rows_remote=63854 dup_rows=24646 dup_rows_remote=20074 dedup_ratio=0.250712 router_tokens_per_expert=[651, 940, 778, 395, 982, 284, 293, 1080, 67, 967, 1521, 361, 634, 802, 765, 1465, 821, 1262, 775, 346, 1524, 296, 1784, 1274, 773, 672, 843, 950, 100, 300, 208, 95, 374, 282, 441, 187, 1115, 1180, 1062, 1068, 2401, 1604, 913, 1209, 394, 1336, 37, 388, 1104, 992, 942, 8, 158, 1769, 222, 1171, 781, 349, 868, 541, 1133, 349, 325, 441] payload_rows_per_dst=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] dup_rows_per_dst=[2298, 1692, 1350, 4136, 3558, 4504, 2536, 4572] combine_rows_per_src=[8508, 6514, 8198, 10362, 10902, 9174, 10196, 9804] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274391040 dispatch_meta_bytes=344064 combine_payload_bytes=274391040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274735104 ingress_bytes=378200064 link_bytes=378200064 token_bytes=549126144 weight_movement_bytes=103809024 total_nvlink_bytes=652935168 weight_to_token_ratio=0.189044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.423136393e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=342472 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274446792 ingress_bytes=377913344 link_bytes=377913344 token_bytes=548551112 weight_movement_bytes=103809024 total_nvlink_bytes=652360136 weight_to_token_ratio=0.189242 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422057489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269885440 dispatch_meta_bytes=346024 combine_payload_bytes=269885440 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=270231464 ingress_bytes=339091456 link_bytes=339091456 token_bytes=540116904 weight_movement_bytes=69206016 total_nvlink_bytes=609322920 weight_to_token_ratio=0.128132 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=14336 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.275973850e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=347552 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268070304 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535793056 weight_movement_bytes=34603008 total_nvlink_bytes=570396064 weight_to_token_ratio=0.064583 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=345808 combine_payload_bytes=264871936 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265217744 ingress_bytes=299474944 link_bytes=299474944 token_bytes=530089680 weight_movement_bytes=34603008 total_nvlink_bytes=564692688 weight_to_token_ratio=0.065278 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=12288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126900105e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=261545984 dispatch_meta_bytes=335712 combine_payload_bytes=261545984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261881696 ingress_bytes=296148992 link_bytes=296148992 token_bytes=523427680 weight_movement_bytes=34603008 total_nvlink_bytes=558030688 weight_to_token_ratio=0.066108 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.114384815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274104320 dispatch_meta_bytes=342472 combine_payload_bytes=274104320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=274446792 ingress_bytes=377913344 link_bytes=377913344 token_bytes=548551112 weight_movement_bytes=103809024 total_nvlink_bytes=652360136 weight_to_token_ratio=0.189242 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422057489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268173312 dispatch_meta_bytes=338120 combine_payload_bytes=268173312 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268511432 ingress_bytes=371982336 link_bytes=371982336 token_bytes=536684744 weight_movement_bytes=103809024 total_nvlink_bytes=640493768 weight_to_token_ratio=0.193426 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=7552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.399739583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75142 payload_rows_remote=65160 dup_rows=23162 dup_rows_remote=19310 dedup_ratio=0.235616 router_tokens_per_expert=[79, 869, 432, 394, 29, 982, 840, 294, 479, 401, 1237, 1239, 1440, 1032, 346, 1431, 1066, 1423, 338, 767, 82, 146, 1134, 1117, 594, 897, 1127, 848, 404, 602, 897, 265, 417, 1106, 950, 586, 1754, 1847, 1592, 419, 939, 147, 343, 248, 1548, 1844, 170, 1388, 1098, 453, 1020, 926, 50, 324, 305, 590, 1107, 378, 104, 441, 169, 1389, 286, 1983] payload_rows_per_dst=[8588, 7930, 9874, 10382, 9982, 9098, 9652, 9636] dup_rows_per_dst=[2758, 1538, 2272, 3766, 3852, 4156, 2742, 2078] combine_rows_per_src=[8588, 7930, 9874, 10382, 9982, 9098, 9652, 9636] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75324 payload_rows_remote=65998 dup_rows=22980 dup_rows_remote=20672 dedup_ratio=0.233765 router_tokens_per_expert=[194, 952, 389, 475, 88, 777, 879, 283, 561, 339, 1182, 1466, 1450, 1596, 565, 1541, 1025, 1289, 429, 803, 100, 332, 1162, 922, 469, 966, 867, 634, 418, 747, 886, 275, 431, 1099, 635, 539, 1747, 1615, 1568, 458, 917, 168, 367, 255, 1626, 1652, 168, 1346, 1300, 400, 816, 767, 102, 399, 366, 662, 1008, 321, 130, 453, 141, 1142, 571, 1922] payload_rows_per_dst=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] dup_rows_per_dst=[2978, 3296, 2530, 2308, 3218, 3984, 2936, 1730] combine_rows_per_src=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76298 payload_rows_remote=67686 dup_rows=22006 dup_rows_remote=18492 dedup_ratio=0.223857 router_tokens_per_expert=[231, 875, 378, 414, 130, 875, 712, 292, 628, 358, 1160, 1166, 1229, 1490, 813, 1902, 973, 1322, 288, 764, 90, 406, 1143, 870, 412, 917, 797, 539, 261, 643, 996, 388, 369, 985, 827, 813, 1543, 1835, 1780, 499, 599, 166, 380, 276, 1721, 1501, 154, 1266, 1288, 622, 1049, 895, 116, 380, 265, 727, 1012, 391, 130, 535, 49, 1363, 486, 1668] payload_rows_per_dst=[8466, 9088, 9654, 9442, 10310, 8612, 10870, 9856] dup_rows_per_dst=[2434, 2142, 2058, 2922, 3906, 3514, 3618, 1412] combine_rows_per_src=[8466, 9088, 9654, 9442, 10310, 8612, 10870, 9856] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=344712 combine_payload_bytes=277241856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277586568 ingress_bytes=381050880 link_bytes=381050880 token_bytes=554828424 weight_movement_bytes=103809024 total_nvlink_bytes=658637448 weight_to_token_ratio=0.187101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433863784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76114 payload_rows_remote=65492 dup_rows=22190 dup_rows_remote=18576 dedup_ratio=0.225728 router_tokens_per_expert=[729, 917, 720, 573, 1116, 269, 362, 1200, 64, 868, 1748, 424, 503, 851, 656, 1503, 728, 1299, 806, 332, 1460, 377, 1775, 1647, 780, 788, 848, 1007, 203, 330, 158, 91, 357, 246, 388, 195, 997, 934, 1012, 982, 1942, 1232, 672, 1130, 357, 1565, 22, 325, 1396, 953, 1293, 3, 127, 1410, 223, 1080, 969, 378, 1072, 551, 1112, 369, 322, 406] payload_rows_per_dst=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] dup_rows_per_dst=[3614, 3752, 4168, 3108, 1470, 1706, 2414, 1958] combine_rows_per_src=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65140 dup_rows=23088 dup_rows_remote=19624 dedup_ratio=0.234863 router_tokens_per_expert=[34, 1143, 325, 492, 13, 927, 887, 186, 518, 331, 1414, 1579, 1216, 1223, 247, 1225, 1037, 1523, 562, 773, 120, 103, 1278, 1172, 537, 1078, 1246, 781, 485, 623, 1156, 121, 308, 963, 490, 342, 2315, 1537, 1329, 418, 1422, 126, 358, 559, 1499, 1807, 198, 1357, 1209, 387, 373, 850, 13, 58, 349, 472, 876, 275, 102, 655, 160, 1661, 176, 2153] payload_rows_per_dst=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] dup_rows_per_dst=[4290, 3464, 2856, 2402, 2174, 4294, 1642, 1966] combine_rows_per_src=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76660 payload_rows_remote=66926 dup_rows=21644 dup_rows_remote=19454 dedup_ratio=0.220174 router_tokens_per_expert=[296, 690, 362, 419, 108, 889, 764, 275, 582, 302, 1187, 1131, 1173, 1790, 708, 2071, 988, 1416, 246, 783, 101, 381, 1141, 906, 413, 926, 750, 457, 290, 629, 972, 384, 363, 932, 783, 733, 1325, 1806, 1734, 467, 588, 147, 354, 249, 1894, 1602, 147, 1441, 1143, 706, 1121, 811, 166, 388, 258, 819, 952, 374, 77, 583, 71, 1227, 585, 1776] payload_rows_per_dst=[8086, 10548, 9734, 8288, 10030, 9066, 11000, 9908] dup_rows_per_dst=[2170, 3198, 2190, 1354, 3606, 3778, 3966, 1382] combine_rows_per_src=[8086, 10548, 9734, 8288, 10030, 9066, 11000, 9908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75324 payload_rows_remote=65998 dup_rows=22980 dup_rows_remote=20672 dedup_ratio=0.233765 router_tokens_per_expert=[194, 952, 389, 475, 88, 777, 879, 283, 561, 339, 1182, 1466, 1450, 1596, 565, 1541, 1025, 1289, 429, 803, 100, 332, 1162, 922, 469, 966, 867, 634, 418, 747, 886, 275, 431, 1099, 635, 539, 1747, 1615, 1568, 458, 917, 168, 367, 255, 1626, 1652, 168, 1346, 1300, 400, 816, 767, 102, 399, 366, 662, 1008, 321, 130, 453, 141, 1142, 571, 1922] payload_rows_per_dst=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] dup_rows_per_dst=[2978, 3296, 2530, 2308, 3218, 3984, 2936, 1730] combine_rows_per_src=[8590, 9912, 9594, 9326, 9472, 9014, 9770, 9646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75142 payload_rows_remote=65160 dup_rows=23162 dup_rows_remote=19310 dedup_ratio=0.235616 router_tokens_per_expert=[79, 869, 432, 394, 29, 982, 840, 294, 479, 401, 1237, 1239, 1440, 1032, 346, 1431, 1066, 1423, 338, 767, 82, 146, 1134, 1117, 594, 897, 1127, 848, 404, 602, 897, 265, 417, 1106, 950, 586, 1754, 1847, 1592, 419, 939, 147, 343, 248, 1548, 1844, 170, 1388, 1098, 453, 1020, 926, 50, 324, 305, 590, 1107, 378, 104, 441, 169, 1389, 286, 1983] payload_rows_per_dst=[8588, 7930, 9874, 10382, 9982, 9098, 9652, 9636] dup_rows_per_dst=[2758, 1538, 2272, 3766, 3852, 4156, 2742, 2078] combine_rows_per_src=[8588, 7930, 9874, 10382, 9982, 9098, 9652, 9636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75428 payload_rows_remote=64092 dup_rows=22876 dup_rows_remote=18210 dedup_ratio=0.232707 router_tokens_per_expert=[49, 1041, 425, 422, 34, 941, 785, 238, 610, 378, 1184, 1381, 1361, 1101, 209, 1510, 1001, 1575, 476, 735, 138, 122, 1236, 1137, 635, 990, 1034, 692, 424, 611, 984, 231, 373, 1057, 817, 487, 1978, 1756, 1389, 439, 921, 137, 336, 332, 1597, 1862, 166, 1370, 1061, 448, 760, 978, 49, 204, 253, 553, 1093, 417, 122, 546, 69, 1514, 244, 2134] payload_rows_per_dst=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] dup_rows_per_dst=[2874, 1752, 2774, 3796, 3056, 1608, 2350, 4666] combine_rows_per_src=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75752 payload_rows_remote=65158 dup_rows=22552 dup_rows_remote=19000 dedup_ratio=0.229411 router_tokens_per_expert=[236, 824, 396, 377, 95, 887, 754, 267, 557, 379, 1008, 1281, 1388, 1363, 478, 1766, 962, 1280, 249, 737, 112, 312, 1155, 971, 506, 942, 846, 583, 217, 663, 980, 307, 419, 1075, 806, 678, 1629, 1743, 1786, 515, 699, 177, 381, 237, 1869, 1516, 164, 1450, 1041, 603, 1186, 909, 101, 337, 331, 799, 1015, 441, 93, 557, 79, 1155, 497, 1986] payload_rows_per_dst=[8558, 8236, 9536, 9806, 10078, 9028, 10594, 9916] dup_rows_per_dst=[2372, 1896, 2020, 3058, 3966, 3408, 3552, 2280] combine_rows_per_src=[8558, 8236, 9536, 9806, 10078, 9028, 10594, 9916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=337880 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267233240 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534128600 weight_movement_bytes=138412032 total_nvlink_bytes=672540632 weight_to_token_ratio=0.259136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268255232 dispatch_meta_bytes=336272 combine_payload_bytes=268255232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268591504 ingress_bytes=302858240 link_bytes=302858240 token_bytes=536846736 weight_movement_bytes=34603008 total_nvlink_bytes=571449744 weight_to_token_ratio=0.064456 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139631175e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75428 payload_rows_remote=64092 dup_rows=22876 dup_rows_remote=18210 dedup_ratio=0.232707 router_tokens_per_expert=[49, 1041, 425, 422, 34, 941, 785, 238, 610, 378, 1184, 1381, 1361, 1101, 209, 1510, 1001, 1575, 476, 735, 138, 122, 1236, 1137, 635, 990, 1034, 692, 424, 611, 984, 231, 373, 1057, 817, 487, 1978, 1756, 1389, 439, 921, 137, 336, 332, 1597, 1862, 166, 1370, 1061, 448, 760, 978, 49, 204, 253, 553, 1093, 417, 122, 546, 69, 1514, 244, 2134] payload_rows_per_dst=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] dup_rows_per_dst=[2874, 1752, 2774, 3796, 3056, 1608, 2350, 4666] combine_rows_per_src=[8952, 7974, 10066, 10128, 9216, 8474, 9282, 11336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=339056 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267152496 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533965936 weight_movement_bytes=103809024 total_nvlink_bytes=637774960 weight_to_token_ratio=0.194411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346680 combine_payload_bytes=270327808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270674488 ingress_bytes=304930816 link_bytes=304930816 token_bytes=541002296 weight_movement_bytes=34603008 total_nvlink_bytes=575605304 weight_to_token_ratio=0.063961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147430112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274128896 dispatch_meta_bytes=345520 combine_payload_bytes=274128896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274474416 ingress_bytes=308731904 link_bytes=308731904 token_bytes=548603312 weight_movement_bytes=34603008 total_nvlink_bytes=583206320 weight_to_token_ratio=0.063075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161733300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=337880 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267233240 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534128600 weight_movement_bytes=138412032 total_nvlink_bytes=672540632 weight_to_token_ratio=0.259136 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=12160 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75216 payload_rows_remote=65140 dup_rows=23088 dup_rows_remote=19624 dedup_ratio=0.234863 router_tokens_per_expert=[34, 1143, 325, 492, 13, 927, 887, 186, 518, 331, 1414, 1579, 1216, 1223, 247, 1225, 1037, 1523, 562, 773, 120, 103, 1278, 1172, 537, 1078, 1246, 781, 485, 623, 1156, 121, 308, 963, 490, 342, 2315, 1537, 1329, 418, 1422, 126, 358, 559, 1499, 1807, 198, 1357, 1209, 387, 373, 850, 13, 58, 349, 472, 876, 275, 102, 655, 160, 1661, 176, 2153] payload_rows_per_dst=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] dup_rows_per_dst=[4290, 3464, 2856, 2402, 2174, 4294, 1642, 1966] combine_rows_per_src=[10012, 10076, 10280, 9652, 6942, 9874, 8230, 10150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75438 payload_rows_remote=64640 dup_rows=22866 dup_rows_remote=18334 dedup_ratio=0.232605 router_tokens_per_expert=[348, 807, 1214, 408, 576, 446, 1150, 786, 1296, 2164, 230, 414, 1286, 504, 905, 468, 1611, 83, 2058, 545, 771, 417, 19, 573, 1034, 974, 649, 621, 518, 1071, 1519, 942, 521, 708, 302, 288, 1184, 771, 2092, 690, 379, 1882, 177, 577, 366, 656, 490, 3138, 92, 358, 335, 973, 738, 1234, 229, 294, 312, 247, 1408, 654, 761, 528, 157, 204] payload_rows_per_dst=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] dup_rows_per_dst=[2510, 1522, 3332, 2376, 3112, 4532, 3384, 2098] combine_rows_per_src=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=270327808 dispatch_meta_bytes=346680 combine_payload_bytes=270327808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=270674488 ingress_bytes=304930816 link_bytes=304930816 token_bytes=541002296 weight_movement_bytes=34603008 total_nvlink_bytes=575605304 weight_to_token_ratio=0.063961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147430112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=274128896 dispatch_meta_bytes=345520 combine_payload_bytes=274128896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274474416 ingress_bytes=308731904 link_bytes=308731904 token_bytes=548603312 weight_movement_bytes=34603008 total_nvlink_bytes=583206320 weight_to_token_ratio=0.063075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161733300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=277241856 dispatch_meta_bytes=344712 combine_payload_bytes=277241856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=277586568 ingress_bytes=381050880 link_bytes=381050880 token_bytes=554828424 weight_movement_bytes=103809024 total_nvlink_bytes=658637448 weight_to_token_ratio=0.187101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.433863784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=336632 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223800 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534110968 weight_movement_bytes=34603008 total_nvlink_bytes=568713976 weight_to_token_ratio=0.064786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=262520832 dispatch_meta_bytes=329208 combine_payload_bytes=262520832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262850040 ingress_bytes=297123840 link_bytes=297123840 token_bytes=525370872 weight_movement_bytes=34603008 total_nvlink_bytes=559973880 weight_to_token_ratio=0.065864 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118053089e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=331896 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265097336 ingress_bytes=437780480 link_bytes=437780480 token_bytes=529862776 weight_movement_bytes=173015040 total_nvlink_bytes=702877816 weight_to_token_ratio=0.326528 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.647332702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75642 payload_rows_remote=65946 dup_rows=22662 dup_rows_remote=20752 dedup_ratio=0.230530 router_tokens_per_expert=[168, 346, 1444, 391, 668, 800, 1218, 1008, 1832, 3101, 247, 383, 794, 674, 1272, 435, 1951, 74, 1904, 424, 920, 500, 1, 452, 1045, 1307, 571, 399, 519, 1165, 1873, 920, 585, 451, 167, 212, 1574, 590, 1848, 718, 64, 1802, 194, 433, 278, 356, 347, 2484, 4, 251, 201, 1053, 777, 860, 170, 186, 461, 286, 1629, 527, 790, 700, 117, 231] payload_rows_per_dst=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] dup_rows_per_dst=[3618, 1910, 3412, 3958, 2618, 1256, 2974, 2916] combine_rows_per_src=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=63278 dup_rows=24062 dup_rows_remote=17850 dedup_ratio=0.244771 router_tokens_per_expert=[20, 1250, 371, 507, 15, 974, 823, 290, 525, 394, 1266, 1514, 1734, 871, 85, 925, 1071, 1684, 503, 700, 130, 97, 1134, 945, 828, 1122, 1120, 792, 345, 542, 962, 126, 419, 1096, 724, 451, 2328, 2010, 1305, 422, 1128, 104, 368, 314, 1573, 1908, 159, 1217, 1127, 125, 512, 1075, 42, 140, 287, 482, 1087, 506, 130, 391, 75, 1666, 118, 2198] payload_rows_per_dst=[10964, 12050, 10156, 9426, 6708, 8188, 6598, 10152] dup_rows_per_dst=[6212, 5204, 2710, 2248, 1788, 2208, 1502, 2190] combine_rows_per_src=[10964, 12050, 10156, 9426, 6708, 8188, 6598, 10152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75864 payload_rows_remote=65896 dup_rows=22440 dup_rows_remote=20008 dedup_ratio=0.228271 router_tokens_per_expert=[316, 674, 1009, 503, 547, 431, 991, 722, 1623, 2343, 164, 415, 958, 725, 1088, 425, 1678, 97, 1823, 518, 643, 361, 21, 526, 1195, 930, 675, 460, 431, 1233, 1644, 1276, 452, 675, 321, 295, 1306, 637, 1886, 667, 387, 1862, 253, 518, 322, 628, 532, 2496, 209, 352, 395, 913, 973, 1067, 400, 500, 364, 295, 1381, 608, 706, 710, 252, 345] payload_rows_per_dst=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] dup_rows_per_dst=[1856, 1570, 2904, 2432, 2838, 4006, 4180, 2654] combine_rows_per_src=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=339056 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267152496 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533965936 weight_movement_bytes=103809024 total_nvlink_bytes=637774960 weight_to_token_ratio=0.194411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76114 payload_rows_remote=65492 dup_rows=22190 dup_rows_remote=18576 dedup_ratio=0.225728 router_tokens_per_expert=[729, 917, 720, 573, 1116, 269, 362, 1200, 64, 868, 1748, 424, 503, 851, 656, 1503, 728, 1299, 806, 332, 1460, 377, 1775, 1647, 780, 788, 848, 1007, 203, 330, 158, 91, 357, 246, 388, 195, 997, 934, 1012, 982, 1942, 1232, 672, 1130, 357, 1565, 22, 325, 1396, 953, 1293, 3, 127, 1410, 223, 1080, 969, 378, 1072, 551, 1112, 369, 322, 406] payload_rows_per_dst=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] dup_rows_per_dst=[3614, 3752, 4168, 3108, 1470, 1706, 2414, 1958] combine_rows_per_src=[10622, 10196, 11390, 8852, 8752, 8060, 9842, 8400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76932 payload_rows_remote=67728 dup_rows=21372 dup_rows_remote=19190 dedup_ratio=0.217407 router_tokens_per_expert=[270, 452, 1550, 283, 746, 668, 1089, 1137, 1754, 2967, 461, 400, 1393, 601, 1168, 277, 1667, 39, 1767, 542, 996, 479, 6, 619, 1024, 1457, 608, 400, 415, 1097, 1471, 734, 586, 634, 209, 184, 1195, 504, 1634, 747, 326, 2432, 218, 447, 295, 485, 321, 2722, 17, 281, 266, 971, 504, 883, 224, 218, 168, 200, 1627, 539, 653, 779, 125, 221] payload_rows_per_dst=[9598, 9950, 8930, 9468, 9204, 10374, 9844, 9564] dup_rows_per_dst=[2792, 2158, 3300, 2002, 2182, 4118, 2818, 2002] combine_rows_per_src=[9598, 9950, 8930, 9468, 9204, 10374, 9844, 9564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75864 payload_rows_remote=65896 dup_rows=22440 dup_rows_remote=20008 dedup_ratio=0.228271 router_tokens_per_expert=[316, 674, 1009, 503, 547, 431, 991, 722, 1623, 2343, 164, 415, 958, 725, 1088, 425, 1678, 97, 1823, 518, 643, 361, 21, 526, 1195, 930, 675, 460, 431, 1233, 1644, 1276, 452, 675, 321, 295, 1306, 637, 1886, 667, 387, 1862, 253, 518, 322, 628, 532, 2496, 209, 352, 395, 913, 973, 1067, 400, 500, 364, 295, 1381, 608, 706, 710, 252, 345] payload_rows_per_dst=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] dup_rows_per_dst=[1856, 1570, 2904, 2432, 2838, 4006, 4180, 2654] combine_rows_per_src=[8530, 9226, 8430, 9968, 9640, 9990, 10124, 9956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=268255232 dispatch_meta_bytes=336272 combine_payload_bytes=268255232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268591504 ingress_bytes=302858240 link_bytes=302858240 token_bytes=536846736 weight_movement_bytes=34603008 total_nvlink_bytes=571449744 weight_to_token_ratio=0.064456 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139631175e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75310 payload_rows_remote=66202 dup_rows=22994 dup_rows_remote=19578 dedup_ratio=0.233907 router_tokens_per_expert=[417, 739, 1269, 337, 573, 428, 1122, 734, 1206, 2245, 238, 402, 1309, 709, 847, 459, 1613, 68, 2283, 545, 804, 353, 24, 572, 1082, 1090, 647, 519, 424, 1037, 1501, 950, 598, 772, 262, 201, 1261, 727, 1982, 758, 482, 2099, 148, 548, 349, 726, 555, 2861, 109, 315, 314, 929, 660, 1039, 309, 346, 239, 332, 1552, 588, 680, 518, 150, 197] payload_rows_per_dst=[8966, 8808, 9108, 10610, 9944, 9120, 9332, 9422] dup_rows_per_dst=[2272, 1532, 3416, 4042, 3178, 3262, 3200, 2092] combine_rows_per_src=[8966, 8808, 9108, 10610, 9944, 9120, 9332, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75836 payload_rows_remote=66214 dup_rows=22468 dup_rows_remote=19162 dedup_ratio=0.228556 router_tokens_per_expert=[367, 740, 1180, 328, 541, 443, 1020, 818, 1512, 2596, 290, 452, 1339, 497, 950, 476, 1665, 64, 2182, 525, 714, 409, 33, 603, 1162, 1098, 665, 584, 445, 995, 1541, 927, 610, 814, 264, 289, 1274, 659, 1686, 730, 527, 2006, 200, 502, 329, 586, 428, 2886, 68, 241, 394, 957, 641, 1046, 236, 285, 221, 237, 1557, 599, 735, 618, 145, 221] payload_rows_per_dst=[8744, 9100, 9092, 9416, 9730, 10546, 9622, 9586] dup_rows_per_dst=[2130, 1932, 3298, 2336, 2922, 4382, 3306, 2162] combine_rows_per_src=[8744, 9100, 9092, 9416, 9730, 10546, 9622, 9586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=346792 combine_payload_bytes=270114816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270461608 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540576424 weight_movement_bytes=69206016 total_nvlink_bytes=609782440 weight_to_token_ratio=0.128023 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269910016 dispatch_meta_bytes=343616 combine_payload_bytes=269910016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270253632 ingress_bytes=373719040 link_bytes=373719040 token_bytes=540163648 weight_movement_bytes=103809024 total_nvlink_bytes=643972672 weight_to_token_ratio=0.192181 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406274661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75438 payload_rows_remote=64640 dup_rows=22866 dup_rows_remote=18334 dedup_ratio=0.232605 router_tokens_per_expert=[348, 807, 1214, 408, 576, 446, 1150, 786, 1296, 2164, 230, 414, 1286, 504, 905, 468, 1611, 83, 2058, 545, 771, 417, 19, 573, 1034, 974, 649, 621, 518, 1071, 1519, 942, 521, 708, 302, 288, 1184, 771, 2092, 690, 379, 1882, 177, 577, 366, 656, 490, 3138, 92, 358, 335, 973, 738, 1234, 229, 294, 312, 247, 1408, 654, 761, 528, 157, 204] payload_rows_per_dst=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] dup_rows_per_dst=[2510, 1522, 3332, 2376, 3112, 4532, 3384, 2098] combine_rows_per_src=[8960, 8684, 8822, 9242, 10000, 10798, 9450, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75836 payload_rows_remote=66214 dup_rows=22468 dup_rows_remote=19162 dedup_ratio=0.228556 router_tokens_per_expert=[367, 740, 1180, 328, 541, 443, 1020, 818, 1512, 2596, 290, 452, 1339, 497, 950, 476, 1665, 64, 2182, 525, 714, 409, 33, 603, 1162, 1098, 665, 584, 445, 995, 1541, 927, 610, 814, 264, 289, 1274, 659, 1686, 730, 527, 2006, 200, 502, 329, 586, 428, 2886, 68, 241, 394, 957, 641, 1046, 236, 285, 221, 237, 1557, 599, 735, 618, 145, 221] payload_rows_per_dst=[8744, 9100, 9092, 9416, 9730, 10546, 9622, 9586] dup_rows_per_dst=[2130, 1932, 3298, 2336, 2922, 4382, 3306, 2162] combine_rows_per_src=[8744, 9100, 9092, 9416, 9730, 10546, 9622, 9586] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76148 payload_rows_remote=65708 dup_rows=22156 dup_rows_remote=19162 dedup_ratio=0.225382 router_tokens_per_expert=[225, 420, 1245, 301, 513, 677, 1248, 905, 1962, 3284, 376, 404, 1423, 489, 1266, 432, 1794, 66, 1953, 492, 919, 589, 3, 481, 1191, 1286, 592, 554, 413, 1088, 1468, 707, 525, 596, 165, 165, 1267, 602, 1789, 732, 174, 2120, 149, 461, 263, 429, 356, 2710, 8, 263, 223, 1115, 681, 849, 148, 150, 269, 225, 1659, 555, 745, 642, 113, 238] payload_rows_per_dst=[8838, 10144, 9208, 8188, 9286, 9710, 10334, 10440] dup_rows_per_dst=[2230, 2560, 3384, 1868, 2396, 3616, 3108, 2994] combine_rows_per_src=[8838, 10144, 9208, 8188, 9286, 9710, 10334, 10440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259186688 dispatch_meta_bytes=324512 combine_payload_bytes=259186688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259511200 ingress_bytes=328392704 link_bytes=328392704 token_bytes=518697888 weight_movement_bytes=69206016 total_nvlink_bytes=587903904 weight_to_token_ratio=0.133423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235715307e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72904 payload_rows_remote=63940 dup_rows=25400 dup_rows_remote=23712 dedup_ratio=0.258382 router_tokens_per_expert=[6, 1895, 1630, 761, 963, 1378, 764, 873, 385, 1142, 2247, 109, 261, 437, 304, 1495, 1470, 369, 133, 85, 287, 567, 253, 401, 903, 418, 1574, 743, 1548, 1064, 901, 29, 279, 30, 51, 612, 1399, 327, 241, 1294, 137, 244, 136, 946, 604, 11, 1695, 5, 298, 695, 390, 682, 444, 1289, 720, 1236, 874, 1161, 100, 1147, 1226, 2668, 1547, 1269] payload_rows_per_dst=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] dup_rows_per_dst=[4826, 3200, 2730, 2806, 2206, 1688, 3072, 4872] combine_rows_per_src=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75642 payload_rows_remote=65946 dup_rows=22662 dup_rows_remote=20752 dedup_ratio=0.230530 router_tokens_per_expert=[168, 346, 1444, 391, 668, 800, 1218, 1008, 1832, 3101, 247, 383, 794, 674, 1272, 435, 1951, 74, 1904, 424, 920, 500, 1, 452, 1045, 1307, 571, 399, 519, 1165, 1873, 920, 585, 451, 167, 212, 1574, 590, 1848, 718, 64, 1802, 194, 433, 278, 356, 347, 2484, 4, 251, 201, 1053, 777, 860, 170, 186, 461, 286, 1629, 527, 790, 700, 117, 231] payload_rows_per_dst=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] dup_rows_per_dst=[3618, 1910, 3412, 3958, 2618, 1256, 2974, 2916] combine_rows_per_src=[10562, 9696, 9040, 10768, 9672, 5692, 9900, 10312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=277413888 dispatch_meta_bytes=347672 combine_payload_bytes=277413888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277761560 ingress_bytes=312016896 link_bytes=312016896 token_bytes=555175448 weight_movement_bytes=34603008 total_nvlink_bytes=589778456 weight_to_token_ratio=0.062328 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=14464 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174094460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271163392 dispatch_meta_bytes=343120 combine_payload_bytes=271163392 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=271506512 ingress_bytes=305766400 link_bytes=305766400 token_bytes=542669904 weight_movement_bytes=34603008 total_nvlink_bytes=577272912 weight_to_token_ratio=0.063764 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150574347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=341504 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271554048 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542766592 weight_movement_bytes=34603008 total_nvlink_bytes=577369600 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269910016 dispatch_meta_bytes=343616 combine_payload_bytes=269910016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=270253632 ingress_bytes=373719040 link_bytes=373719040 token_bytes=540163648 weight_movement_bytes=103809024 total_nvlink_bytes=643972672 weight_to_token_ratio=0.192181 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=8960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.406274661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271163392 dispatch_meta_bytes=343120 combine_payload_bytes=271163392 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=271506512 ingress_bytes=305766400 link_bytes=305766400 token_bytes=542669904 weight_movement_bytes=34603008 total_nvlink_bytes=577272912 weight_to_token_ratio=0.063764 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150574347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264765440 dispatch_meta_bytes=331896 combine_payload_bytes=264765440 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265097336 ingress_bytes=437780480 link_bytes=437780480 token_bytes=529862776 weight_movement_bytes=173015040 total_nvlink_bytes=702877816 weight_to_token_ratio=0.326528 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=16000 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.647332702e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=269139968 dispatch_meta_bytes=339480 combine_payload_bytes=269139968 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269479448 ingress_bytes=338345984 link_bytes=338345984 token_bytes=538619416 weight_movement_bytes=69206016 total_nvlink_bytes=607825432 weight_to_token_ratio=0.128488 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273168699e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261898240 dispatch_meta_bytes=350608 combine_payload_bytes=261898240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262248848 ingress_bytes=296501248 link_bytes=296501248 token_bytes=524147088 weight_movement_bytes=34603008 total_nvlink_bytes=558750096 weight_to_token_ratio=0.066018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115710326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74860 payload_rows_remote=65988 dup_rows=23444 dup_rows_remote=21202 dedup_ratio=0.238485 router_tokens_per_expert=[0, 1094, 1379, 730, 1049, 2492, 593, 1152, 393, 673, 2340, 63, 144, 356, 238, 1350, 1668, 340, 8, 72, 340, 742, 229, 289, 464, 333, 2377, 344, 3047, 699, 597, 0, 248, 0, 0, 1128, 1061, 218, 70, 1345, 60, 213, 11, 1110, 447, 0, 2488, 0, 9, 673, 287, 837, 422, 1608, 956, 1334, 440, 1372, 47, 1322, 965, 1879, 1879, 1128] payload_rows_per_dst=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] dup_rows_per_dst=[3580, 2242, 2902, 3576, 3052, 2970, 3740, 1382] combine_rows_per_src=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73694 payload_rows_remote=64772 dup_rows=24610 dup_rows_remote=21792 dedup_ratio=0.250346 router_tokens_per_expert=[54, 2082, 1589, 423, 959, 1715, 798, 708, 538, 959, 2117, 125, 265, 509, 407, 1256, 1443, 359, 292, 175, 267, 576, 306, 328, 733, 407, 1268, 719, 1664, 1278, 641, 110, 351, 167, 58, 751, 1273, 462, 265, 1482, 191, 422, 566, 1114, 536, 33, 1714, 131, 431, 526, 308, 701, 488, 1329, 757, 907, 887, 1155, 108, 891, 1036, 2360, 1328, 1354] payload_rows_per_dst=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] dup_rows_per_dst=[4506, 3016, 2844, 2818, 2846, 2544, 2772, 3264] combine_rows_per_src=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75306 payload_rows_remote=62906 dup_rows=22998 dup_rows_remote=18700 dedup_ratio=0.233948 router_tokens_per_expert=[182, 378, 1491, 250, 630, 904, 1194, 978, 2324, 3177, 314, 401, 1088, 517, 1292, 436, 1737, 65, 2005, 524, 985, 490, 0, 508, 961, 1492, 587, 420, 335, 1267, 1627, 753, 576, 602, 210, 197, 1402, 596, 1469, 656, 118, 2045, 217, 403, 158, 449, 351, 2342, 0, 288, 215, 1141, 569, 866, 136, 149, 414, 115, 1843, 523, 684, 709, 122, 275] payload_rows_per_dst=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] dup_rows_per_dst=[4298, 6072, 3376, 1968, 2360, 1206, 1038, 2680] combine_rows_per_src=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=270114816 dispatch_meta_bytes=346792 combine_payload_bytes=270114816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=270461608 ingress_bytes=339320832 link_bytes=339320832 token_bytes=540576424 weight_movement_bytes=69206016 total_nvlink_bytes=609782440 weight_to_token_ratio=0.128023 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.276836973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=63278 dup_rows=24062 dup_rows_remote=17850 dedup_ratio=0.244771 router_tokens_per_expert=[20, 1250, 371, 507, 15, 974, 823, 290, 525, 394, 1266, 1514, 1734, 871, 85, 925, 1071, 1684, 503, 700, 130, 97, 1134, 945, 828, 1122, 1120, 792, 345, 542, 962, 126, 419, 1096, 724, 451, 2328, 2010, 1305, 422, 1128, 104, 368, 314, 1573, 1908, 159, 1217, 1127, 125, 512, 1075, 42, 140, 287, 482, 1087, 506, 130, 391, 75, 1666, 118, 2198] payload_rows_per_dst=[10964, 12050, 10156, 9426, 6708, 8188, 6598, 10152] dup_rows_per_dst=[6212, 5204, 2710, 2248, 1788, 2208, 1502, 2190] combine_rows_per_src=[10964, 12050, 10156, 9426, 6708, 8188, 6598, 10152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=271212544 dispatch_meta_bytes=341504 combine_payload_bytes=271212544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=271554048 ingress_bytes=305815552 link_bytes=305815552 token_bytes=542766592 weight_movement_bytes=34603008 total_nvlink_bytes=577369600 weight_to_token_ratio=0.063753 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18560 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.150759302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74862 payload_rows_remote=65712 dup_rows=23442 dup_rows_remote=21178 dedup_ratio=0.238464 router_tokens_per_expert=[0, 1164, 1467, 1165, 853, 1155, 605, 924, 377, 950, 2818, 70, 157, 577, 262, 1025, 1128, 322, 29, 66, 292, 903, 374, 308, 823, 320, 2493, 425, 2820, 732, 484, 0, 270, 2, 4, 1313, 1225, 144, 95, 1499, 49, 200, 7, 958, 477, 0, 2756, 0, 33, 1002, 290, 744, 377, 1779, 663, 1122, 634, 1260, 58, 1188, 1253, 1940, 1447, 1275] payload_rows_per_dst=[8432, 9870, 8614, 8518, 9150, 11518, 9052, 9708] dup_rows_per_dst=[3924, 2602, 2110, 2036, 2264, 3016, 2968, 4522] combine_rows_per_src=[8432, 9870, 8614, 8518, 9150, 11518, 9052, 9708] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73548 payload_rows_remote=63948 dup_rows=24756 dup_rows_remote=22088 dedup_ratio=0.251831 router_tokens_per_expert=[3, 1944, 1788, 718, 970, 1260, 704, 727, 323, 994, 2495, 77, 176, 451, 318, 1376, 1384, 446, 120, 107, 313, 590, 217, 395, 980, 391, 1430, 811, 1370, 1158, 858, 6, 306, 9, 24, 644, 1632, 368, 191, 1238, 102, 257, 194, 877, 665, 0, 1823, 1, 324, 736, 357, 676, 496, 1205, 636, 1344, 978, 1094, 104, 1130, 1224, 2562, 1620, 1435] payload_rows_per_dst=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] dup_rows_per_dst=[4610, 2870, 2668, 4556, 2270, 1620, 2878, 3284] combine_rows_per_src=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73226 payload_rows_remote=65174 dup_rows=25078 dup_rows_remote=22382 dedup_ratio=0.255107 router_tokens_per_expert=[20, 1873, 1593, 727, 915, 1163, 685, 823, 589, 1053, 2295, 82, 174, 471, 326, 1248, 1635, 374, 173, 91, 312, 668, 286, 337, 986, 386, 1511, 625, 1625, 1036, 738, 24, 272, 8, 30, 763, 1572, 306, 186, 1143, 81, 373, 168, 978, 582, 6, 2022, 13, 149, 684, 311, 637, 440, 1343, 570, 1240, 993, 1319, 113, 1160, 1329, 2416, 1597, 1504] payload_rows_per_dst=[8364, 9550, 9652, 8226, 9316, 9646, 8052, 10420] dup_rows_per_dst=[3878, 2926, 2932, 2386, 2600, 2050, 2696, 5610] combine_rows_per_src=[8364, 9550, 9652, 8226, 9316, 9646, 8052, 10420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270286848 dispatch_meta_bytes=348760 combine_payload_bytes=270286848 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270635608 ingress_bytes=304889856 link_bytes=304889856 token_bytes=540922456 weight_movement_bytes=34603008 total_nvlink_bytes=575525464 weight_to_token_ratio=0.063970 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147275982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265306112 dispatch_meta_bytes=346256 combine_payload_bytes=265306112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265652368 ingress_bytes=334512128 link_bytes=334512128 token_bytes=530958480 weight_movement_bytes=69206016 total_nvlink_bytes=600164496 weight_to_token_ratio=0.130342 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258742207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76956 payload_rows_remote=67994 dup_rows=21348 dup_rows_remote=17474 dedup_ratio=0.217163 router_tokens_per_expert=[691, 1736, 1050, 190, 203, 544, 2451, 301, 90, 3, 642, 876, 1451, 0, 6, 1146, 923, 708, 3387, 129, 432, 701, 323, 19, 74, 1753, 815, 138, 144, 188, 3157, 240, 1357, 494, 188, 2030, 583, 345, 1551, 810, 199, 2, 79, 1546, 121, 686, 1528, 825, 1031, 1522, 1235, 622, 245, 631, 1, 1432, 1341, 505, 483, 636, 498, 608, 887, 620] payload_rows_per_dst=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] dup_rows_per_dst=[1554, 3100, 394, 2616, 3606, 3874, 1706, 4498] combine_rows_per_src=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73694 payload_rows_remote=64772 dup_rows=24610 dup_rows_remote=21792 dedup_ratio=0.250346 router_tokens_per_expert=[54, 2082, 1589, 423, 959, 1715, 798, 708, 538, 959, 2117, 125, 265, 509, 407, 1256, 1443, 359, 292, 175, 267, 576, 306, 328, 733, 407, 1268, 719, 1664, 1278, 641, 110, 351, 167, 58, 751, 1273, 462, 265, 1482, 191, 422, 566, 1114, 536, 33, 1714, 131, 431, 526, 308, 701, 488, 1329, 757, 907, 887, 1155, 108, 891, 1036, 2360, 1328, 1354] payload_rows_per_dst=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] dup_rows_per_dst=[4506, 3016, 2844, 2818, 2846, 2544, 2772, 3264] combine_rows_per_src=[8720, 9336, 9368, 8922, 10202, 10198, 8122, 8826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73548 payload_rows_remote=63948 dup_rows=24756 dup_rows_remote=22088 dedup_ratio=0.251831 router_tokens_per_expert=[3, 1944, 1788, 718, 970, 1260, 704, 727, 323, 994, 2495, 77, 176, 451, 318, 1376, 1384, 446, 120, 107, 313, 590, 217, 395, 980, 391, 1430, 811, 1370, 1158, 858, 6, 306, 9, 24, 644, 1632, 368, 191, 1238, 102, 257, 194, 877, 665, 0, 1823, 1, 324, 736, 357, 676, 496, 1205, 636, 1344, 978, 1094, 104, 1130, 1224, 2562, 1620, 1435] payload_rows_per_dst=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] dup_rows_per_dst=[4610, 2870, 2668, 4556, 2270, 1620, 2878, 3284] combine_rows_per_src=[9098, 9550, 9600, 9952, 9074, 8958, 8670, 8646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=259186688 dispatch_meta_bytes=324512 combine_payload_bytes=259186688 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=259511200 ingress_bytes=328392704 link_bytes=328392704 token_bytes=518697888 weight_movement_bytes=69206016 total_nvlink_bytes=587903904 weight_to_token_ratio=0.133423 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.235715307e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72904 payload_rows_remote=63940 dup_rows=25400 dup_rows_remote=23712 dedup_ratio=0.258382 router_tokens_per_expert=[6, 1895, 1630, 761, 963, 1378, 764, 873, 385, 1142, 2247, 109, 261, 437, 304, 1495, 1470, 369, 133, 85, 287, 567, 253, 401, 903, 418, 1574, 743, 1548, 1064, 901, 29, 279, 30, 51, 612, 1399, 327, 241, 1294, 137, 244, 136, 946, 604, 11, 1695, 5, 298, 695, 390, 682, 444, 1289, 720, 1236, 874, 1161, 100, 1147, 1226, 2668, 1547, 1269] payload_rows_per_dst=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] dup_rows_per_dst=[4826, 3200, 2730, 2806, 2206, 1688, 3072, 4872] combine_rows_per_src=[8958, 9560, 9736, 8458, 9016, 8964, 8436, 9776] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75202 payload_rows_remote=64324 dup_rows=23102 dup_rows_remote=17814 dedup_ratio=0.235006 router_tokens_per_expert=[0, 1202, 1428, 1100, 889, 1763, 639, 949, 426, 946, 2862, 69, 148, 361, 261, 1073, 1514, 337, 0, 54, 233, 791, 310, 293, 628, 217, 2283, 381, 2945, 745, 570, 0, 206, 0, 0, 1187, 1062, 91, 62, 1182, 52, 157, 12, 929, 410, 0, 2716, 0, 13, 1058, 247, 679, 402, 1631, 901, 1202, 580, 1400, 57, 1311, 1195, 1946, 1625, 1422] payload_rows_per_dst=[7422, 9686, 8842, 7708, 9996, 11528, 9142, 10878] dup_rows_per_dst=[1602, 2606, 2114, 1940, 3514, 2914, 3124, 5288] combine_rows_per_src=[7422, 9686, 8842, 7708, 9996, 11528, 9142, 10878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=257662976 dispatch_meta_bytes=326424 combine_payload_bytes=257662976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257989400 ingress_bytes=292265984 link_bytes=292265984 token_bytes=515652376 weight_movement_bytes=34603008 total_nvlink_bytes=550255384 weight_to_token_ratio=0.067105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099773368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269156352 dispatch_meta_bytes=347560 combine_payload_bytes=269156352 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=269503912 ingress_bytes=338362368 link_bytes=338362368 token_bytes=538660264 weight_movement_bytes=69206016 total_nvlink_bytes=607866280 weight_to_token_ratio=0.128478 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=13312 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.273230350e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=344144 combine_payload_bytes=261931008 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262275152 ingress_bytes=331137024 link_bytes=331137024 token_bytes=524206160 weight_movement_bytes=69206016 total_nvlink_bytes=593412176 weight_to_token_ratio=0.132021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246041963e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=350224 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267302928 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534255632 weight_movement_bytes=34603008 total_nvlink_bytes=568858640 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74860 payload_rows_remote=65988 dup_rows=23444 dup_rows_remote=21202 dedup_ratio=0.238485 router_tokens_per_expert=[0, 1094, 1379, 730, 1049, 2492, 593, 1152, 393, 673, 2340, 63, 144, 356, 238, 1350, 1668, 340, 8, 72, 340, 742, 229, 289, 464, 333, 2377, 344, 3047, 699, 597, 0, 248, 0, 0, 1128, 1061, 218, 70, 1345, 60, 213, 11, 1110, 447, 0, 2488, 0, 9, 673, 287, 837, 422, 1608, 956, 1334, 440, 1372, 47, 1322, 965, 1879, 1879, 1128] payload_rows_per_dst=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] dup_rows_per_dst=[3580, 2242, 2902, 3576, 3052, 2970, 3740, 1382] combine_rows_per_src=[8414, 8872, 9734, 10064, 10072, 11528, 9758, 6418] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278503424 dispatch_meta_bytes=341872 combine_payload_bytes=278503424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278845296 ingress_bytes=313106432 link_bytes=313106432 token_bytes=557348720 weight_movement_bytes=34603008 total_nvlink_bytes=591951728 weight_to_token_ratio=0.062085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178194296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265306112 dispatch_meta_bytes=346256 combine_payload_bytes=265306112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265652368 ingress_bytes=334512128 link_bytes=334512128 token_bytes=530958480 weight_movement_bytes=69206016 total_nvlink_bytes=600164496 weight_to_token_ratio=0.130342 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258742207e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261931008 dispatch_meta_bytes=344144 combine_payload_bytes=261931008 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=262275152 ingress_bytes=331137024 link_bytes=331137024 token_bytes=524206160 weight_movement_bytes=69206016 total_nvlink_bytes=593412176 weight_to_token_ratio=0.132021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.032500 gemm_active_groups=11 gemm_peak_group_rows=17536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.246041963e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=261898240 dispatch_meta_bytes=350608 combine_payload_bytes=261898240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262248848 ingress_bytes=296501248 link_bytes=296501248 token_bytes=524147088 weight_movement_bytes=34603008 total_nvlink_bytes=558750096 weight_to_token_ratio=0.066018 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=18176 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115710326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73226 payload_rows_remote=65174 dup_rows=25078 dup_rows_remote=22382 dedup_ratio=0.255107 router_tokens_per_expert=[20, 1873, 1593, 727, 915, 1163, 685, 823, 589, 1053, 2295, 82, 174, 471, 326, 1248, 1635, 374, 173, 91, 312, 668, 286, 337, 986, 386, 1511, 625, 1625, 1036, 738, 24, 272, 8, 30, 763, 1572, 306, 186, 1143, 81, 373, 168, 978, 582, 6, 2022, 13, 149, 684, 311, 637, 440, 1343, 570, 1240, 993, 1319, 113, 1160, 1329, 2416, 1597, 1504] payload_rows_per_dst=[8364, 9550, 9652, 8226, 9316, 9646, 8052, 10420] dup_rows_per_dst=[3878, 2926, 2932, 2386, 2600, 2050, 2696, 5610] combine_rows_per_src=[8364, 9550, 9652, 8226, 9316, 9646, 8052, 10420] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=263471104 dispatch_meta_bytes=328552 combine_payload_bytes=263471104 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=263799656 ingress_bytes=436486144 link_bytes=436486144 token_bytes=527270760 weight_movement_bytes=173015040 total_nvlink_bytes=700285800 weight_to_token_ratio=0.328133 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=9472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642462220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=66604 dup_rows=22862 dup_rows_remote=21384 dedup_ratio=0.232564 router_tokens_per_expert=[888, 2161, 672, 156, 86, 367, 2814, 864, 0, 0, 856, 720, 2585, 0, 0, 997, 568, 908, 3946, 91, 428, 1502, 296, 0, 143, 2232, 1271, 138, 58, 2, 2424, 375, 1353, 360, 105, 1920, 367, 315, 1095, 634, 88, 0, 6, 1280, 97, 993, 1239, 727, 915, 1108, 1808, 1162, 133, 485, 0, 1391, 1447, 420, 96, 277, 353, 355, 617, 458] payload_rows_per_dst=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] dup_rows_per_dst=[6014, 1478, 5304, 932, 2564, 3426, 1934, 1210] combine_rows_per_src=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76604 payload_rows_remote=67072 dup_rows=21700 dup_rows_remote=19626 dedup_ratio=0.220744 router_tokens_per_expert=[951, 1850, 1119, 258, 303, 555, 2124, 427, 218, 66, 561, 852, 1514, 5, 53, 1158, 982, 813, 2709, 163, 522, 666, 437, 126, 201, 1294, 889, 204, 213, 258, 2385, 359, 1301, 607, 265, 2509, 499, 553, 1616, 585, 192, 114, 61, 1325, 420, 621, 1683, 972, 1095, 1206, 1124, 812, 344, 568, 72, 1758, 1004, 519, 511, 522, 387, 454, 780, 438] payload_rows_per_dst=[8744, 10240, 10170, 9532, 11502, 9634, 8882, 7900] dup_rows_per_dst=[2182, 2862, 2666, 2074, 4368, 4658, 1560, 1330] combine_rows_per_src=[8744, 10240, 10170, 9532, 11502, 9634, 8882, 7900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=270286848 dispatch_meta_bytes=348760 combine_payload_bytes=270286848 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=270635608 ingress_bytes=304889856 link_bytes=304889856 token_bytes=540922456 weight_movement_bytes=34603008 total_nvlink_bytes=575525464 weight_to_token_ratio=0.063970 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.147275982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=350224 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267302928 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534255632 weight_movement_bytes=34603008 total_nvlink_bytes=568858640 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75306 payload_rows_remote=62906 dup_rows=22998 dup_rows_remote=18700 dedup_ratio=0.233948 router_tokens_per_expert=[182, 378, 1491, 250, 630, 904, 1194, 978, 2324, 3177, 314, 401, 1088, 517, 1292, 436, 1737, 65, 2005, 524, 985, 490, 0, 508, 961, 1492, 587, 420, 335, 1267, 1627, 753, 576, 602, 210, 197, 1402, 596, 1469, 656, 118, 2045, 217, 403, 158, 449, 351, 2342, 0, 288, 215, 1141, 569, 866, 136, 149, 414, 115, 1843, 523, 684, 709, 122, 275] payload_rows_per_dst=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] dup_rows_per_dst=[4298, 6072, 3376, 1968, 2360, 1206, 1038, 2680] combine_rows_per_src=[12400, 13440, 9252, 9662, 9222, 5696, 5690, 9944] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76916 payload_rows_remote=67098 dup_rows=21388 dup_rows_remote=18652 dedup_ratio=0.217570 router_tokens_per_expert=[683, 1770, 631, 243, 88, 276, 2736, 582, 1, 0, 800, 650, 1719, 0, 0, 901, 603, 777, 3706, 73, 456, 1361, 229, 0, 146, 2467, 1184, 125, 59, 1, 3408, 241, 1306, 378, 91, 1553, 613, 312, 1120, 904, 56, 0, 13, 1115, 142, 1203, 1614, 893, 1050, 1131, 1459, 1384, 195, 515, 0, 1597, 1394, 444, 222, 323, 467, 411, 730, 601] payload_rows_per_dst=[7230, 10702, 10894, 11136, 9818, 9450, 9304, 8382] dup_rows_per_dst=[1316, 2912, 2514, 4126, 2736, 3816, 2164, 1804] combine_rows_per_src=[7230, 10702, 10894, 11136, 9818, 9450, 9304, 8382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77018 payload_rows_remote=64878 dup_rows=21286 dup_rows_remote=16882 dedup_ratio=0.216532 router_tokens_per_expert=[555, 1758, 1027, 235, 169, 316, 2724, 300, 59, 0, 723, 857, 1483, 0, 0, 1174, 1068, 796, 3493, 119, 565, 723, 217, 0, 60, 1989, 846, 115, 175, 176, 3551, 331, 1257, 449, 131, 1822, 656, 312, 1449, 811, 319, 0, 84, 1500, 169, 678, 1546, 649, 982, 1323, 1195, 735, 217, 767, 0, 1317, 1331, 452, 445, 492, 525, 640, 739, 556] payload_rows_per_dst=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] dup_rows_per_dst=[2654, 1786, 4404, 2212, 3294, 3936, 1460, 1540] combine_rows_per_src=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65888 dup_rows=24040 dup_rows_remote=20446 dedup_ratio=0.244548 router_tokens_per_expert=[0, 1139, 1435, 926, 812, 2277, 488, 1050, 517, 1196, 2670, 80, 178, 431, 277, 995, 1906, 351, 2, 10, 209, 1024, 321, 269, 692, 99, 1699, 315, 3110, 538, 415, 0, 253, 0, 0, 1635, 887, 39, 34, 1238, 46, 255, 1, 962, 361, 1, 2954, 1, 4, 1108, 227, 697, 405, 2190, 927, 1026, 458, 1327, 46, 1222, 979, 1751, 1301, 1386] payload_rows_per_dst=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] dup_rows_per_dst=[3594, 2820, 3894, 5138, 2914, 622, 4506, 552] combine_rows_per_src=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272809984 dispatch_meta_bytes=351952 combine_payload_bytes=272809984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273161936 ingress_bytes=307412992 link_bytes=307412992 token_bytes=545971920 weight_movement_bytes=34603008 total_nvlink_bytes=580574928 weight_to_token_ratio=0.063379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156770340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77018 payload_rows_remote=64878 dup_rows=21286 dup_rows_remote=16882 dedup_ratio=0.216532 router_tokens_per_expert=[555, 1758, 1027, 235, 169, 316, 2724, 300, 59, 0, 723, 857, 1483, 0, 0, 1174, 1068, 796, 3493, 119, 565, 723, 217, 0, 60, 1989, 846, 115, 175, 176, 3551, 331, 1257, 449, 131, 1822, 656, 312, 1449, 811, 319, 0, 84, 1500, 169, 678, 1546, 649, 982, 1323, 1195, 735, 217, 767, 0, 1317, 1331, 452, 445, 492, 525, 640, 739, 556] payload_rows_per_dst=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] dup_rows_per_dst=[2654, 1786, 4404, 2212, 3294, 3936, 1460, 1540] combine_rows_per_src=[10088, 8232, 12140, 9692, 10480, 8588, 8978, 8820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76604 payload_rows_remote=67072 dup_rows=21700 dup_rows_remote=19626 dedup_ratio=0.220744 router_tokens_per_expert=[951, 1850, 1119, 258, 303, 555, 2124, 427, 218, 66, 561, 852, 1514, 5, 53, 1158, 982, 813, 2709, 163, 522, 666, 437, 126, 201, 1294, 889, 204, 213, 258, 2385, 359, 1301, 607, 265, 2509, 499, 553, 1616, 585, 192, 114, 61, 1325, 420, 621, 1683, 972, 1095, 1206, 1124, 812, 344, 568, 72, 1758, 1004, 519, 511, 522, 387, 454, 780, 438] payload_rows_per_dst=[8744, 10240, 10170, 9532, 11502, 9634, 8882, 7900] dup_rows_per_dst=[2182, 2862, 2666, 2074, 4368, 4658, 1560, 1330] combine_rows_per_src=[8744, 10240, 10170, 9532, 11502, 9634, 8882, 7900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76956 payload_rows_remote=67994 dup_rows=21348 dup_rows_remote=17474 dedup_ratio=0.217163 router_tokens_per_expert=[691, 1736, 1050, 190, 203, 544, 2451, 301, 90, 3, 642, 876, 1451, 0, 6, 1146, 923, 708, 3387, 129, 432, 701, 323, 19, 74, 1753, 815, 138, 144, 188, 3157, 240, 1357, 494, 188, 2030, 583, 345, 1551, 810, 199, 2, 79, 1546, 121, 686, 1528, 825, 1031, 1522, 1235, 622, 245, 631, 1, 1432, 1341, 505, 483, 636, 498, 608, 887, 620] payload_rows_per_dst=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] dup_rows_per_dst=[1554, 3100, 394, 2616, 3606, 3874, 1706, 4498] combine_rows_per_src=[7876, 10230, 6076, 10402, 10630, 8962, 9348, 13432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76156 payload_rows_remote=63812 dup_rows=22148 dup_rows_remote=18314 dedup_ratio=0.225301 router_tokens_per_expert=[596, 1956, 638, 237, 68, 380, 2367, 626, 0, 0, 935, 656, 2343, 0, 0, 972, 504, 916, 3876, 55, 495, 1723, 285, 0, 74, 2475, 1272, 104, 60, 0, 3276, 237, 1278, 434, 85, 1673, 470, 292, 1109, 524, 47, 0, 15, 1148, 106, 1036, 1481, 790, 1035, 1187, 1798, 1248, 191, 501, 0, 1365, 1391, 437, 90, 375, 352, 421, 651, 496] payload_rows_per_dst=[7598, 10966, 7270, 10934, 7170, 7708, 12166, 12344] dup_rows_per_dst=[1404, 3580, 686, 4062, 1214, 1698, 5670, 3834] combine_rows_per_src=[7598, 10966, 7270, 10934, 7170, 7708, 12166, 12344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=257662976 dispatch_meta_bytes=326424 combine_payload_bytes=257662976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257989400 ingress_bytes=292265984 link_bytes=292265984 token_bytes=515652376 weight_movement_bytes=34603008 total_nvlink_bytes=550255384 weight_to_token_ratio=0.067105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=10496 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.099773368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274833408 dispatch_meta_bytes=343000 combine_payload_bytes=274833408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275176408 ingress_bytes=344039424 link_bytes=344039424 token_bytes=550009816 weight_movement_bytes=69206016 total_nvlink_bytes=619215832 weight_to_token_ratio=0.125827 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82630 payload_rows_remote=72744 dup_rows=15674 dup_rows_remote=12624 dedup_ratio=0.159444 router_tokens_per_expert=[910, 134, 1, 3261, 69, 6, 299, 93, 0, 474, 945, 685, 935, 1006, 493, 1081, 7, 2, 659, 27, 21, 508, 419, 2804, 135, 79, 977, 6, 367, 509, 0, 8, 76, 1565, 302, 95, 6, 0, 0, 2155, 1492, 221, 226, 2385, 5139, 108, 2036, 2034, 418, 194, 0, 1636, 319, 629, 5844, 0, 4867, 1, 1, 18, 93, 2, 306, 64] payload_rows_per_dst=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] dup_rows_per_dst=[1136, 2198, 2500, 1760, 1476, 3050, 3302, 252] combine_rows_per_src=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75442 payload_rows_remote=66604 dup_rows=22862 dup_rows_remote=21384 dedup_ratio=0.232564 router_tokens_per_expert=[888, 2161, 672, 156, 86, 367, 2814, 864, 0, 0, 856, 720, 2585, 0, 0, 997, 568, 908, 3946, 91, 428, 1502, 296, 0, 143, 2232, 1271, 138, 58, 2, 2424, 375, 1353, 360, 105, 1920, 367, 315, 1095, 634, 88, 0, 6, 1280, 97, 993, 1239, 727, 915, 1108, 1808, 1162, 133, 485, 0, 1391, 1447, 420, 96, 277, 353, 355, 617, 458] payload_rows_per_dst=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] dup_rows_per_dst=[6014, 1478, 5304, 932, 2564, 3426, 1934, 1210] combine_rows_per_src=[11972, 8838, 13052, 7506, 9734, 8216, 9288, 6836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76786 payload_rows_remote=66094 dup_rows=21518 dup_rows_remote=18324 dedup_ratio=0.218892 router_tokens_per_expert=[789, 1860, 1004, 206, 128, 540, 2610, 459, 42, 6, 717, 793, 1531, 4, 13, 1125, 790, 929, 3357, 80, 530, 806, 313, 20, 96, 2172, 974, 134, 153, 38, 3518, 260, 1066, 493, 125, 1706, 624, 324, 1323, 664, 161, 27, 38, 1352, 127, 743, 1661, 774, 1007, 1308, 1307, 705, 220, 668, 22, 1462, 1262, 470, 464, 502, 531, 596, 821, 602] payload_rows_per_dst=[8202, 10602, 6434, 10948, 7920, 8968, 10692, 13020] dup_rows_per_dst=[1770, 3080, 502, 3742, 1318, 3722, 3194, 4190] combine_rows_per_src=[8202, 10602, 6434, 10948, 7920, 8968, 10692, 13020] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274726912 dispatch_meta_bytes=346792 combine_payload_bytes=274726912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275073704 ingress_bytes=378535936 link_bytes=378535936 token_bytes=549800616 weight_movement_bytes=103809024 total_nvlink_bytes=653609640 weight_to_token_ratio=0.188812 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424400253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274833408 dispatch_meta_bytes=343000 combine_payload_bytes=274833408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=275176408 ingress_bytes=344039424 link_bytes=344039424 token_bytes=550009816 weight_movement_bytes=69206016 total_nvlink_bytes=619215832 weight_to_token_ratio=0.125827 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11008 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.294592655e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76156 payload_rows_remote=63812 dup_rows=22148 dup_rows_remote=18314 dedup_ratio=0.225301 router_tokens_per_expert=[596, 1956, 638, 237, 68, 380, 2367, 626, 0, 0, 935, 656, 2343, 0, 0, 972, 504, 916, 3876, 55, 495, 1723, 285, 0, 74, 2475, 1272, 104, 60, 0, 3276, 237, 1278, 434, 85, 1673, 470, 292, 1109, 524, 47, 0, 15, 1148, 106, 1036, 1481, 790, 1035, 1187, 1798, 1248, 191, 501, 0, 1365, 1391, 437, 90, 375, 352, 421, 651, 496] payload_rows_per_dst=[7598, 10966, 7270, 10934, 7170, 7708, 12166, 12344] dup_rows_per_dst=[1404, 3580, 686, 4062, 1214, 1698, 5670, 3834] combine_rows_per_src=[7598, 10966, 7270, 10934, 7170, 7708, 12166, 12344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=327040 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067328 ingress_bytes=334946304 link_bytes=334946304 token_bytes=531807616 weight_movement_bytes=69206016 total_nvlink_bytes=601013632 weight_to_token_ratio=0.130134 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260375977e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=274726912 dispatch_meta_bytes=346792 combine_payload_bytes=274726912 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=275073704 ingress_bytes=378535936 link_bytes=378535936 token_bytes=549800616 weight_movement_bytes=103809024 total_nvlink_bytes=653609640 weight_to_token_ratio=0.188812 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.424400253e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=278503424 dispatch_meta_bytes=341872 combine_payload_bytes=278503424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=278845296 ingress_bytes=313106432 link_bytes=313106432 token_bytes=557348720 weight_movement_bytes=34603008 total_nvlink_bytes=591951728 weight_to_token_ratio=0.062085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.178194296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261373952 dispatch_meta_bytes=328504 combine_payload_bytes=261373952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261702456 ingress_bytes=295976960 link_bytes=295976960 token_bytes=523076408 weight_movement_bytes=34603008 total_nvlink_bytes=557679416 weight_to_token_ratio=0.066153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113737472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=345336 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270222584 ingress_bytes=408289280 link_bytes=408289280 token_bytes=540099832 weight_movement_bytes=138412032 total_nvlink_bytes=678511864 weight_to_token_ratio=0.256271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.536359691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=272809984 dispatch_meta_bytes=351952 combine_payload_bytes=272809984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=273161936 ingress_bytes=307412992 link_bytes=307412992 token_bytes=545971920 weight_movement_bytes=34603008 total_nvlink_bytes=580574928 weight_to_token_ratio=0.063379 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156770340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=270721024 dispatch_meta_bytes=337672 combine_payload_bytes=270721024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=271058696 ingress_bytes=339927040 link_bytes=339927040 token_bytes=541779720 weight_movement_bytes=69206016 total_nvlink_bytes=610985736 weight_to_token_ratio=0.127738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.279118085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=297959424 dispatch_meta_bytes=341472 combine_payload_bytes=297959424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=298300896 ingress_bytes=436371456 link_bytes=436371456 token_bytes=596260320 weight_movement_bytes=138412032 total_nvlink_bytes=734672352 weight_to_token_ratio=0.232134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642030658e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=261373952 dispatch_meta_bytes=328504 combine_payload_bytes=261373952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=261702456 ingress_bytes=295976960 link_bytes=295976960 token_bytes=523076408 weight_movement_bytes=34603008 total_nvlink_bytes=557679416 weight_to_token_ratio=0.066153 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.113737472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81462 payload_rows_remote=70716 dup_rows=16842 dup_rows_remote=12900 dedup_ratio=0.171326 router_tokens_per_expert=[617, 91, 0, 3715, 92, 0, 192, 76, 0, 589, 870, 581, 906, 366, 383, 658, 0, 0, 879, 1, 0, 657, 432, 1953, 140, 73, 311, 0, 143, 442, 0, 0, 76, 1291, 130, 1, 0, 0, 0, 4211, 1581, 193, 193, 2263, 5489, 231, 1936, 3540, 351, 122, 0, 1005, 229, 1031, 6075, 0, 4572, 0, 0, 6, 0, 0, 401, 58] payload_rows_per_dst=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] dup_rows_per_dst=[3980, 3942, 3730, 440, 3594, 838, 138, 180] combine_rows_per_src=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83238 payload_rows_remote=73402 dup_rows=15066 dup_rows_remote=13572 dedup_ratio=0.153259 router_tokens_per_expert=[678, 99, 0, 4029, 92, 0, 192, 83, 0, 474, 667, 462, 855, 603, 400, 501, 0, 0, 722, 0, 1, 526, 406, 1939, 127, 67, 502, 0, 146, 376, 0, 0, 83, 1283, 140, 21, 0, 0, 0, 4138, 1470, 118, 185, 2328, 5421, 237, 1457, 3496, 360, 98, 0, 1880, 260, 970, 6036, 0, 4770, 0, 0, 0, 0, 0, 385, 69] payload_rows_per_dst=[9428, 6414, 10932, 12492, 9836, 9040, 14824, 10272] dup_rows_per_dst=[918, 1510, 3248, 786, 1494, 2550, 4384, 176] combine_rows_per_src=[9428, 6414, 10932, 12492, 9836, 9040, 14824, 10272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68104 dup_rows=18592 dup_rows_remote=16398 dedup_ratio=0.189128 router_tokens_per_expert=[1083, 195, 47, 2738, 220, 65, 320, 228, 79, 773, 791, 739, 898, 781, 664, 790, 2, 39, 925, 113, 136, 772, 546, 1688, 126, 229, 1113, 39, 682, 611, 25, 74, 115, 1369, 354, 133, 85, 116, 78, 2303, 1543, 276, 315, 2309, 4002, 362, 1461, 2019, 386, 841, 153, 1332, 751, 801, 4960, 254, 3762, 165, 80, 199, 591, 2, 395, 109] payload_rows_per_dst=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] dup_rows_per_dst=[1428, 4196, 2934, 2194, 2290, 1892, 2974, 684] combine_rows_per_src=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80064 payload_rows_remote=69202 dup_rows=18240 dup_rows_remote=15400 dedup_ratio=0.185547 router_tokens_per_expert=[709, 78, 0, 3560, 62, 0, 226, 52, 0, 365, 1107, 725, 961, 977, 514, 987, 0, 0, 676, 1, 5, 463, 484, 2814, 118, 62, 1028, 0, 258, 489, 0, 0, 52, 1361, 221, 10, 0, 0, 0, 2679, 1365, 132, 184, 2403, 5270, 116, 1901, 2408, 403, 98, 0, 1496, 268, 576, 6187, 0, 4951, 0, 0, 3, 83, 0, 227, 37] payload_rows_per_dst=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] dup_rows_per_dst=[1058, 8622, 2840, 1750, 3052, 646, 120, 152] combine_rows_per_src=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=63422 dup_rows=22530 dup_rows_remote=15182 dedup_ratio=0.229187 router_tokens_per_expert=[516, 1797, 659, 277, 55, 383, 2357, 786, 1, 1, 871, 633, 2547, 0, 2, 793, 590, 691, 4011, 46, 386, 1862, 310, 3, 119, 1844, 1165, 128, 68, 33, 3020, 181, 1624, 369, 74, 2013, 511, 335, 1134, 572, 28, 1, 5, 828, 99, 858, 1690, 880, 1206, 999, 1624, 1371, 237, 539, 3, 1435, 1728, 463, 243, 303, 448, 357, 578, 462] payload_rows_per_dst=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] dup_rows_per_dst=[7348, 1116, 3522, 980, 2444, 3340, 2262, 1518] combine_rows_per_src=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78146 payload_rows_remote=63902 dup_rows=20158 dup_rows_remote=12806 dedup_ratio=0.205058 router_tokens_per_expert=[2560, 5078, 14, 276, 703, 104, 64, 206, 5, 459, 178, 390, 60, 325, 410, 1069, 390, 726, 1, 1, 492, 114, 1806, 924, 2032, 3630, 1345, 343, 256, 28, 105, 499, 0, 265, 1541, 1, 67, 2217, 748, 109, 653, 81, 1, 89, 3141, 6399, 433, 1, 4504, 2, 666, 482, 482, 85, 341, 86, 205, 421, 1, 0, 0, 1120, 120, 298] payload_rows_per_dst=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] dup_rows_per_dst=[76, 750, 2944, 2442, 3240, 7352, 1424, 1930] combine_rows_per_src=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289652736 dispatch_meta_bytes=334464 combine_payload_bytes=289652736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289987200 ingress_bytes=324255744 link_bytes=324255744 token_bytes=579639936 weight_movement_bytes=34603008 total_nvlink_bytes=614242944 weight_to_token_ratio=0.059697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220148260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80588 payload_rows_remote=66406 dup_rows=17716 dup_rows_remote=12946 dedup_ratio=0.180216 router_tokens_per_expert=[907, 117, 3, 3507, 97, 8, 243, 89, 1, 492, 823, 606, 687, 980, 519, 830, 0, 1, 622, 17, 20, 528, 431, 2423, 122, 64, 1158, 2, 256, 505, 4, 0, 123, 1386, 255, 12, 3, 26, 24, 2886, 1481, 168, 197, 2233, 5106, 88, 1976, 2630, 393, 109, 13, 1553, 370, 727, 5904, 23, 4964, 7, 23, 28, 28, 3, 282, 69] payload_rows_per_dst=[8744, 7742, 8566, 12406, 7908, 9712, 14182, 11328] dup_rows_per_dst=[1198, 2134, 1592, 1492, 1522, 3464, 4770, 1544] combine_rows_per_src=[8744, 7742, 8566, 12406, 7908, 9712, 14182, 11328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65888 dup_rows=24040 dup_rows_remote=20446 dedup_ratio=0.244548 router_tokens_per_expert=[0, 1139, 1435, 926, 812, 2277, 488, 1050, 517, 1196, 2670, 80, 178, 431, 277, 995, 1906, 351, 2, 10, 209, 1024, 321, 269, 692, 99, 1699, 315, 3110, 538, 415, 0, 253, 0, 0, 1635, 887, 39, 34, 1238, 46, 255, 1, 962, 361, 1, 2954, 1, 4, 1108, 227, 697, 405, 2190, 927, 1026, 458, 1327, 46, 1222, 979, 1751, 1301, 1386] payload_rows_per_dst=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] dup_rows_per_dst=[3594, 2820, 3894, 5138, 2914, 622, 4506, 552] combine_rows_per_src=[8376, 9598, 10564, 11200, 9812, 8540, 11316, 4858] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80064 payload_rows_remote=69202 dup_rows=18240 dup_rows_remote=15400 dedup_ratio=0.185547 router_tokens_per_expert=[709, 78, 0, 3560, 62, 0, 226, 52, 0, 365, 1107, 725, 961, 977, 514, 987, 0, 0, 676, 1, 5, 463, 484, 2814, 118, 62, 1028, 0, 258, 489, 0, 0, 52, 1361, 221, 10, 0, 0, 0, 2679, 1365, 132, 184, 2403, 5270, 116, 1901, 2408, 403, 98, 0, 1496, 268, 576, 6187, 0, 4951, 0, 0, 3, 83, 0, 227, 37] payload_rows_per_dst=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] dup_rows_per_dst=[1058, 8622, 2840, 1750, 3052, 646, 120, 152] combine_rows_per_src=[8316, 15024, 10862, 12700, 10400, 6750, 5562, 10450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79712 payload_rows_remote=68104 dup_rows=18592 dup_rows_remote=16398 dedup_ratio=0.189128 router_tokens_per_expert=[1083, 195, 47, 2738, 220, 65, 320, 228, 79, 773, 791, 739, 898, 781, 664, 790, 2, 39, 925, 113, 136, 772, 546, 1688, 126, 229, 1113, 39, 682, 611, 25, 74, 115, 1369, 354, 133, 85, 116, 78, 2303, 1543, 276, 315, 2309, 4002, 362, 1461, 2019, 386, 841, 153, 1332, 751, 801, 4960, 254, 3762, 165, 80, 199, 591, 2, 395, 109] payload_rows_per_dst=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] dup_rows_per_dst=[1428, 4196, 2934, 2194, 2290, 1892, 2974, 684] combine_rows_per_src=[8364, 11054, 9546, 11608, 9154, 8302, 11762, 9922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82630 payload_rows_remote=72744 dup_rows=15674 dup_rows_remote=12624 dedup_ratio=0.159444 router_tokens_per_expert=[910, 134, 1, 3261, 69, 6, 299, 93, 0, 474, 945, 685, 935, 1006, 493, 1081, 7, 2, 659, 27, 21, 508, 419, 2804, 135, 79, 977, 6, 367, 509, 0, 8, 76, 1565, 302, 95, 6, 0, 0, 2155, 1492, 221, 226, 2385, 5139, 108, 2036, 2034, 418, 194, 0, 1636, 319, 629, 5844, 0, 4867, 1, 1, 18, 93, 2, 306, 64] payload_rows_per_dst=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] dup_rows_per_dst=[1136, 2198, 2500, 1760, 1476, 3050, 3302, 252] combine_rows_per_src=[8410, 9040, 10462, 12680, 6922, 9886, 14778, 10452] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300654592 dispatch_meta_bytes=347896 combine_payload_bytes=300654592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301002488 ingress_bytes=335257600 link_bytes=335257600 token_bytes=601657080 weight_movement_bytes=34603008 total_nvlink_bytes=636260088 weight_to_token_ratio=0.057513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72766 payload_rows_remote=58104 dup_rows=25538 dup_rows_remote=17468 dedup_ratio=0.259786 router_tokens_per_expert=[691, 94, 0, 4163, 87, 0, 177, 78, 0, 480, 619, 454, 896, 540, 412, 611, 0, 0, 589, 1, 0, 489, 418, 1767, 116, 59, 375, 0, 74, 458, 0, 0, 78, 1432, 108, 9, 0, 0, 0, 4412, 1375, 134, 171, 2278, 5612, 161, 1850, 3581, 390, 88, 0, 1297, 213, 949, 6238, 0, 4778, 0, 0, 2, 0, 0, 280, 68] payload_rows_per_dst=[9722, 6464, 5668, 1992, 10620, 14934, 8704, 14662] dup_rows_per_dst=[858, 1560, 860, 172, 1458, 11690, 870, 8070] combine_rows_per_src=[9722, 6464, 5668, 1992, 10620, 14934, 8704, 14662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300654592 dispatch_meta_bytes=347896 combine_payload_bytes=300654592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=301002488 ingress_bytes=335257600 link_bytes=335257600 token_bytes=601657080 weight_movement_bytes=34603008 total_nvlink_bytes=636260088 weight_to_token_ratio=0.057513 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=338008 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279291992 ingress_bytes=313556992 link_bytes=313556992 token_bytes=558245976 weight_movement_bytes=34603008 total_nvlink_bytes=592848984 weight_to_token_ratio=0.061985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179889717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=269877248 dispatch_meta_bytes=345336 combine_payload_bytes=269877248 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=270222584 ingress_bytes=408289280 link_bytes=408289280 token_bytes=540099832 weight_movement_bytes=138412032 total_nvlink_bytes=678511864 weight_to_token_ratio=0.256271 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.536359691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=283451392 dispatch_meta_bytes=338408 combine_payload_bytes=283451392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283789800 ingress_bytes=318054400 link_bytes=318054400 token_bytes=567241192 weight_movement_bytes=34603008 total_nvlink_bytes=601844200 weight_to_token_ratio=0.061002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.196813102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259776512 dispatch_meta_bytes=314416 combine_payload_bytes=259776512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260090928 ingress_bytes=328982528 link_bytes=328982528 token_bytes=519867440 weight_movement_bytes=69206016 total_nvlink_bytes=589073456 weight_to_token_ratio=0.133122 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237934768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=261742592 dispatch_meta_bytes=306832 combine_payload_bytes=261742592 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262049424 ingress_bytes=365551616 link_bytes=365551616 token_bytes=523792016 weight_movement_bytes=103809024 total_nvlink_bytes=627601040 weight_to_token_ratio=0.198187 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375541302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271998976 dispatch_meta_bytes=317408 combine_payload_bytes=271998976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=272316384 ingress_bytes=410411008 link_bytes=410411008 token_bytes=544315360 weight_movement_bytes=138412032 total_nvlink_bytes=682727392 weight_to_token_ratio=0.254286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544343582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80588 payload_rows_remote=66406 dup_rows=17716 dup_rows_remote=12946 dedup_ratio=0.180216 router_tokens_per_expert=[907, 117, 3, 3507, 97, 8, 243, 89, 1, 492, 823, 606, 687, 980, 519, 830, 0, 1, 622, 17, 20, 528, 431, 2423, 122, 64, 1158, 2, 256, 505, 4, 0, 123, 1386, 255, 12, 3, 26, 24, 2886, 1481, 168, 197, 2233, 5106, 88, 1976, 2630, 393, 109, 13, 1553, 370, 727, 5904, 23, 4964, 7, 23, 28, 28, 3, 282, 69] payload_rows_per_dst=[8744, 7742, 8566, 12406, 7908, 9712, 14182, 11328] dup_rows_per_dst=[1198, 2134, 1592, 1492, 1522, 3464, 4770, 1544] combine_rows_per_src=[8744, 7742, 8566, 12406, 7908, 9712, 14182, 11328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81462 payload_rows_remote=70716 dup_rows=16842 dup_rows_remote=12900 dedup_ratio=0.171326 router_tokens_per_expert=[617, 91, 0, 3715, 92, 0, 192, 76, 0, 589, 870, 581, 906, 366, 383, 658, 0, 0, 879, 1, 0, 657, 432, 1953, 140, 73, 311, 0, 143, 442, 0, 0, 76, 1291, 130, 1, 0, 0, 0, 4211, 1581, 193, 193, 2263, 5489, 231, 1936, 3540, 351, 122, 0, 1005, 229, 1031, 6075, 0, 4572, 0, 0, 6, 0, 0, 401, 58] payload_rows_per_dst=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] dup_rows_per_dst=[3980, 3942, 3730, 440, 3594, 838, 138, 180] combine_rows_per_src=[11754, 10746, 11194, 12756, 12350, 7430, 5338, 9894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=278953984 dispatch_meta_bytes=338008 combine_payload_bytes=278953984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279291992 ingress_bytes=313556992 link_bytes=313556992 token_bytes=558245976 weight_movement_bytes=34603008 total_nvlink_bytes=592848984 weight_to_token_ratio=0.061985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=35712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179889717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=297959424 dispatch_meta_bytes=341472 combine_payload_bytes=297959424 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=298300896 ingress_bytes=436371456 link_bytes=436371456 token_bytes=596260320 weight_movement_bytes=138412032 total_nvlink_bytes=734672352 weight_to_token_ratio=0.232134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=11648 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.642030658e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=237993984 dispatch_meta_bytes=302288 combine_payload_bytes=237993984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=238296272 ingress_bytes=272596992 link_bytes=272596992 token_bytes=476290256 weight_movement_bytes=34603008 total_nvlink_bytes=510893264 weight_to_token_ratio=0.072651 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=37376 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.025760535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81156 payload_rows_remote=67336 dup_rows=17148 dup_rows_remote=14482 dedup_ratio=0.174438 router_tokens_per_expert=[3676, 6872, 0, 180, 542, 0, 58, 31, 0, 363, 58, 234, 139, 301, 563, 443, 373, 1140, 0, 0, 312, 85, 387, 447, 4232, 2731, 1254, 235, 7, 0, 64, 536, 0, 250, 636, 0, 92, 3669, 868, 132, 725, 73, 0, 73, 2758, 6142, 128, 0, 4271, 0, 277, 477, 126, 42, 133, 0, 34, 327, 0, 0, 0, 2120, 192, 344] payload_rows_per_dst=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] dup_rows_per_dst=[7842, 2666, 2886, 1234, 1116, 434, 908, 62] combine_rows_per_src=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81622 payload_rows_remote=72796 dup_rows=16682 dup_rows_remote=15220 dedup_ratio=0.169698 router_tokens_per_expert=[3712, 6921, 0, 206, 554, 8, 69, 24, 0, 203, 80, 228, 142, 335, 461, 620, 425, 1065, 0, 0, 275, 95, 898, 675, 3418, 3267, 1457, 314, 0, 0, 139, 489, 0, 349, 608, 1, 92, 2347, 880, 57, 696, 80, 0, 67, 2322, 6499, 79, 0, 4401, 0, 460, 526, 435, 97, 281, 58, 57, 303, 0, 0, 0, 1785, 191, 401] payload_rows_per_dst=[7058, 10546, 10484, 9994, 8826, 8016, 11332, 15366] dup_rows_per_dst=[468, 1974, 3218, 2546, 1462, 1880, 1184, 3950] combine_rows_per_src=[7058, 10546, 10484, 9994, 8826, 8016, 11332, 15366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79664 payload_rows_remote=70006 dup_rows=18640 dup_rows_remote=17042 dedup_ratio=0.189616 router_tokens_per_expert=[2358, 4029, 413, 382, 978, 143, 101, 334, 61, 582, 531, 981, 256, 460, 573, 949, 451, 1293, 338, 231, 558, 140, 769, 1046, 2181, 2401, 1606, 216, 370, 224, 262, 549, 183, 378, 1093, 38, 107, 2724, 1076, 86, 804, 52, 99, 153, 2444, 4611, 503, 172, 2891, 283, 683, 460, 618, 134, 362, 143, 410, 531, 243, 71, 41, 1129, 386, 478] payload_rows_per_dst=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] dup_rows_per_dst=[2768, 4548, 3734, 1598, 1878, 754, 1478, 1882] combine_rows_per_src=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80828 payload_rows_remote=71014 dup_rows=17476 dup_rows_remote=14456 dedup_ratio=0.177775 router_tokens_per_expert=[3027, 5771, 0, 270, 569, 99, 37, 163, 0, 376, 147, 249, 60, 306, 486, 997, 367, 651, 0, 0, 397, 65, 2281, 798, 1858, 3717, 1181, 340, 126, 0, 102, 436, 0, 221, 1562, 0, 62, 1625, 659, 105, 684, 96, 0, 59, 3145, 6674, 357, 0, 5179, 0, 648, 394, 459, 72, 421, 58, 104, 409, 0, 0, 0, 949, 92, 242] payload_rows_per_dst=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] dup_rows_per_dst=[5714, 3690, 3020, 1906, 1058, 592, 1484, 12] combine_rows_per_src=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77790 payload_rows_remote=64456 dup_rows=20514 dup_rows_remote=19838 dedup_ratio=0.208679 router_tokens_per_expert=[577, 507, 3518, 3903, 1201, 0, 6037, 4050, 5, 134, 0, 381, 1, 2018, 0, 181, 323, 343, 1883, 121, 967, 54, 468, 149, 62, 67, 1443, 0, 4, 130, 1, 542, 5729, 1065, 0, 67, 202, 1474, 542, 1, 394, 0, 81, 0, 12, 1, 100, 380, 2005, 1, 11, 332, 12, 1908, 174, 0, 1497, 30, 72, 2456, 1505, 27, 4, 0] payload_rows_per_dst=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] dup_rows_per_dst=[1094, 304, 3844, 4976, 2556, 676, 4786, 2278] combine_rows_per_src=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80430 payload_rows_remote=65526 dup_rows=17874 dup_rows_remote=9788 dedup_ratio=0.181824 router_tokens_per_expert=[767, 115, 0, 3961, 92, 0, 188, 100, 0, 484, 515, 424, 878, 629, 396, 503, 0, 0, 757, 0, 1, 596, 440, 1398, 130, 53, 372, 2, 118, 421, 0, 2, 100, 1384, 90, 22, 0, 3, 2, 3973, 1307, 171, 184, 2540, 5440, 127, 2271, 3493, 337, 95, 1, 1283, 331, 1250, 6272, 1, 4650, 0, 0, 6, 4, 1, 395, 77] payload_rows_per_dst=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] dup_rows_per_dst=[8086, 1516, 2968, 542, 3732, 678, 172, 180] combine_rows_per_src=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=271998976 dispatch_meta_bytes=317408 combine_payload_bytes=271998976 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=272316384 ingress_bytes=410411008 link_bytes=410411008 token_bytes=544315360 weight_movement_bytes=138412032 total_nvlink_bytes=682727392 weight_to_token_ratio=0.254286 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19712 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.544343582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=289652736 dispatch_meta_bytes=334464 combine_payload_bytes=289652736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289987200 ingress_bytes=324255744 link_bytes=324255744 token_bytes=579639936 weight_movement_bytes=34603008 total_nvlink_bytes=614242944 weight_to_token_ratio=0.059697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=11392 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.220148260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77944 payload_rows_remote=66412 dup_rows=20360 dup_rows_remote=18944 dedup_ratio=0.207113 router_tokens_per_expert=[2902, 6218, 62, 253, 613, 67, 75, 84, 10, 412, 186, 366, 89, 353, 463, 1066, 424, 969, 45, 6, 446, 104, 1643, 972, 2452, 3136, 1378, 309, 45, 22, 122, 423, 7, 325, 1212, 2, 79, 2002, 822, 54, 660, 104, 8, 70, 2692, 6161, 355, 31, 4279, 43, 721, 457, 477, 89, 345, 63, 192, 399, 20, 9, 15, 1218, 185, 341] payload_rows_per_dst=[2254, 5130, 7556, 11388, 11758, 13866, 11532, 14460] dup_rows_per_dst=[54, 760, 1662, 4386, 3052, 6296, 1416, 2734] combine_rows_per_src=[2254, 5130, 7556, 11388, 11758, 13866, 11532, 14460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275808256 dispatch_meta_bytes=327272 combine_payload_bytes=275808256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276135528 ingress_bytes=310411264 link_bytes=310411264 token_bytes=551943784 weight_movement_bytes=34603008 total_nvlink_bytes=586546792 weight_to_token_ratio=0.062693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168052596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75774 payload_rows_remote=63422 dup_rows=22530 dup_rows_remote=15182 dedup_ratio=0.229187 router_tokens_per_expert=[516, 1797, 659, 277, 55, 383, 2357, 786, 1, 1, 871, 633, 2547, 0, 2, 793, 590, 691, 4011, 46, 386, 1862, 310, 3, 119, 1844, 1165, 128, 68, 33, 3020, 181, 1624, 369, 74, 2013, 511, 335, 1134, 572, 28, 1, 5, 828, 99, 858, 1690, 880, 1206, 999, 1624, 1371, 237, 539, 3, 1435, 1728, 463, 243, 303, 448, 357, 578, 462] payload_rows_per_dst=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] dup_rows_per_dst=[7348, 1116, 3522, 980, 2444, 3340, 2262, 1518] combine_rows_per_src=[12352, 8580, 12276, 7410, 9506, 8308, 9696, 7646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80828 payload_rows_remote=71014 dup_rows=17476 dup_rows_remote=14456 dedup_ratio=0.177775 router_tokens_per_expert=[3027, 5771, 0, 270, 569, 99, 37, 163, 0, 376, 147, 249, 60, 306, 486, 997, 367, 651, 0, 0, 397, 65, 2281, 798, 1858, 3717, 1181, 340, 126, 0, 102, 436, 0, 221, 1562, 0, 62, 1625, 659, 105, 684, 96, 0, 59, 3145, 6674, 357, 0, 5179, 0, 648, 394, 459, 72, 421, 58, 104, 409, 0, 0, 0, 949, 92, 242] payload_rows_per_dst=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] dup_rows_per_dst=[5714, 3690, 3020, 1906, 1058, 592, 1484, 12] combine_rows_per_src=[14158, 14900, 9814, 9898, 7410, 8090, 12978, 3580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298172416 dispatch_meta_bytes=352064 combine_payload_bytes=298172416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298524480 ingress_bytes=332775424 link_bytes=332775424 token_bytes=596696896 weight_movement_bytes=34603008 total_nvlink_bytes=631299904 weight_to_token_ratio=0.057991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252207130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79664 payload_rows_remote=70006 dup_rows=18640 dup_rows_remote=17042 dedup_ratio=0.189616 router_tokens_per_expert=[2358, 4029, 413, 382, 978, 143, 101, 334, 61, 582, 531, 981, 256, 460, 573, 949, 451, 1293, 338, 231, 558, 140, 769, 1046, 2181, 2401, 1606, 216, 370, 224, 262, 549, 183, 378, 1093, 38, 107, 2724, 1076, 86, 804, 52, 99, 153, 2444, 4611, 503, 172, 2891, 283, 683, 460, 618, 134, 362, 143, 410, 531, 243, 71, 41, 1129, 386, 478] payload_rows_per_dst=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] dup_rows_per_dst=[2768, 4548, 3734, 1598, 1878, 754, 1478, 1882] combine_rows_per_src=[9122, 13460, 10280, 9658, 9492, 7700, 9670, 10282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=259776512 dispatch_meta_bytes=314416 combine_payload_bytes=259776512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=260090928 ingress_bytes=328982528 link_bytes=328982528 token_bytes=519867440 weight_movement_bytes=69206016 total_nvlink_bytes=589073456 weight_to_token_ratio=0.133122 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.237934768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=75616 payload_rows_remote=59982 dup_rows=22688 dup_rows_remote=19018 dedup_ratio=0.230794 router_tokens_per_expert=[3895, 6979, 0, 166, 533, 3, 68, 19, 0, 231, 70, 224, 72, 329, 541, 581, 409, 1293, 0, 0, 248, 90, 681, 738, 4364, 2907, 1707, 297, 2, 0, 117, 475, 0, 293, 664, 0, 87, 2697, 875, 32, 623, 37, 0, 59, 2032, 6328, 60, 0, 4278, 0, 367, 485, 243, 83, 157, 40, 46, 381, 0, 0, 0, 1796, 164, 286] payload_rows_per_dst=[1578, 3386, 5580, 13220, 12756, 12820, 10642, 15634] dup_rows_per_dst=[0, 710, 1338, 6518, 4330, 4120, 2002, 3670] combine_rows_per_src=[1578, 3386, 5580, 13220, 12756, 12820, 10642, 15634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298172416 dispatch_meta_bytes=352064 combine_payload_bytes=298172416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=298524480 ingress_bytes=332775424 link_bytes=332775424 token_bytes=596696896 weight_movement_bytes=34603008 total_nvlink_bytes=631299904 weight_to_token_ratio=0.057991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252207130e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286744576 dispatch_meta_bytes=348192 combine_payload_bytes=286744576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287092768 ingress_bytes=390553600 link_bytes=390553600 token_bytes=573837344 weight_movement_bytes=103809024 total_nvlink_bytes=677646368 weight_to_token_ratio=0.180903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469621755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78146 payload_rows_remote=63902 dup_rows=20158 dup_rows_remote=12806 dedup_ratio=0.205058 router_tokens_per_expert=[2560, 5078, 14, 276, 703, 104, 64, 206, 5, 459, 178, 390, 60, 325, 410, 1069, 390, 726, 1, 1, 492, 114, 1806, 924, 2032, 3630, 1345, 343, 256, 28, 105, 499, 0, 265, 1541, 1, 67, 2217, 748, 109, 653, 81, 1, 89, 3141, 6399, 433, 1, 4504, 2, 666, 482, 482, 85, 341, 86, 205, 421, 1, 0, 0, 1120, 120, 298] payload_rows_per_dst=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] dup_rows_per_dst=[76, 750, 2944, 2442, 3240, 7352, 1424, 1930] combine_rows_per_src=[2658, 5042, 9462, 10536, 11776, 14244, 11872, 12556] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=290873344 dispatch_meta_bytes=341880 combine_payload_bytes=290873344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291215224 ingress_bytes=325476352 link_bytes=325476352 token_bytes=582088568 weight_movement_bytes=34603008 total_nvlink_bytes=616691576 weight_to_token_ratio=0.059446 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.224741310e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=337176 combine_payload_bytes=264011776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264348952 ingress_bytes=298614784 link_bytes=298614784 token_bytes=528360728 weight_movement_bytes=34603008 total_nvlink_bytes=562963736 weight_to_token_ratio=0.065491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123663392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272023552 dispatch_meta_bytes=341424 combine_payload_bytes=272023552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272364976 ingress_bytes=306626560 link_bytes=306626560 token_bytes=544388528 weight_movement_bytes=34603008 total_nvlink_bytes=578991536 weight_to_token_ratio=0.063563 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153811060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268394496 dispatch_meta_bytes=301256 combine_payload_bytes=268394496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268695752 ingress_bytes=302997504 link_bytes=302997504 token_bytes=537090248 weight_movement_bytes=34603008 total_nvlink_bytes=571693256 weight_to_token_ratio=0.064427 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140155214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77944 payload_rows_remote=66412 dup_rows=20360 dup_rows_remote=18944 dedup_ratio=0.207113 router_tokens_per_expert=[2902, 6218, 62, 253, 613, 67, 75, 84, 10, 412, 186, 366, 89, 353, 463, 1066, 424, 969, 45, 6, 446, 104, 1643, 972, 2452, 3136, 1378, 309, 45, 22, 122, 423, 7, 325, 1212, 2, 79, 2002, 822, 54, 660, 104, 8, 70, 2692, 6161, 355, 31, 4279, 43, 721, 457, 477, 89, 345, 63, 192, 399, 20, 9, 15, 1218, 185, 341] payload_rows_per_dst=[2254, 5130, 7556, 11388, 11758, 13866, 11532, 14460] dup_rows_per_dst=[54, 760, 1662, 4386, 3052, 6296, 1416, 2734] combine_rows_per_src=[2254, 5130, 7556, 11388, 11758, 13866, 11532, 14460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=286744576 dispatch_meta_bytes=348192 combine_payload_bytes=286744576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=287092768 ingress_bytes=390553600 link_bytes=390553600 token_bytes=573837344 weight_movement_bytes=103809024 total_nvlink_bytes=677646368 weight_to_token_ratio=0.180903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.469621755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81156 payload_rows_remote=67336 dup_rows=17148 dup_rows_remote=14482 dedup_ratio=0.174438 router_tokens_per_expert=[3676, 6872, 0, 180, 542, 0, 58, 31, 0, 363, 58, 234, 139, 301, 563, 443, 373, 1140, 0, 0, 312, 85, 387, 447, 4232, 2731, 1254, 235, 7, 0, 64, 536, 0, 250, 636, 0, 92, 3669, 868, 132, 725, 73, 0, 73, 2758, 6142, 128, 0, 4271, 0, 277, 477, 126, 42, 133, 0, 34, 327, 0, 0, 0, 2120, 192, 344] payload_rows_per_dst=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] dup_rows_per_dst=[7842, 2666, 2886, 1234, 1116, 434, 908, 62] combine_rows_per_src=[15556, 13820, 11066, 7740, 10178, 7080, 9744, 5972] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=245686272 dispatch_meta_bytes=316000 combine_payload_bytes=245686272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=246002272 ingress_bytes=280289280 link_bytes=280289280 token_bytes=491688544 weight_movement_bytes=34603008 total_nvlink_bytes=526291552 weight_to_token_ratio=0.070376 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.054705996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81722 payload_rows_remote=67680 dup_rows=16582 dup_rows_remote=12950 dedup_ratio=0.168681 router_tokens_per_expert=[589, 330, 3014, 3949, 645, 0, 5984, 2804, 0, 184, 0, 463, 0, 2209, 0, 120, 307, 339, 1654, 102, 884, 60, 352, 134, 58, 92, 1064, 0, 0, 150, 0, 738, 5861, 1524, 0, 92, 405, 3221, 674, 0, 395, 0, 17, 0, 0, 0, 0, 390, 1867, 0, 0, 216, 0, 1048, 292, 0, 718, 0, 120, 3234, 2804, 49, 0, 0] payload_rows_per_dst=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] dup_rows_per_dst=[3552, 3632, 1124, 1422, 2716, 486, 536, 3114] combine_rows_per_src=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=261742592 dispatch_meta_bytes=306832 combine_payload_bytes=261742592 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=262049424 ingress_bytes=365551616 link_bytes=365551616 token_bytes=523792016 weight_movement_bytes=103809024 total_nvlink_bytes=627601040 weight_to_token_ratio=0.198187 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.375541302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80074 payload_rows_remote=69490 dup_rows=18230 dup_rows_remote=13958 dedup_ratio=0.185445 router_tokens_per_expert=[579, 459, 3259, 3383, 1208, 0, 5564, 2833, 0, 184, 0, 497, 0, 2281, 0, 172, 322, 290, 1847, 131, 890, 58, 361, 145, 69, 92, 1082, 0, 0, 161, 0, 568, 5923, 1278, 0, 92, 355, 2531, 599, 0, 427, 0, 11, 0, 0, 0, 0, 493, 2020, 0, 1, 340, 0, 1500, 673, 0, 1140, 0, 41, 3054, 2233, 6, 0, 0] payload_rows_per_dst=[7928, 10476, 11134, 9412, 10584, 12468, 7846, 10226] dup_rows_per_dst=[2230, 2492, 3472, 1298, 4272, 522, 1222, 2722] combine_rows_per_src=[7928, 10476, 11134, 9412, 10584, 12468, 7846, 10226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79406 payload_rows_remote=70194 dup_rows=18898 dup_rows_remote=17144 dedup_ratio=0.192240 router_tokens_per_expert=[547, 586, 3404, 2884, 496, 241, 4810, 2100, 166, 226, 109, 505, 521, 1671, 116, 520, 292, 391, 1238, 133, 833, 58, 754, 293, 79, 240, 1216, 82, 46, 206, 367, 363, 4507, 1369, 88, 227, 230, 1935, 711, 148, 480, 66, 709, 248, 109, 432, 644, 528, 1073, 32, 146, 204, 731, 1407, 598, 382, 902, 455, 101, 2378, 2237, 26, 475, 81] payload_rows_per_dst=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] dup_rows_per_dst=[1306, 3510, 3258, 1754, 2136, 2672, 1520, 2742] combine_rows_per_src=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80660 payload_rows_remote=73000 dup_rows=17644 dup_rows_remote=16552 dedup_ratio=0.179484 router_tokens_per_expert=[548, 523, 3728, 3931, 1358, 0, 5942, 4296, 0, 124, 0, 377, 0, 1699, 0, 101, 314, 420, 2160, 93, 835, 52, 400, 102, 22, 62, 1299, 0, 0, 99, 0, 601, 5929, 1126, 0, 62, 187, 1489, 601, 0, 368, 0, 37, 0, 0, 0, 0, 376, 2148, 0, 12, 558, 2, 1508, 165, 0, 1554, 146, 70, 2294, 1394, 40, 0, 0] payload_rows_per_dst=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] dup_rows_per_dst=[6958, 2560, 1092, 1268, 2164, 576, 1112, 1914] combine_rows_per_src=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83624 payload_rows_remote=71272 dup_rows=14680 dup_rows_remote=10202 dedup_ratio=0.149333 router_tokens_per_expert=[1, 6979, 1043, 19, 1, 68, 709, 4464, 1, 5, 0, 0, 0, 708, 2816, 142, 98, 89, 60, 328, 71, 1, 212, 770, 1512, 346, 0, 468, 253, 63, 197, 2, 73, 7052, 76, 1, 2509, 0, 983, 0, 1240, 0, 26, 0, 2431, 253, 0, 1, 66, 1, 106, 586, 127, 570, 67, 0, 11, 414, 4808, 692, 341, 3057, 1, 2234] payload_rows_per_dst=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] dup_rows_per_dst=[52, 1898, 1666, 2346, 634, 4478, 2012, 1594] combine_rows_per_src=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272023552 dispatch_meta_bytes=341424 combine_payload_bytes=272023552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272364976 ingress_bytes=306626560 link_bytes=306626560 token_bytes=544388528 weight_movement_bytes=34603008 total_nvlink_bytes=578991536 weight_to_token_ratio=0.063563 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=34048 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.153811060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78512 payload_rows_remote=65092 dup_rows=19792 dup_rows_remote=14172 dedup_ratio=0.201335 router_tokens_per_expert=[592, 551, 3661, 3858, 1323, 15, 5735, 3668, 14, 158, 30, 426, 51, 1675, 4, 230, 328, 344, 1926, 119, 894, 55, 458, 145, 69, 96, 1221, 3, 1, 155, 53, 576, 5685, 1168, 3, 82, 205, 2170, 557, 24, 425, 7, 111, 30, 0, 41, 48, 427, 1695, 7, 21, 470, 25, 1328, 259, 30, 1473, 224, 97, 2275, 1787, 11, 24, 9] payload_rows_per_dst=[4360, 4736, 12106, 10560, 10962, 12904, 13420, 9464] dup_rows_per_dst=[602, 440, 3754, 1504, 4792, 744, 5620, 2336] combine_rows_per_src=[4360, 4736, 12106, 10560, 10962, 12904, 13420, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=60796 dup_rows=21526 dup_rows_remote=8510 dedup_ratio=0.218974 router_tokens_per_expert=[4211, 7027, 5, 139, 568, 0, 77, 5, 0, 203, 78, 220, 121, 347, 626, 525, 437, 1285, 4, 0, 201, 107, 520, 652, 4424, 2467, 1810, 231, 4, 0, 130, 445, 0, 398, 441, 0, 92, 3068, 965, 22, 687, 22, 1, 54, 1675, 6341, 91, 0, 4203, 4, 312, 525, 154, 84, 33, 4, 136, 302, 2, 0, 0, 2008, 359, 300] payload_rows_per_dst=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] dup_rows_per_dst=[13016, 2758, 3236, 150, 1108, 286, 884, 88] combine_rows_per_src=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277217280 dispatch_meta_bytes=322520 combine_payload_bytes=277217280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277539800 ingress_bytes=311820288 link_bytes=311820288 token_bytes=554757080 weight_movement_bytes=34603008 total_nvlink_bytes=589360088 weight_to_token_ratio=0.062375 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173354640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275808256 dispatch_meta_bytes=327272 combine_payload_bytes=275808256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=276135528 ingress_bytes=310411264 link_bytes=310411264 token_bytes=551943784 weight_movement_bytes=34603008 total_nvlink_bytes=586546792 weight_to_token_ratio=0.062693 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.168052596e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80430 payload_rows_remote=65526 dup_rows=17874 dup_rows_remote=9788 dedup_ratio=0.181824 router_tokens_per_expert=[767, 115, 0, 3961, 92, 0, 188, 100, 0, 484, 515, 424, 878, 629, 396, 503, 0, 0, 757, 0, 1, 596, 440, 1398, 130, 53, 372, 2, 118, 421, 0, 2, 100, 1384, 90, 22, 0, 3, 2, 3973, 1307, 171, 184, 2540, 5440, 127, 2271, 3493, 337, 95, 1, 1283, 331, 1250, 6272, 1, 4650, 0, 0, 6, 4, 1, 395, 77] payload_rows_per_dst=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] dup_rows_per_dst=[8086, 1516, 2968, 542, 3732, 678, 172, 180] combine_rows_per_src=[14904, 6142, 10402, 12534, 12496, 7442, 6424, 10086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284631040 dispatch_meta_bytes=333792 combine_payload_bytes=284631040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284964832 ingress_bytes=388440064 link_bytes=388440064 token_bytes=569595872 weight_movement_bytes=103809024 total_nvlink_bytes=673404896 weight_to_token_ratio=0.182250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.461668689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80660 payload_rows_remote=73000 dup_rows=17644 dup_rows_remote=16552 dedup_ratio=0.179484 router_tokens_per_expert=[548, 523, 3728, 3931, 1358, 0, 5942, 4296, 0, 124, 0, 377, 0, 1699, 0, 101, 314, 420, 2160, 93, 835, 52, 400, 102, 22, 62, 1299, 0, 0, 99, 0, 601, 5929, 1126, 0, 62, 187, 1489, 601, 0, 368, 0, 37, 0, 0, 0, 0, 376, 2148, 0, 12, 558, 2, 1508, 165, 0, 1554, 146, 70, 2294, 1394, 40, 0, 0] payload_rows_per_dst=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] dup_rows_per_dst=[6958, 2560, 1092, 1268, 2164, 576, 1112, 1914] combine_rows_per_src=[13948, 13900, 7660, 10760, 4766, 12870, 7674, 9082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79406 payload_rows_remote=70194 dup_rows=18898 dup_rows_remote=17144 dedup_ratio=0.192240 router_tokens_per_expert=[547, 586, 3404, 2884, 496, 241, 4810, 2100, 166, 226, 109, 505, 521, 1671, 116, 520, 292, 391, 1238, 133, 833, 58, 754, 293, 79, 240, 1216, 82, 46, 206, 367, 363, 4507, 1369, 88, 227, 230, 1935, 711, 148, 480, 66, 709, 248, 109, 432, 644, 528, 1073, 32, 146, 204, 731, 1407, 598, 382, 902, 455, 101, 2378, 2237, 26, 475, 81] payload_rows_per_dst=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] dup_rows_per_dst=[1306, 3510, 3258, 1754, 2136, 2672, 1520, 2742] combine_rows_per_src=[6778, 13172, 11390, 9212, 7280, 13380, 7626, 10568] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=268394496 dispatch_meta_bytes=301256 combine_payload_bytes=268394496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268695752 ingress_bytes=302997504 link_bytes=302997504 token_bytes=537090248 weight_movement_bytes=34603008 total_nvlink_bytes=571693256 weight_to_token_ratio=0.064427 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140155214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78126 payload_rows_remote=67888 dup_rows=20178 dup_rows_remote=17394 dedup_ratio=0.205261 router_tokens_per_expert=[576, 331, 2962, 3534, 968, 0, 5920, 2782, 0, 172, 0, 413, 0, 2239, 0, 169, 317, 323, 1690, 102, 769, 44, 387, 135, 69, 86, 1009, 0, 0, 155, 0, 666, 5979, 1307, 0, 87, 447, 3430, 578, 0, 415, 0, 19, 0, 0, 0, 0, 408, 1992, 0, 1, 229, 0, 1407, 524, 0, 940, 0, 37, 3049, 2482, 3, 0, 0] payload_rows_per_dst=[3212, 5614, 10274, 9814, 11720, 12996, 14258, 10238] dup_rows_per_dst=[538, 372, 3184, 1224, 5542, 528, 6006, 2784] combine_rows_per_src=[3212, 5614, 10274, 9814, 11720, 12996, 14258, 10238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=287514624 dispatch_meta_bytes=349352 combine_payload_bytes=287514624 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287863976 ingress_bytes=356720640 link_bytes=356720640 token_bytes=575378600 weight_movement_bytes=69206016 total_nvlink_bytes=644584616 weight_to_token_ratio=0.120279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342311050e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=284631040 dispatch_meta_bytes=333792 combine_payload_bytes=284631040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=284964832 ingress_bytes=388440064 link_bytes=388440064 token_bytes=569595872 weight_movement_bytes=103809024 total_nvlink_bytes=673404896 weight_to_token_ratio=0.182250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.461668689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77790 payload_rows_remote=64456 dup_rows=20514 dup_rows_remote=19838 dedup_ratio=0.208679 router_tokens_per_expert=[577, 507, 3518, 3903, 1201, 0, 6037, 4050, 5, 134, 0, 381, 1, 2018, 0, 181, 323, 343, 1883, 121, 967, 54, 468, 149, 62, 67, 1443, 0, 4, 130, 1, 542, 5729, 1065, 0, 67, 202, 1474, 542, 1, 394, 0, 81, 0, 12, 1, 100, 380, 2005, 1, 11, 332, 12, 1908, 174, 0, 1497, 30, 72, 2456, 1505, 27, 4, 0] payload_rows_per_dst=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] dup_rows_per_dst=[1094, 304, 3844, 4976, 2556, 676, 4786, 2278] combine_rows_per_src=[5054, 5136, 11808, 13264, 7776, 13334, 12514, 8904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=299008000 dispatch_meta_bytes=358208 combine_payload_bytes=299008000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299366208 ingress_bytes=333611008 link_bytes=333611008 token_bytes=598374208 weight_movement_bytes=34603008 total_nvlink_bytes=632977216 weight_to_token_ratio=0.057828 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255351365e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78512 payload_rows_remote=65092 dup_rows=19792 dup_rows_remote=14172 dedup_ratio=0.201335 router_tokens_per_expert=[592, 551, 3661, 3858, 1323, 15, 5735, 3668, 14, 158, 30, 426, 51, 1675, 4, 230, 328, 344, 1926, 119, 894, 55, 458, 145, 69, 96, 1221, 3, 1, 155, 53, 576, 5685, 1168, 3, 82, 205, 2170, 557, 24, 425, 7, 111, 30, 0, 41, 48, 427, 1695, 7, 21, 470, 25, 1328, 259, 30, 1473, 224, 97, 2275, 1787, 11, 24, 9] payload_rows_per_dst=[4360, 4736, 12106, 10560, 10962, 12904, 13420, 9464] dup_rows_per_dst=[602, 440, 3754, 1504, 4792, 744, 5620, 2336] combine_rows_per_src=[4360, 4736, 12106, 10560, 10962, 12904, 13420, 9464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291930112 dispatch_meta_bytes=325896 combine_payload_bytes=291930112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292256008 ingress_bytes=326533120 link_bytes=326533120 token_bytes=584186120 weight_movement_bytes=34603008 total_nvlink_bytes=618789128 weight_to_token_ratio=0.059233 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228717842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81722 payload_rows_remote=67680 dup_rows=16582 dup_rows_remote=12950 dedup_ratio=0.168681 router_tokens_per_expert=[589, 330, 3014, 3949, 645, 0, 5984, 2804, 0, 184, 0, 463, 0, 2209, 0, 120, 307, 339, 1654, 102, 884, 60, 352, 134, 58, 92, 1064, 0, 0, 150, 0, 738, 5861, 1524, 0, 92, 405, 3221, 674, 0, 395, 0, 17, 0, 0, 0, 0, 390, 1867, 0, 0, 216, 0, 1048, 292, 0, 718, 0, 120, 3234, 2804, 49, 0, 0] payload_rows_per_dst=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] dup_rows_per_dst=[3552, 3632, 1124, 1422, 2716, 486, 536, 3114] combine_rows_per_src=[11922, 14042, 6540, 10680, 9116, 12376, 6310, 10736] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=287514624 dispatch_meta_bytes=349352 combine_payload_bytes=287514624 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=287863976 ingress_bytes=356720640 link_bytes=356720640 token_bytes=575378600 weight_movement_bytes=69206016 total_nvlink_bytes=644584616 weight_to_token_ratio=0.120279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.342311050e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278069248 dispatch_meta_bytes=341128 combine_payload_bytes=278069248 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=278410376 ingress_bytes=312672256 link_bytes=312672256 token_bytes=556479624 weight_movement_bytes=34603008 total_nvlink_bytes=591082632 weight_to_token_ratio=0.062182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176560527e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84958 payload_rows_remote=76372 dup_rows=13346 dup_rows_remote=12940 dedup_ratio=0.135763 router_tokens_per_expert=[0, 7427, 878, 2, 0, 69, 664, 4105, 0, 6, 0, 0, 0, 842, 3077, 163, 0, 126, 148, 290, 82, 0, 170, 574, 1452, 337, 0, 352, 228, 71, 345, 0, 123, 7136, 92, 0, 3686, 0, 595, 0, 1272, 0, 0, 0, 2619, 252, 0, 0, 92, 0, 204, 501, 14, 437, 90, 0, 0, 373, 4353, 1561, 265, 2726, 0, 1353] payload_rows_per_dst=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] dup_rows_per_dst=[26, 2046, 1602, 2720, 406, 4392, 1710, 444] combine_rows_per_src=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264011776 dispatch_meta_bytes=337176 combine_payload_bytes=264011776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264348952 ingress_bytes=298614784 link_bytes=298614784 token_bytes=528360728 weight_movement_bytes=34603008 total_nvlink_bytes=562963736 weight_to_token_ratio=0.065491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123663392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=317056 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266933888 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533550720 weight_movement_bytes=34603008 total_nvlink_bytes=568153728 weight_to_token_ratio=0.064854 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=277217280 dispatch_meta_bytes=322520 combine_payload_bytes=277217280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277539800 ingress_bytes=311820288 link_bytes=311820288 token_bytes=554757080 weight_movement_bytes=34603008 total_nvlink_bytes=589360088 weight_to_token_ratio=0.062375 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.173354640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76778 payload_rows_remote=60796 dup_rows=21526 dup_rows_remote=8510 dedup_ratio=0.218974 router_tokens_per_expert=[4211, 7027, 5, 139, 568, 0, 77, 5, 0, 203, 78, 220, 121, 347, 626, 525, 437, 1285, 4, 0, 201, 107, 520, 652, 4424, 2467, 1810, 231, 4, 0, 130, 445, 0, 398, 441, 0, 92, 3068, 965, 22, 687, 22, 1, 54, 1675, 6341, 91, 0, 4203, 4, 312, 525, 154, 84, 33, 4, 136, 302, 2, 0, 0, 2008, 359, 300] payload_rows_per_dst=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] dup_rows_per_dst=[13016, 2758, 3236, 150, 1108, 286, 884, 88] combine_rows_per_src=[15982, 14164, 12024, 5090, 8864, 4774, 9754, 6126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=357248 combine_payload_bytes=312819712 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=313176960 ingress_bytes=416628736 link_bytes=416628736 token_bytes=625996672 weight_movement_bytes=103809024 total_nvlink_bytes=729805696 weight_to_token_ratio=0.165830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77270 payload_rows_remote=71170 dup_rows=21034 dup_rows_remote=20470 dedup_ratio=0.213969 router_tokens_per_expert=[0, 7196, 874, 0, 0, 37, 865, 4889, 0, 0, 0, 0, 0, 829, 2560, 107, 28, 77, 60, 309, 52, 0, 169, 824, 1166, 312, 0, 394, 252, 37, 180, 0, 82, 7353, 71, 0, 2108, 0, 958, 0, 1564, 0, 0, 0, 2651, 254, 0, 0, 62, 0, 145, 618, 74, 495, 62, 0, 4, 400, 4831, 550, 221, 3505, 0, 1927] payload_rows_per_dst=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] dup_rows_per_dst=[9492, 1864, 564, 2574, 4328, 908, 906, 398] combine_rows_per_src=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=249020416 dispatch_meta_bytes=277224 combine_payload_bytes=249020416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=249297640 ingress_bytes=352829440 link_bytes=352829440 token_bytes=498318056 weight_movement_bytes=103809024 total_nvlink_bytes=602127080 weight_to_token_ratio=0.208319 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.327668778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=317056 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266933888 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533550720 weight_movement_bytes=34603008 total_nvlink_bytes=568153728 weight_to_token_ratio=0.064854 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=278069248 dispatch_meta_bytes=341128 combine_payload_bytes=278069248 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=278410376 ingress_bytes=312672256 link_bytes=312672256 token_bytes=556479624 weight_movement_bytes=34603008 total_nvlink_bytes=591082632 weight_to_token_ratio=0.062182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=20480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.176560527e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73722 payload_rows_remote=63632 dup_rows=24582 dup_rows_remote=23052 dedup_ratio=0.250061 router_tokens_per_expert=[0, 7428, 1031, 3, 0, 58, 717, 3219, 0, 26, 0, 0, 0, 1052, 2808, 156, 0, 148, 143, 252, 76, 0, 221, 732, 2893, 273, 0, 246, 224, 69, 311, 0, 302, 7095, 93, 0, 3840, 0, 173, 0, 704, 0, 0, 0, 2523, 226, 0, 0, 92, 0, 212, 467, 24, 527, 92, 0, 0, 351, 4139, 2283, 386, 1768, 0, 1769] payload_rows_per_dst=[15542, 10090, 2844, 12546, 15218, 6492, 2452, 8538] dup_rows_per_dst=[9370, 1530, 300, 3764, 7788, 414, 376, 1040] combine_rows_per_src=[15542, 10090, 2844, 12546, 15218, 6492, 2452, 8538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82140 payload_rows_remote=71104 dup_rows=16164 dup_rows_remote=13074 dedup_ratio=0.164429 router_tokens_per_expert=[147, 5736, 2572, 202, 15, 133, 1099, 1688, 12, 757, 225, 58, 102, 1257, 2735, 180, 290, 120, 285, 319, 171, 239, 335, 783, 1366, 370, 274, 573, 251, 103, 385, 57, 392, 5296, 170, 172, 2242, 453, 608, 13, 364, 36, 671, 161, 1889, 329, 65, 556, 98, 524, 193, 486, 93, 596, 395, 708, 125, 408, 3684, 922, 494, 2107, 280, 1783] payload_rows_per_dst=[8006, 11704, 14332, 11036, 7460, 8042, 14286, 7274] dup_rows_per_dst=[1768, 3162, 2224, 3090, 640, 2038, 2492, 750] combine_rows_per_src=[8006, 11704, 14332, 11036, 7460, 8042, 14286, 7274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84958 payload_rows_remote=76372 dup_rows=13346 dup_rows_remote=12940 dedup_ratio=0.135763 router_tokens_per_expert=[0, 7427, 878, 2, 0, 69, 664, 4105, 0, 6, 0, 0, 0, 842, 3077, 163, 0, 126, 148, 290, 82, 0, 170, 574, 1452, 337, 0, 352, 228, 71, 345, 0, 123, 7136, 92, 0, 3686, 0, 595, 0, 1272, 0, 0, 0, 2619, 252, 0, 0, 92, 0, 204, 501, 14, 437, 90, 0, 0, 373, 4353, 1561, 265, 2726, 0, 1353] payload_rows_per_dst=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] dup_rows_per_dst=[26, 2046, 1602, 2720, 406, 4392, 1710, 444] combine_rows_per_src=[3200, 11582, 16032, 11556, 8586, 12104, 15238, 6660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260636672 dispatch_meta_bytes=346736 combine_payload_bytes=260636672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260983408 ingress_bytes=295239680 link_bytes=295239680 token_bytes=521620080 weight_movement_bytes=34603008 total_nvlink_bytes=556223088 weight_to_token_ratio=0.066338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110963147e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77270 payload_rows_remote=71170 dup_rows=21034 dup_rows_remote=20470 dedup_ratio=0.213969 router_tokens_per_expert=[0, 7196, 874, 0, 0, 37, 865, 4889, 0, 0, 0, 0, 0, 829, 2560, 107, 28, 77, 60, 309, 52, 0, 169, 824, 1166, 312, 0, 394, 252, 37, 180, 0, 82, 7353, 71, 0, 2108, 0, 958, 0, 1564, 0, 0, 0, 2651, 254, 0, 0, 62, 0, 145, 618, 74, 495, 62, 0, 4, 400, 4831, 550, 221, 3505, 0, 1927] payload_rows_per_dst=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] dup_rows_per_dst=[9492, 1864, 564, 2574, 4328, 908, 906, 398] combine_rows_per_src=[14604, 12138, 6100, 11770, 12102, 8030, 6720, 5806] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85044 payload_rows_remote=75556 dup_rows=13260 dup_rows_remote=11520 dedup_ratio=0.134888 router_tokens_per_expert=[2, 0, 0, 297, 0, 0, 0, 1, 390, 147, 1515, 1, 529, 320, 1, 79, 68, 0, 14, 0, 0, 71, 2, 0, 1, 3414, 0, 214, 281, 1329, 3, 3, 1189, 3580, 0, 0, 0, 1083, 0, 1393, 2102, 1290, 6064, 0, 633, 632, 957, 0, 0, 119, 1, 2503, 252, 0, 82, 0, 1620, 1, 4532, 5584, 4181, 2538, 0, 134] payload_rows_per_dst=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] dup_rows_per_dst=[4, 988, 18, 1032, 2756, 1740, 2656, 4066] combine_rows_per_src=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=64300 dup_rows=19206 dup_rows_remote=8188 dedup_ratio=0.195374 router_tokens_per_expert=[0, 7433, 857, 0, 0, 68, 706, 3751, 0, 10, 0, 0, 0, 956, 3205, 155, 0, 109, 74, 313, 75, 0, 138, 529, 2040, 330, 0, 368, 235, 68, 252, 0, 202, 7181, 87, 0, 3821, 0, 367, 0, 1360, 0, 0, 0, 2208, 251, 0, 0, 87, 0, 160, 416, 17, 455, 84, 0, 0, 388, 4369, 1821, 278, 2557, 0, 1371] payload_rows_per_dst=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] dup_rows_per_dst=[20, 460, 1290, 1060, 114, 3640, 1604, 11018] combine_rows_per_src=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83624 payload_rows_remote=71272 dup_rows=14680 dup_rows_remote=10202 dedup_ratio=0.149333 router_tokens_per_expert=[1, 6979, 1043, 19, 1, 68, 709, 4464, 1, 5, 0, 0, 0, 708, 2816, 142, 98, 89, 60, 328, 71, 1, 212, 770, 1512, 346, 0, 468, 253, 63, 197, 2, 73, 7052, 76, 1, 2509, 0, 983, 0, 1240, 0, 26, 0, 2431, 253, 0, 1, 66, 1, 106, 586, 127, 570, 67, 0, 11, 414, 4808, 692, 341, 3057, 1, 2234] payload_rows_per_dst=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] dup_rows_per_dst=[52, 1898, 1666, 2346, 634, 4478, 2012, 1594] combine_rows_per_src=[3630, 11560, 15550, 9956, 6650, 12352, 15138, 8788] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72268 payload_rows_remote=57110 dup_rows=26036 dup_rows_remote=12372 dedup_ratio=0.264852 router_tokens_per_expert=[573, 339, 2806, 3504, 864, 1, 6039, 2116, 3, 182, 1, 459, 3, 2344, 0, 142, 307, 348, 1752, 90, 815, 29, 338, 153, 77, 90, 974, 0, 0, 169, 1, 763, 5825, 1323, 1, 92, 420, 3483, 642, 0, 438, 0, 12, 3, 1, 2, 3, 427, 1704, 1, 0, 121, 2, 1533, 563, 3, 938, 2, 31, 3303, 2968, 25, 4, 0] payload_rows_per_dst=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] dup_rows_per_dst=[13664, 3870, 1158, 1330, 2534, 502, 764, 2214] combine_rows_per_src=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=366560 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291878880 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583391200 weight_movement_bytes=34603008 total_nvlink_bytes=617994208 weight_to_token_ratio=0.059314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79896 payload_rows_remote=64646 dup_rows=18408 dup_rows_remote=16406 dedup_ratio=0.187256 router_tokens_per_expert=[11, 7013, 1010, 6, 6, 66, 802, 4255, 0, 83, 25, 0, 6, 806, 2763, 161, 81, 132, 86, 363, 86, 40, 198, 726, 1387, 346, 14, 494, 262, 66, 253, 8, 94, 7081, 88, 29, 2624, 22, 758, 2, 1280, 6, 105, 17, 2429, 269, 9, 67, 79, 45, 145, 546, 24, 555, 92, 59, 0, 383, 4615, 892, 287, 3092, 7, 1896] payload_rows_per_dst=[3738, 7686, 15746, 4538, 6716, 12268, 15250, 13954] dup_rows_per_dst=[64, 226, 1704, 1122, 518, 4606, 2002, 8166] combine_rows_per_src=[3738, 7686, 15746, 4538, 6716, 12268, 15250, 13954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79098 payload_rows_remote=64300 dup_rows=19206 dup_rows_remote=8188 dedup_ratio=0.195374 router_tokens_per_expert=[0, 7433, 857, 0, 0, 68, 706, 3751, 0, 10, 0, 0, 0, 956, 3205, 155, 0, 109, 74, 313, 75, 0, 138, 529, 2040, 330, 0, 368, 235, 68, 252, 0, 202, 7181, 87, 0, 3821, 0, 367, 0, 1360, 0, 0, 0, 2208, 251, 0, 0, 87, 0, 160, 416, 17, 455, 84, 0, 0, 388, 4369, 1821, 278, 2557, 0, 1371] payload_rows_per_dst=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] dup_rows_per_dst=[20, 460, 1290, 1060, 114, 3640, 1604, 11018] combine_rows_per_src=[3242, 8192, 16052, 5526, 4592, 11500, 15196, 14798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73722 payload_rows_remote=63632 dup_rows=24582 dup_rows_remote=23052 dedup_ratio=0.250061 router_tokens_per_expert=[0, 7428, 1031, 3, 0, 58, 717, 3219, 0, 26, 0, 0, 0, 1052, 2808, 156, 0, 148, 143, 252, 76, 0, 221, 732, 2893, 273, 0, 246, 224, 69, 311, 0, 302, 7095, 93, 0, 3840, 0, 173, 0, 704, 0, 0, 0, 2523, 226, 0, 0, 92, 0, 212, 467, 24, 527, 92, 0, 0, 351, 4139, 2283, 386, 1768, 0, 1769] payload_rows_per_dst=[15542, 10090, 2844, 12546, 15218, 6492, 2452, 8538] dup_rows_per_dst=[9370, 1530, 300, 3764, 7788, 414, 376, 1040] combine_rows_per_src=[15542, 10090, 2844, 12546, 15218, 6492, 2452, 8538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291241984 dispatch_meta_bytes=336712 combine_payload_bytes=291241984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291578696 ingress_bytes=325844992 link_bytes=325844992 token_bytes=582820680 weight_movement_bytes=34603008 total_nvlink_bytes=617423688 weight_to_token_ratio=0.059372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24704 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226128472e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=312819712 dispatch_meta_bytes=357248 combine_payload_bytes=312819712 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=313176960 ingress_bytes=416628736 link_bytes=416628736 token_bytes=625996672 weight_movement_bytes=103809024 total_nvlink_bytes=729805696 weight_to_token_ratio=0.165830 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.567740392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 2, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84828 payload_rows_remote=74976 dup_rows=13476 dup_rows_remote=11648 dedup_ratio=0.137085 router_tokens_per_expert=[0, 0, 0, 193, 0, 1, 0, 0, 404, 153, 1920, 0, 572, 344, 0, 92, 67, 0, 0, 0, 0, 83, 0, 0, 0, 3940, 0, 334, 261, 2051, 0, 0, 758, 4514, 0, 0, 0, 805, 0, 1317, 3430, 886, 4767, 0, 551, 1306, 1023, 0, 0, 152, 0, 1597, 61, 0, 67, 0, 1365, 0, 5641, 5664, 3377, 1273, 0, 183] payload_rows_per_dst=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] dup_rows_per_dst=[0, 3754, 0, 1938, 1828, 2916, 1844, 1196] combine_rows_per_src=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291512320 dispatch_meta_bytes=366560 combine_payload_bytes=291512320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291878880 ingress_bytes=326115328 link_bytes=326115328 token_bytes=583391200 weight_movement_bytes=34603008 total_nvlink_bytes=617994208 weight_to_token_ratio=0.059314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36480 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.227145725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=309477376 dispatch_meta_bytes=348304 combine_payload_bytes=309477376 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=309825680 ingress_bytes=447889408 link_bytes=447889408 token_bytes=619303056 weight_movement_bytes=138412032 total_nvlink_bytes=757715088 weight_to_token_ratio=0.223496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.685371784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=277928 combine_payload_bytes=233922560 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=234200488 ingress_bytes=441540608 link_bytes=441540608 token_bytes=468123048 weight_movement_bytes=207618048 total_nvlink_bytes=675741096 weight_to_token_ratio=0.443512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.661481761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264790016 dispatch_meta_bytes=324208 combine_payload_bytes=264790016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114224 ingress_bytes=299393024 link_bytes=299393024 token_bytes=529904240 weight_movement_bytes=34603008 total_nvlink_bytes=564507248 weight_to_token_ratio=0.065300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126591846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=263372800 dispatch_meta_bytes=289952 combine_payload_bytes=263372800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=263662752 ingress_bytes=367181824 link_bytes=367181824 token_bytes=527035552 weight_movement_bytes=103809024 total_nvlink_bytes=630844576 weight_to_token_ratio=0.196968 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.381675643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=68760 dup_rows=16874 dup_rows_remote=11706 dedup_ratio=0.171651 router_tokens_per_expert=[1, 0, 0, 193, 0, 0, 0, 0, 361, 130, 2501, 0, 572, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 0, 4430, 0, 519, 251, 1391, 2, 7, 893, 4945, 0, 0, 0, 759, 0, 1345, 4328, 803, 4141, 0, 544, 1679, 1223, 0, 0, 143, 0, 1748, 0, 0, 73, 0, 1375, 0, 4647, 5459, 2737, 1211, 0, 182] payload_rows_per_dst=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] dup_rows_per_dst=[1652, 5168, 2468, 1568, 866, 2342, 1896, 914] combine_rows_per_src=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79422 payload_rows_remote=68872 dup_rows=18882 dup_rows_remote=16268 dedup_ratio=0.192078 router_tokens_per_expert=[87, 160, 9, 245, 504, 85, 61, 688, 442, 297, 1778, 481, 757, 518, 380, 158, 50, 145, 5, 35, 539, 120, 1, 440, 44, 3067, 39, 487, 790, 1326, 315, 514, 773, 2875, 0, 215, 15, 945, 162, 1655, 3559, 1673, 3848, 397, 713, 968, 1598, 157, 10, 280, 663, 905, 328, 508, 172, 130, 1718, 421, 3335, 2983, 2218, 1222, 6, 133] payload_rows_per_dst=[10578, 11016, 8066, 10550, 9852, 12524, 10842, 5994] dup_rows_per_dst=[796, 4356, 570, 2614, 2114, 5606, 1820, 1006] combine_rows_per_src=[10578, 11016, 8066, 10550, 9852, 12524, 10842, 5994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=281640960 dispatch_meta_bytes=321864 combine_payload_bytes=281640960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281962824 ingress_bytes=316243968 link_bytes=316243968 token_bytes=563603784 weight_movement_bytes=34603008 total_nvlink_bytes=598206792 weight_to_token_ratio=0.061396 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190000592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84828 payload_rows_remote=74976 dup_rows=13476 dup_rows_remote=11648 dedup_ratio=0.137085 router_tokens_per_expert=[0, 0, 0, 193, 0, 1, 0, 0, 404, 153, 1920, 0, 572, 344, 0, 92, 67, 0, 0, 0, 0, 83, 0, 0, 0, 3940, 0, 334, 261, 2051, 0, 0, 758, 4514, 0, 0, 0, 805, 0, 1317, 3430, 886, 4767, 0, 551, 1306, 1023, 0, 0, 152, 0, 1597, 61, 0, 67, 0, 1365, 0, 5641, 5664, 3377, 1273, 0, 183] payload_rows_per_dst=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] dup_rows_per_dst=[0, 3754, 0, 1938, 1828, 2916, 1844, 1196] combine_rows_per_src=[9922, 11460, 11628, 11234, 9852, 11476, 13192, 6064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87540 payload_rows_remote=75686 dup_rows=10764 dup_rows_remote=9428 dedup_ratio=0.109497 router_tokens_per_expert=[0, 63, 287, 97, 74, 1979, 0, 486, 0, 0, 39, 0, 67, 0, 0, 0, 2130, 0, 0, 0, 0, 1365, 0, 941, 0, 65, 17, 853, 0, 1, 0, 63, 5996, 2, 67, 664, 0, 1, 138, 1523, 210, 1, 0, 1349, 71, 1, 1, 1, 352, 14, 4069, 79, 4961, 63, 6304, 1, 54, 3559, 3884, 812, 3841, 2413, 1, 193] payload_rows_per_dst=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] dup_rows_per_dst=[1818, 76, 672, 298, 3306, 1336, 454, 2804] combine_rows_per_src=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84994 payload_rows_remote=73252 dup_rows=13310 dup_rows_remote=13310 dedup_ratio=0.135396 router_tokens_per_expert=[0, 0, 0, 286, 0, 0, 0, 0, 346, 117, 1707, 0, 619, 316, 0, 66, 59, 0, 0, 0, 0, 52, 0, 0, 0, 3523, 0, 121, 221, 1418, 0, 0, 1391, 3545, 0, 0, 0, 977, 0, 1464, 1739, 1075, 6247, 0, 584, 644, 890, 0, 0, 122, 0, 2617, 230, 0, 59, 0, 1226, 0, 4947, 5760, 4816, 1844, 0, 124] payload_rows_per_dst=[13066, 10690, 11742, 9584, 13202, 8426, 12860, 5424] dup_rows_per_dst=[0, 2742, 0, 982, 4094, 1438, 3090, 964] combine_rows_per_src=[13066, 10690, 11742, 9584, 13202, 8426, 12860, 5424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73512 payload_rows_remote=58002 dup_rows=24792 dup_rows_remote=14752 dedup_ratio=0.252197 router_tokens_per_expert=[0, 7405, 900, 8, 1, 78, 778, 3434, 0, 35, 3, 0, 0, 1051, 3568, 170, 0, 178, 118, 281, 97, 4, 106, 365, 1728, 337, 1, 276, 247, 79, 308, 0, 279, 7106, 93, 2, 3719, 0, 247, 1, 1293, 0, 9, 0, 1967, 247, 0, 4, 92, 4, 206, 520, 5, 534, 96, 4, 0, 366, 4374, 2235, 359, 2256, 0, 1578] payload_rows_per_dst=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] dup_rows_per_dst=[10040, 2416, 206, 3010, 7408, 552, 368, 792] combine_rows_per_src=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=291930112 dispatch_meta_bytes=325896 combine_payload_bytes=291930112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=292256008 ingress_bytes=326533120 link_bytes=326533120 token_bytes=584186120 weight_movement_bytes=34603008 total_nvlink_bytes=618789128 weight_to_token_ratio=0.059233 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18816 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.228717842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82126 payload_rows_remote=73834 dup_rows=16178 dup_rows_remote=15098 dedup_ratio=0.164571 router_tokens_per_expert=[0, 7, 0, 201, 32, 2, 0, 68, 419, 177, 1558, 43, 636, 346, 30, 80, 61, 14, 1, 0, 41, 82, 0, 8, 0, 3372, 8, 299, 273, 1621, 35, 66, 1090, 3938, 0, 17, 0, 926, 1, 1384, 2867, 1171, 5723, 33, 570, 941, 1212, 11, 1, 129, 66, 2171, 270, 51, 78, 24, 1393, 65, 4636, 5350, 3721, 1675, 0, 158] payload_rows_per_dst=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] dup_rows_per_dst=[44, 1110, 22, 1566, 2716, 2836, 1080, 6804] combine_rows_per_src=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264790016 dispatch_meta_bytes=324208 combine_payload_bytes=264790016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114224 ingress_bytes=299393024 link_bytes=299393024 token_bytes=529904240 weight_movement_bytes=34603008 total_nvlink_bytes=564507248 weight_to_token_ratio=0.065300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126591846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=260636672 dispatch_meta_bytes=346736 combine_payload_bytes=260636672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=260983408 ingress_bytes=295239680 link_bytes=295239680 token_bytes=521620080 weight_movement_bytes=34603008 total_nvlink_bytes=556223088 weight_to_token_ratio=0.066338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.110963147e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=72268 payload_rows_remote=57110 dup_rows=26036 dup_rows_remote=12372 dedup_ratio=0.264852 router_tokens_per_expert=[573, 339, 2806, 3504, 864, 1, 6039, 2116, 3, 182, 1, 459, 3, 2344, 0, 142, 307, 348, 1752, 90, 815, 29, 338, 153, 77, 90, 974, 0, 0, 169, 1, 763, 5825, 1323, 1, 92, 420, 3483, 642, 0, 438, 0, 12, 3, 1, 2, 3, 427, 1704, 1, 0, 121, 2, 1533, 563, 3, 938, 2, 31, 3303, 2968, 25, 4, 0] payload_rows_per_dst=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] dup_rows_per_dst=[13664, 3870, 1158, 1330, 2534, 502, 764, 2214] combine_rows_per_src=[15158, 14048, 6506, 9826, 9388, 1270, 7090, 8982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307101696 dispatch_meta_bytes=346496 combine_payload_bytes=307101696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=307448192 ingress_bytes=376307712 link_bytes=376307712 token_bytes=614549888 weight_movement_bytes=69206016 total_nvlink_bytes=683755904 weight_to_token_ratio=0.112613 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.416015625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 2, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84994 payload_rows_remote=73252 dup_rows=13310 dup_rows_remote=13310 dedup_ratio=0.135396 router_tokens_per_expert=[0, 0, 0, 286, 0, 0, 0, 0, 346, 117, 1707, 0, 619, 316, 0, 66, 59, 0, 0, 0, 0, 52, 0, 0, 0, 3523, 0, 121, 221, 1418, 0, 0, 1391, 3545, 0, 0, 0, 977, 0, 1464, 1739, 1075, 6247, 0, 584, 644, 890, 0, 0, 122, 0, 2617, 230, 0, 59, 0, 1226, 0, 4947, 5760, 4816, 1844, 0, 124] payload_rows_per_dst=[13066, 10690, 11742, 9584, 13202, 8426, 12860, 5424] dup_rows_per_dst=[0, 2742, 0, 982, 4094, 1438, 3090, 964] combine_rows_per_src=[13066, 10690, 11742, 9584, 13202, 8426, 12860, 5424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=233922560 dispatch_meta_bytes=277928 combine_payload_bytes=233922560 prefetch_bytes=34603008 reduce_bytes=173015040 egress_bytes=234200488 ingress_bytes=441540608 link_bytes=441540608 token_bytes=468123048 weight_movement_bytes=207618048 total_nvlink_bytes=675741096 weight_to_token_ratio=0.443512 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=14976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.661481761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79422 payload_rows_remote=68872 dup_rows=18882 dup_rows_remote=16268 dedup_ratio=0.192078 router_tokens_per_expert=[87, 160, 9, 245, 504, 85, 61, 688, 442, 297, 1778, 481, 757, 518, 380, 158, 50, 145, 5, 35, 539, 120, 1, 440, 44, 3067, 39, 487, 790, 1326, 315, 514, 773, 2875, 0, 215, 15, 945, 162, 1655, 3559, 1673, 3848, 397, 713, 968, 1598, 157, 10, 280, 663, 905, 328, 508, 172, 130, 1718, 421, 3335, 2983, 2218, 1222, 6, 133] payload_rows_per_dst=[10578, 11016, 8066, 10550, 9852, 12524, 10842, 5994] dup_rows_per_dst=[796, 4356, 570, 2614, 2114, 5606, 1820, 1006] combine_rows_per_src=[10578, 11016, 8066, 10550, 9852, 12524, 10842, 5994] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77164 payload_rows_remote=61990 dup_rows=21140 dup_rows_remote=5900 dedup_ratio=0.215047 router_tokens_per_expert=[0, 0, 0, 177, 0, 0, 0, 0, 397, 153, 1733, 0, 566, 337, 0, 86, 59, 0, 0, 0, 0, 78, 0, 0, 0, 3987, 0, 399, 245, 1751, 0, 19, 718, 4711, 0, 0, 0, 763, 0, 1218, 4037, 780, 4835, 0, 564, 1691, 1137, 0, 0, 74, 0, 1709, 110, 0, 59, 0, 1429, 0, 5474, 5589, 2960, 1134, 0, 173] payload_rows_per_dst=[10024, 5300, 11452, 10566, 12640, 8104, 3904, 15174] dup_rows_per_dst=[0, 1244, 0, 1644, 2180, 832, 0, 15240] combine_rows_per_src=[10024, 5300, 11452, 10566, 12640, 8104, 3904, 15174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=307101696 dispatch_meta_bytes=346496 combine_payload_bytes=307101696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=307448192 ingress_bytes=376307712 link_bytes=376307712 token_bytes=614549888 weight_movement_bytes=69206016 total_nvlink_bytes=683755904 weight_to_token_ratio=0.112613 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=12672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.416015625e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=282099712 dispatch_meta_bytes=340560 combine_payload_bytes=282099712 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=282440272 ingress_bytes=316702720 link_bytes=316702720 token_bytes=564539984 weight_movement_bytes=34603008 total_nvlink_bytes=599142992 weight_to_token_ratio=0.061294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191726839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=310009856 dispatch_meta_bytes=340456 combine_payload_bytes=310009856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310350312 ingress_bytes=344612864 link_bytes=344612864 token_bytes=620360168 weight_movement_bytes=34603008 total_nvlink_bytes=654963176 weight_to_token_ratio=0.055779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296750464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85044 payload_rows_remote=75556 dup_rows=13260 dup_rows_remote=11520 dedup_ratio=0.134888 router_tokens_per_expert=[2, 0, 0, 297, 0, 0, 0, 1, 390, 147, 1515, 1, 529, 320, 1, 79, 68, 0, 14, 0, 0, 71, 2, 0, 1, 3414, 0, 214, 281, 1329, 3, 3, 1189, 3580, 0, 0, 0, 1083, 0, 1393, 2102, 1290, 6064, 0, 633, 632, 957, 0, 0, 119, 1, 2503, 252, 0, 82, 0, 1620, 1, 4532, 5584, 4181, 2538, 0, 134] payload_rows_per_dst=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] dup_rows_per_dst=[4, 988, 18, 1032, 2756, 1740, 2656, 4066] combine_rows_per_src=[12724, 4976, 11460, 9458, 11734, 9488, 12322, 12882] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=300040192 dispatch_meta_bytes=346248 combine_payload_bytes=300040192 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=300386440 ingress_bytes=369246208 link_bytes=369246208 token_bytes=600426632 weight_movement_bytes=69206016 total_nvlink_bytes=669632648 weight_to_token_ratio=0.115261 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.025000 gemm_active_groups=11 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389443754e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=237576192 dispatch_meta_bytes=291016 combine_payload_bytes=237576192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237867208 ingress_bytes=341385216 link_bytes=341385216 token_bytes=475443400 weight_movement_bytes=103809024 total_nvlink_bytes=579252424 weight_to_token_ratio=0.218341 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302424064 dispatch_meta_bytes=355728 combine_payload_bytes=302424064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302779792 ingress_bytes=337027072 link_bytes=337027072 token_bytes=605203856 weight_movement_bytes=34603008 total_nvlink_bytes=639806864 weight_to_token_ratio=0.057176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268205739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=253911040 dispatch_meta_bytes=271560 combine_payload_bytes=253911040 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=254182600 ingress_bytes=426926080 link_bytes=426926080 token_bytes=508093640 weight_movement_bytes=173015040 total_nvlink_bytes=681108680 weight_to_token_ratio=0.340518 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.606488469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84786 payload_rows_remote=71122 dup_rows=13518 dup_rows_remote=13438 dedup_ratio=0.137512 router_tokens_per_expert=[0, 58, 227, 300, 34, 1364, 0, 460, 0, 0, 40, 0, 92, 0, 0, 0, 1950, 0, 0, 0, 0, 1634, 0, 1132, 0, 140, 229, 1445, 0, 0, 0, 58, 6009, 3, 92, 756, 0, 0, 168, 827, 226, 0, 0, 1367, 76, 0, 0, 1, 268, 0, 3914, 92, 5677, 58, 6740, 0, 0, 3757, 4306, 884, 2205, 2229, 0, 334] payload_rows_per_dst=[11014, 13664, 14914, 10994, 4878, 13284, 9626, 6412] dup_rows_per_dst=[1386, 80, 6536, 1362, 746, 1410, 1516, 482] combine_rows_per_src=[11014, 13664, 14914, 10994, 4878, 13284, 9626, 6412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291315712 dispatch_meta_bytes=338240 combine_payload_bytes=291315712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291653952 ingress_bytes=325918720 link_bytes=325918720 token_bytes=582969664 weight_movement_bytes=34603008 total_nvlink_bytes=617572672 weight_to_token_ratio=0.059356 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226405905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82126 payload_rows_remote=73834 dup_rows=16178 dup_rows_remote=15098 dedup_ratio=0.164571 router_tokens_per_expert=[0, 7, 0, 201, 32, 2, 0, 68, 419, 177, 1558, 43, 636, 346, 30, 80, 61, 14, 1, 0, 41, 82, 0, 8, 0, 3372, 8, 299, 273, 1621, 35, 66, 1090, 3938, 0, 17, 0, 926, 1, 1384, 2867, 1171, 5723, 33, 570, 941, 1212, 11, 1, 129, 66, 2171, 270, 51, 78, 24, 1393, 65, 4636, 5350, 3721, 1675, 0, 158] payload_rows_per_dst=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] dup_rows_per_dst=[44, 1110, 22, 1566, 2716, 2836, 1080, 6804] combine_rows_per_src=[12022, 5468, 11092, 9782, 11996, 10578, 8292, 12896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81430 payload_rows_remote=68760 dup_rows=16874 dup_rows_remote=11706 dedup_ratio=0.171651 router_tokens_per_expert=[1, 0, 0, 193, 0, 0, 0, 0, 361, 130, 2501, 0, 572, 318, 0, 92, 73, 0, 0, 0, 0, 76, 0, 0, 0, 4430, 0, 519, 251, 1391, 2, 7, 893, 4945, 0, 0, 0, 759, 0, 1345, 4328, 803, 4141, 0, 544, 1679, 1223, 0, 0, 143, 0, 1748, 0, 0, 73, 0, 1375, 0, 4647, 5459, 2737, 1211, 0, 182] payload_rows_per_dst=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] dup_rows_per_dst=[1652, 5168, 2468, 1568, 866, 2342, 1896, 914] combine_rows_per_src=[11226, 12670, 12432, 11632, 6918, 10604, 11326, 4622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=282099712 dispatch_meta_bytes=340560 combine_payload_bytes=282099712 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=282440272 ingress_bytes=316702720 link_bytes=316702720 token_bytes=564539984 weight_movement_bytes=34603008 total_nvlink_bytes=599142992 weight_to_token_ratio=0.061294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191726839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88552 payload_rows_remote=75396 dup_rows=9752 dup_rows_remote=7412 dedup_ratio=0.099202 router_tokens_per_expert=[0, 69, 253, 326, 19, 1522, 0, 446, 0, 0, 65, 0, 92, 0, 0, 0, 1714, 0, 0, 0, 0, 1472, 0, 847, 0, 140, 36, 1482, 0, 0, 0, 69, 5925, 0, 92, 718, 0, 0, 175, 838, 244, 0, 0, 1394, 83, 0, 0, 0, 356, 0, 3618, 91, 5570, 69, 6538, 0, 8, 3793, 4639, 900, 2707, 2508, 0, 334] payload_rows_per_dst=[11222, 13260, 7496, 11616, 13156, 13174, 7884, 10744] dup_rows_per_dst=[1634, 130, 570, 1116, 2340, 1408, 384, 2170] combine_rows_per_src=[11222, 13260, 7496, 11616, 13156, 13174, 7884, 10744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83418 payload_rows_remote=72396 dup_rows=14886 dup_rows_remote=12832 dedup_ratio=0.151428 router_tokens_per_expert=[200, 106, 513, 543, 84, 620, 513, 863, 4, 158, 90, 167, 245, 710, 53, 148, 1173, 7, 45, 0, 174, 879, 146, 989, 104, 358, 813, 1251, 132, 146, 19, 95, 4956, 302, 93, 629, 5, 577, 183, 721, 254, 702, 1217, 1168, 110, 101, 864, 122, 445, 26, 2641, 78, 4556, 68, 5100, 212, 205, 2887, 3620, 941, 2020, 2427, 210, 364] payload_rows_per_dst=[10632, 12398, 6348, 11022, 12618, 13604, 6608, 10188] dup_rows_per_dst=[2026, 952, 478, 2054, 2314, 4584, 332, 2146] combine_rows_per_src=[10632, 12398, 6348, 11022, 12618, 13604, 6608, 10188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84358 payload_rows_remote=71800 dup_rows=13946 dup_rows_remote=9912 dedup_ratio=0.141866 router_tokens_per_expert=[0, 0, 63, 67, 66, 0, 0, 0, 0, 2711, 0, 205, 67, 821, 0, 1374, 0, 6787, 1, 483, 0, 0, 68, 1, 6851, 1, 554, 292, 1, 506, 67, 3998, 6971, 6437, 1, 778, 1, 0, 0, 0, 337, 68, 2303, 0, 0, 1337, 253, 0, 39, 0, 0, 1633, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] dup_rows_per_dst=[392, 1516, 138, 8, 92, 4034, 3030, 4736] combine_rows_per_src=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87510 payload_rows_remote=76118 dup_rows=10794 dup_rows_remote=7710 dedup_ratio=0.109802 router_tokens_per_expert=[0, 37, 257, 69, 63, 1781, 0, 554, 0, 0, 29, 0, 62, 0, 0, 0, 1958, 0, 0, 0, 0, 1315, 0, 870, 0, 60, 7, 841, 0, 0, 0, 37, 6063, 0, 62, 543, 0, 0, 114, 1420, 155, 0, 0, 735, 52, 0, 0, 0, 250, 0, 3969, 65, 5434, 37, 6645, 0, 53, 3750, 4422, 813, 4340, 2107, 0, 183] payload_rows_per_dst=[11484, 13414, 11392, 10418, 8584, 11920, 8868, 11430] dup_rows_per_dst=[1538, 58, 3084, 316, 1630, 832, 730, 2606] combine_rows_per_src=[11484, 13414, 11392, 10418, 8584, 11920, 8868, 11430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=253911040 dispatch_meta_bytes=271560 combine_payload_bytes=253911040 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=254182600 ingress_bytes=426926080 link_bytes=426926080 token_bytes=508093640 weight_movement_bytes=173015040 total_nvlink_bytes=681108680 weight_to_token_ratio=0.340518 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.606488469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=309477376 dispatch_meta_bytes=348304 combine_payload_bytes=309477376 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=309825680 ingress_bytes=447889408 link_bytes=447889408 token_bytes=619303056 weight_movement_bytes=138412032 total_nvlink_bytes=757715088 weight_to_token_ratio=0.223496 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=15872 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.685371784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=67282 dup_rows=18228 dup_rows_remote=14776 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 1, 184, 0, 0, 0, 4, 424, 143, 2086, 3, 583, 337, 1, 90, 52, 0, 0, 0, 3, 100, 0, 3, 0, 4177, 1, 588, 260, 1875, 1, 175, 585, 5014, 0, 1, 0, 874, 0, 1318, 3912, 957, 4022, 0, 520, 1740, 1479, 1, 0, 118, 4, 1606, 6, 1, 53, 0, 1356, 5, 4802, 5670, 2849, 986, 0, 182] payload_rows_per_dst=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] dup_rows_per_dst=[3452, 4528, 4096, 2304, 378, 934, 1772, 764] combine_rows_per_src=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84702 payload_rows_remote=72406 dup_rows=13602 dup_rows_remote=9210 dedup_ratio=0.138367 router_tokens_per_expert=[15, 69, 287, 219, 81, 1657, 46, 494, 0, 6, 60, 14, 89, 72, 8, 8, 1995, 3, 0, 0, 10, 1209, 13, 811, 10, 100, 108, 980, 11, 8, 2, 70, 5909, 60, 79, 651, 0, 53, 161, 967, 237, 66, 99, 1139, 82, 3, 77, 14, 386, 6, 3670, 84, 5336, 66, 6223, 24, 53, 3621, 4439, 896, 3451, 2577, 22, 246] payload_rows_per_dst=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] dup_rows_per_dst=[1692, 174, 472, 752, 2490, 678, 4392, 2952] combine_rows_per_src=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=302424064 dispatch_meta_bytes=355728 combine_payload_bytes=302424064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302779792 ingress_bytes=337027072 link_bytes=337027072 token_bytes=605203856 weight_movement_bytes=34603008 total_nvlink_bytes=639806864 weight_to_token_ratio=0.057176 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268205739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=281640960 dispatch_meta_bytes=321864 combine_payload_bytes=281640960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=281962824 ingress_bytes=316243968 link_bytes=316243968 token_bytes=563603784 weight_movement_bytes=34603008 total_nvlink_bytes=598206792 weight_to_token_ratio=0.061396 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=20608 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.190000592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73512 payload_rows_remote=58002 dup_rows=24792 dup_rows_remote=14752 dedup_ratio=0.252197 router_tokens_per_expert=[0, 7405, 900, 8, 1, 78, 778, 3434, 0, 35, 3, 0, 0, 1051, 3568, 170, 0, 178, 118, 281, 97, 4, 106, 365, 1728, 337, 1, 276, 247, 79, 308, 0, 279, 7106, 93, 2, 3719, 0, 247, 1, 1293, 0, 9, 0, 1967, 247, 0, 4, 92, 4, 206, 520, 5, 534, 96, 4, 0, 366, 4374, 2235, 359, 2256, 0, 1578] payload_rows_per_dst=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] dup_rows_per_dst=[10040, 2416, 206, 3010, 7408, 552, 368, 792] combine_rows_per_src=[15510, 11750, 2092, 11690, 15144, 6488, 2554, 8284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=308822016 dispatch_meta_bytes=331232 combine_payload_bytes=308822016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=309153248 ingress_bytes=378028032 link_bytes=378028032 token_bytes=617975264 weight_movement_bytes=69206016 total_nvlink_bytes=687181280 weight_to_token_ratio=0.111988 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31744 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.422489051e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87510 payload_rows_remote=76118 dup_rows=10794 dup_rows_remote=7710 dedup_ratio=0.109802 router_tokens_per_expert=[0, 37, 257, 69, 63, 1781, 0, 554, 0, 0, 29, 0, 62, 0, 0, 0, 1958, 0, 0, 0, 0, 1315, 0, 870, 0, 60, 7, 841, 0, 0, 0, 37, 6063, 0, 62, 543, 0, 0, 114, 1420, 155, 0, 0, 735, 52, 0, 0, 0, 250, 0, 3969, 65, 5434, 37, 6645, 0, 53, 3750, 4422, 813, 4340, 2107, 0, 183] payload_rows_per_dst=[11484, 13414, 11392, 10418, 8584, 11920, 8868, 11430] dup_rows_per_dst=[1538, 58, 3084, 316, 1630, 832, 730, 2606] combine_rows_per_src=[11484, 13414, 11392, 10418, 8584, 11920, 8868, 11430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=237576192 dispatch_meta_bytes=291016 combine_payload_bytes=237576192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=237867208 ingress_bytes=341385216 link_bytes=341385216 token_bytes=475443400 weight_movement_bytes=103809024 total_nvlink_bytes=579252424 weight_to_token_ratio=0.218341 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20224 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284605084e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83418 payload_rows_remote=72396 dup_rows=14886 dup_rows_remote=12832 dedup_ratio=0.151428 router_tokens_per_expert=[200, 106, 513, 543, 84, 620, 513, 863, 4, 158, 90, 167, 245, 710, 53, 148, 1173, 7, 45, 0, 174, 879, 146, 989, 104, 358, 813, 1251, 132, 146, 19, 95, 4956, 302, 93, 629, 5, 577, 183, 721, 254, 702, 1217, 1168, 110, 101, 864, 122, 445, 26, 2641, 78, 4556, 68, 5100, 212, 205, 2887, 3620, 941, 2020, 2427, 210, 364] payload_rows_per_dst=[10632, 12398, 6348, 11022, 12618, 13604, 6608, 10188] dup_rows_per_dst=[2026, 952, 478, 2054, 2314, 4584, 332, 2146] combine_rows_per_src=[10632, 12398, 6348, 11022, 12618, 13604, 6608, 10188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76078 payload_rows_remote=63986 dup_rows=22226 dup_rows_remote=18974 dedup_ratio=0.226095 router_tokens_per_expert=[0, 68, 252, 283, 16, 1470, 0, 448, 0, 0, 65, 0, 87, 0, 0, 0, 2259, 0, 0, 0, 0, 1742, 0, 970, 0, 52, 87, 1890, 0, 0, 0, 68, 5907, 9, 87, 621, 0, 0, 164, 804, 232, 0, 0, 1518, 77, 0, 0, 0, 246, 0, 3731, 86, 5812, 68, 6403, 0, 4, 3826, 3980, 876, 2334, 2360, 0, 250] payload_rows_per_dst=[8056, 5448, 9040, 10696, 12906, 3500, 14340, 12092] dup_rows_per_dst=[974, 48, 902, 1458, 2278, 154, 13160, 3252] combine_rows_per_src=[8056, 5448, 9040, 10696, 12906, 3500, 14340, 12092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296534016 dispatch_meta_bytes=340912 combine_payload_bytes=296534016 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296874928 ingress_bytes=365740032 link_bytes=365740032 token_bytes=593408944 weight_movement_bytes=69206016 total_nvlink_bytes=662614960 weight_to_token_ratio=0.116624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376250296e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=294092800 dispatch_meta_bytes=326848 combine_payload_bytes=294092800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294419648 ingress_bytes=328695808 link_bytes=328695808 token_bytes=588512448 weight_movement_bytes=34603008 total_nvlink_bytes=623115456 weight_to_token_ratio=0.058797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236855863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=311779328 dispatch_meta_bytes=335312 combine_payload_bytes=311779328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312114640 ingress_bytes=346382336 link_bytes=346382336 token_bytes=623893968 weight_movement_bytes=34603008 total_nvlink_bytes=658496976 weight_to_token_ratio=0.055463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=15488 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303408844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87540 payload_rows_remote=75686 dup_rows=10764 dup_rows_remote=9428 dedup_ratio=0.109497 router_tokens_per_expert=[0, 63, 287, 97, 74, 1979, 0, 486, 0, 0, 39, 0, 67, 0, 0, 0, 2130, 0, 0, 0, 0, 1365, 0, 941, 0, 65, 17, 853, 0, 1, 0, 63, 5996, 2, 67, 664, 0, 1, 138, 1523, 210, 1, 0, 1349, 71, 1, 1, 1, 352, 14, 4069, 79, 4961, 63, 6304, 1, 54, 3559, 3884, 812, 3841, 2413, 1, 193] payload_rows_per_dst=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] dup_rows_per_dst=[1818, 76, 672, 298, 3306, 1336, 454, 2804] combine_rows_per_src=[11272, 12744, 8200, 9468, 13476, 11854, 8702, 11824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82380 payload_rows_remote=73032 dup_rows=15924 dup_rows_remote=13832 dedup_ratio=0.161987 router_tokens_per_expert=[0, 0, 58, 0, 58, 0, 0, 0, 0, 1813, 0, 243, 92, 143, 0, 1460, 1, 6785, 0, 444, 0, 0, 73, 0, 7000, 0, 1667, 227, 0, 734, 92, 4280, 6946, 6806, 0, 835, 0, 0, 0, 0, 96, 72, 869, 0, 0, 1668, 227, 0, 41, 0, 0, 2542, 0, 0, 0, 0, 0, 3723, 84, 0, 0, 0, 0, 73] payload_rows_per_dst=[13892, 9348, 10554, 15944, 14382, 5338, 5162, 7760] dup_rows_per_dst=[232, 2092, 114, 12056, 900, 526, 4, 0] combine_rows_per_src=[13892, 9348, 10554, 15944, 14382, 5338, 5162, 7760] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275587072 dispatch_meta_bytes=328232 combine_payload_bytes=275587072 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275915304 ingress_bytes=344793088 link_bytes=344793088 token_bytes=551502376 weight_movement_bytes=69206016 total_nvlink_bytes=620708392 weight_to_token_ratio=0.125486 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297428632e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=326464 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=296901440 ingress_bytes=400384000 link_bytes=400384000 token_bytes=593476416 weight_movement_bytes=103809024 total_nvlink_bytes=697285440 weight_to_token_ratio=0.174917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506612758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=262086656 dispatch_meta_bytes=331840 combine_payload_bytes=262086656 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=262418496 ingress_bytes=435101696 link_bytes=435101696 token_bytes=524505152 weight_movement_bytes=173015040 total_nvlink_bytes=697520192 weight_to_token_ratio=0.329863 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.637252653e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299139072 dispatch_meta_bytes=347456 combine_payload_bytes=299139072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299486528 ingress_bytes=333742080 link_bytes=333742080 token_bytes=598625600 weight_movement_bytes=34603008 total_nvlink_bytes=633228608 weight_to_token_ratio=0.057804 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255844579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84702 payload_rows_remote=72406 dup_rows=13602 dup_rows_remote=9210 dedup_ratio=0.138367 router_tokens_per_expert=[15, 69, 287, 219, 81, 1657, 46, 494, 0, 6, 60, 14, 89, 72, 8, 8, 1995, 3, 0, 0, 10, 1209, 13, 811, 10, 100, 108, 980, 11, 8, 2, 70, 5909, 60, 79, 651, 0, 53, 161, 967, 237, 66, 99, 1139, 82, 3, 77, 14, 386, 6, 3670, 84, 5336, 66, 6223, 24, 53, 3621, 4439, 896, 3451, 2577, 22, 246] payload_rows_per_dst=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] dup_rows_per_dst=[1692, 174, 472, 752, 2490, 678, 4392, 2952] combine_rows_per_src=[11286, 12786, 7610, 10704, 13270, 5212, 12296, 11538] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84786 payload_rows_remote=71122 dup_rows=13518 dup_rows_remote=13438 dedup_ratio=0.137512 router_tokens_per_expert=[0, 58, 227, 300, 34, 1364, 0, 460, 0, 0, 40, 0, 92, 0, 0, 0, 1950, 0, 0, 0, 0, 1634, 0, 1132, 0, 140, 229, 1445, 0, 0, 0, 58, 6009, 3, 92, 756, 0, 0, 168, 827, 226, 0, 0, 1367, 76, 0, 0, 1, 268, 0, 3914, 92, 5677, 58, 6740, 0, 0, 3757, 4306, 884, 2205, 2229, 0, 334] payload_rows_per_dst=[11014, 13664, 14914, 10994, 4878, 13284, 9626, 6412] dup_rows_per_dst=[1386, 80, 6536, 1362, 746, 1410, 1516, 482] combine_rows_per_src=[11014, 13664, 14914, 10994, 4878, 13284, 9626, 6412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88592 payload_rows_remote=74264 dup_rows=9712 dup_rows_remote=8806 dedup_ratio=0.098796 router_tokens_per_expert=[8, 0, 69, 29, 69, 0, 0, 0, 0, 2798, 0, 242, 91, 153, 0, 1498, 0, 6747, 0, 205, 0, 0, 67, 0, 6981, 0, 790, 252, 0, 662, 92, 4612, 6966, 6833, 0, 784, 0, 0, 0, 0, 304, 67, 979, 0, 0, 1532, 252, 0, 55, 0, 0, 1974, 0, 0, 0, 0, 0, 3967, 6, 0, 0, 0, 0, 68] payload_rows_per_dst=[13932, 8358, 13900, 3464, 14328, 11966, 14562, 8082] dup_rows_per_dst=[350, 1206, 138, 128, 906, 3526, 3458, 0] combine_rows_per_src=[13932, 8358, 13900, 3464, 14328, 11966, 14562, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82108 payload_rows_remote=74266 dup_rows=16196 dup_rows_remote=14872 dedup_ratio=0.164754 router_tokens_per_expert=[238, 68, 83, 109, 71, 113, 67, 0, 123, 2919, 0, 224, 327, 1639, 272, 1099, 57, 5171, 406, 530, 20, 4, 395, 407, 4586, 692, 733, 241, 172, 1645, 116, 3782, 5449, 2384, 638, 869, 342, 205, 3, 308, 573, 132, 1423, 774, 49, 1551, 392, 899, 450, 103, 24, 2171, 88, 25, 128, 56, 370, 1667, 208, 96, 518, 813, 24, 111] payload_rows_per_dst=[11644, 10198, 13166, 7842, 10212, 12118, 10658, 6270] dup_rows_per_dst=[752, 3008, 814, 1324, 1030, 5064, 2860, 1344] combine_rows_per_src=[11644, 10198, 13166, 7842, 10212, 12118, 10658, 6270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86948 payload_rows_remote=75642 dup_rows=11356 dup_rows_remote=11352 dedup_ratio=0.115519 router_tokens_per_expert=[0, 0, 60, 1686, 0, 0, 770, 0, 0, 2847, 789, 0, 1, 1, 794, 1, 168, 461, 0, 2412, 1, 68, 0, 0, 6562, 65, 0, 318, 0, 0, 0, 1, 627, 387, 0, 1, 1450, 2240, 0, 67, 2, 0, 5579, 0, 0, 1, 72, 1, 547, 1, 67, 0, 5564, 57, 16, 4, 6931, 0, 0, 2, 6741, 759, 1031, 0] payload_rows_per_dst=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] dup_rows_per_dst=[162, 1306, 5436, 0, 1080, 4, 904, 2464] combine_rows_per_src=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84878 payload_rows_remote=70406 dup_rows=13426 dup_rows_remote=13308 dedup_ratio=0.136576 router_tokens_per_expert=[0, 0, 37, 55, 37, 0, 0, 0, 0, 3566, 0, 182, 62, 725, 0, 1383, 0, 6874, 0, 362, 0, 0, 59, 0, 6972, 0, 352, 255, 0, 581, 62, 4300, 7410, 6433, 0, 621, 0, 0, 0, 0, 90, 59, 1494, 0, 0, 1189, 254, 0, 82, 0, 0, 1773, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[14820, 10162, 14472, 13466, 14640, 5868, 3684, 7766] dup_rows_per_dst=[258, 1674, 118, 6156, 4890, 304, 26, 0] combine_rows_per_src=[14820, 10162, 14472, 13466, 14640, 5868, 3684, 7766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296534016 dispatch_meta_bytes=340912 combine_payload_bytes=296534016 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=296874928 ingress_bytes=365740032 link_bytes=365740032 token_bytes=593408944 weight_movement_bytes=69206016 total_nvlink_bytes=662614960 weight_to_token_ratio=0.116624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.027500 gemm_active_groups=11 gemm_peak_group_rows=33408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.376250296e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=262086656 dispatch_meta_bytes=331840 combine_payload_bytes=262086656 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=262418496 ingress_bytes=435101696 link_bytes=435101696 token_bytes=524505152 weight_movement_bytes=173015040 total_nvlink_bytes=697520192 weight_to_token_ratio=0.329863 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=19456 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.637252653e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83370 payload_rows_remote=69594 dup_rows=14934 dup_rows_remote=11368 dedup_ratio=0.151917 router_tokens_per_expert=[23, 3, 66, 78, 75, 1, 4, 0, 30, 3395, 0, 187, 105, 798, 17, 1334, 2, 6729, 30, 450, 1, 0, 86, 31, 6624, 76, 462, 261, 41, 721, 83, 4323, 6810, 6041, 46, 772, 34, 23, 0, 13, 378, 70, 1439, 59, 0, 1428, 280, 82, 95, 0, 0, 1937, 6, 0, 4, 5, 35, 3401, 5, 2, 17, 66, 0, 68] payload_rows_per_dst=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] dup_rows_per_dst=[438, 1706, 160, 106, 46, 3854, 3566, 5058] combine_rows_per_src=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=310009856 dispatch_meta_bytes=340456 combine_payload_bytes=310009856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310350312 ingress_bytes=344612864 link_bytes=344612864 token_bytes=620360168 weight_movement_bytes=34603008 total_nvlink_bytes=654963176 weight_to_token_ratio=0.055779 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33024 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296750464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296574976 dispatch_meta_bytes=326464 combine_payload_bytes=296574976 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=296901440 ingress_bytes=400384000 link_bytes=400384000 token_bytes=593476416 weight_movement_bytes=103809024 total_nvlink_bytes=697285440 weight_to_token_ratio=0.174917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=29696 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.506612758e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=291315712 dispatch_meta_bytes=338240 combine_payload_bytes=291315712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291653952 ingress_bytes=325918720 link_bytes=325918720 token_bytes=582969664 weight_movement_bytes=34603008 total_nvlink_bytes=617572672 weight_to_token_ratio=0.059356 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=46592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.226405905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304185344 dispatch_meta_bytes=332280 combine_payload_bytes=304185344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=304517624 ingress_bytes=407994368 link_bytes=407994368 token_bytes=608702968 weight_movement_bytes=103809024 total_nvlink_bytes=712511992 weight_to_token_ratio=0.170541 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=30592 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.535249961e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82968 payload_rows_remote=72298 dup_rows=15336 dup_rows_remote=13736 dedup_ratio=0.156006 router_tokens_per_expert=[1, 77, 247, 348, 42, 1436, 3, 460, 0, 0, 25, 1, 92, 0, 0, 1, 2262, 0, 0, 0, 0, 1363, 0, 1070, 2, 78, 169, 1739, 0, 0, 0, 79, 5600, 6, 92, 649, 0, 4, 190, 743, 267, 4, 5, 1812, 100, 1, 4, 0, 354, 0, 3983, 90, 5484, 77, 6417, 2, 0, 3521, 4054, 801, 2366, 2729, 1, 301] payload_rows_per_dst=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] dup_rows_per_dst=[1600, 46, 5866, 4052, 730, 2258, 366, 418] combine_rows_per_src=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304193536 dispatch_meta_bytes=356552 combine_payload_bytes=304193536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=304550088 ingress_bytes=442605568 link_bytes=442605568 token_bytes=608743624 weight_movement_bytes=138412032 total_nvlink_bytes=747155656 weight_to_token_ratio=0.227373 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665489120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80076 payload_rows_remote=67282 dup_rows=18228 dup_rows_remote=14776 dedup_ratio=0.185425 router_tokens_per_expert=[0, 0, 1, 184, 0, 0, 0, 4, 424, 143, 2086, 3, 583, 337, 1, 90, 52, 0, 0, 0, 3, 100, 0, 3, 0, 4177, 1, 588, 260, 1875, 1, 175, 585, 5014, 0, 1, 0, 874, 0, 1318, 3912, 957, 4022, 0, 520, 1740, 1479, 1, 0, 118, 4, 1606, 6, 1, 53, 0, 1356, 5, 4802, 5670, 2849, 986, 0, 182] payload_rows_per_dst=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] dup_rows_per_dst=[3452, 4528, 4096, 2304, 378, 934, 1772, 764] combine_rows_per_src=[12794, 12834, 13258, 11850, 5178, 8460, 11408, 4294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84878 payload_rows_remote=70406 dup_rows=13426 dup_rows_remote=13308 dedup_ratio=0.136576 router_tokens_per_expert=[0, 0, 37, 55, 37, 0, 0, 0, 0, 3566, 0, 182, 62, 725, 0, 1383, 0, 6874, 0, 362, 0, 0, 59, 0, 6972, 0, 352, 255, 0, 581, 62, 4300, 7410, 6433, 0, 621, 0, 0, 0, 0, 90, 59, 1494, 0, 0, 1189, 254, 0, 82, 0, 0, 1773, 0, 0, 0, 0, 0, 3824, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[14820, 10162, 14472, 13466, 14640, 5868, 3684, 7766] dup_rows_per_dst=[258, 1674, 118, 6156, 4890, 304, 26, 0] combine_rows_per_src=[14820, 10162, 14472, 13466, 14640, 5868, 3684, 7766] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82732 payload_rows_remote=67620 dup_rows=15572 dup_rows_remote=9034 dedup_ratio=0.158407 router_tokens_per_expert=[2, 0, 68, 0, 68, 0, 0, 0, 0, 2291, 0, 171, 86, 154, 0, 1539, 7, 6886, 0, 211, 0, 0, 59, 0, 7156, 0, 1288, 251, 0, 845, 86, 4351, 6942, 6945, 0, 770, 0, 0, 0, 0, 129, 52, 907, 0, 0, 1510, 251, 0, 53, 0, 0, 2194, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] dup_rows_per_dst=[136, 1016, 134, 174, 148, 3332, 4094, 6538] combine_rows_per_src=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=309829632 dispatch_meta_bytes=347976 combine_payload_bytes=309829632 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=310177608 ingress_bytes=344432640 link_bytes=344432640 token_bytes=620007240 weight_movement_bytes=34603008 total_nvlink_bytes=654610248 weight_to_token_ratio=0.055811 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296072295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86900 payload_rows_remote=71038 dup_rows=11404 dup_rows_remote=8526 dedup_ratio=0.116007 router_tokens_per_expert=[0, 0, 143, 1188, 0, 0, 1161, 0, 0, 1182, 855, 0, 0, 0, 715, 0, 160, 412, 0, 1902, 0, 73, 0, 0, 6465, 0, 0, 319, 0, 0, 0, 0, 772, 462, 0, 0, 1510, 2191, 0, 92, 81, 0, 5963, 0, 0, 1, 0, 0, 435, 0, 92, 0, 6566, 92, 76, 13, 7032, 0, 0, 0, 6618, 756, 1825, 0] payload_rows_per_dst=[15202, 15862, 4790, 13568, 10830, 11926, 9952, 4770] dup_rows_per_dst=[3846, 2878, 304, 0, 3124, 164, 696, 392] combine_rows_per_src=[15202, 15862, 4790, 13568, 10830, 11926, 9952, 4770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=288382976 dispatch_meta_bytes=334856 combine_payload_bytes=288382976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=288717832 ingress_bytes=357588992 link_bytes=357588992 token_bytes=577100808 weight_movement_bytes=69206016 total_nvlink_bytes=646306824 weight_to_token_ratio=0.119920 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=44032 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.345578589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=275587072 dispatch_meta_bytes=328232 combine_payload_bytes=275587072 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=275915304 ingress_bytes=344793088 link_bytes=344793088 token_bytes=551502376 weight_movement_bytes=69206016 total_nvlink_bytes=620708392 weight_to_token_ratio=0.125486 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.030000 gemm_active_groups=11 gemm_peak_group_rows=39680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.297428632e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82108 payload_rows_remote=74266 dup_rows=16196 dup_rows_remote=14872 dedup_ratio=0.164754 router_tokens_per_expert=[238, 68, 83, 109, 71, 113, 67, 0, 123, 2919, 0, 224, 327, 1639, 272, 1099, 57, 5171, 406, 530, 20, 4, 395, 407, 4586, 692, 733, 241, 172, 1645, 116, 3782, 5449, 2384, 638, 869, 342, 205, 3, 308, 573, 132, 1423, 774, 49, 1551, 392, 899, 450, 103, 24, 2171, 88, 25, 128, 56, 370, 1667, 208, 96, 518, 813, 24, 111] payload_rows_per_dst=[11644, 10198, 13166, 7842, 10212, 12118, 10658, 6270] dup_rows_per_dst=[752, 3008, 814, 1324, 1030, 5064, 2860, 1344] combine_rows_per_src=[11644, 10198, 13166, 7842, 10212, 12118, 10658, 6270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82732 payload_rows_remote=67620 dup_rows=15572 dup_rows_remote=9034 dedup_ratio=0.158407 router_tokens_per_expert=[2, 0, 68, 0, 68, 0, 0, 0, 0, 2291, 0, 171, 86, 154, 0, 1539, 7, 6886, 0, 211, 0, 0, 59, 0, 7156, 0, 1288, 251, 0, 845, 86, 4351, 6942, 6945, 0, 770, 0, 0, 0, 0, 129, 52, 907, 0, 0, 1510, 251, 0, 53, 0, 0, 2194, 0, 0, 0, 0, 0, 3821, 0, 0, 0, 0, 0, 59] payload_rows_per_dst=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] dup_rows_per_dst=[136, 1016, 134, 174, 148, 3332, 4094, 6538] combine_rows_per_src=[140, 7466, 14192, 4766, 15276, 11068, 14712, 15112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84358 payload_rows_remote=71800 dup_rows=13946 dup_rows_remote=9912 dedup_ratio=0.141866 router_tokens_per_expert=[0, 0, 63, 67, 66, 0, 0, 0, 0, 2711, 0, 205, 67, 821, 0, 1374, 0, 6787, 1, 483, 0, 0, 68, 1, 6851, 1, 554, 292, 1, 506, 67, 3998, 6971, 6437, 1, 778, 1, 0, 0, 0, 337, 68, 2303, 0, 0, 1337, 253, 0, 39, 0, 0, 1633, 0, 0, 0, 0, 0, 3941, 0, 0, 0, 0, 0, 69] payload_rows_per_dst=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] dup_rows_per_dst=[392, 1516, 138, 8, 92, 4034, 3030, 4736] combine_rows_per_src=[13942, 8840, 14542, 2834, 4186, 12558, 14016, 13440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83370 payload_rows_remote=69594 dup_rows=14934 dup_rows_remote=11368 dedup_ratio=0.151917 router_tokens_per_expert=[23, 3, 66, 78, 75, 1, 4, 0, 30, 3395, 0, 187, 105, 798, 17, 1334, 2, 6729, 30, 450, 1, 0, 86, 31, 6624, 76, 462, 261, 41, 721, 83, 4323, 6810, 6041, 46, 772, 34, 23, 0, 13, 378, 70, 1439, 59, 0, 1428, 280, 82, 95, 0, 0, 1937, 6, 0, 4, 5, 35, 3401, 5, 2, 17, 66, 0, 68] payload_rows_per_dst=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] dup_rows_per_dst=[438, 1706, 160, 106, 46, 3854, 3566, 5058] combine_rows_per_src=[13288, 10026, 14498, 3182, 2124, 12264, 13776, 14212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285057024 dispatch_meta_bytes=323848 combine_payload_bytes=285057024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285380872 ingress_bytes=319660032 link_bytes=319660032 token_bytes=570437896 weight_movement_bytes=34603008 total_nvlink_bytes=605040904 weight_to_token_ratio=0.060660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202854966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296132608 dispatch_meta_bytes=344136 combine_payload_bytes=296132608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296476744 ingress_bytes=330735616 link_bytes=330735616 token_bytes=592609352 weight_movement_bytes=34603008 total_nvlink_bytes=627212360 weight_to_token_ratio=0.058391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244531497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=306616 combine_payload_bytes=276971520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277278136 ingress_bytes=311574528 link_bytes=311574528 token_bytes=554249656 weight_movement_bytes=34603008 total_nvlink_bytes=588852664 weight_to_token_ratio=0.062432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172429865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290971648 dispatch_meta_bytes=318256 combine_payload_bytes=290971648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291289904 ingress_bytes=325574656 link_bytes=325574656 token_bytes=582261552 weight_movement_bytes=34603008 total_nvlink_bytes=616864560 weight_to_token_ratio=0.059429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225111220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82380 payload_rows_remote=73032 dup_rows=15924 dup_rows_remote=13832 dedup_ratio=0.161987 router_tokens_per_expert=[0, 0, 58, 0, 58, 0, 0, 0, 0, 1813, 0, 243, 92, 143, 0, 1460, 1, 6785, 0, 444, 0, 0, 73, 0, 7000, 0, 1667, 227, 0, 734, 92, 4280, 6946, 6806, 0, 835, 0, 0, 0, 0, 96, 72, 869, 0, 0, 1668, 227, 0, 41, 0, 0, 2542, 0, 0, 0, 0, 0, 3723, 84, 0, 0, 0, 0, 73] payload_rows_per_dst=[13892, 9348, 10554, 15944, 14382, 5338, 5162, 7760] dup_rows_per_dst=[232, 2092, 114, 12056, 900, 526, 4, 0] combine_rows_per_src=[13892, 9348, 10554, 15944, 14382, 5338, 5162, 7760] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86070 payload_rows_remote=76370 dup_rows=12234 dup_rows_remote=10974 dedup_ratio=0.124451 router_tokens_per_expert=[0, 0, 150, 1261, 0, 0, 1050, 0, 0, 1317, 800, 0, 0, 0, 769, 0, 146, 520, 0, 2075, 0, 67, 0, 0, 6114, 0, 0, 366, 0, 0, 0, 0, 768, 490, 0, 0, 1379, 2751, 0, 92, 39, 0, 6015, 0, 0, 3, 73, 0, 491, 0, 92, 0, 6172, 92, 83, 3, 7017, 0, 0, 0, 6804, 679, 1474, 0] payload_rows_per_dst=[10232, 4498, 14184, 12960, 9700, 12180, 12958, 9358] dup_rows_per_dst=[1942, 1274, 5040, 0, 1260, 80, 908, 1730] combine_rows_per_src=[10232, 4498, 14184, 12960, 9700, 12180, 12958, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82474 payload_rows_remote=70748 dup_rows=15830 dup_rows_remote=15556 dedup_ratio=0.161031 router_tokens_per_expert=[74, 67, 246, 817, 89, 451, 1309, 1, 3, 1138, 1775, 33, 854, 1197, 597, 35, 116, 535, 415, 1685, 99, 66, 19, 76, 4734, 136, 4, 485, 173, 52, 160, 256, 984, 401, 773, 527, 1282, 587, 207, 97, 372, 311, 4834, 0, 204, 195, 256, 1172, 518, 640, 116, 5, 5924, 714, 192, 132, 4257, 15, 242, 245, 4401, 741, 1106, 5] payload_rows_per_dst=[11460, 8294, 11344, 11726, 8046, 12936, 14290, 4378] dup_rows_per_dst=[3162, 2970, 3480, 274, 1670, 1752, 2192, 330] combine_rows_per_src=[11460, 8294, 11344, 11726, 8046, 12936, 14290, 4378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=78954 dup_rows=11796 dup_rows_remote=11454 dedup_ratio=0.119995 router_tokens_per_expert=[3218, 0, 1, 79, 878, 4313, 0, 0, 1445, 71, 1778, 0, 0, 0, 0, 0, 0, 67, 133, 1, 0, 67, 1, 0, 160, 0, 0, 1, 0, 67, 0, 70, 0, 0, 6908, 4, 0, 0, 1343, 793, 0, 6949, 596, 7354, 58, 1, 4616, 67, 0, 0, 0, 67, 65, 0, 0, 5456, 334, 1, 0, 340, 630, 150, 253, 817] payload_rows_per_dst=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] dup_rows_per_dst=[678, 3228, 256, 454, 948, 342, 2040, 3850] combine_rows_per_src=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=304193536 dispatch_meta_bytes=356552 combine_payload_bytes=304193536 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=304550088 ingress_bytes=442605568 link_bytes=442605568 token_bytes=608743624 weight_movement_bytes=138412032 total_nvlink_bytes=747155656 weight_to_token_ratio=0.227373 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=17792 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.665489120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=276971520 dispatch_meta_bytes=306616 combine_payload_bytes=276971520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277278136 ingress_bytes=311574528 link_bytes=311574528 token_bytes=554249656 weight_movement_bytes=34603008 total_nvlink_bytes=588852664 weight_to_token_ratio=0.062432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.172429865e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=285057024 dispatch_meta_bytes=323848 combine_payload_bytes=285057024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=285380872 ingress_bytes=319660032 link_bytes=319660032 token_bytes=570437896 weight_movement_bytes=34603008 total_nvlink_bytes=605040904 weight_to_token_ratio=0.060660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=31360 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.202854966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=294092800 dispatch_meta_bytes=326848 combine_payload_bytes=294092800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=294419648 ingress_bytes=328695808 link_bytes=328695808 token_bytes=588512448 weight_movement_bytes=34603008 total_nvlink_bytes=623115456 weight_to_token_ratio=0.058797 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.236855863e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86422 payload_rows_remote=80016 dup_rows=11882 dup_rows_remote=11434 dedup_ratio=0.120870 router_tokens_per_expert=[0, 0, 60, 1692, 0, 0, 760, 0, 0, 2424, 879, 0, 0, 0, 719, 0, 107, 347, 0, 2595, 0, 59, 0, 0, 6532, 49, 0, 273, 0, 0, 0, 0, 703, 394, 0, 0, 1898, 1597, 0, 62, 0, 0, 5892, 0, 0, 0, 36, 0, 481, 0, 62, 0, 5938, 44, 6, 33, 6999, 0, 0, 0, 6772, 759, 980, 0] payload_rows_per_dst=[15294, 15730, 6406, 13706, 8060, 11856, 12194, 3176] dup_rows_per_dst=[3728, 5220, 448, 2, 1248, 0, 934, 302] combine_rows_per_src=[15294, 15730, 6406, 13706, 8060, 11856, 12194, 3176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=299139072 dispatch_meta_bytes=347456 combine_payload_bytes=299139072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=299486528 ingress_bytes=333742080 link_bytes=333742080 token_bytes=598625600 weight_movement_bytes=34603008 total_nvlink_bytes=633228608 weight_to_token_ratio=0.057804 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=21760 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255844579e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=312811520 dispatch_meta_bytes=349376 combine_payload_bytes=312811520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313160896 ingress_bytes=347414528 link_bytes=347414528 token_bytes=625972416 weight_movement_bytes=34603008 total_nvlink_bytes=660575424 weight_to_token_ratio=0.055279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.307292900e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86862 payload_rows_remote=73842 dup_rows=11442 dup_rows_remote=10424 dedup_ratio=0.116394 router_tokens_per_expert=[5, 9, 91, 1099, 10, 9, 1187, 0, 0, 2074, 953, 2, 74, 107, 707, 3, 144, 507, 46, 2137, 17, 61, 3, 10, 6269, 10, 0, 406, 25, 4, 0, 52, 744, 420, 44, 19, 1661, 1881, 16, 79, 77, 52, 5853, 0, 6, 6, 135, 98, 507, 61, 80, 0, 6103, 113, 88, 67, 6706, 0, 9, 2, 6531, 698, 1075, 0] payload_rows_per_dst=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] dup_rows_per_dst=[192, 1458, 4914, 20, 1212, 254, 1018, 2374] combine_rows_per_src=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=289783808 dispatch_meta_bytes=345216 combine_payload_bytes=289783808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290129024 ingress_bytes=393592832 link_bytes=393592832 token_bytes=579912832 weight_movement_bytes=103809024 total_nvlink_bytes=683721856 weight_to_token_ratio=0.179008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481058140e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78794 payload_rows_remote=65036 dup_rows=19510 dup_rows_remote=19180 dedup_ratio=0.198466 router_tokens_per_expert=[4, 0, 77, 10, 78, 0, 0, 0, 0, 2190, 0, 214, 92, 184, 1, 1342, 1, 6821, 2, 199, 0, 0, 53, 2, 6943, 4, 1353, 247, 3, 803, 90, 4409, 6875, 6769, 0, 799, 4, 0, 0, 0, 262, 52, 1189, 4, 0, 1483, 249, 3, 51, 1, 0, 2296, 0, 0, 0, 0, 0, 3879, 57, 0, 1, 3, 0, 53] payload_rows_per_dst=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] dup_rows_per_dst=[330, 6432, 28, 11240, 832, 624, 22, 2] combine_rows_per_src=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323395584 dispatch_meta_bytes=361632 combine_payload_bytes=323395584 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=323757216 ingress_bytes=496410624 link_bytes=496410624 token_bytes=647152800 weight_movement_bytes=173015040 total_nvlink_bytes=820167840 weight_to_token_ratio=0.267348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.867953214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73482 payload_rows_remote=57554 dup_rows=24822 dup_rows_remote=6408 dedup_ratio=0.252502 router_tokens_per_expert=[0, 0, 74, 1229, 0, 0, 1129, 0, 0, 1261, 803, 0, 0, 0, 793, 0, 152, 502, 0, 1739, 0, 59, 0, 0, 5960, 0, 0, 365, 0, 0, 0, 0, 749, 412, 0, 0, 1513, 2702, 0, 86, 20, 0, 5967, 0, 0, 0, 56, 0, 454, 0, 86, 0, 6475, 86, 78, 0, 7044, 0, 0, 0, 7012, 687, 1659, 0] payload_rows_per_dst=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] dup_rows_per_dst=[158, 1262, 1386, 0, 1264, 42, 2296, 18414] combine_rows_per_src=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85270 payload_rows_remote=79452 dup_rows=13034 dup_rows_remote=11802 dedup_ratio=0.132589 router_tokens_per_expert=[3101, 0, 0, 92, 757, 5564, 0, 0, 1661, 75, 1789, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 256, 0, 0, 0, 3, 92, 0, 74, 0, 0, 7057, 0, 0, 0, 1395, 768, 0, 6977, 656, 7566, 58, 0, 3549, 4, 0, 0, 0, 92, 139, 0, 0, 4353, 332, 0, 0, 340, 609, 216, 227, 1032] payload_rows_per_dst=[14582, 5818, 14290, 15284, 14820, 8054, 9168, 3254] dup_rows_per_dst=[4446, 1232, 300, 698, 3620, 480, 0, 2258] combine_rows_per_src=[14582, 5818, 14290, 15284, 14820, 8054, 9168, 3254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82968 payload_rows_remote=72298 dup_rows=15336 dup_rows_remote=13736 dedup_ratio=0.156006 router_tokens_per_expert=[1, 77, 247, 348, 42, 1436, 3, 460, 0, 0, 25, 1, 92, 0, 0, 1, 2262, 0, 0, 0, 0, 1363, 0, 1070, 2, 78, 169, 1739, 0, 0, 0, 79, 5600, 6, 92, 649, 0, 4, 190, 743, 267, 4, 5, 1812, 100, 1, 4, 0, 354, 0, 3983, 90, 5484, 77, 6417, 2, 0, 3521, 4054, 801, 2366, 2729, 1, 301] payload_rows_per_dst=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] dup_rows_per_dst=[1600, 46, 5866, 4052, 730, 2258, 366, 418] combine_rows_per_src=[10670, 13026, 14724, 11978, 3582, 13096, 8646, 7246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=327745536 dispatch_meta_bytes=365800 combine_payload_bytes=327745536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=328111336 ingress_bytes=362348544 link_bytes=362348544 token_bytes=655856872 weight_movement_bytes=34603008 total_nvlink_bytes=690459880 weight_to_token_ratio=0.052760 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.363488400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=296132608 dispatch_meta_bytes=344136 combine_payload_bytes=296132608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=296476744 ingress_bytes=330735616 link_bytes=330735616 token_bytes=592609352 weight_movement_bytes=34603008 total_nvlink_bytes=627212360 weight_to_token_ratio=0.058391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.244531497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82474 payload_rows_remote=70748 dup_rows=15830 dup_rows_remote=15556 dedup_ratio=0.161031 router_tokens_per_expert=[74, 67, 246, 817, 89, 451, 1309, 1, 3, 1138, 1775, 33, 854, 1197, 597, 35, 116, 535, 415, 1685, 99, 66, 19, 76, 4734, 136, 4, 485, 173, 52, 160, 256, 984, 401, 773, 527, 1282, 587, 207, 97, 372, 311, 4834, 0, 204, 195, 256, 1172, 518, 640, 116, 5, 5924, 714, 192, 132, 4257, 15, 242, 245, 4401, 741, 1106, 5] payload_rows_per_dst=[11460, 8294, 11344, 11726, 8046, 12936, 14290, 4378] dup_rows_per_dst=[3162, 2970, 3480, 274, 1670, 1752, 2192, 330] combine_rows_per_src=[11460, 8294, 11344, 11726, 8046, 12936, 14290, 4378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73482 payload_rows_remote=57554 dup_rows=24822 dup_rows_remote=6408 dedup_ratio=0.252502 router_tokens_per_expert=[0, 0, 74, 1229, 0, 0, 1129, 0, 0, 1261, 803, 0, 0, 0, 793, 0, 152, 502, 0, 1739, 0, 59, 0, 0, 5960, 0, 0, 365, 0, 0, 0, 0, 749, 412, 0, 0, 1513, 2702, 0, 86, 20, 0, 5967, 0, 0, 0, 56, 0, 454, 0, 86, 0, 6475, 86, 78, 0, 7044, 0, 0, 0, 7012, 687, 1659, 0] payload_rows_per_dst=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] dup_rows_per_dst=[158, 1262, 1386, 0, 1264, 42, 2296, 18414] combine_rows_per_src=[4706, 4452, 7994, 5006, 9660, 12044, 13692, 15928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86862 payload_rows_remote=73842 dup_rows=11442 dup_rows_remote=10424 dedup_ratio=0.116394 router_tokens_per_expert=[5, 9, 91, 1099, 10, 9, 1187, 0, 0, 2074, 953, 2, 74, 107, 707, 3, 144, 507, 46, 2137, 17, 61, 3, 10, 6269, 10, 0, 406, 25, 4, 0, 52, 744, 420, 44, 19, 1661, 1881, 16, 79, 77, 52, 5853, 0, 6, 6, 135, 98, 507, 61, 80, 0, 6103, 113, 88, 67, 6706, 0, 9, 2, 6531, 698, 1075, 0] payload_rows_per_dst=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] dup_rows_per_dst=[192, 1458, 4914, 20, 1212, 254, 1018, 2374] combine_rows_per_src=[4628, 6382, 13998, 13512, 8516, 12200, 13020, 14606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86948 payload_rows_remote=75642 dup_rows=11356 dup_rows_remote=11352 dedup_ratio=0.115519 router_tokens_per_expert=[0, 0, 60, 1686, 0, 0, 770, 0, 0, 2847, 789, 0, 1, 1, 794, 1, 168, 461, 0, 2412, 1, 68, 0, 0, 6562, 65, 0, 318, 0, 0, 0, 1, 627, 387, 0, 1, 1450, 2240, 0, 67, 2, 0, 5579, 0, 0, 1, 72, 1, 547, 1, 67, 0, 5564, 57, 16, 4, 6931, 0, 0, 2, 6741, 759, 1031, 0] payload_rows_per_dst=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] dup_rows_per_dst=[162, 1306, 5436, 0, 1080, 4, 904, 2464] combine_rows_per_src=[4870, 7560, 14266, 13892, 8464, 11306, 11608, 14982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86900 payload_rows_remote=71038 dup_rows=11404 dup_rows_remote=8526 dedup_ratio=0.116007 router_tokens_per_expert=[0, 0, 143, 1188, 0, 0, 1161, 0, 0, 1182, 855, 0, 0, 0, 715, 0, 160, 412, 0, 1902, 0, 73, 0, 0, 6465, 0, 0, 319, 0, 0, 0, 0, 772, 462, 0, 0, 1510, 2191, 0, 92, 81, 0, 5963, 0, 0, 1, 0, 0, 435, 0, 92, 0, 6566, 92, 76, 13, 7032, 0, 0, 0, 6618, 756, 1825, 0] payload_rows_per_dst=[15202, 15862, 4790, 13568, 10830, 11926, 9952, 4770] dup_rows_per_dst=[3846, 2878, 304, 0, 3124, 164, 696, 392] combine_rows_per_src=[15202, 15862, 4790, 13568, 10830, 11926, 9952, 4770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87654 payload_rows_remote=76362 dup_rows=10650 dup_rows_remote=8152 dedup_ratio=0.108337 router_tokens_per_expert=[2653, 0, 0, 93, 788, 5453, 0, 0, 1185, 83, 1357, 0, 0, 0, 0, 0, 0, 92, 154, 0, 0, 92, 0, 0, 284, 0, 0, 0, 0, 92, 0, 83, 0, 0, 6943, 0, 0, 0, 1200, 802, 0, 6922, 865, 7463, 66, 0, 4452, 9, 0, 0, 0, 92, 110, 0, 0, 4998, 332, 0, 0, 318, 593, 218, 252, 1108] payload_rows_per_dst=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] dup_rows_per_dst=[688, 2970, 326, 752, 2498, 648, 0, 2768] combine_rows_per_src=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=289783808 dispatch_meta_bytes=345216 combine_payload_bytes=289783808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290129024 ingress_bytes=393592832 link_bytes=393592832 token_bytes=579912832 weight_movement_bytes=103809024 total_nvlink_bytes=683721856 weight_to_token_ratio=0.179008 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481058140e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=235741184 dispatch_meta_bytes=255848 combine_payload_bytes=235741184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=235997032 ingress_bytes=374153216 link_bytes=374153216 token_bytes=471738216 weight_movement_bytes=138412032 total_nvlink_bytes=610150248 weight_to_token_ratio=0.293409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=302456832 dispatch_meta_bytes=337064 combine_payload_bytes=302456832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=302793896 ingress_bytes=371662848 link_bytes=371662848 token_bytes=605250728 weight_movement_bytes=69206016 total_nvlink_bytes=674456744 weight_to_token_ratio=0.114343 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40832 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398537376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86332 payload_rows_remote=72050 dup_rows=11972 dup_rows_remote=11774 dedup_ratio=0.121785 router_tokens_per_expert=[3282, 0, 0, 94, 804, 4890, 0, 0, 1322, 52, 1805, 0, 0, 0, 0, 0, 0, 62, 89, 0, 0, 62, 0, 0, 151, 0, 0, 0, 0, 62, 0, 49, 0, 0, 6856, 0, 0, 0, 1261, 743, 0, 7027, 494, 7402, 26, 0, 4679, 61, 0, 0, 0, 62, 60, 0, 0, 5154, 299, 0, 0, 422, 658, 119, 256, 849] payload_rows_per_dst=[14060, 5246, 14282, 14902, 14488, 10162, 10552, 2640] dup_rows_per_dst=[4080, 1112, 198, 426, 3232, 358, 0, 2566] combine_rows_per_src=[14060, 5246, 14282, 14902, 14488, 10162, 10552, 2640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=309829632 dispatch_meta_bytes=347976 combine_payload_bytes=309829632 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=310177608 ingress_bytes=344432640 link_bytes=344432640 token_bytes=620007240 weight_movement_bytes=34603008 total_nvlink_bytes=654610248 weight_to_token_ratio=0.055811 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.032500 gemm_active_groups=8 gemm_peak_group_rows=45824 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296072295e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=290971648 dispatch_meta_bytes=318256 combine_payload_bytes=290971648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=291289904 ingress_bytes=325574656 link_bytes=325574656 token_bytes=582261552 weight_movement_bytes=34603008 total_nvlink_bytes=616864560 weight_to_token_ratio=0.059429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.225111220e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78794 payload_rows_remote=65036 dup_rows=19510 dup_rows_remote=19180 dedup_ratio=0.198466 router_tokens_per_expert=[4, 0, 77, 10, 78, 0, 0, 0, 0, 2190, 0, 214, 92, 184, 1, 1342, 1, 6821, 2, 199, 0, 0, 53, 2, 6943, 4, 1353, 247, 3, 803, 90, 4409, 6875, 6769, 0, 799, 4, 0, 0, 0, 262, 52, 1189, 4, 0, 1483, 249, 3, 51, 1, 0, 2296, 0, 0, 0, 0, 0, 3879, 57, 0, 1, 3, 0, 53] payload_rows_per_dst=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] dup_rows_per_dst=[330, 6432, 28, 11240, 832, 624, 22, 2] combine_rows_per_src=[13758, 15256, 1336, 15614, 14312, 5860, 4674, 7984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=312778752 dispatch_meta_bytes=338056 combine_payload_bytes=312778752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313116808 ingress_bytes=381984768 link_bytes=381984768 token_bytes=625895560 weight_movement_bytes=69206016 total_nvlink_bytes=695101576 weight_to_token_ratio=0.110571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437377930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295116800 dispatch_meta_bytes=335296 combine_payload_bytes=295116800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295452096 ingress_bytes=329719808 link_bytes=329719808 token_bytes=590568896 weight_movement_bytes=34603008 total_nvlink_bytes=625171904 weight_to_token_ratio=0.058593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240709093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=336864 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266724320 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533111776 weight_movement_bytes=34603008 total_nvlink_bytes=567714784 weight_to_token_ratio=0.064908 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=235741184 dispatch_meta_bytes=255848 combine_payload_bytes=235741184 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=235997032 ingress_bytes=374153216 link_bytes=374153216 token_bytes=471738216 weight_movement_bytes=138412032 total_nvlink_bytes=610150248 weight_to_token_ratio=0.293409 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24192 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407908430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=365016 combine_payload_bytes=325435392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325800408 ingress_bytes=360038400 link_bytes=360038400 token_bytes=651235800 weight_movement_bytes=34603008 total_nvlink_bytes=685838808 weight_to_token_ratio=0.053134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354795514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87654 payload_rows_remote=76362 dup_rows=10650 dup_rows_remote=8152 dedup_ratio=0.108337 router_tokens_per_expert=[2653, 0, 0, 93, 788, 5453, 0, 0, 1185, 83, 1357, 0, 0, 0, 0, 0, 0, 92, 154, 0, 0, 92, 0, 0, 284, 0, 0, 0, 0, 92, 0, 83, 0, 0, 6943, 0, 0, 0, 1200, 802, 0, 6922, 865, 7463, 66, 0, 4452, 9, 0, 0, 0, 92, 110, 0, 0, 4998, 332, 0, 0, 318, 593, 218, 252, 1108] payload_rows_per_dst=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] dup_rows_per_dst=[688, 2970, 326, 752, 2498, 648, 0, 2768] combine_rows_per_src=[6380, 13186, 14194, 15092, 11292, 10136, 10400, 6974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68630 payload_rows_remote=53402 dup_rows=29674 dup_rows_remote=25360 dedup_ratio=0.301860 router_tokens_per_expert=[2840, 0, 0, 87, 804, 5782, 0, 0, 1392, 77, 1243, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 274, 0, 0, 0, 1, 87, 0, 78, 0, 0, 7012, 0, 0, 0, 1120, 750, 0, 7009, 779, 7596, 65, 0, 3984, 1, 0, 0, 0, 87, 65, 0, 0, 4941, 316, 0, 0, 331, 586, 133, 251, 1142] payload_rows_per_dst=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] dup_rows_per_dst=[652, 3446, 0, 2, 860, 15264, 5136, 4314] combine_rows_per_src=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81064 payload_rows_remote=70474 dup_rows=17240 dup_rows_remote=16100 dedup_ratio=0.175374 router_tokens_per_expert=[1329, 64, 1378, 88, 759, 5451, 413, 173, 2032, 233, 499, 15, 96, 84, 79, 58, 2, 65, 944, 972, 294, 106, 536, 210, 497, 35, 71, 368, 52, 102, 27, 245, 471, 1, 3391, 1, 22, 276, 1884, 497, 4, 4892, 625, 4468, 94, 776, 3589, 131, 155, 500, 93, 78, 37, 155, 433, 1890, 1348, 1228, 62, 955, 817, 189, 506, 2307] payload_rows_per_dst=[9500, 10888, 13688, 10590, 10538, 9890, 6354, 9616] dup_rows_per_dst=[1998, 3116, 2354, 1140, 2548, 548, 328, 5208] combine_rows_per_src=[9500, 10888, 13688, 10590, 10538, 9890, 6354, 9616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83646 payload_rows_remote=79452 dup_rows=14658 dup_rows_remote=14656 dedup_ratio=0.149109 router_tokens_per_expert=[5369, 822, 0, 0, 0, 0, 1704, 67, 0, 3274, 0, 4024, 27, 0, 1240, 0, 0, 134, 4066, 2, 0, 403, 6, 1, 0, 0, 1502, 808, 992, 0, 312, 332, 1, 0, 0, 0, 1, 0, 0, 5373, 0, 1, 268, 97, 1564, 167, 7155, 1, 6728, 51, 0, 146, 0, 67, 18, 44, 71, 0, 0, 1, 1232, 67, 0, 1014] payload_rows_per_dst=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] dup_rows_per_dst=[824, 800, 5542, 2956, 2, 2, 392, 4140] combine_rows_per_src=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=62950 dup_rows=22216 dup_rows_remote=16202 dedup_ratio=0.225993 router_tokens_per_expert=[2665, 4, 117, 85, 788, 5449, 25, 16, 1468, 109, 1514, 0, 10, 3, 3, 9, 0, 78, 179, 100, 40, 79, 48, 19, 209, 0, 6, 41, 4, 79, 0, 95, 32, 0, 6477, 0, 1, 18, 1344, 774, 0, 6823, 680, 7080, 66, 75, 4454, 80, 14, 18, 13, 79, 83, 12, 34, 4869, 405, 112, 3, 355, 625, 143, 281, 960] payload_rows_per_dst=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] dup_rows_per_dst=[728, 3758, 352, 312, 888, 6148, 6014, 4016] combine_rows_per_src=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86508 payload_rows_remote=78954 dup_rows=11796 dup_rows_remote=11454 dedup_ratio=0.119995 router_tokens_per_expert=[3218, 0, 1, 79, 878, 4313, 0, 0, 1445, 71, 1778, 0, 0, 0, 0, 0, 0, 67, 133, 1, 0, 67, 1, 0, 160, 0, 0, 1, 0, 67, 0, 70, 0, 0, 6908, 4, 0, 0, 1343, 793, 0, 6949, 596, 7354, 58, 1, 4616, 67, 0, 0, 0, 67, 65, 0, 0, 5456, 334, 1, 0, 340, 630, 150, 253, 817] payload_rows_per_dst=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] dup_rows_per_dst=[678, 3228, 256, 454, 948, 342, 2040, 3850] combine_rows_per_src=[7674, 11986, 14180, 14850, 3332, 7554, 11916, 15016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=312778752 dispatch_meta_bytes=338056 combine_payload_bytes=312778752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=313116808 ingress_bytes=381984768 link_bytes=381984768 token_bytes=625895560 weight_movement_bytes=69206016 total_nvlink_bytes=695101576 weight_to_token_ratio=0.110571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437377930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86332 payload_rows_remote=72050 dup_rows=11972 dup_rows_remote=11774 dedup_ratio=0.121785 router_tokens_per_expert=[3282, 0, 0, 94, 804, 4890, 0, 0, 1322, 52, 1805, 0, 0, 0, 0, 0, 0, 62, 89, 0, 0, 62, 0, 0, 151, 0, 0, 0, 0, 62, 0, 49, 0, 0, 6856, 0, 0, 0, 1261, 743, 0, 7027, 494, 7402, 26, 0, 4679, 61, 0, 0, 0, 62, 60, 0, 0, 5154, 299, 0, 0, 422, 658, 119, 256, 849] payload_rows_per_dst=[14060, 5246, 14282, 14902, 14488, 10162, 10552, 2640] dup_rows_per_dst=[4080, 1112, 198, 426, 3232, 358, 0, 2566] combine_rows_per_src=[14060, 5246, 14282, 14902, 14488, 10162, 10552, 2640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85270 payload_rows_remote=79452 dup_rows=13034 dup_rows_remote=11802 dedup_ratio=0.132589 router_tokens_per_expert=[3101, 0, 0, 92, 757, 5564, 0, 0, 1661, 75, 1789, 0, 0, 0, 0, 0, 0, 92, 134, 0, 0, 92, 0, 0, 256, 0, 0, 0, 3, 92, 0, 74, 0, 0, 7057, 0, 0, 0, 1395, 768, 0, 6977, 656, 7566, 58, 0, 3549, 4, 0, 0, 0, 92, 139, 0, 0, 4353, 332, 0, 0, 340, 609, 216, 227, 1032] payload_rows_per_dst=[14582, 5818, 14290, 15284, 14820, 8054, 9168, 3254] dup_rows_per_dst=[4446, 1232, 300, 698, 3620, 480, 0, 2258] combine_rows_per_src=[14582, 5818, 14290, 15284, 14820, 8054, 9168, 3254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82888 payload_rows_remote=73732 dup_rows=15416 dup_rows_remote=15416 dedup_ratio=0.156820 router_tokens_per_expert=[5445, 1580, 0, 0, 0, 0, 2430, 91, 0, 2251, 0, 4060, 0, 0, 971, 0, 0, 184, 3551, 0, 0, 350, 0, 0, 0, 0, 2257, 611, 1130, 0, 397, 349, 0, 0, 0, 0, 0, 0, 0, 4578, 0, 0, 406, 93, 1935, 111, 7000, 0, 6514, 53, 0, 160, 0, 92, 0, 0, 83, 0, 0, 0, 1114, 89, 0, 1267] payload_rows_per_dst=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] dup_rows_per_dst=[1586, 2462, 5116, 1198, 0, 1288, 430, 3336] combine_rows_per_src=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218734592 dispatch_meta_bytes=315048 combine_payload_bytes=218734592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219049640 ingress_bytes=253337600 link_bytes=253337600 token_bytes=437784232 weight_movement_bytes=34603008 total_nvlink_bytes=472387240 weight_to_token_ratio=0.079041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.532889935e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=288661504 dispatch_meta_bytes=346296 combine_payload_bytes=288661504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=289007800 ingress_bytes=323264512 link_bytes=323264512 token_bytes=577669304 weight_movement_bytes=34603008 total_nvlink_bytes=612272312 weight_to_token_ratio=0.059901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=45056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.216418334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76088 payload_rows_remote=62950 dup_rows=22216 dup_rows_remote=16202 dedup_ratio=0.225993 router_tokens_per_expert=[2665, 4, 117, 85, 788, 5449, 25, 16, 1468, 109, 1514, 0, 10, 3, 3, 9, 0, 78, 179, 100, 40, 79, 48, 19, 209, 0, 6, 41, 4, 79, 0, 95, 32, 0, 6477, 0, 1, 18, 1344, 774, 0, 6823, 680, 7080, 66, 75, 4454, 80, 14, 18, 13, 79, 83, 12, 34, 4869, 405, 112, 3, 355, 625, 143, 281, 960] payload_rows_per_dst=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] dup_rows_per_dst=[728, 3758, 352, 312, 888, 6148, 6014, 4016] combine_rows_per_src=[6672, 13372, 8606, 7010, 3450, 9134, 13138, 14706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=77770 dup_rows=11870 dup_rows_remote=11870 dedup_ratio=0.120748 router_tokens_per_expert=[5842, 682, 0, 0, 0, 0, 1595, 62, 0, 2545, 0, 4446, 0, 0, 1390, 0, 0, 124, 3872, 0, 0, 336, 0, 0, 0, 0, 1895, 681, 1099, 0, 287, 316, 0, 0, 0, 0, 0, 0, 0, 5223, 0, 0, 182, 70, 2133, 131, 7237, 0, 6806, 29, 0, 125, 0, 62, 0, 0, 52, 0, 0, 0, 856, 62, 0, 1012] payload_rows_per_dst=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] dup_rows_per_dst=[2692, 3438, 0, 862, 0, 4258, 308, 312] combine_rows_per_src=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=257843200 dispatch_meta_bytes=316608 combine_payload_bytes=257843200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258159808 ingress_bytes=292446208 link_bytes=292446208 token_bytes=516003008 weight_movement_bytes=34603008 total_nvlink_bytes=550606016 weight_to_token_ratio=0.067060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100451537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=323395584 dispatch_meta_bytes=361632 combine_payload_bytes=323395584 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=323757216 ingress_bytes=496410624 link_bytes=496410624 token_bytes=647152800 weight_movement_bytes=173015040 total_nvlink_bytes=820167840 weight_to_token_ratio=0.267348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.867953214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81668 payload_rows_remote=66556 dup_rows=16636 dup_rows_remote=13026 dedup_ratio=0.169230 router_tokens_per_expert=[0, 1, 118, 828, 0, 5, 1382, 0, 0, 1199, 927, 0, 2, 6, 702, 0, 161, 444, 5, 1494, 0, 53, 0, 0, 6138, 0, 0, 388, 0, 0, 0, 1, 827, 457, 2, 3, 1322, 2929, 0, 90, 92, 3, 5936, 0, 0, 8, 29, 4, 488, 2, 90, 0, 6455, 91, 100, 50, 7027, 0, 2, 1, 6743, 801, 1746, 0] payload_rows_per_dst=[15112, 15918, 3914, 13160, 16056, 11382, 1362, 4764] dup_rows_per_dst=[3610, 3240, 400, 478, 8114, 178, 280, 336] combine_rows_per_src=[15112, 15918, 3914, 13160, 16056, 11382, 1362, 4764] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=376432 combine_payload_bytes=325435392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325811824 ingress_bytes=429244416 link_bytes=429244416 token_bytes=651247216 weight_movement_bytes=103809024 total_nvlink_bytes=755056240 weight_to_token_ratio=0.159400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.615212180e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=68630 payload_rows_remote=53402 dup_rows=29674 dup_rows_remote=25360 dedup_ratio=0.301860 router_tokens_per_expert=[2840, 0, 0, 87, 804, 5782, 0, 0, 1392, 77, 1243, 0, 0, 0, 0, 0, 0, 87, 146, 0, 0, 86, 0, 0, 274, 0, 0, 0, 1, 87, 0, 78, 0, 0, 7012, 0, 0, 0, 1120, 750, 0, 7009, 779, 7596, 65, 0, 3984, 1, 0, 0, 0, 87, 65, 0, 0, 4941, 316, 0, 0, 331, 586, 133, 251, 1142] payload_rows_per_dst=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] dup_rows_per_dst=[652, 3446, 0, 2, 860, 15264, 5136, 4314] combine_rows_per_src=[6412, 13542, 638, 878, 2880, 16034, 13018, 15228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85516 payload_rows_remote=74430 dup_rows=12788 dup_rows_remote=10842 dedup_ratio=0.130086 router_tokens_per_expert=[4843, 1328, 0, 0, 0, 0, 2867, 92, 0, 2314, 0, 3252, 0, 0, 950, 0, 0, 184, 4238, 0, 0, 358, 0, 0, 0, 0, 1668, 578, 1600, 0, 415, 399, 0, 0, 0, 0, 0, 0, 0, 4868, 0, 0, 480, 121, 1777, 111, 7172, 0, 5971, 12, 0, 170, 0, 92, 0, 0, 76, 0, 0, 0, 1889, 92, 0, 1235] payload_rows_per_dst=[13240, 11086, 9560, 8094, 9736, 15636, 12136, 6028] dup_rows_per_dst=[5020, 1946, 0, 1226, 0, 3686, 354, 556] combine_rows_per_src=[13240, 11086, 9560, 8094, 9736, 15636, 12136, 6028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=302006272 dispatch_meta_bytes=356592 combine_payload_bytes=302006272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302362864 ingress_bytes=336609280 link_bytes=336609280 token_bytes=604369136 weight_movement_bytes=34603008 total_nvlink_bytes=638972144 weight_to_token_ratio=0.057255 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266633621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=365016 combine_payload_bytes=325435392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=325800408 ingress_bytes=360038400 link_bytes=360038400 token_bytes=651235800 weight_movement_bytes=34603008 total_nvlink_bytes=685838808 weight_to_token_ratio=0.053134 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.354795514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318545920 dispatch_meta_bytes=358560 combine_payload_bytes=318545920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318904480 ingress_bytes=353148928 link_bytes=353148928 token_bytes=637450400 weight_movement_bytes=34603008 total_nvlink_bytes=672053408 weight_to_token_ratio=0.054283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328870985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=295116800 dispatch_meta_bytes=335296 combine_payload_bytes=295116800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=295452096 ingress_bytes=329719808 link_bytes=329719808 token_bytes=590568896 weight_movement_bytes=34603008 total_nvlink_bytes=625171904 weight_to_token_ratio=0.058593 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=43776 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.240709093e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=257843200 dispatch_meta_bytes=316608 combine_payload_bytes=257843200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=258159808 ingress_bytes=292446208 link_bytes=292446208 token_bytes=516003008 weight_movement_bytes=34603008 total_nvlink_bytes=550606016 weight_to_token_ratio=0.067060 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=36736 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.100451537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=272613376 dispatch_meta_bytes=318328 combine_payload_bytes=272613376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272931704 ingress_bytes=307216384 link_bytes=307216384 token_bytes=545545080 weight_movement_bytes=34603008 total_nvlink_bytes=580148088 weight_to_token_ratio=0.063428 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.156030520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=218734592 dispatch_meta_bytes=315048 combine_payload_bytes=218734592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=219049640 ingress_bytes=253337600 link_bytes=253337600 token_bytes=437784232 weight_movement_bytes=34603008 total_nvlink_bytes=472387240 weight_to_token_ratio=0.079041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.532889935e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82888 payload_rows_remote=73732 dup_rows=15416 dup_rows_remote=15416 dedup_ratio=0.156820 router_tokens_per_expert=[5445, 1580, 0, 0, 0, 0, 2430, 91, 0, 2251, 0, 4060, 0, 0, 971, 0, 0, 184, 3551, 0, 0, 350, 0, 0, 0, 0, 2257, 611, 1130, 0, 397, 349, 0, 0, 0, 0, 0, 0, 0, 4578, 0, 0, 406, 93, 1935, 111, 7000, 0, 6514, 53, 0, 160, 0, 92, 0, 0, 83, 0, 0, 0, 1114, 89, 0, 1267] payload_rows_per_dst=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] dup_rows_per_dst=[1586, 2462, 5116, 1198, 0, 1288, 430, 3336] combine_rows_per_src=[6616, 12102, 13944, 8290, 9156, 7990, 13212, 11578] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304865280 dispatch_meta_bytes=341088 combine_payload_bytes=304865280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305206368 ingress_bytes=374071296 link_bytes=374071296 token_bytes=610071648 weight_movement_bytes=69206016 total_nvlink_bytes=679277664 weight_to_token_ratio=0.113439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407600172e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81334 payload_rows_remote=66430 dup_rows=16970 dup_rows_remote=11522 dedup_ratio=0.172628 router_tokens_per_expert=[5399, 1367, 0, 0, 0, 0, 2582, 86, 0, 2108, 0, 3779, 0, 0, 992, 0, 0, 173, 3747, 0, 0, 335, 0, 0, 0, 0, 2069, 638, 1259, 0, 336, 352, 0, 0, 0, 0, 0, 0, 0, 4628, 0, 0, 424, 136, 1786, 105, 7155, 0, 6432, 1, 0, 156, 0, 86, 0, 0, 78, 0, 0, 0, 1789, 86, 0, 1068] payload_rows_per_dst=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] dup_rows_per_dst=[1322, 84, 0, 3802, 5734, 274, 306, 5448] combine_rows_per_src=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78358 payload_rows_remote=68888 dup_rows=19946 dup_rows_remote=18212 dedup_ratio=0.202901 router_tokens_per_expert=[3903, 2830, 38, 13, 278, 305, 1252, 150, 221, 1639, 526, 1950, 245, 108, 612, 3, 273, 221, 1899, 251, 233, 391, 112, 321, 53, 244, 1354, 764, 1465, 370, 425, 927, 1346, 915, 132, 6, 1095, 63, 11, 2216, 111, 1182, 1175, 112, 2890, 279, 4377, 679, 4594, 101, 35, 258, 0, 95, 116, 392, 82, 76, 15, 1341, 935, 154, 92, 901] payload_rows_per_dst=[7988, 8874, 11184, 9470, 8774, 14682, 10656, 6730] dup_rows_per_dst=[2888, 1734, 2880, 1734, 2794, 6928, 526, 462] combine_rows_per_src=[7988, 8874, 11184, 9470, 8774, 14682, 10656, 6730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82840 payload_rows_remote=66930 dup_rows=15464 dup_rows_remote=8026 dedup_ratio=0.157308 router_tokens_per_expert=[0, 67, 1608, 0, 0, 0, 3466, 0, 0, 0, 180, 1395, 7310, 0, 1, 3669, 0, 0, 0, 0, 0, 2620, 1, 1, 1, 0, 1, 7832, 0, 1, 2601, 0, 8, 0, 131, 0, 0, 4964, 0, 0, 0, 0, 3564, 0, 0, 1, 0, 277, 0, 0, 372, 0, 3, 1561, 0, 67, 1, 6719, 0, 0, 648, 81, 0, 1] payload_rows_per_dst=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] dup_rows_per_dst=[1060, 936, 5092, 2, 16, 7438, 654, 266] combine_rows_per_src=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=302006272 dispatch_meta_bytes=356592 combine_payload_bytes=302006272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302362864 ingress_bytes=336609280 link_bytes=336609280 token_bytes=604369136 weight_movement_bytes=34603008 total_nvlink_bytes=638972144 weight_to_token_ratio=0.057255 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=35968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266633621e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86434 payload_rows_remote=77770 dup_rows=11870 dup_rows_remote=11870 dedup_ratio=0.120748 router_tokens_per_expert=[5842, 682, 0, 0, 0, 0, 1595, 62, 0, 2545, 0, 4446, 0, 0, 1390, 0, 0, 124, 3872, 0, 0, 336, 0, 0, 0, 0, 1895, 681, 1099, 0, 287, 316, 0, 0, 0, 0, 0, 0, 0, 5223, 0, 0, 182, 70, 2133, 131, 7237, 0, 6806, 29, 0, 125, 0, 62, 0, 0, 52, 0, 0, 0, 856, 62, 0, 1012] payload_rows_per_dst=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] dup_rows_per_dst=[2692, 3438, 0, 862, 0, 4258, 308, 312] combine_rows_per_src=[13670, 13324, 8664, 7694, 10446, 15248, 13736, 3652] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=348400 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282513648 ingress_bytes=316768256 link_bytes=316768256 token_bytes=564678896 weight_movement_bytes=34603008 total_nvlink_bytes=599281904 weight_to_token_ratio=0.061279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191973445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80984 payload_rows_remote=67538 dup_rows=17320 dup_rows_remote=16844 dedup_ratio=0.176188 router_tokens_per_expert=[5600, 1276, 1, 0, 10, 9, 1792, 89, 9, 2378, 46, 4172, 13, 8, 1123, 0, 25, 164, 3201, 17, 16, 359, 10, 17, 0, 24, 1858, 660, 1419, 64, 453, 357, 114, 68, 10, 0, 101, 16, 0, 4365, 10, 97, 370, 102, 1905, 156, 6861, 46, 6576, 54, 4, 161, 0, 79, 8, 79, 82, 0, 0, 125, 1367, 80, 4, 1142] payload_rows_per_dst=[5140, 6950, 9698, 13606, 12502, 4960, 13446, 14682] dup_rows_per_dst=[1214, 204, 1138, 4408, 4828, 412, 476, 4640] combine_rows_per_src=[5140, 6950, 9698, 13606, 12502, 4960, 13446, 14682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81326 payload_rows_remote=66392 dup_rows=16978 dup_rows_remote=12158 dedup_ratio=0.172709 router_tokens_per_expert=[3083, 1, 6, 92, 763, 5931, 1, 0, 1444, 101, 1156, 0, 1, 0, 0, 0, 0, 92, 177, 5, 0, 90, 3, 1, 258, 0, 0, 3, 0, 92, 0, 101, 0, 0, 6971, 0, 0, 1, 1012, 671, 0, 6952, 881, 7503, 73, 4, 3610, 8, 2, 0, 0, 92, 19, 1, 1, 5005, 354, 6, 0, 350, 631, 171, 248, 1185] payload_rows_per_dst=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] dup_rows_per_dst=[4820, 766, 342, 696, 7940, 74, 2, 2338] combine_rows_per_src=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=274145280 dispatch_meta_bytes=299824 combine_payload_bytes=274145280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274445104 ingress_bytes=308748288 link_bytes=308748288 token_bytes=548590384 weight_movement_bytes=34603008 total_nvlink_bytes=583193392 weight_to_token_ratio=0.063076 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161794951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81334 payload_rows_remote=66430 dup_rows=16970 dup_rows_remote=11522 dedup_ratio=0.172628 router_tokens_per_expert=[5399, 1367, 0, 0, 0, 0, 2582, 86, 0, 2108, 0, 3779, 0, 0, 992, 0, 0, 173, 3747, 0, 0, 335, 0, 0, 0, 0, 2069, 638, 1259, 0, 336, 352, 0, 0, 0, 0, 0, 0, 0, 4628, 0, 0, 424, 136, 1786, 105, 7155, 0, 6432, 1, 0, 156, 0, 86, 0, 0, 78, 0, 0, 0, 1789, 86, 0, 1068] payload_rows_per_dst=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] dup_rows_per_dst=[1322, 84, 0, 3802, 5734, 274, 306, 5448] combine_rows_per_src=[6748, 6116, 8510, 13064, 14320, 4628, 13044, 14904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=318545920 dispatch_meta_bytes=358560 combine_payload_bytes=318545920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=318904480 ingress_bytes=353148928 link_bytes=353148928 token_bytes=637450400 weight_movement_bytes=34603008 total_nvlink_bytes=672053408 weight_to_token_ratio=0.054283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=27904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.328870985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78666 payload_rows_remote=63204 dup_rows=19638 dup_rows_remote=15456 dedup_ratio=0.199768 router_tokens_per_expert=[0, 5, 2427, 0, 0, 0, 4514, 0, 0, 0, 150, 639, 7079, 0, 0, 1954, 1, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 7815, 0, 0, 3377, 0, 0, 0, 150, 0, 0, 6363, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 611, 0, 0, 256, 0, 0, 3449, 0, 92, 0, 6491, 0, 0, 496, 447, 0, 0] payload_rows_per_dst=[11726, 15462, 2930, 15814, 13026, 3920, 13902, 1886] dup_rows_per_dst=[2166, 4182, 0, 6570, 0, 46, 6674, 0] combine_rows_per_src=[11726, 15462, 2930, 15814, 13026, 3920, 13902, 1886] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=276635648 dispatch_meta_bytes=337528 combine_payload_bytes=276635648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276973176 ingress_bytes=345841664 link_bytes=345841664 token_bytes=553608824 weight_movement_bytes=69206016 total_nvlink_bytes=622814840 weight_to_token_ratio=0.125009 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301374339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=271941632 dispatch_meta_bytes=314200 combine_payload_bytes=271941632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272255832 ingress_bytes=341147648 link_bytes=341147648 token_bytes=544197464 weight_movement_bytes=69206016 total_nvlink_bytes=613403480 weight_to_token_ratio=0.127171 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283711135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83646 payload_rows_remote=79452 dup_rows=14658 dup_rows_remote=14656 dedup_ratio=0.149109 router_tokens_per_expert=[5369, 822, 0, 0, 0, 0, 1704, 67, 0, 3274, 0, 4024, 27, 0, 1240, 0, 0, 134, 4066, 2, 0, 403, 6, 1, 0, 0, 1502, 808, 992, 0, 312, 332, 1, 0, 0, 0, 1, 0, 0, 5373, 0, 1, 268, 97, 1564, 167, 7155, 1, 6728, 51, 0, 146, 0, 67, 18, 44, 71, 0, 0, 1, 1232, 67, 0, 1014] payload_rows_per_dst=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] dup_rows_per_dst=[824, 800, 5542, 2956, 2, 2, 392, 4140] combine_rows_per_src=[4362, 8706, 14420, 12560, 10748, 4194, 13716, 14940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86782 payload_rows_remote=73582 dup_rows=11522 dup_rows_remote=11522 dedup_ratio=0.117208 router_tokens_per_expert=[0, 73, 2480, 0, 0, 0, 3606, 0, 0, 0, 161, 1057, 7102, 0, 0, 2349, 0, 0, 0, 0, 0, 1465, 2, 0, 0, 0, 0, 7779, 0, 0, 3738, 0, 0, 0, 161, 0, 0, 6437, 0, 2, 0, 0, 1900, 0, 0, 0, 0, 321, 0, 0, 308, 0, 0, 2787, 0, 92, 0, 6379, 0, 0, 620, 333, 0, 0] payload_rows_per_dst=[10984, 6822, 14426, 7476, 13200, 15888, 9488, 8498] dup_rows_per_dst=[1334, 312, 2712, 0, 0, 4112, 2976, 76] combine_rows_per_src=[10984, 6822, 14426, 7476, 13200, 15888, 9488, 8498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85516 payload_rows_remote=74430 dup_rows=12788 dup_rows_remote=10842 dedup_ratio=0.130086 router_tokens_per_expert=[4843, 1328, 0, 0, 0, 0, 2867, 92, 0, 2314, 0, 3252, 0, 0, 950, 0, 0, 184, 4238, 0, 0, 358, 0, 0, 0, 0, 1668, 578, 1600, 0, 415, 399, 0, 0, 0, 0, 0, 0, 0, 4868, 0, 0, 480, 121, 1777, 111, 7172, 0, 5971, 12, 0, 170, 0, 92, 0, 0, 76, 0, 0, 0, 1889, 92, 0, 1235] payload_rows_per_dst=[13240, 11086, 9560, 8094, 9736, 15636, 12136, 6028] dup_rows_per_dst=[5020, 1946, 0, 1226, 0, 3686, 354, 556] combine_rows_per_src=[13240, 11086, 9560, 8094, 9736, 15636, 12136, 6028] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=272097280 dispatch_meta_bytes=311808 combine_payload_bytes=272097280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=272409088 ingress_bytes=306700288 link_bytes=306700288 token_bytes=544506368 weight_movement_bytes=34603008 total_nvlink_bytes=579109376 weight_to_token_ratio=0.063549 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.154088492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=258883584 dispatch_meta_bytes=314640 combine_payload_bytes=258883584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=259198224 ingress_bytes=362692608 link_bytes=362692608 token_bytes=518081808 weight_movement_bytes=103809024 total_nvlink_bytes=621890832 weight_to_token_ratio=0.200372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364783085e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87722 payload_rows_remote=80230 dup_rows=10582 dup_rows_remote=10326 dedup_ratio=0.107646 router_tokens_per_expert=[40, 221, 3768, 328, 47, 40, 2013, 9, 3, 74, 157, 444, 4962, 31, 1120, 1899, 272, 13, 26, 120, 83, 652, 240, 991, 38, 58, 95, 4974, 195, 1451, 1955, 82, 177, 64, 133, 6, 100, 5224, 572, 119, 46, 0, 676, 11, 349, 1607, 797, 271, 19, 766, 144, 249, 76, 572, 1288, 109, 132, 3371, 1, 337, 567, 3792, 1055, 121] payload_rows_per_dst=[10810, 7228, 13260, 7492, 12244, 14466, 11468, 10754] dup_rows_per_dst=[2122, 228, 1458, 256, 546, 2996, 1720, 1256] combine_rows_per_src=[10810, 7228, 13260, 7492, 12244, 14466, 11468, 10754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=74972 dup_rows=8936 dup_rows_remote=8788 dedup_ratio=0.090902 router_tokens_per_expert=[44, 0, 0, 0, 1, 67, 413, 0, 0, 71, 89, 0, 0, 103, 2644, 0, 4503, 1, 99, 7197, 7042, 0, 1, 0, 0, 657, 488, 0, 7373, 0, 63, 0, 3674, 1, 1, 0, 253, 3396, 6859, 67, 0, 0, 0, 1, 0, 5, 0, 69, 0, 1286, 0, 629, 0, 1, 0, 0, 0, 0, 0, 254, 1090, 1, 379, 330] payload_rows_per_dst=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] dup_rows_per_dst=[144, 14, 0, 1358, 2772, 148, 1824, 2676] combine_rows_per_src=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=301391872 dispatch_meta_bytes=340416 combine_payload_bytes=301391872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301732288 ingress_bytes=370597888 link_bytes=370597888 token_bytes=603124160 weight_movement_bytes=69206016 total_nvlink_bytes=672330176 weight_to_token_ratio=0.114746 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=282165248 dispatch_meta_bytes=348400 combine_payload_bytes=282165248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=282513648 ingress_bytes=316768256 link_bytes=316768256 token_bytes=564678896 weight_movement_bytes=34603008 total_nvlink_bytes=599281904 weight_to_token_ratio=0.061279 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=15104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.191973445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79704 payload_rows_remote=68048 dup_rows=18600 dup_rows_remote=14486 dedup_ratio=0.189209 router_tokens_per_expert=[0, 35, 1751, 0, 0, 0, 2879, 0, 0, 0, 103, 1599, 7334, 0, 0, 3431, 0, 0, 0, 0, 0, 3011, 0, 0, 0, 0, 0, 7864, 0, 0, 2641, 0, 0, 0, 100, 0, 0, 5329, 0, 0, 0, 0, 2897, 0, 0, 0, 0, 266, 0, 0, 417, 0, 0, 1678, 0, 62, 0, 6944, 0, 0, 729, 82, 0, 0] payload_rows_per_dst=[8630, 11354, 11656, 7928, 10858, 13330, 14334, 1614] dup_rows_per_dst=[700, 3832, 4114, 1294, 0, 4784, 3868, 8] combine_rows_per_src=[8630, 11354, 11656, 7928, 10858, 13330, 14334, 1614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=362224 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=328984304 ingress_bytes=432431104 link_bytes=432431104 token_bytes=657606384 weight_movement_bytes=103809024 total_nvlink_bytes=761415408 weight_to_token_ratio=0.157859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.627203431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79604 payload_rows_remote=75646 dup_rows=18700 dup_rows_remote=18020 dedup_ratio=0.190226 router_tokens_per_expert=[1, 89, 2402, 25, 0, 2, 2945, 1, 0, 0, 144, 1791, 7030, 2, 131, 3519, 8, 1, 3, 13, 7, 1915, 52, 76, 18, 0, 15, 7574, 11, 130, 2722, 8, 7, 12, 156, 0, 5, 5661, 26, 11, 1, 0, 1950, 0, 12, 136, 78, 348, 1, 59, 368, 23, 9, 1677, 103, 79, 18, 6420, 0, 38, 732, 471, 72, 44] payload_rows_per_dst=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] dup_rows_per_dst=[1044, 1018, 3672, 54, 44, 1162, 680, 11026] combine_rows_per_src=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=276635648 dispatch_meta_bytes=337528 combine_payload_bytes=276635648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=276973176 ingress_bytes=345841664 link_bytes=345841664 token_bytes=553608824 weight_movement_bytes=69206016 total_nvlink_bytes=622814840 weight_to_token_ratio=0.125009 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=46208 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301374339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81326 payload_rows_remote=66392 dup_rows=16978 dup_rows_remote=12158 dedup_ratio=0.172709 router_tokens_per_expert=[3083, 1, 6, 92, 763, 5931, 1, 0, 1444, 101, 1156, 0, 1, 0, 0, 0, 0, 92, 177, 5, 0, 90, 3, 1, 258, 0, 0, 3, 0, 92, 0, 101, 0, 0, 6971, 0, 0, 1, 1012, 671, 0, 6952, 881, 7503, 73, 4, 3610, 8, 2, 0, 0, 92, 19, 1, 1, 5005, 354, 6, 0, 350, 631, 171, 248, 1185] payload_rows_per_dst=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] dup_rows_per_dst=[4820, 766, 342, 696, 7940, 74, 2, 2338] combine_rows_per_src=[14934, 4638, 14298, 15218, 14970, 3478, 10238, 3552] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307085312 dispatch_meta_bytes=335040 combine_payload_bytes=307085312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307420352 ingress_bytes=341688320 link_bytes=341688320 token_bytes=614505664 weight_movement_bytes=34603008 total_nvlink_bytes=649108672 weight_to_token_ratio=0.056310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285745640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76602 payload_rows_remote=62308 dup_rows=21702 dup_rows_remote=11002 dedup_ratio=0.220764 router_tokens_per_expert=[4970, 2290, 0, 0, 1, 0, 2474, 90, 1, 1705, 1, 3627, 2, 0, 894, 0, 2, 182, 3289, 0, 0, 351, 0, 0, 0, 4, 2417, 583, 886, 0, 411, 369, 6, 3, 1, 0, 4, 0, 0, 4618, 0, 5, 438, 164, 1998, 95, 7028, 3, 6538, 6, 1, 146, 0, 90, 5, 6, 100, 0, 0, 6, 2090, 92, 0, 1160] payload_rows_per_dst=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] dup_rows_per_dst=[10700, 6418, 0, 1220, 14, 2582, 208, 560] combine_rows_per_src=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=325435392 dispatch_meta_bytes=376432 combine_payload_bytes=325435392 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=325811824 ingress_bytes=429244416 link_bytes=429244416 token_bytes=651247216 weight_movement_bytes=103809024 total_nvlink_bytes=755056240 weight_to_token_ratio=0.159400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=25344 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.615212180e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=301391872 dispatch_meta_bytes=340416 combine_payload_bytes=301391872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301732288 ingress_bytes=370597888 link_bytes=370597888 token_bytes=603124160 weight_movement_bytes=69206016 total_nvlink_bytes=672330176 weight_to_token_ratio=0.114746 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=46720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=304865280 dispatch_meta_bytes=341088 combine_payload_bytes=304865280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=305206368 ingress_bytes=374071296 link_bytes=374071296 token_bytes=610071648 weight_movement_bytes=69206016 total_nvlink_bytes=679277664 weight_to_token_ratio=0.113439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=18304 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.407600172e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=278724608 dispatch_meta_bytes=330136 combine_payload_bytes=278724608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=279054744 ingress_bytes=313327616 link_bytes=313327616 token_bytes=557779352 weight_movement_bytes=34603008 total_nvlink_bytes=592382360 weight_to_token_ratio=0.062037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=33536 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.179026594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=271941632 dispatch_meta_bytes=314200 combine_payload_bytes=271941632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=272255832 ingress_bytes=341147648 link_bytes=341147648 token_bytes=544197464 weight_movement_bytes=69206016 total_nvlink_bytes=613403480 weight_to_token_ratio=0.127171 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22016 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.283711135e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76814 payload_rows_remote=60548 dup_rows=21490 dup_rows_remote=7254 dedup_ratio=0.218608 router_tokens_per_expert=[0, 11, 2547, 0, 0, 0, 3741, 0, 0, 0, 154, 855, 7158, 0, 0, 2265, 0, 0, 0, 0, 0, 1246, 3, 0, 0, 0, 0, 7787, 0, 0, 3840, 0, 0, 0, 154, 0, 0, 6924, 0, 0, 0, 0, 1542, 0, 0, 0, 0, 366, 0, 0, 309, 0, 0, 2699, 0, 87, 0, 6441, 0, 0, 517, 506, 0, 0] payload_rows_per_dst=[11046, 5914, 6326, 13838, 14234, 3770, 5420, 16266] dup_rows_per_dst=[1552, 208, 514, 3816, 348, 46, 770, 14236] combine_rows_per_src=[11046, 5914, 6326, 13838, 14234, 3770, 5420, 16266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76070 payload_rows_remote=61066 dup_rows=22234 dup_rows_remote=14942 dedup_ratio=0.226176 router_tokens_per_expert=[36, 0, 0, 0, 0, 89, 314, 0, 0, 76, 231, 0, 0, 58, 3612, 0, 4912, 104, 184, 7133, 6933, 0, 0, 0, 0, 330, 948, 0, 7171, 0, 58, 0, 4140, 0, 39, 0, 220, 1597, 6706, 92, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1327, 0, 862, 0, 0, 0, 0, 0, 0, 0, 238, 959, 0, 370, 340] payload_rows_per_dst=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] dup_rows_per_dst=[0, 7292, 11690, 240, 428, 144, 1504, 936] combine_rows_per_src=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=309846016 dispatch_meta_bytes=374664 combine_payload_bytes=309846016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310220680 ingress_bytes=344449024 link_bytes=344449024 token_bytes=620066696 weight_movement_bytes=34603008 total_nvlink_bytes=654669704 weight_to_token_ratio=0.055805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296133947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87722 payload_rows_remote=80230 dup_rows=10582 dup_rows_remote=10326 dedup_ratio=0.107646 router_tokens_per_expert=[40, 221, 3768, 328, 47, 40, 2013, 9, 3, 74, 157, 444, 4962, 31, 1120, 1899, 272, 13, 26, 120, 83, 652, 240, 991, 38, 58, 95, 4974, 195, 1451, 1955, 82, 177, 64, 133, 6, 100, 5224, 572, 119, 46, 0, 676, 11, 349, 1607, 797, 271, 19, 766, 144, 249, 76, 572, 1288, 109, 132, 3371, 1, 337, 567, 3792, 1055, 121] payload_rows_per_dst=[10810, 7228, 13260, 7492, 12244, 14466, 11468, 10754] dup_rows_per_dst=[2122, 228, 1458, 256, 546, 2996, 1720, 1256] combine_rows_per_src=[10810, 7228, 13260, 7492, 12244, 14466, 11468, 10754] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255213568 dispatch_meta_bytes=293240 combine_payload_bytes=255213568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=255506808 ingress_bytes=359022592 link_bytes=359022592 token_bytes=510720376 weight_movement_bytes=103809024 total_nvlink_bytes=614529400 weight_to_token_ratio=0.203260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350973110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89986 payload_rows_remote=78558 dup_rows=8318 dup_rows_remote=6036 dedup_ratio=0.084615 router_tokens_per_expert=[3, 0, 0, 0, 0, 78, 316, 0, 0, 83, 242, 0, 0, 76, 3280, 0, 4499, 81, 184, 7096, 6937, 0, 0, 0, 0, 293, 860, 0, 6961, 0, 69, 0, 4509, 0, 36, 0, 176, 2042, 6829, 92, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1193, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 226, 964, 0, 380, 318] payload_rows_per_dst=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] dup_rows_per_dst=[16, 132, 0, 1136, 2282, 134, 2282, 2336] combine_rows_per_src=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79604 payload_rows_remote=75646 dup_rows=18700 dup_rows_remote=18020 dedup_ratio=0.190226 router_tokens_per_expert=[1, 89, 2402, 25, 0, 2, 2945, 1, 0, 0, 144, 1791, 7030, 2, 131, 3519, 8, 1, 3, 13, 7, 1915, 52, 76, 18, 0, 15, 7574, 11, 130, 2722, 8, 7, 12, 156, 0, 5, 5661, 26, 11, 1, 0, 1950, 0, 12, 136, 78, 348, 1, 59, 368, 23, 9, 1677, 103, 79, 18, 6420, 0, 38, 732, 471, 72, 44] payload_rows_per_dst=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] dup_rows_per_dst=[1044, 1018, 3672, 54, 44, 1162, 680, 11026] combine_rows_per_src=[9886, 10156, 14538, 5754, 11712, 7832, 3958, 15768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=248004608 dispatch_meta_bytes=271208 combine_payload_bytes=248004608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=248275816 ingress_bytes=317210624 link_bytes=317210624 token_bytes=496280424 weight_movement_bytes=69206016 total_nvlink_bytes=565486440 weight_to_token_ratio=0.139449 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.193638041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79704 payload_rows_remote=67552 dup_rows=18600 dup_rows_remote=13360 dedup_ratio=0.189209 router_tokens_per_expert=[88, 2, 1154, 1, 1553, 85, 485, 122, 58, 82, 381, 247, 2, 287, 1724, 33, 2599, 617, 245, 4730, 3314, 7, 962, 157, 0, 1593, 2467, 0, 3508, 503, 1377, 61, 2397, 332, 723, 186, 206, 462, 2490, 134, 2, 291, 62, 1225, 69, 28, 2, 64, 120, 350, 171, 4182, 6, 1260, 0, 4, 553, 108, 785, 306, 1175, 1489, 1111, 415] payload_rows_per_dst=[9878, 5654, 8364, 12152, 7534, 12594, 13040, 10488] dup_rows_per_dst=[2082, 1600, 198, 5240, 1346, 352, 4344, 3438] combine_rows_per_src=[9878, 5654, 8364, 12152, 7534, 12594, 13040, 10488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=250126336 dispatch_meta_bytes=304032 combine_payload_bytes=250126336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250430368 ingress_bytes=284729344 link_bytes=284729344 token_bytes=500556704 weight_movement_bytes=34603008 total_nvlink_bytes=535159712 weight_to_token_ratio=0.069129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.071413599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86950 payload_rows_remote=71512 dup_rows=11354 dup_rows_remote=10346 dedup_ratio=0.115499 router_tokens_per_expert=[0, 0, 1, 663, 0, 1, 0, 0, 0, 0, 1, 70, 0, 69, 924, 0, 0, 0, 2943, 2973, 7454, 649, 1, 67, 7649, 0, 0, 7146, 60, 63, 0, 0, 0, 1, 126, 0, 63, 0, 0, 6231, 0, 487, 0, 1, 0, 0, 86, 0, 0, 0, 2, 210, 2826, 1, 774, 190, 1242, 67, 13, 0, 1, 5438, 16, 643] payload_rows_per_dst=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] dup_rows_per_dst=[1094, 1114, 910, 0, 126, 1008, 1192, 5910] combine_rows_per_src=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=321773568 dispatch_meta_bytes=338376 combine_payload_bytes=321773568 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=322111944 ingress_bytes=425582592 link_bytes=425582592 token_bytes=643885512 weight_movement_bytes=103809024 total_nvlink_bytes=747694536 weight_to_token_ratio=0.161223 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601433031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=328622080 dispatch_meta_bytes=362224 combine_payload_bytes=328622080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=328984304 ingress_bytes=432431104 link_bytes=432431104 token_bytes=657606384 weight_movement_bytes=103809024 total_nvlink_bytes=761415408 weight_to_token_ratio=0.157859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.627203431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82840 payload_rows_remote=66930 dup_rows=15464 dup_rows_remote=8026 dedup_ratio=0.157308 router_tokens_per_expert=[0, 67, 1608, 0, 0, 0, 3466, 0, 0, 0, 180, 1395, 7310, 0, 1, 3669, 0, 0, 0, 0, 0, 2620, 1, 1, 1, 0, 1, 7832, 0, 1, 2601, 0, 8, 0, 131, 0, 0, 4964, 0, 0, 0, 0, 3564, 0, 0, 1, 0, 277, 0, 0, 372, 0, 3, 1561, 0, 67, 1, 6719, 0, 0, 648, 81, 0, 1] payload_rows_per_dst=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] dup_rows_per_dst=[1060, 936, 5092, 2, 16, 7438, 654, 266] combine_rows_per_src=[9222, 9554, 14772, 5206, 10190, 15910, 3352, 14634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89986 payload_rows_remote=78558 dup_rows=8318 dup_rows_remote=6036 dedup_ratio=0.084615 router_tokens_per_expert=[3, 0, 0, 0, 0, 78, 316, 0, 0, 83, 242, 0, 0, 76, 3280, 0, 4499, 81, 184, 7096, 6937, 0, 0, 0, 0, 293, 860, 0, 6961, 0, 69, 0, 4509, 0, 36, 0, 176, 2042, 6829, 92, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1193, 0, 1262, 0, 0, 0, 0, 0, 0, 0, 226, 964, 0, 380, 318] payload_rows_per_dst=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] dup_rows_per_dst=[16, 132, 0, 1136, 2282, 134, 2282, 2336] combine_rows_per_src=[14436, 7230, 530, 15230, 11428, 14192, 11626, 15314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78666 payload_rows_remote=63204 dup_rows=19638 dup_rows_remote=15456 dedup_ratio=0.199768 router_tokens_per_expert=[0, 5, 2427, 0, 0, 0, 4514, 0, 0, 0, 150, 639, 7079, 0, 0, 1954, 1, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 7815, 0, 0, 3377, 0, 0, 0, 150, 0, 0, 6363, 0, 0, 0, 0, 1372, 0, 0, 0, 0, 611, 0, 0, 256, 0, 0, 3449, 0, 92, 0, 6491, 0, 0, 496, 447, 0, 0] payload_rows_per_dst=[11726, 15462, 2930, 15814, 13026, 3920, 13902, 1886] dup_rows_per_dst=[2166, 4182, 0, 6570, 0, 46, 6674, 0] combine_rows_per_src=[11726, 15462, 2930, 15814, 13026, 3920, 13902, 1886] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87642 payload_rows_remote=73422 dup_rows=10662 dup_rows_remote=10662 dedup_ratio=0.108459 router_tokens_per_expert=[2, 0, 0, 0, 0, 62, 479, 0, 0, 52, 147, 0, 0, 55, 2664, 0, 4209, 8, 71, 7143, 7031, 0, 0, 0, 0, 470, 555, 0, 7330, 0, 37, 0, 3809, 0, 1, 0, 253, 3543, 6893, 62, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1392, 0, 771, 0, 0, 0, 0, 0, 0, 0, 211, 1122, 0, 299, 422] payload_rows_per_dst=[14752, 14954, 14220, 6018, 9378, 14286, 10790, 3244] dup_rows_per_dst=[120, 5542, 0, 366, 1698, 118, 1954, 864] combine_rows_per_src=[14752, 14954, 14220, 6018, 9378, 14286, 10790, 3244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=323648 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277016640 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553709632 weight_movement_bytes=69206016 total_nvlink_bytes=622915648 weight_to_token_ratio=0.124986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=309846016 dispatch_meta_bytes=374664 combine_payload_bytes=309846016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=310220680 ingress_bytes=344449024 link_bytes=344449024 token_bytes=620066696 weight_movement_bytes=34603008 total_nvlink_bytes=654669704 weight_to_token_ratio=0.055805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=26368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.296133947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76602 payload_rows_remote=62308 dup_rows=21702 dup_rows_remote=11002 dedup_ratio=0.220764 router_tokens_per_expert=[4970, 2290, 0, 0, 1, 0, 2474, 90, 1, 1705, 1, 3627, 2, 0, 894, 0, 2, 182, 3289, 0, 0, 351, 0, 0, 0, 4, 2417, 583, 886, 0, 411, 369, 6, 3, 1, 0, 4, 0, 0, 4618, 0, 5, 438, 164, 1998, 95, 7028, 3, 6538, 6, 1, 146, 0, 90, 5, 6, 100, 0, 0, 6, 2090, 92, 0, 1160] payload_rows_per_dst=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] dup_rows_per_dst=[10700, 6418, 0, 1220, 14, 2582, 208, 560] combine_rows_per_src=[14294, 14060, 7648, 8120, 9250, 11536, 5358, 6336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87060 payload_rows_remote=76128 dup_rows=11244 dup_rows_remote=9040 dedup_ratio=0.114380 router_tokens_per_expert=[65, 0, 96, 0, 130, 79, 419, 9, 0, 82, 144, 14, 0, 110, 2482, 1, 4013, 219, 165, 6953, 6639, 2, 84, 12, 0, 607, 833, 0, 6916, 40, 147, 0, 3511, 33, 80, 7, 259, 3199, 6449, 79, 0, 27, 2, 101, 21, 0, 0, 61, 0, 1194, 16, 1231, 1, 113, 0, 0, 56, 3, 69, 260, 1173, 130, 465, 351] payload_rows_per_dst=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] dup_rows_per_dst=[302, 146, 10, 1660, 3860, 130, 2204, 2932] combine_rows_per_src=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76806 payload_rows_remote=64362 dup_rows=21498 dup_rows_remote=17920 dedup_ratio=0.218689 router_tokens_per_expert=[0, 85, 2760, 0, 0, 0, 4258, 0, 0, 0, 167, 892, 7113, 1, 6, 2370, 0, 0, 0, 0, 0, 1411, 3, 3, 1, 0, 0, 7766, 1, 7, 3627, 0, 0, 1, 167, 0, 0, 6681, 1, 1, 1, 0, 1024, 0, 1, 6, 3, 267, 0, 3, 286, 0, 0, 2715, 3, 92, 1, 6421, 0, 0, 437, 566, 3, 1] payload_rows_per_dst=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] dup_rows_per_dst=[3578, 5452, 0, 7068, 0, 14, 5378, 8] combine_rows_per_src=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292913152 dispatch_meta_bytes=327432 combine_payload_bytes=292913152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293240584 ingress_bytes=327516160 link_bytes=327516160 token_bytes=586153736 weight_movement_bytes=34603008 total_nvlink_bytes=620756744 weight_to_token_ratio=0.059034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232416943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=274145280 dispatch_meta_bytes=299824 combine_payload_bytes=274145280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=274445104 ingress_bytes=308748288 link_bytes=308748288 token_bytes=548590384 weight_movement_bytes=34603008 total_nvlink_bytes=583193392 weight_to_token_ratio=0.063076 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=28544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.161794951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=321773568 dispatch_meta_bytes=338376 combine_payload_bytes=321773568 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=322111944 ingress_bytes=425582592 link_bytes=425582592 token_bytes=643885512 weight_movement_bytes=103809024 total_nvlink_bytes=747694536 weight_to_token_ratio=0.161223 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.601433031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=258883584 dispatch_meta_bytes=314640 combine_payload_bytes=258883584 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=259198224 ingress_bytes=362692608 link_bytes=362692608 token_bytes=518081808 weight_movement_bytes=103809024 total_nvlink_bytes=621890832 weight_to_token_ratio=0.200372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=21248 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.364783085e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=300736512 dispatch_meta_bytes=336336 combine_payload_bytes=300736512 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=301072848 ingress_bytes=439148544 link_bytes=439148544 token_bytes=601809360 weight_movement_bytes=138412032 total_nvlink_bytes=740221392 weight_to_token_ratio=0.229993 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.652480617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=255213568 dispatch_meta_bytes=293240 combine_payload_bytes=255213568 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=255506808 ingress_bytes=359022592 link_bytes=359022592 token_bytes=510720376 weight_movement_bytes=103809024 total_nvlink_bytes=614529400 weight_to_token_ratio=0.203260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.350973110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80754 payload_rows_remote=65284 dup_rows=17550 dup_rows_remote=15036 dedup_ratio=0.178528 router_tokens_per_expert=[17, 0, 0, 0, 0, 73, 308, 0, 0, 73, 232, 0, 0, 68, 3422, 0, 4665, 34, 173, 7217, 7016, 0, 0, 0, 0, 254, 896, 0, 7174, 0, 68, 0, 4544, 0, 60, 0, 205, 1809, 6711, 87, 0, 0, 0, 0, 0, 0, 0, 59, 0, 1120, 0, 891, 0, 0, 0, 0, 0, 0, 0, 219, 1072, 0, 354, 331] payload_rows_per_dst=[794, 7466, 414, 15600, 15064, 14434, 11512, 15470] dup_rows_per_dst=[2, 124, 0, 1184, 11768, 118, 1840, 2514] combine_rows_per_src=[794, 7466, 414, 15600, 15064, 14434, 11512, 15470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82456 payload_rows_remote=67774 dup_rows=15848 dup_rows_remote=15014 dedup_ratio=0.161214 router_tokens_per_expert=[0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 709, 0, 0, 0, 2380, 1929, 7347, 637, 0, 92, 7183, 0, 0, 6865, 143, 58, 0, 0, 0, 0, 350, 0, 58, 0, 0, 6419, 0, 676, 0, 0, 0, 0, 92, 0, 0, 0, 0, 236, 2631, 0, 2035, 293, 1522, 92, 11, 0, 0, 5539, 71, 1000] payload_rows_per_dst=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] dup_rows_per_dst=[864, 834, 1110, 10330, 116, 0, 2134, 460] combine_rows_per_src=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311820288 dispatch_meta_bytes=340672 combine_payload_bytes=311820288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312160960 ingress_bytes=346423296 link_bytes=346423296 token_bytes=623981248 weight_movement_bytes=34603008 total_nvlink_bytes=658584256 weight_to_token_ratio=0.055455 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303562973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=263626752 dispatch_meta_bytes=329128 combine_payload_bytes=263626752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263955880 ingress_bytes=298229760 link_bytes=298229760 token_bytes=527582632 weight_movement_bytes=34603008 total_nvlink_bytes=562185640 weight_to_token_ratio=0.065588 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122214577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79704 payload_rows_remote=67552 dup_rows=18600 dup_rows_remote=13360 dedup_ratio=0.189209 router_tokens_per_expert=[88, 2, 1154, 1, 1553, 85, 485, 122, 58, 82, 381, 247, 2, 287, 1724, 33, 2599, 617, 245, 4730, 3314, 7, 962, 157, 0, 1593, 2467, 0, 3508, 503, 1377, 61, 2397, 332, 723, 186, 206, 462, 2490, 134, 2, 291, 62, 1225, 69, 28, 2, 64, 120, 350, 171, 4182, 6, 1260, 0, 4, 553, 108, 785, 306, 1175, 1489, 1111, 415] payload_rows_per_dst=[9878, 5654, 8364, 12152, 7534, 12594, 13040, 10488] dup_rows_per_dst=[2082, 1600, 198, 5240, 1346, 352, 4344, 3438] combine_rows_per_src=[9878, 5654, 8364, 12152, 7534, 12594, 13040, 10488] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87060 payload_rows_remote=76128 dup_rows=11244 dup_rows_remote=9040 dedup_ratio=0.114380 router_tokens_per_expert=[65, 0, 96, 0, 130, 79, 419, 9, 0, 82, 144, 14, 0, 110, 2482, 1, 4013, 219, 165, 6953, 6639, 2, 84, 12, 0, 607, 833, 0, 6916, 40, 147, 0, 3511, 33, 80, 7, 259, 3199, 6449, 79, 0, 27, 2, 101, 21, 0, 0, 61, 0, 1194, 16, 1231, 1, 113, 0, 0, 56, 3, 69, 260, 1173, 130, 465, 351] payload_rows_per_dst=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] dup_rows_per_dst=[302, 146, 10, 1660, 3860, 130, 2204, 2932] combine_rows_per_src=[12610, 5520, 954, 15426, 12058, 14200, 10932, 15360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89620 payload_rows_remote=76342 dup_rows=8684 dup_rows_remote=8546 dedup_ratio=0.088338 router_tokens_per_expert=[0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 812, 0, 0, 0, 2543, 2565, 7309, 634, 0, 87, 7503, 0, 0, 7028, 150, 69, 0, 0, 0, 0, 262, 0, 68, 5, 0, 6373, 0, 611, 0, 0, 0, 0, 92, 0, 0, 0, 0, 210, 2696, 0, 1552, 310, 1452, 92, 0, 0, 0, 5104, 15, 839] payload_rows_per_dst=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] dup_rows_per_dst=[790, 1026, 772, 0, 138, 1184, 1760, 3014] combine_rows_per_src=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=276692992 dispatch_meta_bytes=323648 combine_payload_bytes=276692992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=277016640 ingress_bytes=345899008 link_bytes=345899008 token_bytes=553709632 weight_movement_bytes=69206016 total_nvlink_bytes=622915648 weight_to_token_ratio=0.124986 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.301590120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=321280 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267724544 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535127808 weight_movement_bytes=34603008 total_nvlink_bytes=569730816 weight_to_token_ratio=0.064663 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28672 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78928 payload_rows_remote=75628 dup_rows=19376 dup_rows_remote=19242 dedup_ratio=0.197103 router_tokens_per_expert=[4, 0, 1097, 2421, 432, 533, 1, 1, 62, 73, 1, 111, 581, 33, 1459, 694, 31, 116, 740, 1070, 3367, 571, 300, 96, 4387, 692, 12, 2120, 164, 161, 57, 24, 0, 1105, 727, 1484, 65, 33, 0, 4233, 996, 735, 81, 1436, 79, 2, 135, 22, 237, 172, 340, 396, 5333, 286, 3465, 469, 1358, 275, 576, 13, 1395, 607, 414, 1272] payload_rows_per_dst=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] dup_rows_per_dst=[2612, 1430, 604, 134, 2144, 2736, 8104, 1612] combine_rows_per_src=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277602304 dispatch_meta_bytes=331152 combine_payload_bytes=277602304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933456 ingress_bytes=312205312 link_bytes=312205312 token_bytes=555535760 weight_movement_bytes=34603008 total_nvlink_bytes=590138768 weight_to_token_ratio=0.062288 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174803454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88362 payload_rows_remote=72508 dup_rows=9942 dup_rows_remote=6304 dedup_ratio=0.101135 router_tokens_per_expert=[0, 4221, 2923, 766, 7470, 7708, 0, 0, 0, 0, 0, 253, 0, 0, 1, 0, 1, 0, 0, 15, 0, 1671, 0, 0, 0, 0, 1390, 562, 67, 0, 1, 0, 0, 0, 79, 0, 0, 6736, 232, 510, 0, 104, 2171, 0, 1, 0, 0, 0, 0, 1831, 12, 0, 0, 1, 90, 4419, 203, 5645, 0, 1, 67, 0, 1, 0] payload_rows_per_dst=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] dup_rows_per_dst=[296, 506, 1670, 246, 890, 3638, 2300, 396] combine_rows_per_src=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=312696832 dispatch_meta_bytes=339552 combine_payload_bytes=312696832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=313036384 ingress_bytes=381902848 link_bytes=381902848 token_bytes=625733216 weight_movement_bytes=69206016 total_nvlink_bytes=694939232 weight_to_token_ratio=0.110600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437069671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89368 payload_rows_remote=74972 dup_rows=8936 dup_rows_remote=8788 dedup_ratio=0.090902 router_tokens_per_expert=[44, 0, 0, 0, 1, 67, 413, 0, 0, 71, 89, 0, 0, 103, 2644, 0, 4503, 1, 99, 7197, 7042, 0, 1, 0, 0, 657, 488, 0, 7373, 0, 63, 0, 3674, 1, 1, 0, 253, 3396, 6859, 67, 0, 0, 0, 1, 0, 5, 0, 69, 0, 1286, 0, 629, 0, 1, 0, 0, 0, 0, 0, 254, 1090, 1, 379, 330] payload_rows_per_dst=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] dup_rows_per_dst=[144, 14, 0, 1358, 2772, 148, 1824, 2676] combine_rows_per_src=[14624, 5800, 202, 15804, 12012, 14396, 11014, 15516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89620 payload_rows_remote=76342 dup_rows=8684 dup_rows_remote=8546 dedup_ratio=0.088338 router_tokens_per_expert=[0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 92, 0, 91, 812, 0, 0, 0, 2543, 2565, 7309, 634, 0, 87, 7503, 0, 0, 7028, 150, 69, 0, 0, 0, 0, 262, 0, 68, 5, 0, 6373, 0, 611, 0, 0, 0, 0, 92, 0, 0, 0, 0, 210, 2696, 0, 1552, 310, 1452, 92, 0, 0, 0, 5104, 15, 839] payload_rows_per_dst=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] dup_rows_per_dst=[790, 1026, 772, 0, 138, 1184, 1760, 3014] combine_rows_per_src=[15004, 15020, 7330, 5970, 13278, 15228, 7776, 10014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=311820288 dispatch_meta_bytes=340672 combine_payload_bytes=311820288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=312160960 ingress_bytes=346423296 link_bytes=346423296 token_bytes=623981248 weight_movement_bytes=34603008 total_nvlink_bytes=658584256 weight_to_token_ratio=0.055455 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303562973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76070 payload_rows_remote=61066 dup_rows=22234 dup_rows_remote=14942 dedup_ratio=0.226176 router_tokens_per_expert=[36, 0, 0, 0, 0, 89, 314, 0, 0, 76, 231, 0, 0, 58, 3612, 0, 4912, 104, 184, 7133, 6933, 0, 0, 0, 0, 330, 948, 0, 7171, 0, 58, 0, 4140, 0, 39, 0, 220, 1597, 6706, 92, 0, 0, 0, 0, 0, 0, 0, 73, 0, 1327, 0, 862, 0, 0, 0, 0, 0, 0, 0, 238, 959, 0, 370, 340] payload_rows_per_dst=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] dup_rows_per_dst=[0, 7292, 11690, 240, 428, 144, 1504, 936] combine_rows_per_src=[14290, 15004, 15332, 4068, 3468, 12934, 8096, 2878] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89962 payload_rows_remote=79440 dup_rows=8342 dup_rows_remote=6508 dedup_ratio=0.084859 router_tokens_per_expert=[0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 62, 0, 62, 1018, 0, 0, 0, 2468, 2939, 7540, 709, 0, 62, 7579, 0, 0, 6988, 60, 37, 0, 0, 0, 0, 122, 0, 33, 0, 0, 6361, 0, 497, 0, 0, 0, 0, 62, 0, 0, 0, 0, 206, 3217, 0, 786, 159, 1239, 62, 0, 0, 0, 5668, 0, 628] payload_rows_per_dst=[15108, 15080, 10522, 12070, 12966, 13314, 7444, 3458] dup_rows_per_dst=[1148, 1180, 1834, 1548, 66, 874, 1292, 400] combine_rows_per_src=[15108, 15080, 10522, 12070, 12966, 13314, 7444, 3458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309772288 dispatch_meta_bytes=379480 combine_payload_bytes=309772288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=310151768 ingress_bytes=413581312 link_bytes=413581312 token_bytes=619924056 weight_movement_bytes=103809024 total_nvlink_bytes=723733080 weight_to_token_ratio=0.167454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556273181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=296992768 dispatch_meta_bytes=315248 combine_payload_bytes=296992768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297308016 ingress_bytes=331595776 link_bytes=331595776 token_bytes=594300784 weight_movement_bytes=34603008 total_nvlink_bytes=628903792 weight_to_token_ratio=0.058225 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247768209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57756 payload_rows_remote=43544 dup_rows=40548 dup_rows_remote=40506 dedup_ratio=0.412476 router_tokens_per_expert=[39, 0, 3, 0, 6, 30, 422, 0, 0, 100, 165, 0, 0, 77, 3684, 0, 4785, 80, 182, 7093, 6921, 0, 4, 4, 0, 268, 1062, 0, 6887, 1, 80, 0, 4126, 4, 200, 0, 242, 1066, 6627, 92, 0, 2, 0, 4, 0, 3, 0, 53, 0, 1198, 1, 1467, 0, 5, 0, 0, 2, 0, 2, 225, 1184, 6, 399, 351] payload_rows_per_dst=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] dup_rows_per_dst=[42, 7280, 30706, 6, 572, 2, 730, 1210] combine_rows_per_src=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83762 payload_rows_remote=73756 dup_rows=14542 dup_rows_remote=12288 dedup_ratio=0.147929 router_tokens_per_expert=[0, 0, 87, 781, 63, 52, 0, 0, 4, 0, 2, 78, 34, 74, 1347, 62, 1, 3, 2055, 2628, 6937, 652, 24, 79, 7108, 73, 0, 6259, 82, 76, 1, 1, 0, 107, 262, 118, 66, 4, 0, 6096, 93, 634, 9, 126, 3, 0, 102, 0, 22, 12, 54, 256, 4224, 20, 1298, 244, 1396, 86, 77, 0, 136, 4175, 49, 920] payload_rows_per_dst=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] dup_rows_per_dst=[90, 660, 4902, 14, 308, 1342, 2254, 4972] combine_rows_per_src=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=76806 payload_rows_remote=64362 dup_rows=21498 dup_rows_remote=17920 dedup_ratio=0.218689 router_tokens_per_expert=[0, 85, 2760, 0, 0, 0, 4258, 0, 0, 0, 167, 892, 7113, 1, 6, 2370, 0, 0, 0, 0, 0, 1411, 3, 3, 1, 0, 0, 7766, 1, 7, 3627, 0, 0, 1, 167, 0, 0, 6681, 1, 1, 1, 0, 1024, 0, 1, 6, 3, 267, 0, 3, 286, 0, 0, 2715, 3, 92, 1, 6421, 0, 0, 437, 566, 3, 1] payload_rows_per_dst=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] dup_rows_per_dst=[3578, 5452, 0, 7068, 0, 14, 5378, 8] combine_rows_per_src=[12444, 15646, 2834, 15736, 11886, 2590, 13662, 2008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=312696832 dispatch_meta_bytes=339552 combine_payload_bytes=312696832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=313036384 ingress_bytes=381902848 link_bytes=381902848 token_bytes=625733216 weight_movement_bytes=69206016 total_nvlink_bytes=694939232 weight_to_token_ratio=0.110600 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.032500 gemm_active_groups=7 gemm_peak_group_rows=43520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.437069671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=307085312 dispatch_meta_bytes=335040 combine_payload_bytes=307085312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307420352 ingress_bytes=341688320 link_bytes=341688320 token_bytes=614505664 weight_movement_bytes=34603008 total_nvlink_bytes=649108672 weight_to_token_ratio=0.056310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=46848 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.285745640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=325386240 dispatch_meta_bytes=343792 combine_payload_bytes=325386240 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=325730032 ingress_bytes=429195264 link_bytes=429195264 token_bytes=651116272 weight_movement_bytes=103809024 total_nvlink_bytes=754925296 weight_to_token_ratio=0.159432 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=17280 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.615027225e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=250126336 dispatch_meta_bytes=304032 combine_payload_bytes=250126336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=250430368 ingress_bytes=284729344 link_bytes=284729344 token_bytes=500556704 weight_movement_bytes=34603008 total_nvlink_bytes=535159712 weight_to_token_ratio=0.069129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=23552 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.071413599e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=263626752 dispatch_meta_bytes=329128 combine_payload_bytes=263626752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263955880 ingress_bytes=298229760 link_bytes=298229760 token_bytes=527582632 weight_movement_bytes=34603008 total_nvlink_bytes=562185640 weight_to_token_ratio=0.065588 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=27520 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122214577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81226 payload_rows_remote=66424 dup_rows=17078 dup_rows_remote=11320 dedup_ratio=0.173726 router_tokens_per_expert=[0, 0, 0, 592, 0, 0, 0, 0, 0, 0, 0, 87, 0, 87, 840, 0, 0, 0, 2142, 2405, 7337, 651, 0, 86, 7345, 0, 0, 6853, 74, 68, 0, 0, 0, 0, 180, 0, 67, 1, 0, 6370, 0, 725, 0, 0, 0, 0, 87, 0, 0, 0, 0, 220, 2828, 0, 2004, 228, 1466, 86, 5, 0, 0, 5332, 70, 916] payload_rows_per_dst=[1184, 1854, 15290, 13990, 6810, 15200, 12096, 14802] dup_rows_per_dst=[0, 174, 5142, 0, 70, 1114, 4820, 5758] combine_rows_per_src=[1184, 1854, 15290, 13990, 6810, 15200, 12096, 14802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80680 payload_rows_remote=76528 dup_rows=17624 dup_rows_remote=17514 dedup_ratio=0.179281 router_tokens_per_expert=[0, 3832, 2241, 1110, 7503, 7633, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 0, 0, 0, 0, 1394, 2562, 92, 0, 0, 0, 0, 0, 114, 0, 0, 6032, 397, 176, 0, 0, 1675, 0, 0, 76, 65, 0, 0, 2290, 294, 0, 0, 0, 76, 2282, 222, 6599, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[13440, 4152, 10186, 15902, 12606, 3350, 7662, 13382] dup_rows_per_dst=[4718, 110, 1816, 7200, 832, 282, 2222, 444] combine_rows_per_src=[13440, 4152, 10186, 15902, 12606, 3350, 7662, 13382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=178356224 dispatch_meta_bytes=336200 combine_payload_bytes=178356224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=178692424 ingress_bytes=212959232 link_bytes=212959232 token_bytes=357048648 weight_movement_bytes=34603008 total_nvlink_bytes=391651656 weight_to_token_ratio=0.096914 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.013484454e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=302104576 dispatch_meta_bytes=344176 combine_payload_bytes=302104576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302448752 ingress_bytes=336707584 link_bytes=336707584 token_bytes=604553328 weight_movement_bytes=34603008 total_nvlink_bytes=639156336 weight_to_token_ratio=0.057237 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267003531e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78928 payload_rows_remote=75628 dup_rows=19376 dup_rows_remote=19242 dedup_ratio=0.197103 router_tokens_per_expert=[4, 0, 1097, 2421, 432, 533, 1, 1, 62, 73, 1, 111, 581, 33, 1459, 694, 31, 116, 740, 1070, 3367, 571, 300, 96, 4387, 692, 12, 2120, 164, 161, 57, 24, 0, 1105, 727, 1484, 65, 33, 0, 4233, 996, 735, 81, 1436, 79, 2, 135, 22, 237, 172, 340, 396, 5333, 286, 3465, 469, 1358, 275, 576, 13, 1395, 607, 414, 1272] payload_rows_per_dst=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] dup_rows_per_dst=[2612, 1430, 604, 134, 2144, 2736, 8104, 1612] combine_rows_per_src=[13100, 8838, 5244, 3300, 13150, 13010, 13292, 8994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=83762 payload_rows_remote=73756 dup_rows=14542 dup_rows_remote=12288 dedup_ratio=0.147929 router_tokens_per_expert=[0, 0, 87, 781, 63, 52, 0, 0, 4, 0, 2, 78, 34, 74, 1347, 62, 1, 3, 2055, 2628, 6937, 652, 24, 79, 7108, 73, 0, 6259, 82, 76, 1, 1, 0, 107, 262, 118, 66, 4, 0, 6096, 93, 634, 9, 126, 3, 0, 102, 0, 22, 12, 54, 256, 4224, 20, 1298, 244, 1396, 86, 77, 0, 136, 4175, 49, 920] payload_rows_per_dst=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] dup_rows_per_dst=[90, 660, 4902, 14, 308, 1342, 2254, 4972] combine_rows_per_src=[1876, 5334, 14600, 10178, 12998, 14808, 10006, 13962] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88250 payload_rows_remote=75342 dup_rows=10054 dup_rows_remote=9086 dedup_ratio=0.102275 router_tokens_per_expert=[0, 3575, 3002, 1046, 7551, 7624, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 1380, 1638, 92, 0, 0, 0, 0, 0, 112, 0, 0, 6208, 439, 179, 0, 0, 1651, 0, 0, 17, 112, 0, 0, 2345, 322, 0, 0, 0, 89, 2380, 224, 6348, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] dup_rows_per_dst=[478, 504, 1990, 458, 968, 2930, 2278, 448] combine_rows_per_src=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81338 payload_rows_remote=69058 dup_rows=16966 dup_rows_remote=11734 dedup_ratio=0.172587 router_tokens_per_expert=[15, 1407, 1354, 730, 4811, 4879, 277, 0, 63, 101, 198, 205, 431, 139, 579, 410, 1571, 505, 46, 57, 1056, 3048, 28, 0, 31, 1, 1426, 2791, 135, 4, 844, 67, 48, 79, 775, 53, 18, 3414, 621, 178, 294, 56, 804, 18, 830, 195, 416, 1509, 283, 1066, 755, 0, 146, 1701, 137, 1635, 152, 4727, 103, 994, 88, 109, 91, 648] payload_rows_per_dst=[4508, 12708, 11576, 12280, 9184, 9260, 9116, 12706] dup_rows_per_dst=[244, 1302, 3860, 5232, 1188, 1692, 2330, 1118] combine_rows_per_src=[4508, 12708, 11576, 12280, 9184, 9260, 9116, 12706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=272072704 dispatch_meta_bytes=310976 combine_payload_bytes=272072704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=272383680 ingress_bytes=341278720 link_bytes=341278720 token_bytes=544456384 weight_movement_bytes=69206016 total_nvlink_bytes=613662400 weight_to_token_ratio=0.127110 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=17408 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.284204348e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=313458688 dispatch_meta_bytes=376168 combine_payload_bytes=313458688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313834856 ingress_bytes=348061696 link_bytes=348061696 token_bytes=627293544 weight_movement_bytes=34603008 total_nvlink_bytes=661896552 weight_to_token_ratio=0.055162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309728141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88232 payload_rows_remote=72512 dup_rows=10072 dup_rows_remote=9568 dedup_ratio=0.102458 router_tokens_per_expert=[3, 0, 0, 0, 0, 4, 0, 4641, 540, 1, 230, 0, 0, 0, 395, 7025, 0, 0, 0, 3274, 283, 883, 0, 323, 1, 0, 0, 0, 0, 1, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 2386, 0, 1, 0, 7859, 0, 1, 251, 0, 0, 0, 0, 3559, 1, 7827, 0, 1, 0, 0, 0, 5902, 138, 407, 0, 67] payload_rows_per_dst=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] dup_rows_per_dst=[10, 134, 7534, 2, 1072, 504, 2, 814] combine_rows_per_src=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=308600832 dispatch_meta_bytes=337712 combine_payload_bytes=308600832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=308938544 ingress_bytes=377806848 link_bytes=377806848 token_bytes=617539376 weight_movement_bytes=69206016 total_nvlink_bytes=686745392 weight_to_token_ratio=0.112067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421656753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=309772288 dispatch_meta_bytes=379480 combine_payload_bytes=309772288 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=310151768 ingress_bytes=413581312 link_bytes=413581312 token_bytes=619924056 weight_movement_bytes=103809024 total_nvlink_bytes=723733080 weight_to_token_ratio=0.167454 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=19584 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.556273181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88250 payload_rows_remote=75342 dup_rows=10054 dup_rows_remote=9086 dedup_ratio=0.102275 router_tokens_per_expert=[0, 3575, 3002, 1046, 7551, 7624, 0, 0, 0, 0, 0, 252, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2474, 0, 0, 0, 0, 1380, 1638, 92, 0, 0, 0, 0, 0, 112, 0, 0, 6208, 439, 179, 0, 0, 1651, 0, 0, 17, 112, 0, 0, 2345, 322, 0, 0, 0, 89, 2380, 224, 6348, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] dup_rows_per_dst=[478, 504, 1990, 458, 968, 2930, 2278, 448] combine_rows_per_src=[7618, 15248, 10108, 5762, 12908, 15732, 7994, 12880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=302104576 dispatch_meta_bytes=344176 combine_payload_bytes=302104576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=302448752 ingress_bytes=336707584 link_bytes=336707584 token_bytes=604553328 weight_movement_bytes=34603008 total_nvlink_bytes=639156336 weight_to_token_ratio=0.057237 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267003531e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86950 payload_rows_remote=71512 dup_rows=11354 dup_rows_remote=10346 dedup_ratio=0.115499 router_tokens_per_expert=[0, 0, 1, 663, 0, 1, 0, 0, 0, 0, 1, 70, 0, 69, 924, 0, 0, 0, 2943, 2973, 7454, 649, 1, 67, 7649, 0, 0, 7146, 60, 63, 0, 0, 0, 1, 126, 0, 63, 0, 0, 6231, 0, 487, 0, 1, 0, 0, 86, 0, 0, 0, 2, 210, 2826, 1, 774, 190, 1242, 67, 13, 0, 1, 5438, 16, 643] payload_rows_per_dst=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] dup_rows_per_dst=[1094, 1114, 910, 0, 126, 1008, 1192, 5910] combine_rows_per_src=[14606, 15306, 6948, 246, 12716, 15438, 6814, 14876] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=89800 payload_rows_remote=79550 dup_rows=8504 dup_rows_remote=6722 dedup_ratio=0.086507 router_tokens_per_expert=[0, 4212, 2750, 1081, 7510, 7794, 0, 0, 0, 0, 0, 254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1804, 0, 0, 0, 0, 1380, 435, 62, 0, 0, 0, 0, 0, 85, 0, 0, 6679, 205, 253, 0, 0, 2450, 0, 0, 0, 0, 0, 0, 1722, 16, 0, 0, 0, 62, 4121, 154, 6061, 0, 0, 62, 0, 0, 0] payload_rows_per_dst=[7376, 15588, 10250, 16038, 13682, 4900, 9720, 12246] dup_rows_per_dst=[286, 508, 1782, 2736, 762, 0, 2122, 308] combine_rows_per_src=[7376, 15588, 10250, 16038, 13682, 4900, 9720, 12246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282861568 dispatch_meta_bytes=323168 combine_payload_bytes=282861568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283184736 ingress_bytes=317464576 link_bytes=317464576 token_bytes=566046304 weight_movement_bytes=34603008 total_nvlink_bytes=600649312 weight_to_token_ratio=0.061131 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.194593641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=82456 payload_rows_remote=67774 dup_rows=15848 dup_rows_remote=15014 dedup_ratio=0.161214 router_tokens_per_expert=[0, 0, 0, 600, 0, 0, 0, 0, 0, 0, 0, 92, 0, 92, 709, 0, 0, 0, 2380, 1929, 7347, 637, 0, 92, 7183, 0, 0, 6865, 143, 58, 0, 0, 0, 0, 350, 0, 58, 0, 0, 6419, 0, 676, 0, 0, 0, 0, 92, 0, 0, 0, 0, 236, 2631, 0, 2035, 293, 1522, 92, 11, 0, 0, 5539, 71, 1000] payload_rows_per_dst=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] dup_rows_per_dst=[864, 834, 1110, 10330, 116, 0, 2134, 460] combine_rows_per_src=[15030, 14682, 8966, 15516, 13538, 1536, 8256, 4932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=297009152 dispatch_meta_bytes=328320 combine_payload_bytes=297009152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=297337472 ingress_bytes=400818176 link_bytes=400818176 token_bytes=594346624 weight_movement_bytes=103809024 total_nvlink_bytes=698155648 weight_to_token_ratio=0.174661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508246528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80662 payload_rows_remote=65734 dup_rows=17642 dup_rows_remote=16772 dedup_ratio=0.179464 router_tokens_per_expert=[0, 0, 4, 618, 2, 6, 0, 0, 0, 0, 0, 92, 2, 92, 833, 2, 0, 0, 1998, 2541, 7269, 671, 2, 92, 7407, 3, 0, 6865, 118, 77, 0, 0, 0, 6, 276, 4, 77, 0, 0, 6338, 3, 655, 1, 6, 1, 0, 90, 0, 0, 0, 2, 234, 3078, 2, 1931, 285, 1464, 91, 1, 0, 6, 4899, 56, 952] payload_rows_per_dst=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] dup_rows_per_dst=[870, 996, 3330, 9518, 104, 10, 2406, 408] combine_rows_per_src=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=79010 dup_rows=10264 dup_rows_remote=8004 dedup_ratio=0.104411 router_tokens_per_expert=[0, 3725, 2569, 1043, 6991, 7480, 20, 0, 1, 6, 24, 258, 17, 16, 66, 17, 141, 29, 6, 0, 85, 2283, 3, 0, 2, 0, 1386, 1045, 79, 0, 96, 1, 5, 5, 302, 0, 3, 6155, 472, 332, 7, 1, 1862, 0, 61, 65, 108, 127, 27, 1767, 240, 0, 2, 146, 106, 3357, 205, 6147, 2, 129, 76, 11, 12, 31] payload_rows_per_dst=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] dup_rows_per_dst=[366, 544, 2136, 336, 1062, 3078, 2260, 482] combine_rows_per_src=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=308600832 dispatch_meta_bytes=337712 combine_payload_bytes=308600832 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=308938544 ingress_bytes=377806848 link_bytes=377806848 token_bytes=617539376 weight_movement_bytes=69206016 total_nvlink_bytes=686745392 weight_to_token_ratio=0.112067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=41728 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.421656753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=57756 payload_rows_remote=43544 dup_rows=40548 dup_rows_remote=40506 dedup_ratio=0.412476 router_tokens_per_expert=[39, 0, 3, 0, 6, 30, 422, 0, 0, 100, 165, 0, 0, 77, 3684, 0, 4785, 80, 182, 7093, 6921, 0, 4, 4, 0, 268, 1062, 0, 6887, 1, 80, 0, 4126, 4, 200, 0, 242, 1066, 6627, 92, 0, 2, 0, 4, 0, 3, 0, 53, 0, 1198, 1, 1467, 0, 5, 0, 0, 2, 0, 2, 225, 1184, 6, 399, 351] payload_rows_per_dst=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] dup_rows_per_dst=[42, 7280, 30706, 6, 572, 2, 730, 1210] combine_rows_per_src=[14212, 14546, 15684, 2816, 2636, 122, 4612, 3128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=292913152 dispatch_meta_bytes=327432 combine_payload_bytes=292913152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=293240584 ingress_bytes=327516160 link_bytes=327516160 token_bytes=586153736 weight_movement_bytes=34603008 total_nvlink_bytes=620756744 weight_to_token_ratio=0.059034 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=40576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.232416943e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=325836800 dispatch_meta_bytes=345088 combine_payload_bytes=325836800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=326181888 ingress_bytes=360439808 link_bytes=360439808 token_bytes=652018688 weight_movement_bytes=34603008 total_nvlink_bytes=686621696 weight_to_token_ratio=0.053071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=26880 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.356305980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=277602304 dispatch_meta_bytes=331152 combine_payload_bytes=277602304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=277933456 ingress_bytes=312205312 link_bytes=312205312 token_bytes=555535760 weight_movement_bytes=34603008 total_nvlink_bytes=590138768 weight_to_token_ratio=0.062288 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=38528 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.174803454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=84616 payload_rows_remote=71280 dup_rows=13688 dup_rows_remote=12248 dedup_ratio=0.139242 router_tokens_per_expert=[0, 3592, 2519, 829, 7691, 7740, 0, 0, 0, 0, 0, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 2, 0, 0, 0, 1367, 1881, 86, 0, 0, 0, 0, 0, 128, 0, 0, 6259, 677, 178, 0, 0, 1710, 0, 0, 30, 14, 0, 0, 2134, 294, 0, 0, 0, 84, 2372, 193, 6515, 0, 0, 86, 0, 0, 0] payload_rows_per_dst=[6280, 15480, 10132, 6352, 2600, 15784, 14652, 13336] dup_rows_per_dst=[416, 502, 2096, 316, 90, 3106, 5722, 1440] combine_rows_per_src=[6280, 15480, 10132, 6352, 2600, 15784, 14652, 13336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86118 payload_rows_remote=71416 dup_rows=12186 dup_rows_remote=10324 dedup_ratio=0.123962 router_tokens_per_expert=[11, 0, 0, 0, 0, 0, 0, 3921, 454, 0, 379, 0, 0, 0, 344, 7105, 0, 0, 0, 1520, 271, 476, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 7769, 0, 0, 293, 0, 0, 0, 0, 3698, 0, 7710, 0, 0, 0, 0, 0, 6998, 168, 432, 0, 92] payload_rows_per_dst=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] dup_rows_per_dst=[4928, 1862, 116, 0, 2524, 208, 1890, 658] combine_rows_per_src=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=330024 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269576488 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538822952 weight_movement_bytes=34603008 total_nvlink_bytes=573425960 weight_to_token_ratio=0.064220 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323624960 dispatch_meta_bytes=348056 combine_payload_bytes=323624960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323973016 ingress_bytes=358227968 link_bytes=358227968 token_bytes=647597976 weight_movement_bytes=34603008 total_nvlink_bytes=682200984 weight_to_token_ratio=0.053433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347983004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=178356224 dispatch_meta_bytes=336200 combine_payload_bytes=178356224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=178692424 ingress_bytes=212959232 link_bytes=212959232 token_bytes=357048648 weight_movement_bytes=34603008 total_nvlink_bytes=391651656 weight_to_token_ratio=0.096914 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=8.013484454e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81338 payload_rows_remote=69058 dup_rows=16966 dup_rows_remote=11734 dedup_ratio=0.172587 router_tokens_per_expert=[15, 1407, 1354, 730, 4811, 4879, 277, 0, 63, 101, 198, 205, 431, 139, 579, 410, 1571, 505, 46, 57, 1056, 3048, 28, 0, 31, 1, 1426, 2791, 135, 4, 844, 67, 48, 79, 775, 53, 18, 3414, 621, 178, 294, 56, 804, 18, 830, 195, 416, 1509, 283, 1066, 755, 0, 146, 1701, 137, 1635, 152, 4727, 103, 994, 88, 109, 91, 648] payload_rows_per_dst=[4508, 12708, 11576, 12280, 9184, 9260, 9116, 12706] dup_rows_per_dst=[244, 1302, 3860, 5232, 1188, 1692, 2330, 1118] combine_rows_per_src=[4508, 12708, 11576, 12280, 9184, 9260, 9116, 12706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88040 payload_rows_remote=79010 dup_rows=10264 dup_rows_remote=8004 dedup_ratio=0.104411 router_tokens_per_expert=[0, 3725, 2569, 1043, 6991, 7480, 20, 0, 1, 6, 24, 258, 17, 16, 66, 17, 141, 29, 6, 0, 85, 2283, 3, 0, 2, 0, 1386, 1045, 79, 0, 96, 1, 5, 5, 302, 0, 3, 6155, 472, 332, 7, 1, 1862, 0, 61, 65, 108, 127, 27, 1767, 240, 0, 2, 146, 106, 3357, 205, 6147, 2, 129, 76, 11, 12, 31] payload_rows_per_dst=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] dup_rows_per_dst=[366, 544, 2136, 336, 1062, 3078, 2260, 482] combine_rows_per_src=[6898, 15226, 10408, 4882, 13486, 15366, 9030, 12744] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=82458 dup_rows=5550 dup_rows_remote=4150 dedup_ratio=0.056458 router_tokens_per_expert=[45, 0, 0, 0, 0, 0, 0, 4068, 490, 0, 347, 0, 0, 0, 309, 6948, 0, 0, 0, 1959, 283, 522, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 7784, 0, 0, 306, 0, 0, 0, 0, 4645, 0, 7678, 0, 0, 0, 0, 0, 6725, 175, 410, 0, 92] payload_rows_per_dst=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] dup_rows_per_dst=[82, 1260, 1390, 0, 1400, 598, 0, 820] combine_rows_per_src=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78344 payload_rows_remote=69942 dup_rows=19960 dup_rows_remote=19730 dedup_ratio=0.203044 router_tokens_per_expert=[3227, 1422, 61, 119, 31, 84, 179, 1244, 438, 1654, 3870, 87, 115, 20, 487, 4532, 21, 471, 102, 287, 309, 273, 93, 634, 34, 48, 30, 32, 73, 198, 94, 308, 3814, 1377, 227, 3, 37, 0, 0, 1261, 15, 1599, 4, 3192, 799, 1506, 168, 63, 46, 66, 13, 2289, 626, 3499, 0, 1006, 40, 351, 63, 5637, 176, 480, 147, 71] payload_rows_per_dst=[10478, 14926, 3878, 8402, 11044, 10186, 6854, 12576] dup_rows_per_dst=[2256, 7480, 502, 230, 2394, 4506, 1238, 1354] combine_rows_per_src=[10478, 14926, 3878, 8402, 11044, 10186, 6854, 12576] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=291962880 dispatch_meta_bytes=334112 combine_payload_bytes=291962880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292296992 ingress_bytes=361168896 link_bytes=361168896 token_bytes=584259872 weight_movement_bytes=69206016 total_nvlink_bytes=653465888 weight_to_token_ratio=0.118451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.359049479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292519936 dispatch_meta_bytes=326960 combine_payload_bytes=292519936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292846896 ingress_bytes=361725952 link_bytes=361725952 token_bytes=585366832 weight_movement_bytes=69206016 total_nvlink_bytes=654572848 weight_to_token_ratio=0.118227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361145636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93670 payload_rows_remote=84024 dup_rows=4634 dup_rows_remote=4036 dedup_ratio=0.047139 router_tokens_per_expert=[0, 0, 429, 0, 0, 3, 0, 0, 1, 237, 0, 0, 4666, 0, 5141, 0, 0, 340, 65, 0, 0, 0, 481, 0, 1048, 0, 1, 0, 0, 7, 0, 6226, 7295, 0, 203, 0, 1, 0, 478, 0, 721, 4332, 0, 0, 1, 7342, 4142, 68, 1, 498, 0, 0, 253, 0, 67, 0, 0, 1, 5104, 0, 0, 0, 0, 0] payload_rows_per_dst=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] dup_rows_per_dst=[682, 0, 608, 1218, 862, 598, 666, 0] combine_rows_per_src=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=92754 payload_rows_remote=82458 dup_rows=5550 dup_rows_remote=4150 dedup_ratio=0.056458 router_tokens_per_expert=[45, 0, 0, 0, 0, 0, 0, 4068, 490, 0, 347, 0, 0, 0, 309, 6948, 0, 0, 0, 1959, 283, 522, 0, 518, 0, 0, 0, 0, 0, 0, 0, 0, 2520, 0, 0, 0, 0, 0, 0, 3328, 0, 0, 0, 7784, 0, 0, 306, 0, 0, 0, 0, 4645, 0, 7678, 0, 0, 0, 0, 0, 6725, 175, 410, 0, 92] payload_rows_per_dst=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] dup_rows_per_dst=[82, 1260, 1390, 0, 1400, 598, 0, 820] combine_rows_per_src=[8144, 11374, 8728, 15356, 10296, 15582, 9290, 13984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88362 payload_rows_remote=72508 dup_rows=9942 dup_rows_remote=6304 dedup_ratio=0.101135 router_tokens_per_expert=[0, 4221, 2923, 766, 7470, 7708, 0, 0, 0, 0, 0, 253, 0, 0, 1, 0, 1, 0, 0, 15, 0, 1671, 0, 0, 0, 0, 1390, 562, 67, 0, 1, 0, 0, 0, 79, 0, 0, 6736, 232, 510, 0, 104, 2171, 0, 1, 0, 0, 0, 0, 1831, 12, 0, 0, 1, 90, 4419, 203, 5645, 0, 1, 67, 0, 1, 0] payload_rows_per_dst=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] dup_rows_per_dst=[296, 506, 1670, 246, 890, 3638, 2300, 396] combine_rows_per_src=[7082, 15418, 10146, 3794, 14224, 15854, 10406, 11438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=282861568 dispatch_meta_bytes=323168 combine_payload_bytes=282861568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=283184736 ingress_bytes=317464576 link_bytes=317464576 token_bytes=566046304 weight_movement_bytes=34603008 total_nvlink_bytes=600649312 weight_to_token_ratio=0.061131 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=23424 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.194593641e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93882 payload_rows_remote=85460 dup_rows=4422 dup_rows_remote=4074 dedup_ratio=0.044983 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 4386, 494, 0, 218, 0, 0, 0, 359, 7002, 0, 0, 0, 2921, 246, 759, 0, 459, 0, 0, 0, 0, 0, 0, 0, 0, 3312, 0, 0, 0, 0, 0, 0, 2033, 0, 0, 0, 7870, 0, 0, 250, 0, 0, 0, 0, 4550, 0, 7847, 0, 0, 0, 0, 0, 5784, 114, 484, 0, 62] payload_rows_per_dst=[8772, 14356, 8422, 15694, 9872, 15746, 9100, 11920] dup_rows_per_dst=[4, 1790, 348, 0, 818, 494, 0, 968] combine_rows_per_src=[8772, 14356, 8422, 15694, 9872, 15746, 9100, 11920] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=337747968 dispatch_meta_bytes=346432 combine_payload_bytes=337747968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338094400 ingress_bytes=372350976 link_bytes=372350976 token_bytes=675842368 weight_movement_bytes=34603008 total_nvlink_bytes=710445376 weight_to_token_ratio=0.051200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401126746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=286482432 dispatch_meta_bytes=358688 combine_payload_bytes=286482432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286841120 ingress_bytes=321085440 link_bytes=321085440 token_bytes=573323552 weight_movement_bytes=34603008 total_nvlink_bytes=607926560 weight_to_token_ratio=0.060355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208218661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80680 payload_rows_remote=76528 dup_rows=17624 dup_rows_remote=17514 dedup_ratio=0.179281 router_tokens_per_expert=[0, 3832, 2241, 1110, 7503, 7633, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2169, 0, 0, 0, 0, 1394, 2562, 92, 0, 0, 0, 0, 0, 114, 0, 0, 6032, 397, 176, 0, 0, 1675, 0, 0, 76, 65, 0, 0, 2290, 294, 0, 0, 0, 76, 2282, 222, 6599, 0, 0, 92, 0, 0, 0] payload_rows_per_dst=[13440, 4152, 10186, 15902, 12606, 3350, 7662, 13382] dup_rows_per_dst=[4718, 110, 1816, 7200, 832, 282, 2222, 444] combine_rows_per_src=[13440, 4152, 10186, 15902, 12606, 3350, 7662, 13382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=323624960 dispatch_meta_bytes=348056 combine_payload_bytes=323624960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=323973016 ingress_bytes=358227968 link_bytes=358227968 token_bytes=647597976 weight_movement_bytes=34603008 total_nvlink_bytes=682200984 weight_to_token_ratio=0.053433 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=22912 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.347983004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66794 payload_rows_remote=50606 dup_rows=31510 dup_rows_remote=9400 dedup_ratio=0.320536 router_tokens_per_expert=[0, 3606, 2513, 990, 7461, 7691, 0, 0, 0, 0, 1, 247, 0, 1, 4, 1, 6, 0, 0, 0, 4, 2780, 2, 0, 0, 0, 1428, 2031, 90, 0, 6, 0, 0, 0, 187, 0, 0, 6057, 454, 213, 0, 0, 1441, 0, 4, 100, 118, 3, 2, 2516, 335, 0, 0, 7, 110, 2275, 191, 6178, 1, 6, 90, 0, 1, 1] payload_rows_per_dst=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] dup_rows_per_dst=[22110, 0, 1560, 3950, 920, 436, 2288, 246] combine_rows_per_src=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87106 payload_rows_remote=77658 dup_rows=11198 dup_rows_remote=9694 dedup_ratio=0.113912 router_tokens_per_expert=[355, 98, 1, 7, 1, 5, 7, 3956, 519, 152, 791, 7, 25, 0, 742, 6482, 2, 70, 9, 2050, 259, 508, 15, 594, 2, 1, 0, 0, 0, 8, 0, 21, 3364, 109, 61, 0, 1, 0, 0, 2804, 0, 146, 0, 7092, 70, 132, 266, 1, 3, 0, 0, 3962, 50, 7416, 0, 95, 0, 19, 7, 6192, 165, 418, 14, 78] payload_rows_per_dst=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] dup_rows_per_dst=[248, 836, 5000, 4, 1858, 874, 1504, 874] combine_rows_per_src=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=344162304 dispatch_meta_bytes=352240 combine_payload_bytes=344162304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=344514544 ingress_bytes=482574336 link_bytes=482574336 token_bytes=688676848 weight_movement_bytes=138412032 total_nvlink_bytes=827088880 weight_to_token_ratio=0.200983 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.815888376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=337747968 dispatch_meta_bytes=346432 combine_payload_bytes=337747968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=338094400 ingress_bytes=372350976 link_bytes=372350976 token_bytes=675842368 weight_movement_bytes=34603008 total_nvlink_bytes=710445376 weight_to_token_ratio=0.051200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.027500 gemm_active_groups=8 gemm_peak_group_rows=23680 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.401126746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=296992768 dispatch_meta_bytes=315248 combine_payload_bytes=296992768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297308016 ingress_bytes=331595776 link_bytes=331595776 token_bytes=594300784 weight_movement_bytes=34603008 total_nvlink_bytes=628903792 weight_to_token_ratio=0.058225 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=31104 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.247768209e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80662 payload_rows_remote=65734 dup_rows=17642 dup_rows_remote=16772 dedup_ratio=0.179464 router_tokens_per_expert=[0, 0, 4, 618, 2, 6, 0, 0, 0, 0, 0, 92, 2, 92, 833, 2, 0, 0, 1998, 2541, 7269, 671, 2, 92, 7407, 3, 0, 6865, 118, 77, 0, 0, 0, 6, 276, 4, 77, 0, 0, 6338, 3, 655, 1, 6, 1, 0, 90, 0, 0, 0, 2, 234, 3078, 2, 1931, 285, 1464, 91, 1, 0, 6, 4899, 56, 952] payload_rows_per_dst=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] dup_rows_per_dst=[870, 996, 3330, 9518, 104, 10, 2406, 408] combine_rows_per_src=[14928, 14776, 10800, 15490, 9776, 1502, 8658, 4732] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=350044160 dispatch_meta_bytes=358136 combine_payload_bytes=350044160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=350402296 ingress_bytes=384647168 link_bytes=384647168 token_bytes=700446456 weight_movement_bytes=34603008 total_nvlink_bytes=735049464 weight_to_token_ratio=0.049401 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=22272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.447396327e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=207282176 dispatch_meta_bytes=240024 combine_payload_bytes=207282176 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=207522200 ingress_bytes=380297216 link_bytes=380297216 token_bytes=414804376 weight_movement_bytes=173015040 total_nvlink_bytes=587819416 weight_to_token_ratio=0.417100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431027807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=318087168 dispatch_meta_bytes=349408 combine_payload_bytes=318087168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318436576 ingress_bytes=387293184 link_bytes=387293184 token_bytes=636523744 weight_movement_bytes=69206016 total_nvlink_bytes=705729760 weight_to_token_ratio=0.108725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457353072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=73490 dup_rows=12578 dup_rows_remote=8862 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4469, 0, 5200, 0, 0, 340, 58, 0, 0, 0, 459, 0, 2804, 0, 0, 0, 0, 0, 0, 6036, 6009, 0, 126, 0, 0, 0, 247, 0, 658, 3968, 0, 0, 0, 7258, 5017, 73, 0, 704, 0, 0, 227, 0, 92, 0, 0, 0, 4975, 0, 0, 0, 0, 0] payload_rows_per_dst=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] dup_rows_per_dst=[680, 3716, 4768, 1438, 676, 210, 1090, 0] combine_rows_per_src=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=313458688 dispatch_meta_bytes=376168 combine_payload_bytes=313458688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=313834856 ingress_bytes=348061696 link_bytes=348061696 token_bytes=627293544 weight_movement_bytes=34603008 total_nvlink_bytes=661896552 weight_to_token_ratio=0.055162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=45184 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.309728141e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=80024 payload_rows_remote=65320 dup_rows=18280 dup_rows_remote=15550 dedup_ratio=0.185954 router_tokens_per_expert=[38, 0, 0, 0, 0, 0, 0, 4198, 492, 0, 266, 0, 0, 0, 424, 6922, 0, 0, 0, 1678, 263, 429, 0, 555, 0, 0, 0, 0, 0, 0, 0, 0, 2595, 0, 0, 0, 0, 0, 0, 3515, 0, 0, 0, 7655, 0, 0, 278, 0, 0, 0, 0, 4496, 0, 7738, 0, 0, 0, 0, 0, 6941, 164, 418, 0, 87] payload_rows_per_dst=[8402, 1962, 15468, 0, 10786, 13226, 15476, 14704] dup_rows_per_dst=[70, 402, 4226, 0, 1434, 426, 8992, 2730] combine_rows_per_src=[8402, 1962, 15468, 0, 10786, 13226, 15476, 14704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90872 payload_rows_remote=79224 dup_rows=7432 dup_rows_remote=6652 dedup_ratio=0.075602 router_tokens_per_expert=[0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4889, 0, 5331, 0, 0, 318, 69, 0, 0, 0, 689, 0, 2208, 0, 0, 0, 0, 0, 0, 6230, 5676, 0, 194, 0, 0, 0, 344, 0, 927, 3720, 0, 0, 0, 7216, 5072, 67, 0, 704, 0, 0, 252, 0, 92, 0, 0, 0, 4745, 0, 0, 0, 0, 0] payload_rows_per_dst=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] dup_rows_per_dst=[634, 0, 924, 2916, 780, 988, 1190, 0] combine_rows_per_src=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81966 payload_rows_remote=70962 dup_rows=16338 dup_rows_remote=13938 dedup_ratio=0.166199 router_tokens_per_expert=[166, 86, 697, 219, 29, 41, 142, 89, 1087, 203, 91, 5, 1487, 40, 3532, 47, 353, 340, 85, 19, 0, 64, 221, 96, 4476, 332, 332, 22, 127, 58, 18, 1337, 1022, 37, 114, 0, 1575, 1, 1332, 22, 3091, 3689, 208, 703, 1703, 4959, 1884, 64, 1758, 677, 6, 477, 2119, 3, 75, 6, 34, 1708, 4283, 1455, 114, 72, 48, 72] payload_rows_per_dst=[11516, 5736, 5296, 11004, 7878, 14466, 13430, 12640] dup_rows_per_dst=[1340, 184, 828, 2400, 328, 4450, 3876, 2932] combine_rows_per_src=[11516, 5736, 5296, 11004, 7878, 14466, 13430, 12640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=269246464 dispatch_meta_bytes=330024 combine_payload_bytes=269246464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=269576488 ingress_bytes=303849472 link_bytes=303849472 token_bytes=538822952 weight_movement_bytes=34603008 total_nvlink_bytes=573425960 weight_to_token_ratio=0.064220 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=40064 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.143361101e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78344 payload_rows_remote=69942 dup_rows=19960 dup_rows_remote=19730 dedup_ratio=0.203044 router_tokens_per_expert=[3227, 1422, 61, 119, 31, 84, 179, 1244, 438, 1654, 3870, 87, 115, 20, 487, 4532, 21, 471, 102, 287, 309, 273, 93, 634, 34, 48, 30, 32, 73, 198, 94, 308, 3814, 1377, 227, 3, 37, 0, 0, 1261, 15, 1599, 4, 3192, 799, 1506, 168, 63, 46, 66, 13, 2289, 626, 3499, 0, 1006, 40, 351, 63, 5637, 176, 480, 147, 71] payload_rows_per_dst=[10478, 14926, 3878, 8402, 11044, 10186, 6854, 12576] dup_rows_per_dst=[2256, 7480, 502, 230, 2394, 4506, 1238, 1354] combine_rows_per_src=[10478, 14926, 3878, 8402, 11044, 10186, 6854, 12576] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=301015040 dispatch_meta_bytes=329408 combine_payload_bytes=301015040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301344448 ingress_bytes=370221056 link_bytes=370221056 token_bytes=602359488 weight_movement_bytes=69206016 total_nvlink_bytes=671565504 weight_to_token_ratio=0.114892 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87564 payload_rows_remote=84288 dup_rows=10740 dup_rows_remote=10604 dedup_ratio=0.109253 router_tokens_per_expert=[0, 280, 0, 1, 1, 1, 331, 340, 67, 7250, 0, 0, 0, 67, 0, 137, 68, 0, 0, 66, 1, 218, 1, 0, 0, 0, 0, 0, 546, 7316, 0, 0, 0, 1, 0, 0, 0, 7110, 0, 8124, 67, 0, 0, 1570, 2, 0, 67, 0, 319, 0, 0, 0, 0, 5840, 0, 0, 0, 0, 838, 7839, 0, 0, 0, 684] payload_rows_per_dst=[10730, 14642, 6596, 14746, 10080, 3276, 11682, 15812] dup_rows_per_dst=[1444, 400, 282, 978, 3954, 136, 636, 2910] combine_rows_per_src=[10730, 14642, 6596, 14746, 10080, 3276, 11682, 15812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87106 payload_rows_remote=77658 dup_rows=11198 dup_rows_remote=9694 dedup_ratio=0.113912 router_tokens_per_expert=[355, 98, 1, 7, 1, 5, 7, 3956, 519, 152, 791, 7, 25, 0, 742, 6482, 2, 70, 9, 2050, 259, 508, 15, 594, 2, 1, 0, 0, 0, 8, 0, 21, 3364, 109, 61, 0, 1, 0, 0, 2804, 0, 146, 0, 7092, 70, 132, 266, 1, 3, 0, 0, 3962, 50, 7416, 0, 95, 0, 19, 7, 6192, 165, 418, 14, 78] payload_rows_per_dst=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] dup_rows_per_dst=[248, 836, 5000, 4, 1858, 874, 1504, 874] combine_rows_per_src=[8612, 3636, 14978, 12160, 10820, 14540, 9448, 12912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267550720 dispatch_meta_bytes=323480 combine_payload_bytes=267550720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267874200 ingress_bytes=371359744 link_bytes=371359744 token_bytes=535424920 weight_movement_bytes=103809024 total_nvlink_bytes=639233944 weight_to_token_ratio=0.193882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=41984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397396820e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=286482432 dispatch_meta_bytes=358688 combine_payload_bytes=286482432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=286841120 ingress_bytes=321085440 link_bytes=321085440 token_bytes=573323552 weight_movement_bytes=34603008 total_nvlink_bytes=607926560 weight_to_token_ratio=0.060355 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=48384 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.208218661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90872 payload_rows_remote=79224 dup_rows=7432 dup_rows_remote=6652 dedup_ratio=0.075602 router_tokens_per_expert=[0, 0, 409, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4889, 0, 5331, 0, 0, 318, 69, 0, 0, 0, 689, 0, 2208, 0, 0, 0, 0, 0, 0, 6230, 5676, 0, 194, 0, 0, 0, 344, 0, 927, 3720, 0, 0, 0, 7216, 5072, 67, 0, 704, 0, 0, 252, 0, 92, 0, 0, 0, 4745, 0, 0, 0, 0, 0] payload_rows_per_dst=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] dup_rows_per_dst=[634, 0, 924, 2916, 780, 988, 1190, 0] combine_rows_per_src=[14616, 9778, 11372, 13960, 11648, 8440, 11568, 9490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=88232 payload_rows_remote=72512 dup_rows=10072 dup_rows_remote=9568 dedup_ratio=0.102458 router_tokens_per_expert=[3, 0, 0, 0, 0, 4, 0, 4641, 540, 1, 230, 0, 0, 0, 395, 7025, 0, 0, 0, 3274, 283, 883, 0, 323, 1, 0, 0, 0, 0, 1, 0, 0, 3148, 0, 0, 0, 0, 0, 0, 2386, 0, 1, 0, 7859, 0, 1, 251, 0, 0, 0, 0, 3559, 1, 7827, 0, 1, 0, 0, 0, 5902, 138, 407, 0, 67] payload_rows_per_dst=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] dup_rows_per_dst=[10, 134, 7534, 2, 1072, 504, 2, 814] combine_rows_per_src=[9286, 2198, 16042, 15656, 9996, 15720, 7120, 12214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93628 payload_rows_remote=84626 dup_rows=4676 dup_rows_remote=4072 dedup_ratio=0.047567 router_tokens_per_expert=[0, 0, 483, 0, 0, 0, 0, 0, 0, 227, 0, 0, 4959, 0, 4770, 0, 0, 422, 37, 0, 0, 0, 536, 0, 899, 0, 0, 0, 0, 0, 0, 6256, 7324, 0, 168, 0, 0, 0, 445, 0, 808, 4589, 0, 0, 0, 7420, 3808, 59, 0, 489, 0, 0, 254, 0, 62, 0, 0, 0, 5137, 0, 0, 0, 0, 0] payload_rows_per_dst=[14966, 10372, 9002, 13158, 15102, 10290, 10464, 10274] dup_rows_per_dst=[840, 0, 604, 1152, 772, 622, 686, 0] combine_rows_per_src=[14966, 10372, 9002, 13158, 15102, 10290, 10464, 10274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324501504 dispatch_meta_bytes=343504 combine_payload_bytes=324501504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324845008 ingress_bytes=393707520 link_bytes=393707520 token_bytes=649346512 weight_movement_bytes=69206016 total_nvlink_bytes=718552528 weight_to_token_ratio=0.106578 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481489702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=290660352 dispatch_meta_bytes=339600 combine_payload_bytes=290660352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290999952 ingress_bytes=394469376 link_bytes=394469376 token_bytes=581660304 weight_movement_bytes=103809024 total_nvlink_bytes=685469328 weight_to_token_ratio=0.178470 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484356504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77632 payload_rows_remote=61750 dup_rows=20672 dup_rows_remote=12734 dedup_ratio=0.210286 router_tokens_per_expert=[34, 3, 0, 1, 0, 0, 0, 4225, 477, 7, 259, 0, 0, 0, 378, 7042, 0, 4, 0, 1696, 275, 472, 0, 605, 0, 0, 0, 0, 0, 1, 0, 0, 1930, 4, 1, 0, 0, 0, 0, 4090, 0, 7, 0, 7647, 0, 6, 300, 0, 0, 0, 0, 3944, 4, 7689, 0, 5, 0, 2, 0, 7322, 190, 440, 0, 92] payload_rows_per_dst=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] dup_rows_per_dst=[7938, 6276, 164, 0, 1288, 12, 4810, 184] combine_rows_per_src=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86118 payload_rows_remote=71416 dup_rows=12186 dup_rows_remote=10324 dedup_ratio=0.123962 router_tokens_per_expert=[11, 0, 0, 0, 0, 0, 0, 3921, 454, 0, 379, 0, 0, 0, 344, 7105, 0, 0, 0, 1520, 271, 476, 0, 575, 0, 0, 0, 0, 0, 0, 0, 0, 2886, 0, 0, 0, 0, 0, 0, 4050, 0, 0, 0, 7769, 0, 0, 293, 0, 0, 0, 0, 3698, 0, 7710, 0, 0, 0, 0, 0, 6998, 168, 432, 0, 92] payload_rows_per_dst=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] dup_rows_per_dst=[4928, 1862, 116, 0, 2524, 208, 1890, 658] combine_rows_per_src=[13192, 14702, 5568, 15420, 11940, 5068, 9900, 10328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=318087168 dispatch_meta_bytes=349408 combine_payload_bytes=318087168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=318436576 ingress_bytes=387293184 link_bytes=387293184 token_bytes=636523744 weight_movement_bytes=69206016 total_nvlink_bytes=705729760 weight_to_token_ratio=0.108725 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.457353072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=345243648 dispatch_meta_bytes=379568 combine_payload_bytes=345243648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345623216 ingress_bytes=379846656 link_bytes=379846656 token_bytes=690866864 weight_movement_bytes=34603008 total_nvlink_bytes=725469872 weight_to_token_ratio=0.050086 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429332386e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87044 payload_rows_remote=75036 dup_rows=11260 dup_rows_remote=10296 dedup_ratio=0.114543 router_tokens_per_expert=[21, 2, 477, 25, 14, 0, 10, 0, 132, 221, 1, 1, 4519, 4, 5271, 4, 25, 340, 68, 0, 0, 0, 438, 19, 2278, 25, 18, 0, 1, 7, 0, 5715, 5917, 0, 205, 0, 146, 0, 704, 0, 1436, 3816, 10, 46, 151, 6994, 3785, 61, 157, 527, 0, 20, 433, 0, 78, 0, 0, 149, 4747, 116, 4, 5, 4, 5] payload_rows_per_dst=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] dup_rows_per_dst=[404, 8, 674, 2620, 936, 5394, 964, 260] combine_rows_per_src=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=297009152 dispatch_meta_bytes=328320 combine_payload_bytes=297009152 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=297337472 ingress_bytes=400818176 link_bytes=400818176 token_bytes=594346624 weight_movement_bytes=103809024 total_nvlink_bytes=698155648 weight_to_token_ratio=0.174661 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=42752 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.508246528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=66794 payload_rows_remote=50606 dup_rows=31510 dup_rows_remote=9400 dedup_ratio=0.320536 router_tokens_per_expert=[0, 3606, 2513, 990, 7461, 7691, 0, 0, 0, 0, 1, 247, 0, 1, 4, 1, 6, 0, 0, 0, 4, 2780, 2, 0, 0, 0, 1428, 2031, 90, 0, 6, 0, 0, 0, 187, 0, 0, 6057, 454, 213, 0, 0, 1441, 0, 4, 100, 118, 3, 2, 2516, 335, 0, 0, 7, 110, 2275, 191, 6178, 1, 6, 90, 0, 1, 1] payload_rows_per_dst=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] dup_rows_per_dst=[22110, 0, 1560, 3950, 920, 436, 2288, 246] combine_rows_per_src=[16188, 508, 8588, 12820, 12902, 2896, 8202, 4690] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=324501504 dispatch_meta_bytes=343504 combine_payload_bytes=324501504 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=324845008 ingress_bytes=393707520 link_bytes=393707520 token_bytes=649346512 weight_movement_bytes=69206016 total_nvlink_bytes=718552528 weight_to_token_ratio=0.106578 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=42112 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.481489702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=346628096 dispatch_meta_bytes=354792 combine_payload_bytes=346628096 prefetch_bytes=69206016 reduce_bytes=0 egress_bytes=346982888 ingress_bytes=415834112 link_bytes=415834112 token_bytes=693610984 weight_movement_bytes=69206016 total_nvlink_bytes=762817000 weight_to_token_ratio=0.099776 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=34432 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.564750286e-04 prefetch_bytes_per_gemm_flop=2.604166667e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=252928000 dispatch_meta_bytes=297936 combine_payload_bytes=252928000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253225936 ingress_bytes=287531008 link_bytes=287531008 token_bytes=506153936 weight_movement_bytes=34603008 total_nvlink_bytes=540756944 weight_to_token_ratio=0.068365 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.081956035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13556462120 prefetch_bytes=622854144 reduce_bytes=1141899264 weight_movement_bytes=1764753408 egress_bytes=6782136872 ingress_bytes=8539078656 link_bytes=8539078656 total_nvlink_bytes=15321215528 weight_to_token_ratio=0.130178 prefetch_share_of_ingress=0.072942 dedup_ratio=0.157213 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=33 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.397037632e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90556 payload_rows_remote=75538 dup_rows=7748 dup_rows_remote=7240 dedup_ratio=0.078817 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 307, 339, 92, 7433, 0, 0, 0, 92, 0, 146, 73, 0, 0, 410, 0, 193, 0, 0, 0, 0, 0, 0, 570, 7255, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 8100, 92, 0, 0, 1360, 8, 0, 92, 0, 319, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 464, 7815, 0, 0, 0, 1104] payload_rows_per_dst=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] dup_rows_per_dst=[1664, 508, 1352, 854, 0, 2736, 634, 0] combine_rows_per_src=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=307347456 dispatch_meta_bytes=341328 combine_payload_bytes=307347456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307688784 ingress_bytes=341950464 link_bytes=341950464 token_bytes=615036240 weight_movement_bytes=34603008 total_nvlink_bytes=649639248 weight_to_token_ratio=0.056262 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286732067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=292519936 dispatch_meta_bytes=326960 combine_payload_bytes=292519936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=292846896 ingress_bytes=361725952 link_bytes=361725952 token_bytes=585366832 weight_movement_bytes=69206016 total_nvlink_bytes=654572848 weight_to_token_ratio=0.118227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=30976 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.361145636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=78752 payload_rows_remote=63906 dup_rows=19552 dup_rows_remote=12304 dedup_ratio=0.198893 router_tokens_per_expert=[0, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4534, 0, 5259, 0, 0, 331, 68, 0, 0, 0, 552, 0, 2231, 0, 0, 0, 0, 0, 0, 6205, 5811, 0, 116, 0, 0, 0, 275, 0, 895, 3838, 0, 0, 0, 7300, 5342, 59, 0, 739, 0, 0, 251, 0, 87, 0, 0, 0, 4842, 0, 0, 0, 0, 0] payload_rows_per_dst=[834, 9068, 11780, 4462, 10278, 15990, 11494, 14846] dup_rows_per_dst=[0, 0, 806, 0, 642, 9678, 1178, 7248] combine_rows_per_src=[834, 9068, 11780, 4462, 10278, 15990, 11494, 14846] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90908 payload_rows_remote=90908 dup_rows=7396 dup_rows_remote=7396 dedup_ratio=0.075236 router_tokens_per_expert=[0, 248, 0, 0, 0, 0, 320, 323, 92, 7407, 0, 0, 0, 92, 0, 166, 67, 0, 0, 336, 0, 192, 0, 0, 0, 0, 0, 0, 592, 7272, 0, 0, 0, 0, 0, 0, 0, 6985, 0, 8100, 92, 0, 0, 1103, 9, 0, 100, 0, 344, 0, 0, 0, 0, 6008, 0, 0, 0, 0, 608, 7816, 0, 0, 0, 880] payload_rows_per_dst=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] dup_rows_per_dst=[1634, 534, 1190, 890, 0, 1430, 678, 1040] combine_rows_per_src=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79286 payload_rows_remote=67652 dup_rows=19018 dup_rows_remote=18168 dedup_ratio=0.193461 router_tokens_per_expert=[3, 208, 1035, 1563, 1836, 989, 289, 521, 76, 4646, 33, 8, 3, 671, 138, 114, 78, 309, 0, 355, 1533, 254, 1059, 0, 20, 389, 61, 73, 927, 3405, 1361, 6, 341, 1770, 0, 103, 295, 2433, 192, 6081, 73, 14, 118, 2002, 490, 161, 1816, 83, 2724, 44, 53, 15, 16, 451, 5, 0, 60, 3, 806, 3894, 54, 42, 75, 2975] payload_rows_per_dst=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] dup_rows_per_dst=[7618, 738, 3686, 850, 712, 5014, 198, 202] combine_rows_per_src=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=81966 payload_rows_remote=70962 dup_rows=16338 dup_rows_remote=13938 dedup_ratio=0.166199 router_tokens_per_expert=[166, 86, 697, 219, 29, 41, 142, 89, 1087, 203, 91, 5, 1487, 40, 3532, 47, 353, 340, 85, 19, 0, 64, 221, 96, 4476, 332, 332, 22, 127, 58, 18, 1337, 1022, 37, 114, 0, 1575, 1, 1332, 22, 3091, 3689, 208, 703, 1703, 4959, 1884, 64, 1758, 677, 6, 477, 2119, 3, 75, 6, 34, 1708, 4283, 1455, 114, 72, 48, 72] payload_rows_per_dst=[11516, 5736, 5296, 11004, 7878, 14466, 13430, 12640] dup_rows_per_dst=[1340, 184, 828, 2400, 328, 4450, 3876, 2932] combine_rows_per_src=[11516, 5736, 5296, 11004, 7878, 14466, 13430, 12640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=207282176 dispatch_meta_bytes=240024 combine_payload_bytes=207282176 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=207522200 ingress_bytes=380297216 link_bytes=380297216 token_bytes=414804376 weight_movement_bytes=173015040 total_nvlink_bytes=587819416 weight_to_token_ratio=0.417100 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=19968 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.431027807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309403648 dispatch_meta_bytes=331112 combine_payload_bytes=309403648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309734760 ingress_bytes=413212672 link_bytes=413212672 token_bytes=619138408 weight_movement_bytes=103809024 total_nvlink_bytes=722947432 weight_to_token_ratio=0.167667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554886018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87044 payload_rows_remote=75036 dup_rows=11260 dup_rows_remote=10296 dedup_ratio=0.114543 router_tokens_per_expert=[21, 2, 477, 25, 14, 0, 10, 0, 132, 221, 1, 1, 4519, 4, 5271, 4, 25, 340, 68, 0, 0, 0, 438, 19, 2278, 25, 18, 0, 1, 7, 0, 5715, 5917, 0, 205, 0, 146, 0, 704, 0, 1436, 3816, 10, 46, 151, 6994, 3785, 61, 157, 527, 0, 20, 433, 0, 78, 0, 0, 149, 4747, 116, 4, 5, 4, 5] payload_rows_per_dst=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] dup_rows_per_dst=[404, 8, 674, 2620, 936, 5394, 964, 260] combine_rows_per_src=[6852, 9756, 8676, 13414, 13008, 13476, 12008, 9854] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=261758976 dispatch_meta_bytes=304840 combine_payload_bytes=261758976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=262063816 ingress_bytes=296361984 link_bytes=296361984 token_bytes=523822792 weight_movement_bytes=34603008 total_nvlink_bytes=558425800 weight_to_token_ratio=0.066059 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.115186287e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372359168 dispatch_meta_bytes=393216 combine_payload_bytes=372359168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372752384 ingress_bytes=476168192 link_bytes=476168192 token_bytes=745111552 weight_movement_bytes=103809024 total_nvlink_bytes=848920576 weight_to_token_ratio=0.139320 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791782572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=290660352 dispatch_meta_bytes=339600 combine_payload_bytes=290660352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=290999952 ingress_bytes=394469376 link_bytes=394469376 token_bytes=581660304 weight_movement_bytes=103809024 total_nvlink_bytes=685469328 weight_to_token_ratio=0.178470 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.484356504e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=93670 payload_rows_remote=84024 dup_rows=4634 dup_rows_remote=4036 dedup_ratio=0.047139 router_tokens_per_expert=[0, 0, 429, 0, 0, 3, 0, 0, 1, 237, 0, 0, 4666, 0, 5141, 0, 0, 340, 65, 0, 0, 0, 481, 0, 1048, 0, 1, 0, 0, 7, 0, 6226, 7295, 0, 203, 0, 1, 0, 478, 0, 721, 4332, 0, 0, 1, 7342, 4142, 68, 1, 498, 0, 0, 253, 0, 67, 0, 0, 1, 5104, 0, 0, 0, 0, 0] payload_rows_per_dst=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] dup_rows_per_dst=[682, 0, 608, 1218, 862, 598, 666, 0] combine_rows_per_src=[14866, 9808, 9448, 13346, 15092, 9646, 11254, 10210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90908 payload_rows_remote=90908 dup_rows=7396 dup_rows_remote=7396 dedup_ratio=0.075236 router_tokens_per_expert=[0, 248, 0, 0, 0, 0, 320, 323, 92, 7407, 0, 0, 0, 92, 0, 166, 67, 0, 0, 336, 0, 192, 0, 0, 0, 0, 0, 0, 592, 7272, 0, 0, 0, 0, 0, 0, 0, 6985, 0, 8100, 92, 0, 0, 1103, 9, 0, 100, 0, 344, 0, 0, 0, 0, 6008, 0, 0, 0, 0, 608, 7816, 0, 0, 0, 880] payload_rows_per_dst=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] dup_rows_per_dst=[1634, 534, 1190, 890, 0, 1430, 678, 1040] combine_rows_per_src=[14118, 14980, 16200, 14838, 0, 11316, 12026, 7430] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=91252 payload_rows_remote=74992 dup_rows=7052 dup_rows_remote=6474 dedup_ratio=0.071737 router_tokens_per_expert=[0, 217, 0, 0, 0, 0, 311, 422, 62, 7346, 0, 0, 0, 62, 0, 114, 59, 0, 0, 46, 0, 184, 0, 0, 0, 0, 0, 0, 512, 7332, 0, 0, 0, 0, 0, 0, 0, 7121, 0, 8130, 62, 0, 0, 1370, 0, 0, 66, 0, 318, 0, 0, 0, 0, 5868, 0, 0, 0, 0, 829, 7884, 0, 0, 0, 837] payload_rows_per_dst=[14316, 14796, 16260, 14784, 0, 16024, 11740, 3332] dup_rows_per_dst=[1826, 372, 578, 904, 0, 2740, 632, 0] combine_rows_per_src=[14316, 14796, 16260, 14784, 0, 16024, 11740, 3332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77388 payload_rows_remote=62724 dup_rows=20916 dup_rows_remote=20206 dedup_ratio=0.212769 router_tokens_per_expert=[0, 0, 440, 1, 0, 0, 1, 0, 5, 3, 0, 0, 4280, 0, 5778, 0, 6, 350, 77, 0, 0, 0, 415, 1, 2632, 0, 3, 0, 0, 0, 0, 6208, 5047, 0, 153, 0, 7, 0, 259, 0, 774, 3690, 0, 3, 7, 7245, 5327, 53, 7, 737, 0, 0, 251, 0, 92, 0, 0, 6, 5290, 3, 0, 0, 0, 1] payload_rows_per_dst=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] dup_rows_per_dst=[710, 12570, 5440, 750, 534, 6, 900, 6] combine_rows_per_src=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277102592 dispatch_meta_bytes=343280 combine_payload_bytes=277102592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=277445872 ingress_bytes=346308608 link_bytes=346308608 token_bytes=554548464 weight_movement_bytes=69206016 total_nvlink_bytes=623754480 weight_to_token_ratio=0.124797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303131412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=85726 payload_rows_remote=73490 dup_rows=12578 dup_rows_remote=8862 dedup_ratio=0.127950 router_tokens_per_expert=[0, 0, 432, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4469, 0, 5200, 0, 0, 340, 58, 0, 0, 0, 459, 0, 2804, 0, 0, 0, 0, 0, 0, 6036, 6009, 0, 126, 0, 0, 0, 247, 0, 658, 3968, 0, 0, 0, 7258, 5017, 73, 0, 704, 0, 0, 227, 0, 92, 0, 0, 0, 4975, 0, 0, 0, 0, 0] payload_rows_per_dst=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] dup_rows_per_dst=[680, 3716, 4768, 1438, 676, 210, 1090, 0] combine_rows_per_src=[14700, 12236, 13786, 9228, 12088, 2382, 11356, 9950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=307347456 dispatch_meta_bytes=341328 combine_payload_bytes=307347456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307688784 ingress_bytes=341950464 link_bytes=341950464 token_bytes=615036240 weight_movement_bytes=34603008 total_nvlink_bytes=649639248 weight_to_token_ratio=0.056262 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=36992 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286732067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13067929672 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6537840712 ingress_bytes=7775797248 link_bytes=7775797248 total_nvlink_bytes=14313637960 weight_to_token_ratio=0.095326 prefetch_share_of_ingress=0.093452 dedup_ratio=0.177958 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.272160828e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67476 payload_rows_remote=55116 dup_rows=30828 dup_rows_remote=24404 dedup_ratio=0.313599 router_tokens_per_expert=[0, 269, 92, 145, 157, 79, 310, 396, 78, 7093, 5, 0, 0, 106, 13, 116, 63, 25, 0, 296, 141, 183, 113, 0, 1, 38, 0, 0, 578, 6756, 119, 0, 4, 154, 0, 7, 54, 6562, 7, 7944, 78, 3, 5, 2212, 90, 12, 253, 5, 554, 3, 4, 1, 1, 4131, 0, 0, 0, 0, 864, 7501, 1, 0, 8, 1522] payload_rows_per_dst=[1932, 4960, 1420, 10234, 16252, 4924, 12360, 15394] dup_rows_per_dst=[964, 298, 222, 348, 17614, 392, 6424, 4566] combine_rows_per_src=[1932, 4960, 1420, 10234, 16252, 4924, 12360, 15394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13916168648 prefetch_bytes=588251136 reduce_bytes=968884224 weight_movement_bytes=1557135360 egress_bytes=6962061768 ingress_bytes=8511242240 link_bytes=8511242240 total_nvlink_bytes=15473304008 weight_to_token_ratio=0.111894 prefetch_share_of_ingress=0.069115 dedup_ratio=0.145411 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=28 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=27432 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.392483449e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77632 payload_rows_remote=61750 dup_rows=20672 dup_rows_remote=12734 dedup_ratio=0.210286 router_tokens_per_expert=[34, 3, 0, 1, 0, 0, 0, 4225, 477, 7, 259, 0, 0, 0, 378, 7042, 0, 4, 0, 1696, 275, 472, 0, 605, 0, 0, 0, 0, 0, 1, 0, 0, 1930, 4, 1, 0, 0, 0, 0, 4090, 0, 7, 0, 7647, 0, 6, 300, 0, 0, 0, 0, 3944, 4, 7689, 0, 5, 0, 2, 0, 7322, 190, 440, 0, 92] payload_rows_per_dst=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] dup_rows_per_dst=[7938, 6276, 164, 0, 1288, 12, 4810, 184] combine_rows_per_src=[15882, 15110, 5940, 15380, 10762, 614, 12680, 1264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=344162304 dispatch_meta_bytes=352240 combine_payload_bytes=344162304 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=344514544 ingress_bytes=482574336 link_bytes=482574336 token_bytes=688676848 weight_movement_bytes=138412032 total_nvlink_bytes=827088880 weight_to_token_ratio=0.200983 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.815888376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=372359168 dispatch_meta_bytes=393216 combine_payload_bytes=372359168 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=372752384 ingress_bytes=476168192 link_bytes=476168192 token_bytes=745111552 weight_movement_bytes=103809024 total_nvlink_bytes=848920576 weight_to_token_ratio=0.139320 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=21120 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.791782572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=307167232 dispatch_meta_bytes=325864 combine_payload_bytes=307167232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=307493096 ingress_bytes=341770240 link_bytes=341770240 token_bytes=614660328 weight_movement_bytes=34603008 total_nvlink_bytes=649263336 weight_to_token_ratio=0.056296 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.030000 gemm_active_groups=8 gemm_peak_group_rows=41600 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.286053898e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13140420784 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=6574188720 ingress_bytes=8192573440 link_bytes=8192573440 total_nvlink_bytes=14766762160 weight_to_token_ratio=0.123766 prefetch_share_of_ingress=0.101369 dedup_ratio=0.191568 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.340347578e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256917504 dispatch_meta_bytes=331720 combine_payload_bytes=256917504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257249224 ingress_bytes=291520512 link_bytes=291520512 token_bytes=514166728 weight_movement_bytes=34603008 total_nvlink_bytes=548769736 weight_to_token_ratio=0.067299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096968217e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=225755136 dispatch_meta_bytes=318080 combine_payload_bytes=225755136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=226073216 ingress_bytes=260358144 link_bytes=260358144 token_bytes=451828352 weight_movement_bytes=34603008 total_nvlink_bytes=486431360 weight_to_token_ratio=0.076584 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.797067353e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=301015040 dispatch_meta_bytes=329408 combine_payload_bytes=301015040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=301344448 ingress_bytes=370221056 link_bytes=370221056 token_bytes=602359488 weight_movement_bytes=69206016 total_nvlink_bytes=671565504 weight_to_token_ratio=0.114892 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=33920 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=51170 payload_rows_remote=34786 dup_rows=47134 dup_rows_remote=30100 dedup_ratio=0.479472 router_tokens_per_expert=[0, 229, 0, 0, 0, 0, 295, 333, 86, 7427, 0, 0, 0, 86, 0, 157, 59, 0, 0, 383, 0, 177, 0, 0, 0, 0, 0, 0, 539, 7347, 0, 0, 0, 0, 0, 0, 0, 7041, 0, 8106, 87, 0, 0, 1370, 7, 0, 104, 0, 338, 0, 0, 0, 0, 5699, 0, 0, 0, 0, 484, 7842, 0, 0, 0, 956] payload_rows_per_dst=[1392, 328, 1238, 1078, 16384, 2944, 11422, 16384] dup_rows_per_dst=[322, 330, 0, 0, 28604, 192, 652, 17034] combine_rows_per_src=[1392, 328, 1238, 1078, 16384, 2944, 11422, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=79286 payload_rows_remote=67652 dup_rows=19018 dup_rows_remote=18168 dedup_ratio=0.193461 router_tokens_per_expert=[3, 208, 1035, 1563, 1836, 989, 289, 521, 76, 4646, 33, 8, 3, 671, 138, 114, 78, 309, 0, 355, 1533, 254, 1059, 0, 20, 389, 61, 73, 927, 3405, 1361, 6, 341, 1770, 0, 103, 295, 2433, 192, 6081, 73, 14, 118, 2002, 490, 161, 1816, 83, 2724, 44, 53, 15, 16, 451, 5, 0, 60, 3, 806, 3894, 54, 42, 75, 2975] payload_rows_per_dst=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] dup_rows_per_dst=[7618, 738, 3686, 850, 712, 5014, 198, 202] combine_rows_per_src=[10136, 10640, 15652, 11634, 4690, 12288, 6418, 7828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=252928000 dispatch_meta_bytes=297936 combine_payload_bytes=252928000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=253225936 ingress_bytes=287531008 link_bytes=287531008 token_bytes=506153936 weight_movement_bytes=34603008 total_nvlink_bytes=540756944 weight_to_token_ratio=0.068365 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.081956035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13916168648 prefetch_bytes=588251136 reduce_bytes=968884224 weight_movement_bytes=1557135360 egress_bytes=6962061768 ingress_bytes=8511242240 link_bytes=8511242240 total_nvlink_bytes=15473304008 weight_to_token_ratio=0.111894 prefetch_share_of_ingress=0.069115 dedup_ratio=0.145411 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=28 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=27432 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.392483449e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13733814392 prefetch_bytes=865075200 reduce_bytes=380633088 weight_movement_bytes=1245708288 egress_bytes=6870868088 ingress_bytes=8108654592 link_bytes=8108654592 total_nvlink_bytes=14979522680 weight_to_token_ratio=0.090704 prefetch_share_of_ingress=0.106685 dedup_ratio=0.151874 prefetch_slots=50 unused_prefetch_slots=318 slot_utilization=0.135870 reduce_slots=11 remote_demand=25 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.326618019e-04 prefetch_bytes_per_gemm_flop=1.415307971e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13014483064 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6511124600 ingress_bytes=7645257728 link_bytes=7645257728 total_nvlink_bytes=14156382328 weight_to_token_ratio=0.087741 prefetch_share_of_ingress=0.099574 dedup_ratio=0.185047 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.250803884e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=67476 payload_rows_remote=55116 dup_rows=30828 dup_rows_remote=24404 dedup_ratio=0.313599 router_tokens_per_expert=[0, 269, 92, 145, 157, 79, 310, 396, 78, 7093, 5, 0, 0, 106, 13, 116, 63, 25, 0, 296, 141, 183, 113, 0, 1, 38, 0, 0, 578, 6756, 119, 0, 4, 154, 0, 7, 54, 6562, 7, 7944, 78, 3, 5, 2212, 90, 12, 253, 5, 554, 3, 4, 1, 1, 4131, 0, 0, 0, 0, 864, 7501, 1, 0, 8, 1522] payload_rows_per_dst=[1932, 4960, 1420, 10234, 16252, 4924, 12360, 15394] dup_rows_per_dst=[964, 298, 222, 348, 17614, 392, 6424, 4566] combine_rows_per_src=[1932, 4960, 1420, 10234, 16252, 4924, 12360, 15394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=142483456 dispatch_meta_bytes=259544 combine_payload_bytes=142483456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=142743000 ingress_bytes=211689472 link_bytes=211689472 token_bytes=285226456 weight_movement_bytes=69206016 total_nvlink_bytes=354432472 weight_to_token_ratio=0.242635 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=25984 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=7.965704407e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86876 payload_rows_remote=72634 dup_rows=11428 dup_rows_remote=9680 dedup_ratio=0.116252 router_tokens_per_expert=[0, 256, 4, 6, 7, 4, 335, 360, 90, 7417, 0, 0, 0, 90, 1, 170, 53, 0, 0, 370, 7, 183, 6, 0, 0, 1, 0, 0, 555, 7236, 5, 0, 0, 7, 0, 0, 0, 7023, 0, 8095, 92, 0, 0, 1420, 22, 0, 100, 0, 347, 0, 0, 0, 0, 5459, 0, 0, 0, 0, 521, 7813, 0, 0, 0, 1097] payload_rows_per_dst=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] dup_rows_per_dst=[1748, 4322, 1224, 600, 0, 2884, 650, 0] combine_rows_per_src=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=277102592 dispatch_meta_bytes=343280 combine_payload_bytes=277102592 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=277445872 ingress_bytes=346308608 link_bytes=346308608 token_bytes=554548464 weight_movement_bytes=69206016 total_nvlink_bytes=623754480 weight_to_token_ratio=0.124797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=42368 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.303131412e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=87564 payload_rows_remote=84288 dup_rows=10740 dup_rows_remote=10604 dedup_ratio=0.109253 router_tokens_per_expert=[0, 280, 0, 1, 1, 1, 331, 340, 67, 7250, 0, 0, 0, 67, 0, 137, 68, 0, 0, 66, 1, 218, 1, 0, 0, 0, 0, 0, 546, 7316, 0, 0, 0, 1, 0, 0, 0, 7110, 0, 8124, 67, 0, 0, 1570, 2, 0, 67, 0, 319, 0, 0, 0, 0, 5840, 0, 0, 0, 0, 838, 7839, 0, 0, 0, 684] payload_rows_per_dst=[10730, 14642, 6596, 14746, 10080, 3276, 11682, 15812] dup_rows_per_dst=[1444, 400, 282, 978, 3954, 136, 636, 2910] combine_rows_per_src=[10730, 14642, 6596, 14746, 10080, 3276, 11682, 15812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=225755136 dispatch_meta_bytes=318080 combine_payload_bytes=225755136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=226073216 ingress_bytes=260358144 link_bytes=260358144 token_bytes=451828352 weight_movement_bytes=34603008 total_nvlink_bytes=486431360 weight_to_token_ratio=0.076584 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=39168 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.797067353e-05 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=90556 payload_rows_remote=75538 dup_rows=7748 dup_rows_remote=7240 dedup_ratio=0.078817 router_tokens_per_expert=[0, 244, 0, 0, 0, 0, 307, 339, 92, 7433, 0, 0, 0, 92, 0, 146, 73, 0, 0, 410, 0, 193, 0, 0, 0, 0, 0, 0, 570, 7255, 0, 0, 0, 0, 0, 0, 0, 7051, 0, 8100, 92, 0, 0, 1360, 8, 0, 92, 0, 319, 0, 0, 0, 0, 5593, 0, 0, 0, 0, 464, 7815, 0, 0, 0, 1104] payload_rows_per_dst=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] dup_rows_per_dst=[1664, 508, 1352, 854, 0, 2736, 634, 0] combine_rows_per_src=[14218, 15018, 16200, 14796, 0, 15998, 11190, 3136] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11759756584 prefetch_bytes=726663168 reduce_bytes=968884224 weight_movement_bytes=1695547392 egress_bytes=5883462952 ingress_bytes=7571841024 link_bytes=7571841024 total_nvlink_bytes=13455303976 weight_to_token_ratio=0.144182 prefetch_share_of_ingress=0.095969 dedup_ratio=0.226170 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=28 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238792530e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13140420784 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=6574188720 ingress_bytes=8192573440 link_bytes=8192573440 total_nvlink_bytes=14766762160 weight_to_token_ratio=0.123766 prefetch_share_of_ingress=0.101369 dedup_ratio=0.191568 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.340347578e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=77388 payload_rows_remote=62724 dup_rows=20916 dup_rows_remote=20206 dedup_ratio=0.212769 router_tokens_per_expert=[0, 0, 440, 1, 0, 0, 1, 0, 5, 3, 0, 0, 4280, 0, 5778, 0, 6, 350, 77, 0, 0, 0, 415, 1, 2632, 0, 3, 0, 0, 0, 0, 6208, 5047, 0, 153, 0, 7, 0, 259, 0, 774, 3690, 0, 3, 7, 7245, 5327, 53, 7, 737, 0, 0, 251, 0, 92, 0, 0, 6, 5290, 3, 0, 0, 0, 1] payload_rows_per_dst=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] dup_rows_per_dst=[710, 12570, 5440, 750, 534, 6, 900, 6] combine_rows_per_src=[14664, 15072, 14292, 7238, 7680, 1668, 6180, 10594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13014483064 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6511124600 ingress_bytes=7645257728 link_bytes=7645257728 total_nvlink_bytes=14156382328 weight_to_token_ratio=0.087741 prefetch_share_of_ingress=0.099574 dedup_ratio=0.185047 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.250803884e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=345243648 dispatch_meta_bytes=379568 combine_payload_bytes=345243648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=345623216 ingress_bytes=379846656 link_bytes=379846656 token_bytes=690866864 weight_movement_bytes=34603008 total_nvlink_bytes=725469872 weight_to_token_ratio=0.050086 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.429332386e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=309403648 dispatch_meta_bytes=331112 combine_payload_bytes=309403648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=309734760 ingress_bytes=413212672 link_bytes=413212672 token_bytes=619138408 weight_movement_bytes=103809024 total_nvlink_bytes=722947432 weight_to_token_ratio=0.167667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=43904 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.554886018e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=256917504 dispatch_meta_bytes=331720 combine_payload_bytes=256917504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=257249224 ingress_bytes=291520512 link_bytes=291520512 token_bytes=514166728 weight_movement_bytes=34603008 total_nvlink_bytes=548769736 weight_to_token_ratio=0.067299 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=44544 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.096968217e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13556462120 prefetch_bytes=622854144 reduce_bytes=1141899264 weight_movement_bytes=1764753408 egress_bytes=6782136872 ingress_bytes=8539078656 link_bytes=8539078656 total_nvlink_bytes=15321215528 weight_to_token_ratio=0.130178 prefetch_share_of_ingress=0.072942 dedup_ratio=0.157213 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=33 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28067 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.397037632e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=13067929672 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6537840712 ingress_bytes=7775797248 link_bytes=7775797248 total_nvlink_bytes=14313637960 weight_to_token_ratio=0.095326 prefetch_share_of_ingress=0.093452 dedup_ratio=0.177958 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.272160828e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=86876 payload_rows_remote=72634 dup_rows=11428 dup_rows_remote=9680 dedup_ratio=0.116252 router_tokens_per_expert=[0, 256, 4, 6, 7, 4, 335, 360, 90, 7417, 0, 0, 0, 90, 1, 170, 53, 0, 0, 370, 7, 183, 6, 0, 0, 1, 0, 0, 555, 7236, 5, 0, 0, 7, 0, 0, 0, 7023, 0, 8095, 92, 0, 0, 1420, 22, 0, 100, 0, 347, 0, 0, 0, 0, 5459, 0, 0, 0, 0, 521, 7813, 0, 0, 0, 1097] payload_rows_per_dst=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] dup_rows_per_dst=[1748, 4322, 1224, 600, 0, 2884, 650, 0] combine_rows_per_src=[14242, 15438, 16204, 10770, 14, 16010, 10962, 3236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=297508864 dispatch_meta_bytes=329256 combine_payload_bytes=297508864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=297838120 ingress_bytes=332111872 link_bytes=332111872 token_bytes=595346984 weight_movement_bytes=34603008 total_nvlink_bytes=629949992 weight_to_token_ratio=0.058122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=35840 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.249710237e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=11878436448 prefetch_bytes=830472192 reduce_bytes=795869184 weight_movement_bytes=1626341376 egress_bytes=5942849120 ingress_bytes=7561928704 link_bytes=7561928704 total_nvlink_bytes=13504777824 weight_to_token_ratio=0.136915 prefetch_share_of_ingress=0.109823 dedup_ratio=0.221790 prefetch_slots=48 unused_prefetch_slots=320 slot_utilization=0.130435 reduce_slots=23 remote_demand=24 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237170823e-04 prefetch_bytes_per_gemm_flop=1.358695652e-05 anomalies=none - [2026-08-26 16:55:30] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 801.0 | throughput per GPU (TFLOP/s/GPU): 159.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.611391E+00 | loss scale: 1.0 | grad norm: 75.148 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 16:55:30 -[rank0]:[W826 16:55:31.645659731 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 16:55:32.650311033 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 16:55:32.650948323 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 16:55:32.734645186 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 16:55:33.123947355 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 16:55:33.138094648 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 16:55:33.144549635 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 16:55:33.154661187 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 16:55:39.378000 284717 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:55:39.379000 284717 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 284810 closing signal SIGTERM -W0826 16:55:39.379000 284717 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 284816 closing signal SIGTERM + [2026-08-26 18:07:37] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 799.6 | throughput per GPU (TFLOP/s/GPU): 159.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.611391E+00 | loss scale: 1.0 | grad norm: 75.148 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:07:37 +[rank0]:[W826 18:07:39.113512323 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 18:07:39.820297021 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 18:07:39.828136787 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 18:07:40.320149145 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 18:07:40.329776473 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 18:07:40.678944141 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 18:07:40.699245287 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 18:07:40.699940831 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:07:47.023000 457134 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:07:47.024000 457134 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 457232 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7028,4 +7027,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 284717 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 457134 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv index 888eabce7a851315d12b2906c6d292ded6abef4e..c77fe73efcf591637e6d822334ab81097dd42da9 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/mem_samples.csv @@ -39,21 +39,21 @@ 6, 4 7, 4 0, 4 -1, 4 -2, 4 -3, 4 -4, 4 +1, 629 +2, 629 +3, 629 +4, 629 5, 629 6, 629 -7, 4 -0, 845 -1, 837 -2, 835 -3, 843 -4, 847 -5, 845 -6, 845 -7, 845 +7, 629 +0, 907 +1, 923 +2, 915 +3, 917 +4, 927 +5, 915 +6, 923 +7, 921 0, 1955 1, 1955 2, 1955 @@ -62,38 +62,38 @@ 5, 1955 6, 1955 7, 1955 -0, 3759 -1, 3761 -2, 3757 -3, 3771 -4, 3761 -5, 3761 -6, 3769 -7, 3761 -0, 32465 -1, 32619 -2, 32531 -3, 32509 -4, 32575 -5, 32575 -6, 32575 -7, 32597 -0, 35965 -1, 36055 -2, 35925 -3, 35985 -4, 35937 -5, 35985 -6, 35973 -7, 36055 -0, 45899 -1, 46011 -2, 45877 -3, 45949 -4, 45903 -5, 45947 -6, 45929 -7, 46019 +0, 3837 +1, 3803 +2, 3811 +3, 3813 +4, 3821 +5, 3823 +6, 3815 +7, 3831 +0, 34625 +1, 34761 +2, 34571 +3, 34509 +4, 34619 +5, 34703 +6, 34643 +7, 34601 +0, 41335 +1, 41455 +2, 41329 +3, 41387 +4, 41339 +5, 41385 +6, 41377 +7, 41455 +0, 46287 +1, 46407 +2, 46281 +3, 46339 +4, 46291 +5, 46337 +6, 46329 +7, 46407 0, 46289 1, 46409 2, 46283 @@ -126,14 +126,6 @@ 5, 139399 6, 140177 7, 139405 -0, 139745 -1, 139713 -2, 139755 -3, 139761 -4, 139737 -5, 139721 -6, 140499 -7, 139749 0, 139801 1, 139793 2, 139795 @@ -142,22 +134,30 @@ 5, 139787 6, 140565 7, 139793 -0, 142111 -1, 142103 -2, 142105 -3, 142101 -4, 142117 -5, 142095 -6, 142875 -7, 142103 -0, 143093 -1, 143109 -2, 143157 -3, 143169 -4, 143129 -5, 143121 -6, 143881 -7, 143155 +0, 141851 +1, 141843 +2, 141845 +3, 141841 +4, 141857 +5, 141837 +6, 142615 +7, 141843 +0, 142499 +1, 142491 +2, 142493 +3, 142489 +4, 142505 +5, 142485 +6, 143263 +7, 142491 +0, 143159 +1, 143185 +2, 143183 +3, 143219 +4, 143173 +5, 143165 +6, 143949 +7, 143199 0, 157149 1, 157133 2, 157143 @@ -190,38 +190,38 @@ 5, 157127 6, 156339 7, 157133 -0, 157085 -1, 156871 -2, 157143 -3, 156873 -4, 156831 -5, 157127 -6, 156339 +0, 156887 +1, 157133 +2, 156817 +3, 156553 +4, 157151 +5, 156801 +6, 155757 7, 157133 -0, 155267 -1, 154989 -2, 155261 -3, 155379 -4, 155657 -5, 155245 -6, 154751 -7, 154989 -0, 152553 -1, 152989 -2, 153581 -3, 153375 -4, 154235 -5, 153623 -6, 152777 -7, 153245 -0, 4 -1, 4 -2, 143043 -3, 4 -4, 143447 -5, 4 -6, 142239 -7, 4 +0, 155005 +1, 155251 +2, 154935 +3, 154671 +4, 155269 +5, 154919 +6, 153875 +7, 155251 +0, 152873 +1, 152925 +2, 152547 +3, 152991 +4, 153589 +5, 153239 +6, 151807 +7, 153301 +0, 4360 +1, 4360 +2, 4360 +3, 4360 +4, 147803 +5, 4360 +6, 4360 +7, 147785 0, 0 1, 0 2, 0 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log index 5ae22597e1a3b456bbfc73099713c859e9157b92..22cda9488be2fd61db62bd2afb507738077d56f8 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/train.log @@ -1,5 +1,5 @@ -[16:55:41] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:55:41] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[18:07:48] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:07:48] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== DSV2_LITE_L24 / moonep (slots/rank=8 policy=matched) E=64 K=6 H'=1408 m=8 @@ -8,10 +8,19 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 16:55:42.894000 287395 torch/distributed/run.py:766] -W0826 16:55:42.894000 287395 torch/distributed/run.py:766] ***************************************** -W0826 16:55:42.894000 287395 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:55:42.894000 287395 torch/distributed/run.py:766] ***************************************** +W0826 18:07:49.555000 459728 torch/distributed/run.py:766] +W0826 18:07:49.555000 459728 torch/distributed/run.py:766] ***************************************** +W0826 18:07:49.555000 459728 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:07:49.555000 459728 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -562,6 +571,8 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... @@ -569,62 +580,51 @@ WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 5[Gloo] Rank [Gloo] Rank [Gloo] Rank is connected to 23677 is connected to is connected to is connected to peer ranks. is connected to 777Expected number of connected peer ranks is : 7 peer ranks. peer ranks. peer ranks. 7 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : -Expected number of connected peer ranks is : 7777 - - - -[Gloo] Rank [Gloo] Rank 32 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 - +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : -7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank6]:[W826 16:56:01.647103401 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:56:01.647122829 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:56:01.647176426 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:56:01.647200380 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:56:01.647271423 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:56:01.647290877 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:56:01.647314668 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:08:08.226120562 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:08:08.226135120 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:08:08.227176662 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:08:08.227467936 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:08:08.227647951 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:08:08.231679034 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:08:08.232828197 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.033 seconds +>>> done with dataset index builder. Compilation time: 0.036 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:56:01.851996225 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.660 seconds -time to initialize megatron (seconds): 7.241 -[after megatron is initialized] datetime: 2026-08-26 16:56:07 +[rank0]:[W826 18:08:08.393740912 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.491 seconds +time to initialize megatron (seconds): 6.837 +[after megatron is initialized] datetime: 2026-08-26 18:08:14 building GPT model ... /workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_config.py:1542: UserWarning: MLA config max_position_embeddings is overridden by customer input, please use the original_max_position_embeddings instead for YaRN!max_position_embeddings will be deprecated in the future.Assigned original_max_position_embeddings to max_position_embeddings if not set,and assigned max_position_embeddings back to the original value. warnings.warn( @@ -665,22 +665,21 @@ Params for bucket 1 (103022592 elements, 103022592 padded size): module.output_layer.weight Params for bucket 2 (48503296 elements, 48503296 padded size): module.decoder.final_layernorm.weight + module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.23.mlp.router.weight + module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.23.input_layernorm.weight + module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight module.decoder.layers.23.mlp.shared_experts.linear_fc1.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.weight module.decoder.layers.23.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.23.self_attention.linear_kv_down_proj.weight module.decoder.layers.23.self_attention.linear_q_proj.weight - module.decoder.layers.23.self_attention.linear_proj.weight - module.decoder.layers.23.input_layernorm.weight - module.decoder.layers.22.mlp.shared_experts.linear_fc2.weight module.decoder.layers.22.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.23.mlp.router.weight - module.decoder.layers.23.pre_mlp_layernorm.weight Params for bucket 3 (40901632 elements, 40901632 padded size): - module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.22.self_attention.linear_kv_down_proj.weight - module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.mlp.router.weight + module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.22.self_attention.linear_q_proj.weight module.decoder.layers.22.self_attention.linear_proj.weight module.decoder.layers.22.input_layernorm.weight @@ -688,134 +687,129 @@ Params for bucket 3 (40901632 elements, 40901632 padded size): module.decoder.layers.21.mlp.shared_experts.linear_fc1.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.weight module.decoder.layers.21.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.22.mlp.router.weight - module.decoder.layers.22.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.weight + module.decoder.layers.22.self_attention.linear_kv_down_proj.weight + module.decoder.layers.22.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.21.mlp.router.weight module.decoder.layers.21.pre_mlp_layernorm.weight - module.decoder.layers.21.self_attention.linear_kv_down_proj.weight module.decoder.layers.21.self_attention.linear_q_proj.weight Params for bucket 4 (41163264 elements, 41163264 padded size): + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.self_attention.linear_proj.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight module.decoder.layers.21.input_layernorm.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.20.mlp.shared_experts.linear_fc1.weight module.decoder.layers.20.pre_mlp_layernorm.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.weight module.decoder.layers.20.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.20.self_attention.linear_proj.weight - module.decoder.layers.20.input_layernorm.weight - module.decoder.layers.19.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.20.mlp.router.weight module.decoder.layers.20.self_attention.linear_kv_down_proj.weight module.decoder.layers.20.self_attention.linear_q_proj.weight + module.decoder.layers.20.input_layernorm.weight Params for bucket 5 (42734080 elements, 42734080 padded size): - module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.weight - module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.19.pre_mlp_layernorm.weight + module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.self_attention.linear_q_proj.weight + module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.19.input_layernorm.weight module.decoder.layers.18.mlp.shared_experts.linear_fc2.weight module.decoder.layers.18.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.19.mlp.router.weight + module.decoder.layers.19.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.weight + module.decoder.layers.19.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.19.self_attention.linear_kv_down_proj.weight - module.decoder.layers.19.self_attention.linear_proj.weight Params for bucket 6 (40901632 elements, 40901632 padded size): - module.decoder.layers.18.pre_mlp_layernorm.weight - module.decoder.layers.17.mlp.router.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.linear_q_proj.weight - module.decoder.layers.18.mlp.router.weight - module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.18.self_attention.linear_kv_up_proj.weight + module.decoder.layers.18.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.18.self_attention.linear_kv_down_proj.weight - module.decoder.layers.18.self_attention.linear_q_proj.weight module.decoder.layers.18.self_attention.linear_proj.weight module.decoder.layers.18.input_layernorm.weight module.decoder.layers.17.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.mlp.shared_experts.linear_fc1.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.weight - module.decoder.layers.17.self_attention.linear_kv_down_proj.weight module.decoder.layers.17.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.17.self_attention.linear_kv_down_proj.weight + module.decoder.layers.17.self_attention.linear_q_proj.weight + module.decoder.layers.18.mlp.router.weight + module.decoder.layers.18.pre_mlp_layernorm.weight + module.decoder.layers.18.self_attention.linear_q_proj.weight + module.decoder.layers.17.mlp.router.weight + module.decoder.layers.17.pre_mlp_layernorm.weight Params for bucket 7 (41163264 elements, 41163264 padded size): - module.decoder.layers.17.self_attention.linear_proj.weight - module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.input_layernorm.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight module.decoder.layers.17.input_layernorm.weight + module.decoder.layers.16.mlp.shared_experts.linear_fc2.weight module.decoder.layers.16.mlp.shared_experts.linear_fc1.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.weight module.decoder.layers.16.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.16.self_attention.linear_kv_down_proj.weight module.decoder.layers.16.self_attention.linear_q_proj.weight + module.decoder.layers.16.input_layernorm.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight + module.decoder.layers.16.pre_mlp_layernorm.weight module.decoder.layers.16.self_attention.linear_proj.weight Params for bucket 8 (42734080 elements, 42734080 padded size): - module.decoder.layers.15.mlp.router.weight + module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight module.decoder.layers.15.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.15.self_attention.linear_kv_up_proj.weight module.decoder.layers.15.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.15.mlp.router.weight module.decoder.layers.15.self_attention.linear_q_proj.weight module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.15.input_layernorm.weight module.decoder.layers.14.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.14.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.15.pre_mlp_layernorm.weight Params for bucket 9 (40901632 elements, 40901632 padded size): - module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_kv_up_proj.weight module.decoder.layers.14.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.14.self_attention.linear_q_proj.weight module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.14.input_layernorm.weight module.decoder.layers.13.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.13.mlp.router.weight - module.decoder.layers.13.pre_mlp_layernorm.weight - module.decoder.layers.13.self_attention.linear_q_proj.weight - module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.13.mlp.shared_experts.linear_fc1.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.weight module.decoder.layers.13.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.13.self_attention.linear_kv_down_proj.weight + module.decoder.layers.14.mlp.router.weight + module.decoder.layers.14.pre_mlp_layernorm.weight + module.decoder.layers.13.mlp.router.weight + module.decoder.layers.13.pre_mlp_layernorm.weight + module.decoder.layers.13.self_attention.linear_q_proj.weight Params for bucket 10 (41163264 elements, 41163264 padded size): - module.decoder.layers.12.mlp.router.weight + module.decoder.layers.13.self_attention.linear_proj.weight module.decoder.layers.12.mlp.shared_experts.linear_fc2.weight module.decoder.layers.13.input_layernorm.weight - module.decoder.layers.12.self_attention.linear_kv_down_proj.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.13.self_attention.linear_proj.weight module.decoder.layers.12.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.weight module.decoder.layers.12.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.12.self_attention.linear_kv_down_proj.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.12.self_attention.linear_q_proj.weight module.decoder.layers.12.input_layernorm.weight module.decoder.layers.11.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (42734080 elements, 42734080 padded size): - module.decoder.layers.11.mlp.router.weight - module.decoder.layers.11.self_attention.linear_kv_down_proj.weight - module.decoder.layers.11.self_attention.linear_proj.weight - module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.weight - module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.linear_q_proj.weight module.decoder.layers.11.input_layernorm.weight module.decoder.layers.10.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.11.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.weight + module.decoder.layers.11.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.11.self_attention.linear_kv_down_proj.weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.linear_kv_down_proj.weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.10.self_attention.linear_kv_up_proj.weight - module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_q_proj.weight - module.decoder.layers.10.self_attention.linear_proj.weight module.decoder.layers.10.input_layernorm.weight module.decoder.layers.9.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight @@ -823,7 +817,19 @@ Params for bucket 12 (40901632 elements, 40901632 padded size): module.decoder.layers.9.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.9.self_attention.linear_kv_down_proj.weight module.decoder.layers.9.self_attention.linear_q_proj.weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.10.self_attention.linear_kv_up_proj.weight + module.decoder.layers.10.self_attention.linear_kv_down_proj.weight + module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight Params for bucket 13 (41163264 elements, 41163264 padded size): + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.9.input_layernorm.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.weight + module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.8.self_attention.linear_kv_down_proj.weight module.decoder.layers.9.self_attention.linear_proj.weight module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight @@ -831,139 +837,133 @@ Params for bucket 13 (41163264 elements, 41163264 padded size): module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.8.input_layernorm.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.9.input_layernorm.weight - module.decoder.layers.8.self_attention.linear_kv_up_proj.weight - module.decoder.layers.8.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.8.self_attention.linear_kv_down_proj.weight Params for bucket 14 (42734080 elements, 42734080 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.7.self_attention.linear_kv_up_proj.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.7.self_attention.linear_kv_down_proj.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.linear_q_proj.weight module.decoder.layers.7.input_layernorm.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 15 (40901632 elements, 40901632 padded size): - module.decoder.layers.6.self_attention.linear_kv_up_proj.weight - module.decoder.layers.6.self_attention.linear_kv_down_proj.weight - module.decoder.layers.6.pre_mlp_layernorm.weight - module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.5.self_attention.linear_kv_up_proj.weight - module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.5.self_attention.linear_kv_down_proj.weight module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.linear_q_proj.weight module.decoder.layers.6.self_attention.linear_proj.weight module.decoder.layers.6.input_layernorm.weight module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.weight module.decoder.layers.5.self_attention.linear_q_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.weight + module.decoder.layers.6.self_attention.linear_kv_down_proj.weight + module.decoder.layers.6.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight + module.decoder.layers.5.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.5.self_attention.linear_kv_down_proj.weight Params for bucket 16 (41163264 elements, 41163264 padded size): + module.decoder.layers.5.self_attention.linear_proj.weight + module.decoder.layers.5.input_layernorm.weight module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.pre_mlp_layernorm.weight - module.decoder.layers.4.self_attention.linear_kv_up_proj.weight + module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight module.decoder.layers.4.self_attention.linear_q_proj.weight module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.4.input_layernorm.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.5.input_layernorm.weight - module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_kv_up_proj.weight module.decoder.layers.4.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.4.self_attention.linear_kv_down_proj.weight Params for bucket 17 (42734080 elements, 42734080 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight module.decoder.layers.3.self_attention.linear_kv_up_proj.weight - module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_kv_up_proj.layer_norm_weight module.decoder.layers.3.self_attention.linear_kv_down_proj.weight - module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_q_proj.weight module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.input_layernorm.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight module.decoder.layers.2.mlp.shared_experts.linear_fc1.weight Params for bucket 18 (40901632 elements, 40901632 padded size): - module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.self_attention.linear_q_proj.weight module.decoder.layers.2.self_attention.linear_proj.weight module.decoder.layers.2.input_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_kv_down_proj.weight + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_kv_up_proj.weight module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight - module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.weight module.decoder.layers.1.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.layer_norm_weight - module.decoder.layers.2.self_attention.linear_kv_down_proj.weight - module.decoder.layers.2.self_attention.linear_kv_up_proj.weight - module.decoder.layers.2.self_attention.linear_q_proj.weight - module.decoder.layers.1.self_attention.linear_kv_down_proj.weight module.decoder.layers.1.self_attention.linear_q_proj.weight Params for bucket 19 (71436288 elements, 71436288 padded size): module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.0.mlp.linear_fc2.weight - module.decoder.layers.1.input_layernorm.weight module.decoder.layers.0.mlp.linear_fc1.weight + module.decoder.layers.1.input_layernorm.weight + module.decoder.layers.0.mlp.linear_fc2.weight Params for bucket 20 (116789760 elements, 116789760 padded size): - module.decoder.layers.0.self_attention.linear_kv_up_proj.weight - module.decoder.layers.0.self_attention.linear_q_proj.weight - module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight - module.embedding.word_embeddings.weight module.decoder.layers.0.mlp.linear_fc1.layer_norm_weight - module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.layer_norm_weight + module.decoder.layers.0.self_attention.linear_q_proj.weight module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.input_layernorm.weight + module.decoder.layers.0.self_attention.linear_kv_up_proj.weight + module.decoder.layers.0.self_attention.linear_kv_down_proj.weight + module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 40 Params for bucket 1 (40370176 elements, 40370176 padded size): + module.decoder.layers.23.mlp.experts.linear_fc2.weight6 + module.decoder.layers.23.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.23.mlp.experts.linear_fc2.weight1 + module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc2.weight7 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 module.decoder.layers.23.mlp.experts.linear_fc1.weight7 module.decoder.layers.23.mlp.experts.linear_fc1.weight5 - module.decoder.layers.23.mlp.experts.linear_fc2.weight6 - module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc2.weight2 - module.decoder.layers.23.mlp.experts.linear_fc2.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight6 Params for bucket 2 (40370176 elements, 40370176 padded size): - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc2.weight6 - module.decoder.layers.22.mlp.experts.linear_fc2.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc1.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc2.weight4 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight5 Params for bucket 3 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight3 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 - module.decoder.layers.22.mlp.experts.linear_fc2.weight0 module.decoder.layers.22.mlp.experts.linear_fc1.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 + module.decoder.layers.22.mlp.experts.linear_fc2.weight3 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 Params for bucket 4 (40370176 elements, 40370176 padded size): - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.22.mlp.experts.linear_fc1.weight0 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.22.mlp.experts.linear_fc1.weight0 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 module.decoder.layers.21.mlp.experts.linear_fc2.weight2 @@ -974,157 +974,157 @@ Params for bucket 5 (40370176 elements, 40370176 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc1.weight1 module.decoder.layers.21.mlp.experts.linear_fc1.weight4 - module.decoder.layers.21.mlp.experts.linear_fc1.weight6 module.decoder.layers.21.mlp.experts.linear_fc1.weight3 + module.decoder.layers.21.mlp.experts.linear_fc1.weight6 Params for bucket 6 (40370176 elements, 40370176 padded size): module.decoder.layers.20.mlp.experts.linear_fc2.weight7 module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 + module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight6 module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 - module.decoder.layers.20.mlp.experts.linear_fc2.weight2 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 - module.decoder.layers.20.mlp.experts.linear_fc1.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 Params for bucket 7 (40370176 elements, 40370176 padded size): - module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.20.mlp.experts.linear_fc1.weight1 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 - module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 module.decoder.layers.19.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 Params for bucket 8 (40370176 elements, 40370176 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight2 - module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight5 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 Params for bucket 9 (40370176 elements, 40370176 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight3 - module.decoder.layers.18.mlp.experts.linear_fc2.weight2 Params for bucket 10 (40370176 elements, 40370176 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight1 - module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 - module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.18.mlp.experts.linear_fc1.weight3 Params for bucket 11 (40370176 elements, 40370176 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.17.mlp.experts.linear_fc2.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc2.weight4 module.decoder.layers.17.mlp.experts.linear_fc2.weight2 module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 - module.decoder.layers.17.mlp.experts.linear_fc2.weight6 module.decoder.layers.18.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc2.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 Params for bucket 12 (40370176 elements, 40370176 padded size): + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc1.weight6 - module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight0 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 Params for bucket 13 (40370176 elements, 40370176 padded size): - module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.16.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight1 - module.decoder.layers.16.mlp.experts.linear_fc2.weight0 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc2.weight7 - module.decoder.layers.16.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight7 module.decoder.layers.16.mlp.experts.linear_fc1.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight7 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight0 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 Params for bucket 14 (40370176 elements, 40370176 padded size): + module.decoder.layers.16.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc2.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight4 module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.16.mlp.experts.linear_fc1.weight2 - module.decoder.layers.16.mlp.experts.linear_fc1.weight1 - module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc2.weight4 Params for bucket 15 (40370176 elements, 40370176 padded size): + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 module.decoder.layers.15.mlp.experts.linear_fc2.weight3 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 module.decoder.layers.15.mlp.experts.linear_fc1.weight4 module.decoder.layers.15.mlp.experts.linear_fc1.weight3 - module.decoder.layers.15.mlp.experts.linear_fc2.weight1 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight5 Params for bucket 16 (40370176 elements, 40370176 padded size): - module.decoder.layers.14.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight0 - module.decoder.layers.14.mlp.experts.linear_fc2.weight4 - module.decoder.layers.14.mlp.experts.linear_fc2.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.14.mlp.experts.linear_fc2.weight7 module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight7 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc2.weight3 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight0 + module.decoder.layers.14.mlp.experts.linear_fc2.weight4 + module.decoder.layers.14.mlp.experts.linear_fc2.weight2 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 Params for bucket 17 (40370176 elements, 40370176 padded size): + module.decoder.layers.14.mlp.experts.linear_fc1.weight3 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 module.decoder.layers.14.mlp.experts.linear_fc1.weight7 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight2 module.decoder.layers.14.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 - module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 Params for bucket 18 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight0 module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 - module.decoder.layers.13.mlp.experts.linear_fc2.weight0 - module.decoder.layers.13.mlp.experts.linear_fc1.weight6 - module.decoder.layers.14.mlp.experts.linear_fc1.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight7 module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc2.weight0 + module.decoder.layers.13.mlp.experts.linear_fc1.weight6 Params for bucket 19 (40370176 elements, 40370176 padded size): - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 - module.decoder.layers.13.mlp.experts.linear_fc1.weight3 - module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 + module.decoder.layers.13.mlp.experts.linear_fc1.weight3 + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 + module.decoder.layers.13.mlp.experts.linear_fc1.weight0 module.decoder.layers.12.mlp.experts.linear_fc2.weight7 Params for bucket 20 (40370176 elements, 40370176 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 module.decoder.layers.12.mlp.experts.linear_fc2.weight4 module.decoder.layers.12.mlp.experts.linear_fc2.weight2 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 module.decoder.layers.12.mlp.experts.linear_fc1.weight5 module.decoder.layers.12.mlp.experts.linear_fc2.weight5 @@ -1134,8 +1134,8 @@ Params for bucket 20 (40370176 elements, 40370176 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight4 Params for bucket 21 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight7 - module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.12.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc1.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight3 @@ -1149,86 +1149,82 @@ Params for bucket 22 (40370176 elements, 40370176 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 Params for bucket 23 (40370176 elements, 40370176 padded size): - module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.10.mlp.experts.linear_fc2.weight7 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 Params for bucket 24 (40370176 elements, 40370176 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight5 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40370176 elements, 40370176 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 - module.decoder.layers.9.mlp.experts.linear_fc1.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 Params for bucket 26 (40370176 elements, 40370176 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 Params for bucket 27 (40370176 elements, 40370176 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc1.weight5 Params for bucket 28 (40370176 elements, 40370176 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (40370176 elements, 40370176 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 30 (40370176 elements, 40370176 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 @@ -1236,24 +1232,28 @@ Params for bucket 30 (40370176 elements, 40370176 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 Params for bucket 31 (40370176 elements, 40370176 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight6 Params for bucket 32 (40370176 elements, 40370176 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight3 - module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 33 (40370176 elements, 40370176 padded size): @@ -1262,9 +1262,9 @@ Params for bucket 33 (40370176 elements, 40370176 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 Params for bucket 34 (40370176 elements, 40370176 padded size): @@ -1272,69 +1272,69 @@ Params for bucket 34 (40370176 elements, 40370176 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 Params for bucket 35 (40370176 elements, 40370176 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight7 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 - module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 36 (40370176 elements, 40370176 padded size): - module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight5 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 37 (40370176 elements, 40370176 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight5 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 Params for bucket 38 (40370176 elements, 40370176 padded size): - module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 39 (40370176 elements, 40370176 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight4 module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 40 (17301504 elements, 17301504 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:56:11 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:08:17 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1342,7 +1342,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1355,14 +1355,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:56:11 +[after dataloaders are built] datetime: 2026-08-26 18:08:17 done with setup ... training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (3597.21, 3612.97) - train/valid/test-data-iterators-setup ..........: (61.09, 69.51) -[before the start of training step] datetime: 2026-08-26 16:56:11 + model-and-optimizer-setup ......................: (3633.32, 3641.59) + train/valid/test-data-iterators-setup ..........: (61.75, 67.58) +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 18:08:17 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=2048, E=64, K=6, NvS=51184, NvS_padded=51200 sm_count : num_sms=32, num_sms_dedup=148 @@ -1371,5635 +1371,5636 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 2.01 MiB total_size : 204.01 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 16:56:33] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22414.4 | throughput per GPU (TFLOP/s/GPU): 5.7 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092128E+01 | loss scale: 1.0 | grad norm: 30.251 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 13.53 - + [2026-08-26 18:08:39] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21929.8 | throughput per GPU (TFLOP/s/GPU): 5.8 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.092128E+01 | loss scale: 1.0 | grad norm: 30.251 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 13.53 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 13.74 Number of parameters in most loaded shard in billions: 13.7356 Theoretical memory footprints: weight and optimizer=98244.96 MB [Rank 0] (after 1 iterations) memory (MB) | allocated: 66075.978515625 | max allocated: 124945.7734375 | reserved: 127502.0 | max reserved: 127502.0 - [2026-08-26 16:56:34] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 797.9 | throughput per GPU (TFLOP/s/GPU): 159.9 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091997E+01 | loss scale: 1.0 | grad norm: 30.182 | number of skipped iterations: 0 | number of nan iterations: 0 | + [2026-08-26 18:08:40] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 800.3 | throughput per GPU (TFLOP/s/GPU): 159.4 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.091997E+01 | loss scale: 1.0 | grad norm: 30.182 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65098 dup_rows=23836 dup_rows_remote=20824 dedup_ratio=0.242472 router_tokens_per_expert=[751, 759, 810, 729, 763, 836, 747, 766, 779, 758, 819, 816, 754, 754, 771, 790, 793, 714, 765, 757, 764, 781, 799, 763, 786, 723, 752, 747, 757, 762, 744, 781, 752, 757, 764, 757, 776, 798, 751, 770, 807, 744, 794, 792, 745, 774, 758, 777, 795, 772, 703, 788, 775, 772, 739, 751, 812, 790, 764, 782, 762, 748, 741, 752] payload_rows_per_dst=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] dup_rows_per_dst=[3014, 3028, 2986, 2892, 3004, 3012, 2920, 2980] combine_rows_per_src=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64900 dup_rows=24108 dup_rows_remote=21060 dedup_ratio=0.245239 router_tokens_per_expert=[761, 800, 809, 777, 773, 790, 787, 758, 753, 737, 800, 764, 746, 756, 838, 765, 779, 721, 725, 775, 795, 749, 776, 792, 786, 729, 738, 762, 801, 801, 777, 778, 832, 744, 777, 820, 752, 750, 780, 721, 763, 745, 746, 783, 819, 750, 745, 763, 730, 731, 795, 780, 763, 782, 776, 746, 771, 753, 723, 743, 758, 778, 789, 746] payload_rows_per_dst=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] dup_rows_per_dst=[3110, 3058, 2974, 3048, 3048, 2998, 2930, 2942] combine_rows_per_src=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64792 dup_rows=24064 dup_rows_remote=20924 dedup_ratio=0.244792 router_tokens_per_expert=[722, 789, 748, 730, 800, 759, 765, 760, 791, 795, 762, 768, 798, 794, 796, 740, 766, 815, 801, 764, 785, 778, 756, 807, 782, 766, 760, 741, 791, 775, 765, 765, 767, 768, 682, 748, 714, 764, 721, 735, 745, 851, 779, 738, 759, 767, 737, 742, 733, 808, 753, 811, 756, 758, 737, 825, 732, 782, 806, 842, 767, 770, 771, 750] payload_rows_per_dst=[8862, 8974, 9122, 9286, 9486, 9324, 9738, 9448] dup_rows_per_dst=[2642, 2754, 2798, 2944, 3296, 2972, 3518, 3140] combine_rows_per_src=[8862, 8974, 9122, 9286, 9486, 9324, 9738, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64922 dup_rows=24166 dup_rows_remote=21086 dedup_ratio=0.245829 router_tokens_per_expert=[766, 792, 779, 806, 822, 754, 776, 737, 781, 750, 827, 778, 783, 749, 714, 766, 775, 789, 756, 768, 773, 743, 805, 802, 753, 782, 790, 778, 767, 773, 821, 750, 783, 748, 682, 764, 709, 821, 770, 772, 795, 709, 729, 780, 767, 766, 773, 744, 731, 793, 770, 734, 784, 744, 814, 780, 801, 731, 776, 743, 790, 761, 744, 739] payload_rows_per_dst=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] dup_rows_per_dst=[3136, 3080, 3046, 3070, 2880, 2930, 3034, 2990] combine_rows_per_src=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21016 dedup_ratio=0.244629 router_tokens_per_expert=[745, 717, 776, 802, 751, 792, 740, 758, 800, 804, 754, 791, 764, 792, 800, 766, 751, 746, 807, 712, 804, 738, 780, 781, 752, 778, 806, 740, 760, 782, 730, 736, 740, 741, 757, 808, 798, 780, 786, 780, 741, 819, 782, 725, 767, 751, 779, 776, 743, 766, 740, 724, 787, 790, 779, 740, 737, 815, 775, 784, 783, 751, 781, 772] payload_rows_per_dst=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] dup_rows_per_dst=[3020, 3086, 2954, 2972, 3032, 2978, 2940, 3066] combine_rows_per_src=[9142, 9456, 9284, 9196, 9348, 9302, 9198, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64900 dup_rows=24108 dup_rows_remote=21060 dedup_ratio=0.245239 router_tokens_per_expert=[761, 800, 809, 777, 773, 790, 787, 758, 753, 737, 800, 764, 746, 756, 838, 765, 779, 721, 725, 775, 795, 749, 776, 792, 786, 729, 738, 762, 801, 801, 777, 778, 832, 744, 777, 820, 752, 750, 780, 721, 763, 745, 746, 783, 819, 750, 745, 763, 730, 731, 795, 780, 763, 782, 776, 746, 771, 753, 723, 743, 758, 778, 789, 746] payload_rows_per_dst=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] dup_rows_per_dst=[3110, 3058, 2974, 3048, 3048, 2998, 2930, 2942] combine_rows_per_src=[9400, 9260, 9250, 9296, 9304, 9230, 9276, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65098 dup_rows=23836 dup_rows_remote=20824 dedup_ratio=0.242472 router_tokens_per_expert=[751, 759, 810, 729, 763, 836, 747, 766, 779, 758, 819, 816, 754, 754, 771, 790, 793, 714, 765, 757, 764, 781, 799, 763, 786, 723, 752, 747, 757, 762, 744, 781, 752, 757, 764, 757, 776, 798, 751, 770, 807, 744, 794, 792, 745, 774, 758, 777, 795, 772, 703, 788, 775, 772, 739, 751, 812, 790, 764, 782, 762, 748, 741, 752] payload_rows_per_dst=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] dup_rows_per_dst=[3014, 3028, 2986, 2892, 3004, 3012, 2920, 2980] combine_rows_per_src=[9308, 9454, 9286, 9212, 9246, 9370, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64892 dup_rows=24128 dup_rows_remote=21062 dedup_ratio=0.245443 router_tokens_per_expert=[782, 754, 773, 735, 755, 868, 802, 770, 801, 774, 745, 748, 721, 751, 730, 809, 776, 726, 786, 784, 812, 744, 798, 749, 779, 751, 787, 793, 787, 760, 770, 786, 748, 800, 812, 713, 742, 732, 777, 731, 805, 790, 774, 797, 776, 755, 761, 731, 748, 778, 758, 758, 778, 758, 794, 706, 725, 786, 753, 801, 767, 764, 755, 773] payload_rows_per_dst=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] dup_rows_per_dst=[3086, 3026, 3066, 3072, 2958, 3112, 2902, 2906] combine_rows_per_src=[9392, 9132, 9284, 9354, 9152, 9266, 9254, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64922 dup_rows=24166 dup_rows_remote=21086 dedup_ratio=0.245829 router_tokens_per_expert=[766, 792, 779, 806, 822, 754, 776, 737, 781, 750, 827, 778, 783, 749, 714, 766, 775, 789, 756, 768, 773, 743, 805, 802, 753, 782, 790, 778, 767, 773, 821, 750, 783, 748, 682, 764, 709, 821, 770, 772, 795, 709, 729, 780, 767, 766, 773, 744, 731, 793, 770, 734, 784, 744, 814, 780, 801, 731, 776, 743, 790, 761, 744, 739] payload_rows_per_dst=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] dup_rows_per_dst=[3136, 3080, 3046, 3070, 2880, 2930, 3034, 2990] combine_rows_per_src=[9328, 9216, 9376, 9358, 9218, 9196, 9266, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64974 dup_rows=24242 dup_rows_remote=21426 dedup_ratio=0.246602 router_tokens_per_expert=[769, 777, 782, 788, 743, 768, 751, 761, 715, 764, 817, 793, 769, 756, 784, 808, 811, 768, 774, 820, 762, 756, 783, 753, 749, 801, 806, 779, 733, 748, 778, 743, 750, 760, 749, 724, 750, 827, 804, 782, 788, 782, 776, 774, 789, 768, 760, 754, 710, 774, 771, 725, 728, 762, 768, 714, 729, 762, 776, 793, 746, 750, 781, 817] payload_rows_per_dst=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] dup_rows_per_dst=[3002, 3030, 3048, 3068, 3040, 3166, 2816, 3072] combine_rows_per_src=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343688 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985096 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626504 weight_movement_bytes=34603008 total_nvlink_bytes=568229512 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342864 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730896 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118928 weight_movement_bytes=34603008 total_nvlink_bytes=565721936 weight_to_token_ratio=0.065151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343840 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266174240 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004640 weight_movement_bytes=69206016 total_nvlink_bytes=601210656 weight_to_token_ratio=0.130085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342864 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730896 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118928 weight_movement_bytes=34603008 total_nvlink_bytes=565721936 weight_to_token_ratio=0.065151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343696 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206864 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070032 weight_movement_bytes=34603008 total_nvlink_bytes=566673040 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343840 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266174240 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004640 weight_movement_bytes=69206016 total_nvlink_bytes=601210656 weight_to_token_ratio=0.130085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343816 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266141448 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531939080 weight_movement_bytes=69206016 total_nvlink_bytes=601145096 weight_to_token_ratio=0.130101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343688 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985096 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626504 weight_movement_bytes=34603008 total_nvlink_bytes=568229512 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344032 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266264544 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532185056 weight_movement_bytes=69206016 total_nvlink_bytes=601391072 weight_to_token_ratio=0.130041 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=345600 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266479104 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532612608 weight_movement_bytes=34603008 total_nvlink_bytes=567215616 weight_to_token_ratio=0.064968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343816 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266141448 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531939080 weight_movement_bytes=69206016 total_nvlink_bytes=601145096 weight_to_token_ratio=0.130101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64978 dup_rows=23860 dup_rows_remote=20740 dedup_ratio=0.242716 router_tokens_per_expert=[763, 802, 813, 769, 785, 774, 791, 783, 798, 769, 769, 785, 726, 766, 783, 788, 799, 774, 744, 785, 792, 785, 770, 763, 758, 721, 835, 750, 783, 810, 742, 779, 803, 725, 774, 723, 788, 729, 758, 720, 735, 815, 751, 781, 755, 719, 789, 771, 804, 744, 723, 757, 761, 809, 716, 763, 757, 770, 722, 727, 769, 772, 817, 751] payload_rows_per_dst=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] dup_rows_per_dst=[3120, 3074, 3048, 3114, 2786, 2892, 2866, 2960] combine_rows_per_src=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64974 dup_rows=24242 dup_rows_remote=21426 dedup_ratio=0.246602 router_tokens_per_expert=[769, 777, 782, 788, 743, 768, 751, 761, 715, 764, 817, 793, 769, 756, 784, 808, 811, 768, 774, 820, 762, 756, 783, 753, 749, 801, 806, 779, 733, 748, 778, 743, 750, 760, 749, 724, 750, 827, 804, 782, 788, 782, 776, 774, 789, 768, 760, 754, 710, 774, 771, 725, 728, 762, 768, 714, 729, 762, 776, 793, 746, 750, 781, 817] payload_rows_per_dst=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] dup_rows_per_dst=[3002, 3030, 3048, 3068, 3040, 3166, 2816, 3072] combine_rows_per_src=[9276, 9382, 9406, 9206, 9252, 9216, 9088, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65270 dup_rows=23858 dup_rows_remote=20816 dedup_ratio=0.242696 router_tokens_per_expert=[768, 801, 754, 873, 739, 743, 788, 768, 760, 727, 832, 733, 700, 798, 743, 772, 822, 711, 821, 764, 773, 759, 751, 754, 780, 769, 748, 794, 758, 760, 765, 734, 806, 775, 793, 740, 797, 765, 802, 805, 775, 718, 759, 794, 746, 784, 755, 778, 733, 770, 730, 774, 754, 760, 769, 755, 767, 766, 782, 798, 744, 756, 787, 753] payload_rows_per_dst=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] dup_rows_per_dst=[3032, 2956, 2998, 2864, 3064, 3042, 2904, 2998] combine_rows_per_src=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64952 dup_rows=24198 dup_rows_remote=21242 dedup_ratio=0.246155 router_tokens_per_expert=[786, 754, 783, 814, 779, 805, 763, 821, 781, 765, 781, 788, 767, 760, 779, 764, 794, 741, 735, 733, 786, 755, 771, 786, 756, 757, 740, 767, 774, 743, 753, 765, 775, 734, 818, 721, 764, 805, 759, 766, 749, 754, 788, 759, 740, 767, 747, 844, 750, 756, 771, 783, 729, 724, 787, 774, 789, 776, 723, 727, 810, 761, 800, 756] payload_rows_per_dst=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] dup_rows_per_dst=[3118, 3120, 2988, 2956, 2990, 3026, 2962, 3038] combine_rows_per_src=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65034 dup_rows=23906 dup_rows_remote=20782 dedup_ratio=0.243184 router_tokens_per_expert=[749, 829, 788, 739, 749, 734, 775, 806, 784, 766, 757, 750, 697, 777, 750, 756, 794, 746, 744, 815, 827, 742, 726, 775, 789, 715, 750, 749, 773, 729, 845, 732, 800, 770, 793, 776, 772, 780, 760, 778, 712, 754, 798, 796, 780, 779, 778, 720, 788, 714, 776, 795, 772, 752, 852, 796, 775, 783, 733, 712, 809, 776, 765, 751] payload_rows_per_dst=[9114, 9228, 9354, 9184, 9396, 9246, 9512, 9364] dup_rows_per_dst=[2808, 2846, 2984, 2932, 3110, 2864, 3238, 3124] combine_rows_per_src=[9114, 9228, 9354, 9184, 9396, 9246, 9512, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=342872 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266492760 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532642648 weight_movement_bytes=138412032 total_nvlink_bytes=671054680 weight_to_token_ratio=0.259859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65084 dup_rows=23878 dup_rows_remote=20968 dedup_ratio=0.242900 router_tokens_per_expert=[759, 767, 796, 757, 792, 774, 736, 784, 793, 731, 733, 753, 769, 787, 770, 715, 759, 800, 791, 766, 761, 738, 761, 775, 753, 813, 796, 759, 753, 765, 716, 815, 749, 765, 748, 801, 797, 757, 752, 757, 773, 801, 748, 805, 802, 771, 792, 729, 738, 748, 769, 749, 754, 781, 793, 779, 786, 762, 731, 809, 761, 804, 738, 766] payload_rows_per_dst=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] dup_rows_per_dst=[3046, 2926, 3026, 3008, 2910, 3022, 2978, 2962] combine_rows_per_src=[9284, 9176, 9276, 9332, 9342, 9420, 9244, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344344 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267690264 ingress_bytes=371154944 link_bytes=371154944 token_bytes=535036184 weight_movement_bytes=103809024 total_nvlink_bytes=638845208 weight_to_token_ratio=0.194022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396626174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=345600 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266479104 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532612608 weight_movement_bytes=34603008 total_nvlink_bytes=567215616 weight_to_token_ratio=0.064968 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65036 dup_rows=24140 dup_rows_remote=21236 dedup_ratio=0.245565 router_tokens_per_expert=[726, 756, 715, 751, 777, 748, 836, 775, 772, 763, 737, 761, 751, 796, 746, 776, 769, 777, 733, 785, 749, 730, 722, 751, 766, 783, 726, 769, 773, 759, 770, 768, 788, 745, 759, 755, 739, 774, 799, 805, 806, 801, 793, 760, 740, 744, 807, 759, 806, 799, 784, 764, 797, 777, 810, 753, 789, 745, 763, 766, 813, 791, 761, 744] payload_rows_per_dst=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] dup_rows_per_dst=[2940, 2934, 2904, 3018, 3026, 3062, 3162, 3094] combine_rows_per_src=[9228, 9270, 9128, 9210, 9302, 9358, 9418, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65270 dup_rows=23858 dup_rows_remote=20816 dedup_ratio=0.242696 router_tokens_per_expert=[768, 801, 754, 873, 739, 743, 788, 768, 760, 727, 832, 733, 700, 798, 743, 772, 822, 711, 821, 764, 773, 759, 751, 754, 780, 769, 748, 794, 758, 760, 765, 734, 806, 775, 793, 740, 797, 765, 802, 805, 775, 718, 759, 794, 746, 784, 755, 778, 733, 770, 730, 774, 754, 760, 769, 755, 767, 766, 782, 798, 744, 756, 787, 753] payload_rows_per_dst=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] dup_rows_per_dst=[3032, 2956, 2998, 2864, 3064, 3042, 2904, 2998] combine_rows_per_src=[9436, 9174, 9312, 9352, 9502, 9176, 9186, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64952 dup_rows=24198 dup_rows_remote=21242 dedup_ratio=0.246155 router_tokens_per_expert=[786, 754, 783, 814, 779, 805, 763, 821, 781, 765, 781, 788, 767, 760, 779, 764, 794, 741, 735, 733, 786, 755, 771, 786, 756, 757, 740, 767, 774, 743, 753, 765, 775, 734, 818, 721, 764, 805, 759, 766, 749, 754, 788, 759, 740, 767, 747, 844, 750, 756, 771, 783, 729, 724, 787, 774, 789, 776, 723, 727, 810, 761, 800, 756] payload_rows_per_dst=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] dup_rows_per_dst=[3118, 3120, 2988, 2956, 2990, 3026, 2962, 3038] combine_rows_per_src=[9492, 9250, 9214, 9154, 9294, 9270, 9186, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65124 dup_rows=24024 dup_rows_remote=20920 dedup_ratio=0.244385 router_tokens_per_expert=[714, 774, 762, 750, 810, 818, 770, 774, 773, 777, 744, 751, 755, 752, 784, 794, 824, 790, 718, 739, 776, 784, 779, 732, 743, 753, 774, 763, 823, 788, 783, 816, 759, 775, 791, 708, 751, 785, 770, 735, 802, 766, 751, 765, 780, 814, 770, 800, 775, 793, 749, 742, 729, 749, 817, 767, 758, 732, 755, 710, 743, 734, 808, 782] payload_rows_per_dst=[9316, 9156, 9310, 9436, 9230, 9402, 9200, 9230] dup_rows_per_dst=[3028, 3104, 2974, 3050, 2918, 3094, 3042, 2814] combine_rows_per_src=[9316, 9156, 9310, 9436, 9230, 9402, 9200, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344776 combine_payload_bytes=266043392 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266388168 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532431560 weight_movement_bytes=69206016 total_nvlink_bytes=601637576 weight_to_token_ratio=0.129981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343264 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266722528 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533101792 weight_movement_bytes=34603008 total_nvlink_bytes=567704800 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65238 dup_rows=23924 dup_rows_remote=20994 dedup_ratio=0.243368 router_tokens_per_expert=[759, 790, 739, 777, 787, 727, 789, 777, 754, 746, 741, 779, 759, 783, 812, 719, 787, 770, 758, 736, 784, 767, 764, 812, 788, 767, 758, 747, 779, 793, 778, 786, 741, 747, 761, 810, 817, 805, 719, 812, 796, 737, 775, 788, 755, 801, 791, 744, 764, 731, 764, 774, 739, 766, 727, 771, 783, 732, 769, 765, 779, 740, 774, 763] payload_rows_per_dst=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] dup_rows_per_dst=[3016, 2978, 3010, 2956, 2996, 3082, 2930, 2956] combine_rows_per_src=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344208 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928272 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512336 weight_movement_bytes=69206016 total_nvlink_bytes=602718352 weight_to_token_ratio=0.129718 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344208 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928272 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512336 weight_movement_bytes=69206016 total_nvlink_bytes=602718352 weight_to_token_ratio=0.129718 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=345088 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732544 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533120000 weight_movement_bytes=34603008 total_nvlink_bytes=567723008 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344344 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267690264 ingress_bytes=371154944 link_bytes=371154944 token_bytes=535036184 weight_movement_bytes=103809024 total_nvlink_bytes=638845208 weight_to_token_ratio=0.194022 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396626174e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344176 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092080 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839984 weight_movement_bytes=34603008 total_nvlink_bytes=568442992 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344776 combine_payload_bytes=266043392 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266388168 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532431560 weight_movement_bytes=69206016 total_nvlink_bytes=601637576 weight_to_token_ratio=0.129981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344928 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559776 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774624 weight_movement_bytes=34603008 total_nvlink_bytes=569377632 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64888 dup_rows=24146 dup_rows_remote=21136 dedup_ratio=0.245626 router_tokens_per_expert=[734, 750, 769, 775, 724, 809, 748, 747, 736, 803, 773, 754, 780, 743, 746, 752, 806, 805, 792, 750, 767, 780, 784, 758, 776, 763, 733, 781, 787, 766, 730, 754, 787, 780, 759, 786, 785, 756, 744, 778, 756, 733, 791, 808, 764, 758, 744, 786, 749, 769, 807, 772, 779, 737, 789, 806, 791, 765, 739, 797, 780, 794, 719, 769] payload_rows_per_dst=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] dup_rows_per_dst=[2900, 3026, 3192, 2968, 2990, 3010, 3056, 3004] combine_rows_per_src=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65200 dup_rows=23856 dup_rows_remote=20932 dedup_ratio=0.242676 router_tokens_per_expert=[790, 795, 750, 738, 750, 748, 758, 744, 750, 767, 795, 791, 757, 761, 779, 800, 773, 761, 785, 744, 763, 728, 777, 779, 759, 762, 769, 805, 797, 781, 770, 778, 800, 795, 786, 784, 742, 821, 737, 722, 733, 800, 817, 787, 773, 721, 695, 778, 766, 710, 775, 758, 749, 787, 775, 780, 769, 731, 770, 772, 786, 770, 775, 784] payload_rows_per_dst=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] dup_rows_per_dst=[2924, 3296, 3008, 3032, 2828, 2812, 2926, 3030] combine_rows_per_src=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64812 dup_rows=24104 dup_rows_remote=21102 dedup_ratio=0.245199 router_tokens_per_expert=[736, 756, 776, 783, 774, 793, 794, 774, 781, 699, 783, 749, 776, 743, 745, 788, 733, 799, 750, 707, 792, 699, 755, 777, 796, 774, 793, 778, 750, 779, 765, 760, 782, 733, 805, 755, 807, 778, 770, 771, 749, 791, 862, 786, 769, 745, 775, 741, 760, 756, 742, 738, 745, 748, 758, 781, 819, 768, 766, 740, 800, 790, 790, 775] payload_rows_per_dst=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] dup_rows_per_dst=[3098, 2974, 2928, 3002, 3120, 3018, 2788, 3176] combine_rows_per_src=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65238 dup_rows=23924 dup_rows_remote=20994 dedup_ratio=0.243368 router_tokens_per_expert=[759, 790, 739, 777, 787, 727, 789, 777, 754, 746, 741, 779, 759, 783, 812, 719, 787, 770, 758, 736, 784, 767, 764, 812, 788, 767, 758, 747, 779, 793, 778, 786, 741, 747, 761, 810, 817, 805, 719, 812, 796, 737, 775, 788, 755, 801, 791, 744, 764, 731, 764, 774, 739, 766, 727, 771, 783, 732, 769, 765, 779, 740, 774, 763] payload_rows_per_dst=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] dup_rows_per_dst=[3016, 2978, 3010, 2956, 2996, 3082, 2930, 2956] combine_rows_per_src=[9274, 9208, 9346, 9436, 9428, 9292, 9142, 9254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64790 dup_rows=24250 dup_rows_remote=21198 dedup_ratio=0.246684 router_tokens_per_expert=[729, 794, 808, 771, 738, 809, 784, 784, 765, 730, 743, 722, 817, 750, 724, 776, 777, 745, 766, 806, 780, 735, 756, 760, 795, 764, 749, 767, 725, 762, 808, 769, 791, 761, 732, 744, 775, 779, 778, 736, 785, 733, 811, 800, 775, 780, 780, 728, 755, 763, 790, 753, 775, 768, 809, 789, 773, 783, 763, 734, 785, 779, 766, 771] payload_rows_per_dst=[9036, 9190, 9446, 9282, 9276, 9192, 9368, 9264] dup_rows_per_dst=[2850, 2958, 3258, 2996, 2908, 2950, 3278, 3052] combine_rows_per_src=[9036, 9190, 9446, 9282, 9276, 9192, 9368, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65148 dup_rows=23942 dup_rows_remote=21106 dedup_ratio=0.243551 router_tokens_per_expert=[760, 780, 789, 776, 798, 764, 780, 739, 809, 795, 786, 757, 750, 802, 764, 787, 754, 747, 727, 784, 778, 748, 796, 792, 758, 790, 748, 757, 776, 773, 740, 776, 739, 723, 783, 763, 765, 738, 757, 757, 790, 774, 791, 794, 718, 745, 731, 766, 758, 759, 822, 787, 724, 768, 771, 758, 757, 797, 807, 740, 813, 735, 789, 753] payload_rows_per_dst=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] dup_rows_per_dst=[3070, 3130, 3046, 2982, 2836, 2924, 2940, 3014] combine_rows_per_src=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=344096 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266125344 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906592 weight_movement_bytes=69206016 total_nvlink_bytes=601112608 weight_to_token_ratio=0.130109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65148 dup_rows=23942 dup_rows_remote=21106 dedup_ratio=0.243551 router_tokens_per_expert=[760, 780, 789, 776, 798, 764, 780, 739, 809, 795, 786, 757, 750, 802, 764, 787, 754, 747, 727, 784, 778, 748, 796, 792, 758, 790, 748, 757, 776, 773, 740, 776, 739, 723, 783, 763, 765, 738, 757, 757, 790, 774, 791, 794, 718, 745, 731, 766, 758, 759, 822, 787, 724, 768, 771, 758, 757, 797, 807, 740, 813, 735, 789, 753] payload_rows_per_dst=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] dup_rows_per_dst=[3070, 3130, 3046, 2982, 2836, 2924, 2940, 3014] combine_rows_per_src=[9302, 9370, 9206, 9254, 9214, 9294, 9354, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65012 dup_rows=24028 dup_rows_remote=21080 dedup_ratio=0.244425 router_tokens_per_expert=[775, 778, 744, 777, 777, 768, 810, 763, 759, 771, 815, 748, 782, 729, 767, 817, 776, 785, 765, 724, 741, 776, 791, 748, 796, 740, 765, 757, 801, 750, 794, 788, 759, 701, 757, 743, 789, 774, 791, 744, 783, 792, 792, 773, 819, 767, 724, 779, 774, 768, 771, 774, 754, 695, 807, 765, 754, 741, 764, 763, 789, 780, 741, 748] payload_rows_per_dst=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] dup_rows_per_dst=[3136, 3026, 2948, 2982, 2918, 3052, 3042, 2924] combine_rows_per_src=[9248, 9350, 9264, 9400, 9198, 9406, 9174, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343952 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723792 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103632 weight_movement_bytes=69206016 total_nvlink_bytes=600309648 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343656 combine_payload_bytes=265469952 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265813608 ingress_bytes=265469952 link_bytes=265813608 token_bytes=531283560 weight_movement_bytes=0 total_nvlink_bytes=531283560 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000235207e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344528 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267403728 ingress_bytes=370868224 link_bytes=370868224 token_bytes=534462928 weight_movement_bytes=103809024 total_nvlink_bytes=638271952 weight_to_token_ratio=0.194231 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395547270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64956 dup_rows=24154 dup_rows_remote=21096 dedup_ratio=0.245707 router_tokens_per_expert=[820, 810, 763, 763, 796, 757, 753, 772, 726, 760, 803, 751, 762, 759, 765, 800, 713, 768, 756, 753, 730, 718, 741, 778, 805, 779, 786, 763, 758, 748, 786, 759, 774, 768, 749, 796, 756, 745, 760, 824, 846, 771, 739, 770, 767, 791, 763, 720, 783, 776, 760, 773, 759, 768, 771, 765, 761, 749, 795, 794, 757, 752, 788, 761] payload_rows_per_dst=[9364, 9194, 9098, 9362, 9348, 9334, 9196, 9254] dup_rows_per_dst=[3104, 3058, 2816, 3006, 2996, 3000, 3114, 3060] combine_rows_per_src=[9364, 9194, 9098, 9362, 9348, 9334, 9196, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64888 dup_rows=24146 dup_rows_remote=21136 dedup_ratio=0.245626 router_tokens_per_expert=[734, 750, 769, 775, 724, 809, 748, 747, 736, 803, 773, 754, 780, 743, 746, 752, 806, 805, 792, 750, 767, 780, 784, 758, 776, 763, 733, 781, 787, 766, 730, 754, 787, 780, 759, 786, 785, 756, 744, 778, 756, 733, 791, 808, 764, 758, 744, 786, 749, 769, 807, 772, 779, 737, 789, 806, 791, 765, 739, 797, 780, 794, 719, 769] payload_rows_per_dst=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] dup_rows_per_dst=[2900, 3026, 3192, 2968, 2990, 3010, 3056, 3004] combine_rows_per_src=[9212, 9148, 9292, 9212, 9360, 9270, 9360, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65208 dup_rows=23848 dup_rows_remote=20904 dedup_ratio=0.242594 router_tokens_per_expert=[775, 778, 765, 800, 738, 752, 739, 792, 804, 767, 785, 805, 755, 762, 787, 768, 774, 743, 800, 764, 765, 772, 809, 787, 788, 807, 761, 757, 792, 744, 728, 756, 790, 789, 786, 755, 822, 747, 762, 756, 749, 746, 744, 747, 754, 753, 764, 766, 776, 762, 798, 742, 727, 786, 748, 757, 766, 746, 770, 766, 761, 778, 723, 797] payload_rows_per_dst=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] dup_rows_per_dst=[2996, 3102, 2990, 2938, 3034, 2940, 2944, 2904] combine_rows_per_src=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64812 dup_rows=24104 dup_rows_remote=21102 dedup_ratio=0.245199 router_tokens_per_expert=[736, 756, 776, 783, 774, 793, 794, 774, 781, 699, 783, 749, 776, 743, 745, 788, 733, 799, 750, 707, 792, 699, 755, 777, 796, 774, 793, 778, 750, 779, 765, 760, 782, 733, 805, 755, 807, 778, 770, 771, 749, 791, 862, 786, 769, 745, 775, 741, 760, 756, 742, 738, 745, 748, 758, 781, 819, 768, 766, 740, 800, 790, 790, 775] payload_rows_per_dst=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] dup_rows_per_dst=[3098, 2974, 2928, 3002, 3120, 3018, 2788, 3176] combine_rows_per_src=[9274, 9154, 9096, 9388, 9282, 9418, 9268, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343952 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723792 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103632 weight_movement_bytes=69206016 total_nvlink_bytes=600309648 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344928 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559776 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774624 weight_movement_bytes=34603008 total_nvlink_bytes=569377632 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=345016 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037432 weight_movement_bytes=69206016 total_nvlink_bytes=603243448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=345016 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037432 weight_movement_bytes=69206016 total_nvlink_bytes=603243448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344368 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633520 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922672 weight_movement_bytes=34603008 total_nvlink_bytes=567525680 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344208 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403984 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463760 weight_movement_bytes=34603008 total_nvlink_bytes=567066768 weight_to_token_ratio=0.064987 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=344096 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266125344 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906592 weight_movement_bytes=69206016 total_nvlink_bytes=601112608 weight_to_token_ratio=0.130109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343656 combine_payload_bytes=265469952 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265813608 ingress_bytes=265469952 link_bytes=265813608 token_bytes=531283560 weight_movement_bytes=0 total_nvlink_bytes=531283560 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.000235207e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64978 dup_rows=23860 dup_rows_remote=20740 dedup_ratio=0.242716 router_tokens_per_expert=[763, 802, 813, 769, 785, 774, 791, 783, 798, 769, 769, 785, 726, 766, 783, 788, 799, 774, 744, 785, 792, 785, 770, 763, 758, 721, 835, 750, 783, 810, 742, 779, 803, 725, 774, 723, 788, 729, 758, 720, 735, 815, 751, 781, 755, 719, 789, 771, 804, 744, 723, 757, 761, 809, 716, 763, 757, 770, 722, 727, 769, 772, 817, 751] payload_rows_per_dst=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] dup_rows_per_dst=[3120, 3074, 3048, 3114, 2786, 2892, 2866, 2960] combine_rows_per_src=[9466, 9294, 9376, 9216, 9254, 9340, 9288, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65110 dup_rows=23780 dup_rows_remote=20764 dedup_ratio=0.241903 router_tokens_per_expert=[749, 785, 759, 751, 761, 747, 775, 762, 769, 721, 757, 767, 773, 746, 752, 811, 762, 737, 769, 758, 708, 748, 782, 775, 797, 816, 790, 828, 761, 771, 760, 759, 791, 779, 746, 795, 760, 733, 787, 743, 782, 789, 822, 796, 741, 771, 769, 745, 785, 797, 757, 795, 750, 780, 740, 764, 779, 744, 765, 758, 749, 778, 786, 770] payload_rows_per_dst=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] dup_rows_per_dst=[2912, 2946, 2852, 3136, 2972, 3016, 3030, 2916] combine_rows_per_src=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65038 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[755, 747, 759, 766, 776, 752, 787, 775, 759, 772, 775, 761, 771, 806, 748, 768, 742, 725, 758, 772, 797, 779, 821, 780, 775, 775, 792, 770, 747, 759, 763, 794, 735, 789, 773, 785, 770, 757, 759, 810, 783, 778, 775, 791, 776, 779, 757, 754, 778, 775, 728, 734, 782, 789, 761, 744, 760, 766, 756, 739, 753, 762, 766, 762] payload_rows_per_dst=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] dup_rows_per_dst=[2974, 2988, 3050, 2992, 3038, 3066, 2870, 2930] combine_rows_per_src=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64914 dup_rows=24086 dup_rows_remote=21034 dedup_ratio=0.245015 router_tokens_per_expert=[786, 731, 766, 747, 750, 770, 765, 730, 788, 765, 796, 740, 799, 803, 821, 790, 754, 758, 748, 767, 792, 774, 755, 734, 793, 736, 779, 768, 770, 768, 716, 801, 740, 774, 769, 760, 776, 754, 798, 773, 767, 779, 771, 766, 757, 765, 771, 732, 765, 768, 774, 737, 751, 845, 770, 756, 721, 764, 718, 828, 785, 790, 802, 766] payload_rows_per_dst=[9208, 8940, 9618, 9244, 9314, 9320, 9270, 9304] dup_rows_per_dst=[2882, 2768, 3442, 3018, 2974, 2896, 3054, 3052] combine_rows_per_src=[9208, 8940, 9618, 9244, 9314, 9320, 9270, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343496 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267034056 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724616 weight_movement_bytes=34603008 total_nvlink_bytes=568327624 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65208 dup_rows=23848 dup_rows_remote=20904 dedup_ratio=0.242594 router_tokens_per_expert=[775, 778, 765, 800, 738, 752, 739, 792, 804, 767, 785, 805, 755, 762, 787, 768, 774, 743, 800, 764, 765, 772, 809, 787, 788, 807, 761, 757, 792, 744, 728, 756, 790, 789, 786, 755, 822, 747, 762, 756, 749, 746, 744, 747, 754, 753, 764, 766, 776, 762, 798, 742, 727, 786, 748, 757, 766, 746, 770, 766, 761, 778, 723, 797] payload_rows_per_dst=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] dup_rows_per_dst=[2996, 3102, 2990, 2938, 3034, 2940, 2944, 2904] combine_rows_per_src=[9282, 9364, 9438, 9328, 9380, 9106, 9248, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64932 dup_rows=23982 dup_rows_remote=20940 dedup_ratio=0.243958 router_tokens_per_expert=[749, 796, 780, 751, 784, 801, 771, 777, 732, 756, 748, 767, 763, 788, 825, 751, 760, 793, 780, 757, 737, 772, 782, 762, 799, 771, 748, 755, 787, 718, 749, 775, 746, 775, 771, 774, 818, 780, 755, 810, 769, 774, 750, 765, 759, 842, 801, 711, 770, 789, 784, 709, 772, 733, 731, 799, 758, 798, 757, 771, 766, 724, 801, 706] payload_rows_per_dst=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] dup_rows_per_dst=[3042, 2978, 3020, 2998, 3110, 2950, 2984, 2900] combine_rows_per_src=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65046 dup_rows=24028 dup_rows_remote=21004 dedup_ratio=0.244425 router_tokens_per_expert=[768, 788, 784, 758, 788, 810, 799, 724, 774, 736, 747, 754, 747, 776, 725, 825, 767, 790, 825, 750, 757, 749, 768, 711, 804, 793, 691, 719, 788, 822, 755, 769, 714, 781, 762, 729, 818, 807, 763, 753, 770, 779, 746, 785, 803, 782, 795, 767, 771, 777, 756, 747, 754, 708, 796, 750, 753, 752, 779, 750, 785, 799, 744, 816] payload_rows_per_dst=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] dup_rows_per_dst=[3112, 3000, 3036, 2954, 3024, 3024, 2870, 3008] combine_rows_per_src=[9326, 9168, 9198, 9328, 9230, 9430, 9248, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343792 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231536 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119280 weight_movement_bytes=34603008 total_nvlink_bytes=566722288 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=65120 dup_rows=24116 dup_rows_remote=21116 dedup_ratio=0.245321 router_tokens_per_expert=[715, 786, 761, 776, 721, 765, 806, 773, 744, 722, 752, 755, 780, 827, 783, 797, 780, 755, 703, 750, 790, 738, 767, 751, 783, 770, 796, 717, 807, 724, 794, 709, 761, 778, 788, 759, 777, 780, 792, 747, 799, 733, 784, 780, 750, 766, 769, 718, 818, 752, 800, 788, 847, 781, 756, 796, 757, 752, 753, 783, 763, 778, 779, 771] payload_rows_per_dst=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] dup_rows_per_dst=[2968, 3064, 3000, 2910, 3084, 2904, 3294, 2892] combine_rows_per_src=[9238, 9256, 9068, 9290, 9280, 9294, 9382, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343816 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739464 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135112 weight_movement_bytes=69206016 total_nvlink_bytes=602341128 weight_to_token_ratio=0.129810 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64964 dup_rows=24014 dup_rows_remote=20976 dedup_ratio=0.244283 router_tokens_per_expert=[784, 796, 745, 768, 746, 782, 776, 770, 803, 773, 793, 715, 773, 740, 807, 778, 771, 790, 743, 741, 799, 767, 769, 773, 725, 739, 742, 751, 767, 759, 796, 761, 734, 776, 754, 786, 781, 754, 772, 786, 801, 732, 774, 790, 785, 780, 772, 752, 764, 758, 750, 769, 758, 769, 751, 723, 764, 801, 747, 753, 821, 795, 780, 778] payload_rows_per_dst=[9324, 9326, 9324, 9208, 9344, 9298, 9064, 9402] dup_rows_per_dst=[3010, 3038, 2982, 2872, 2942, 3074, 3020, 3076] combine_rows_per_src=[9324, 9326, 9324, 9208, 9344, 9298, 9064, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64784 dup_rows=24240 dup_rows_remote=21238 dedup_ratio=0.246582 router_tokens_per_expert=[783, 783, 753, 776, 781, 785, 745, 772, 805, 733, 694, 787, 790, 765, 769, 737, 735, 783, 736, 791, 727, 757, 754, 782, 724, 779, 816, 753, 801, 787, 787, 805, 770, 720, 768, 757, 729, 797, 755, 756, 785, 776, 736, 753, 794, 791, 773, 728, 771, 821, 788, 739, 726, 782, 738, 776, 799, 782, 784, 857, 762, 734, 748, 782] payload_rows_per_dst=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] dup_rows_per_dst=[3076, 3046, 2922, 3152, 2926, 3008, 3002, 3108] combine_rows_per_src=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65110 dup_rows=23780 dup_rows_remote=20764 dedup_ratio=0.241903 router_tokens_per_expert=[749, 785, 759, 751, 761, 747, 775, 762, 769, 721, 757, 767, 773, 746, 752, 811, 762, 737, 769, 758, 708, 748, 782, 775, 797, 816, 790, 828, 761, 771, 760, 759, 791, 779, 746, 795, 760, 733, 787, 743, 782, 789, 822, 796, 741, 771, 769, 745, 785, 797, 757, 795, 750, 780, 740, 764, 779, 744, 765, 758, 749, 778, 786, 770] payload_rows_per_dst=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] dup_rows_per_dst=[2912, 2946, 2852, 3136, 2972, 3016, 3030, 2916] combine_rows_per_src=[9266, 9246, 9226, 9428, 9296, 9414, 9306, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65038 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[755, 747, 759, 766, 776, 752, 787, 775, 759, 772, 775, 761, 771, 806, 748, 768, 742, 725, 758, 772, 797, 779, 821, 780, 775, 775, 792, 770, 747, 759, 763, 794, 735, 789, 773, 785, 770, 757, 759, 810, 783, 778, 775, 791, 776, 779, 757, 754, 778, 775, 728, 734, 782, 789, 761, 744, 760, 766, 756, 739, 753, 762, 766, 762] payload_rows_per_dst=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] dup_rows_per_dst=[2974, 2988, 3050, 2992, 3038, 3066, 2870, 2930] combine_rows_per_src=[9260, 9332, 9298, 9358, 9318, 9320, 9312, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343792 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231536 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119280 weight_movement_bytes=34603008 total_nvlink_bytes=566722288 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344200 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772616 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201032 weight_movement_bytes=34603008 total_nvlink_bytes=567804040 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=342872 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266492760 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532642648 weight_movement_bytes=138412032 total_nvlink_bytes=671054680 weight_to_token_ratio=0.259859 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343488 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266304960 ingress_bytes=369770496 link_bytes=369770496 token_bytes=532266432 weight_movement_bytes=103809024 total_nvlink_bytes=636075456 weight_to_token_ratio=0.195032 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391416607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344944 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076464 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807984 weight_movement_bytes=34603008 total_nvlink_bytes=568410992 weight_to_token_ratio=0.064823 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344200 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772616 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201032 weight_movement_bytes=34603008 total_nvlink_bytes=567804040 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64944 dup_rows=24092 dup_rows_remote=21080 dedup_ratio=0.245076 router_tokens_per_expert=[753, 742, 740, 775, 733, 685, 812, 806, 798, 764, 815, 720, 776, 843, 795, 774, 759, 771, 776, 755, 742, 790, 757, 762, 777, 822, 788, 740, 758, 800, 740, 755, 775, 816, 738, 774, 766, 769, 794, 715, 800, 797, 755, 773, 705, 768, 802, 740, 709, 765, 754, 765, 769, 788, 749, 776, 763, 811, 737, 772, 777, 745, 763, 799] payload_rows_per_dst=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] dup_rows_per_dst=[2892, 3128, 3020, 3052, 3052, 3012, 2920, 3016] combine_rows_per_src=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343760 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436304 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528848 weight_movement_bytes=69206016 total_nvlink_bytes=601734864 weight_to_token_ratio=0.129957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344088 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265699352 ingress_bytes=369164288 link_bytes=369164288 token_bytes=531054616 weight_movement_bytes=103809024 total_nvlink_bytes=634863640 weight_to_token_ratio=0.195477 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389135496e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343496 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267034056 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724616 weight_movement_bytes=34603008 total_nvlink_bytes=568327624 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343816 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739464 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135112 weight_movement_bytes=69206016 total_nvlink_bytes=602341128 weight_to_token_ratio=0.129810 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65070 dup_rows=23914 dup_rows_remote=20842 dedup_ratio=0.243266 router_tokens_per_expert=[753, 727, 763, 752, 778, 769, 760, 743, 744, 852, 785, 736, 772, 780, 782, 727, 761, 788, 742, 775, 751, 776, 773, 765, 775, 784, 759, 739, 740, 795, 821, 783, 754, 771, 778, 817, 799, 784, 758, 776, 763, 730, 769, 774, 762, 758, 759, 764, 758, 764, 750, 768, 782, 712, 746, 747, 765, 823, 833, 754, 785, 782, 771, 746] payload_rows_per_dst=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] dup_rows_per_dst=[2966, 3054, 2962, 3072, 3032, 2842, 2940, 3046] combine_rows_per_src=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344096 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354720 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365344 weight_movement_bytes=69206016 total_nvlink_bytes=601571360 weight_to_token_ratio=0.129997 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65282 dup_rows=23692 dup_rows_remote=20708 dedup_ratio=0.241007 router_tokens_per_expert=[728, 776, 769, 797, 764, 753, 775, 746, 818, 761, 784, 773, 756, 783, 787, 779, 758, 743, 733, 748, 740, 744, 731, 745, 794, 764, 763, 790, 800, 723, 788, 761, 716, 750, 801, 733, 811, 745, 743, 753, 790, 771, 751, 771, 790, 801, 828, 734, 823, 785, 820, 749, 761, 780, 750, 769, 755, 787, 799, 782, 776, 770, 720, 764] payload_rows_per_dst=[9266, 9476, 9110, 9398, 9226, 9370, 9436, 9330] dup_rows_per_dst=[2950, 2998, 2774, 2968, 2878, 3102, 3038, 2984] combine_rows_per_src=[9266, 9476, 9110, 9398, 9226, 9370, 9436, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343648 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870368 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397088 weight_movement_bytes=69206016 total_nvlink_bytes=602603104 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64784 dup_rows=24240 dup_rows_remote=21238 dedup_ratio=0.246582 router_tokens_per_expert=[783, 783, 753, 776, 781, 785, 745, 772, 805, 733, 694, 787, 790, 765, 769, 737, 735, 783, 736, 791, 727, 757, 754, 782, 724, 779, 816, 753, 801, 787, 787, 805, 770, 720, 768, 757, 729, 797, 755, 756, 785, 776, 736, 753, 794, 791, 773, 728, 771, 821, 788, 739, 726, 782, 738, 776, 799, 782, 784, 857, 762, 734, 748, 782] payload_rows_per_dst=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] dup_rows_per_dst=[3076, 3046, 2922, 3152, 2926, 3008, 3002, 3108] combine_rows_per_src=[9280, 9114, 9208, 9352, 9178, 9264, 9280, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73940 payload_rows_remote=64594 dup_rows=24364 dup_rows_remote=21242 dedup_ratio=0.247843 router_tokens_per_expert=[799, 734, 782, 777, 762, 783, 787, 791, 802, 809, 771, 773, 777, 758, 831, 800, 771, 776, 755, 731, 752, 772, 740, 743, 752, 765, 775, 772, 723, 764, 808, 816, 781, 709, 761, 805, 778, 760, 771, 763, 758, 745, 719, 773, 747, 769, 763, 778, 783, 771, 790, 765, 805, 816, 747, 764, 744, 760, 732, 797, 709, 798, 729, 711] payload_rows_per_dst=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] dup_rows_per_dst=[3122, 3400, 2928, 2970, 3070, 2920, 3090, 2864] combine_rows_per_src=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65060 dup_rows=23926 dup_rows_remote=21068 dedup_ratio=0.243388 router_tokens_per_expert=[745, 796, 804, 720, 771, 777, 752, 728, 741, 803, 750, 779, 732, 746, 771, 790, 772, 740, 780, 782, 780, 791, 714, 729, 818, 808, 780, 745, 773, 774, 790, 767, 796, 745, 747, 712, 756, 764, 750, 762, 760, 735, 754, 763, 804, 791, 747, 852, 814, 758, 757, 768, 743, 763, 772, 753, 795, 796, 749, 801, 763, 765, 772, 797] payload_rows_per_dst=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] dup_rows_per_dst=[2996, 3004, 2858, 3134, 2858, 3050, 2974, 3052] combine_rows_per_src=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65074 dup_rows=24024 dup_rows_remote=21110 dedup_ratio=0.244385 router_tokens_per_expert=[763, 739, 740, 775, 752, 745, 777, 783, 759, 822, 737, 794, 789, 813, 766, 749, 763, 714, 776, 759, 771, 752, 783, 814, 754, 748, 738, 782, 789, 766, 803, 779, 795, 762, 730, 754, 774, 755, 728, 762, 758, 813, 744, 767, 745, 807, 776, 769, 818, 788, 762, 759, 750, 815, 758, 776, 734, 736, 778, 744, 757, 805, 778, 761] payload_rows_per_dst=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] dup_rows_per_dst=[2914, 2982, 3034, 3100, 2914, 3042, 3050, 2988] combine_rows_per_src=[9234, 9476, 9230, 9218, 9206, 9316, 9402, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=343960 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267739032 ingress_bytes=371204096 link_bytes=371204096 token_bytes=535134104 weight_movement_bytes=103809024 total_nvlink_bytes=638943128 weight_to_token_ratio=0.193987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396811129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65052 dup_rows=23848 dup_rows_remote=20744 dedup_ratio=0.242594 router_tokens_per_expert=[785, 763, 779, 747, 776, 744, 766, 795, 828, 784, 821, 768, 744, 771, 766, 772, 732, 721, 767, 782, 785, 677, 759, 734, 741, 790, 777, 778, 708, 749, 800, 793, 761, 774, 797, 783, 769, 792, 835, 787, 774, 711, 754, 744, 813, 772, 756, 812, 701, 766, 749, 798, 776, 776, 795, 780, 769, 762, 757, 782, 767, 717, 769, 752] payload_rows_per_dst=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] dup_rows_per_dst=[2990, 3104, 2862, 2976, 2990, 3026, 2962, 2938] combine_rows_per_src=[9320, 9404, 9052, 9296, 9606, 9246, 9320, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65060 dup_rows=23926 dup_rows_remote=21068 dedup_ratio=0.243388 router_tokens_per_expert=[745, 796, 804, 720, 771, 777, 752, 728, 741, 803, 750, 779, 732, 746, 771, 790, 772, 740, 780, 782, 780, 791, 714, 729, 818, 808, 780, 745, 773, 774, 790, 767, 796, 745, 747, 712, 756, 764, 750, 762, 760, 735, 754, 763, 804, 791, 747, 852, 814, 758, 757, 768, 743, 763, 772, 753, 795, 796, 749, 801, 763, 765, 772, 797] payload_rows_per_dst=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] dup_rows_per_dst=[2996, 3004, 2858, 3134, 2858, 3050, 2974, 3052] combine_rows_per_src=[9190, 9220, 9318, 9376, 9206, 9362, 9282, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344736 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887840 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430944 weight_movement_bytes=34603008 total_nvlink_bytes=568033952 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64824 dup_rows=24186 dup_rows_remote=21104 dedup_ratio=0.246033 router_tokens_per_expert=[756, 784, 775, 772, 757, 730, 770, 776, 751, 739, 784, 748, 738, 747, 715, 805, 765, 757, 798, 766, 733, 806, 751, 811, 804, 813, 744, 772, 762, 767, 798, 796, 773, 758, 740, 770, 734, 812, 810, 776, 742, 731, 754, 804, 752, 776, 751, 754, 783, 768, 807, 746, 749, 779, 789, 767, 734, 787, 754, 787, 760, 774, 794, 747] payload_rows_per_dst=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] dup_rows_per_dst=[3054, 2874, 3010, 3134, 3096, 2890, 3082, 3046] combine_rows_per_src=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64944 dup_rows=24092 dup_rows_remote=21080 dedup_ratio=0.245076 router_tokens_per_expert=[753, 742, 740, 775, 733, 685, 812, 806, 798, 764, 815, 720, 776, 843, 795, 774, 759, 771, 776, 755, 742, 790, 757, 762, 777, 822, 788, 740, 758, 800, 740, 755, 775, 816, 738, 774, 766, 769, 794, 715, 800, 797, 755, 773, 705, 768, 802, 740, 709, 765, 754, 765, 769, 788, 749, 776, 763, 811, 737, 772, 777, 745, 763, 799] payload_rows_per_dst=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] dup_rows_per_dst=[2892, 3128, 3020, 3052, 3052, 3012, 2920, 3016] combine_rows_per_src=[9200, 9442, 9204, 9308, 9242, 9268, 9230, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65070 dup_rows=23914 dup_rows_remote=20842 dedup_ratio=0.243266 router_tokens_per_expert=[753, 727, 763, 752, 778, 769, 760, 743, 744, 852, 785, 736, 772, 780, 782, 727, 761, 788, 742, 775, 751, 776, 773, 765, 775, 784, 759, 739, 740, 795, 821, 783, 754, 771, 778, 817, 799, 784, 758, 776, 763, 730, 769, 774, 762, 758, 759, 764, 758, 764, 750, 768, 782, 712, 746, 747, 765, 823, 833, 754, 785, 782, 771, 746] payload_rows_per_dst=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] dup_rows_per_dst=[2966, 3054, 2962, 3072, 3032, 2842, 2940, 3046] combine_rows_per_src=[9124, 9302, 9300, 9320, 9442, 9316, 9114, 9472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344088 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265699352 ingress_bytes=369164288 link_bytes=369164288 token_bytes=531054616 weight_movement_bytes=103809024 total_nvlink_bytes=634863640 weight_to_token_ratio=0.195477 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389135496e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65012 dup_rows=23964 dup_rows_remote=20876 dedup_ratio=0.243774 router_tokens_per_expert=[734, 811, 808, 824, 750, 770, 791, 741, 761, 819, 750, 739, 779, 739, 827, 692, 715, 738, 770, 767, 804, 777, 817, 769, 766, 760, 775, 764, 731, 777, 762, 789, 783, 768, 755, 720, 787, 774, 774, 756, 761, 756, 822, 791, 798, 730, 774, 776, 746, 749, 814, 787, 759, 750, 756, 754, 738, 734, 797, 775, 759, 731, 773, 789] payload_rows_per_dst=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] dup_rows_per_dst=[3144, 2966, 2946, 2938, 2980, 3088, 3012, 2890] combine_rows_per_src=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=343344 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264920368 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529497392 weight_movement_bytes=34603008 total_nvlink_bytes=564100400 weight_to_token_ratio=0.065351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344512 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266830272 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533316032 weight_movement_bytes=34603008 total_nvlink_bytes=567919040 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344736 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887840 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430944 weight_movement_bytes=34603008 total_nvlink_bytes=568033952 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343184 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796176 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249168 weight_movement_bytes=103809024 total_nvlink_bytes=637058192 weight_to_token_ratio=0.194673 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343648 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870368 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397088 weight_movement_bytes=69206016 total_nvlink_bytes=602603104 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344096 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354720 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365344 weight_movement_bytes=69206016 total_nvlink_bytes=601571360 weight_to_token_ratio=0.129997 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65200 dup_rows=23856 dup_rows_remote=20932 dedup_ratio=0.242676 router_tokens_per_expert=[790, 795, 750, 738, 750, 748, 758, 744, 750, 767, 795, 791, 757, 761, 779, 800, 773, 761, 785, 744, 763, 728, 777, 779, 759, 762, 769, 805, 797, 781, 770, 778, 800, 795, 786, 784, 742, 821, 737, 722, 733, 800, 817, 787, 773, 721, 695, 778, 766, 710, 775, 758, 749, 787, 775, 780, 769, 731, 770, 772, 786, 770, 775, 784] payload_rows_per_dst=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] dup_rows_per_dst=[2924, 3296, 3008, 3032, 2828, 2812, 2926, 3030] combine_rows_per_src=[9248, 9682, 9472, 9384, 8968, 9136, 9274, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65324 dup_rows=23660 dup_rows_remote=20758 dedup_ratio=0.240682 router_tokens_per_expert=[796, 750, 759, 757, 779, 793, 755, 752, 704, 827, 725, 763, 790, 774, 770, 774, 704, 762, 747, 770, 752, 772, 806, 763, 766, 766, 781, 780, 734, 765, 757, 762, 774, 805, 758, 767, 742, 736, 771, 794, 764, 787, 807, 758, 747, 804, 786, 770, 761, 809, 767, 769, 754, 769, 764, 789, 790, 789, 774, 769, 760, 777, 739, 747] payload_rows_per_dst=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] dup_rows_per_dst=[2908, 2932, 2894, 2902, 2944, 3118, 3022, 2940] combine_rows_per_src=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343552 combine_payload_bytes=266289152 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266632704 ingress_bytes=404701184 link_bytes=404701184 token_bytes=532921856 weight_movement_bytes=138412032 total_nvlink_bytes=671333888 weight_to_token_ratio=0.259723 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522857974e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64700 dup_rows=24208 dup_rows_remote=21012 dedup_ratio=0.246257 router_tokens_per_expert=[788, 709, 765, 801, 765, 743, 734, 768, 770, 764, 778, 778, 800, 812, 769, 774, 820, 741, 765, 744, 717, 781, 738, 787, 768, 792, 735, 764, 739, 784, 777, 727, 786, 787, 799, 749, 769, 758, 767, 773, 760, 774, 771, 740, 777, 807, 777, 726, 775, 773, 778, 806, 760, 776, 783, 757, 775, 768, 767, 746, 756, 787, 777, 751] payload_rows_per_dst=[9170, 9394, 9220, 9026, 9250, 9150, 9490, 9396] dup_rows_per_dst=[2976, 3096, 2966, 2808, 3126, 2784, 3256, 3196] combine_rows_per_src=[9170, 9394, 9220, 9026, 9250, 9150, 9490, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65214 dup_rows=23760 dup_rows_remote=20768 dedup_ratio=0.241699 router_tokens_per_expert=[748, 769, 742, 780, 767, 760, 780, 742, 788, 760, 768, 776, 743, 753, 769, 802, 738, 787, 763, 756, 799, 770, 743, 760, 784, 773, 781, 767, 737, 756, 765, 812, 769, 770, 789, 784, 726, 796, 738, 789, 759, 801, 797, 797, 768, 809, 756, 782, 780, 766, 772, 796, 755, 758, 736, 761, 737, 750, 777, 762, 766, 782, 722, 764] payload_rows_per_dst=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] dup_rows_per_dst=[2952, 2938, 3000, 2976, 2992, 3070, 2874, 2958] combine_rows_per_src=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267567104 dispatch_meta_bytes=344328 combine_payload_bytes=267567104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267911432 ingress_bytes=371376128 link_bytes=371376128 token_bytes=535478536 weight_movement_bytes=103809024 total_nvlink_bytes=639287560 weight_to_token_ratio=0.193862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397458471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342848 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265354048 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530365248 weight_movement_bytes=34603008 total_nvlink_bytes=564968256 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344528 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267403728 ingress_bytes=370868224 link_bytes=370868224 token_bytes=534462928 weight_movement_bytes=103809024 total_nvlink_bytes=638271952 weight_to_token_ratio=0.194231 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395547270e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65030 dup_rows=23988 dup_rows_remote=21046 dedup_ratio=0.244019 router_tokens_per_expert=[765, 775, 773, 765, 758, 741, 772, 754, 776, 750, 797, 760, 780, 744, 749, 758, 793, 762, 807, 802, 755, 812, 817, 762, 773, 805, 798, 803, 745, 749, 764, 787, 750, 711, 772, 735, 744, 782, 808, 716, 743, 807, 790, 769, 791, 736, 763, 765, 764, 781, 749, 777, 778, 773, 736, 767, 785, 772, 769, 762, 751, 760, 792, 703] payload_rows_per_dst=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] dup_rows_per_dst=[2924, 2942, 3234, 3002, 2966, 2948, 2958, 3014] combine_rows_per_src=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64824 dup_rows=24186 dup_rows_remote=21104 dedup_ratio=0.246033 router_tokens_per_expert=[756, 784, 775, 772, 757, 730, 770, 776, 751, 739, 784, 748, 738, 747, 715, 805, 765, 757, 798, 766, 733, 806, 751, 811, 804, 813, 744, 772, 762, 767, 798, 796, 773, 758, 740, 770, 734, 812, 810, 776, 742, 731, 754, 804, 752, 776, 751, 754, 783, 768, 807, 746, 749, 779, 789, 767, 734, 787, 754, 787, 760, 774, 794, 747] payload_rows_per_dst=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] dup_rows_per_dst=[3054, 2874, 3010, 3134, 3096, 2890, 3082, 3046] combine_rows_per_src=[9186, 9180, 9364, 9378, 9250, 9238, 9294, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64638 dup_rows=24056 dup_rows_remote=20644 dedup_ratio=0.244710 router_tokens_per_expert=[768, 717, 737, 781, 778, 711, 841, 797, 809, 783, 762, 768, 788, 758, 756, 785, 789, 750, 750, 739, 758, 723, 781, 747, 739, 814, 739, 790, 781, 777, 778, 771, 731, 809, 788, 761, 721, 738, 803, 742, 766, 783, 776, 754, 784, 781, 755, 751, 789, 758, 753, 751, 791, 786, 757, 747, 772, 773, 742, 789, 794, 762, 804, 776] payload_rows_per_dst=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] dup_rows_per_dst=[3412, 2700, 3432, 2622, 3032, 2970, 2990, 2898] combine_rows_per_src=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65202 dup_rows=24004 dup_rows_remote=21120 dedup_ratio=0.244181 router_tokens_per_expert=[804, 727, 769, 760, 782, 756, 795, 756, 766, 746, 754, 736, 792, 786, 785, 763, 758, 706, 775, 751, 778, 726, 743, 754, 781, 784, 752, 840, 779, 825, 759, 779, 812, 789, 746, 794, 779, 744, 722, 805, 780, 758, 737, 786, 802, 759, 737, 789, 748, 752, 756, 789, 749, 748, 776, 750, 771, 767, 739, 799, 741, 794, 767, 800] payload_rows_per_dst=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] dup_rows_per_dst=[3080, 3046, 2884, 3068, 2984, 2930, 2906, 3106] combine_rows_per_src=[9218, 9210, 9098, 9530, 9398, 9366, 9230, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65200 dup_rows=23756 dup_rows_remote=20952 dedup_ratio=0.241659 router_tokens_per_expert=[820, 796, 754, 784, 777, 737, 738, 756, 753, 824, 728, 753, 773, 762, 749, 779, 791, 755, 766, 762, 741, 773, 797, 747, 756, 817, 771, 775, 818, 815, 791, 765, 755, 776, 741, 757, 790, 789, 732, 752, 782, 784, 761, 769, 789, 768, 736, 767, 788, 764, 774, 772, 744, 719, 728, 787, 774, 786, 768, 770, 722, 761, 763, 761] payload_rows_per_dst=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] dup_rows_per_dst=[3012, 3044, 2954, 3110, 2924, 2994, 2804, 2914] combine_rows_per_src=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=343928 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460472 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577016 weight_movement_bytes=34603008 total_nvlink_bytes=569180024 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65324 dup_rows=23660 dup_rows_remote=20758 dedup_ratio=0.240682 router_tokens_per_expert=[796, 750, 759, 757, 779, 793, 755, 752, 704, 827, 725, 763, 790, 774, 770, 774, 704, 762, 747, 770, 752, 772, 806, 763, 766, 766, 781, 780, 734, 765, 757, 762, 774, 805, 758, 767, 742, 736, 771, 794, 764, 787, 807, 758, 747, 804, 786, 770, 761, 809, 767, 769, 754, 769, 764, 789, 790, 789, 774, 769, 760, 777, 739, 747] payload_rows_per_dst=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] dup_rows_per_dst=[2908, 2932, 2894, 2902, 2944, 3118, 3022, 2940] combine_rows_per_src=[9374, 9322, 9258, 9320, 9350, 9328, 9342, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65012 dup_rows=23964 dup_rows_remote=20876 dedup_ratio=0.243774 router_tokens_per_expert=[734, 811, 808, 824, 750, 770, 791, 741, 761, 819, 750, 739, 779, 739, 827, 692, 715, 738, 770, 767, 804, 777, 817, 769, 766, 760, 775, 764, 731, 777, 762, 789, 783, 768, 755, 720, 787, 774, 774, 756, 761, 756, 822, 791, 798, 730, 774, 776, 746, 749, 814, 787, 759, 750, 756, 754, 738, 734, 797, 775, 759, 731, 773, 789] payload_rows_per_dst=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] dup_rows_per_dst=[3144, 2966, 2946, 2938, 2980, 3088, 3012, 2890] combine_rows_per_src=[9314, 9246, 9368, 9310, 9254, 9328, 9218, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65214 dup_rows=23760 dup_rows_remote=20768 dedup_ratio=0.241699 router_tokens_per_expert=[748, 769, 742, 780, 767, 760, 780, 742, 788, 760, 768, 776, 743, 753, 769, 802, 738, 787, 763, 756, 799, 770, 743, 760, 784, 773, 781, 767, 737, 756, 765, 812, 769, 770, 789, 784, 726, 796, 738, 789, 759, 801, 797, 797, 768, 809, 756, 782, 780, 766, 772, 796, 755, 758, 736, 761, 737, 750, 777, 762, 766, 782, 722, 764] payload_rows_per_dst=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] dup_rows_per_dst=[2952, 2938, 3000, 2976, 2992, 3070, 2874, 2958] combine_rows_per_src=[9224, 9380, 9232, 9374, 9330, 9468, 9374, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65030 dup_rows=23988 dup_rows_remote=21046 dedup_ratio=0.244019 router_tokens_per_expert=[765, 775, 773, 765, 758, 741, 772, 754, 776, 750, 797, 760, 780, 744, 749, 758, 793, 762, 807, 802, 755, 812, 817, 762, 773, 805, 798, 803, 745, 749, 764, 787, 750, 711, 772, 735, 744, 782, 808, 716, 743, 807, 790, 769, 791, 736, 763, 765, 764, 781, 749, 777, 778, 773, 736, 767, 785, 772, 769, 762, 751, 760, 792, 703] payload_rows_per_dst=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] dup_rows_per_dst=[2924, 2942, 3234, 3002, 2966, 2948, 2958, 3014] combine_rows_per_src=[9282, 9286, 9386, 9446, 9070, 9380, 9292, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344304 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707184 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070064 weight_movement_bytes=34603008 total_nvlink_bytes=567673072 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64960 dup_rows=24024 dup_rows_remote=20988 dedup_ratio=0.244385 router_tokens_per_expert=[753, 732, 746, 792, 753, 748, 767, 797, 831, 778, 799, 767, 762, 738, 754, 746, 799, 754, 781, 820, 807, 765, 742, 719, 766, 749, 771, 781, 768, 745, 791, 760, 762, 774, 679, 733, 744, 763, 803, 775, 803, 801, 750, 734, 759, 754, 781, 796, 812, 782, 823, 781, 768, 791, 751, 756, 722, 739, 746, 792, 809, 753, 786, 749] payload_rows_per_dst=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] dup_rows_per_dst=[2986, 3042, 3038, 3010, 2870, 3036, 3094, 2948] combine_rows_per_src=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264757248 dispatch_meta_bytes=341128 combine_payload_bytes=264757248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265098376 ingress_bytes=299360256 link_bytes=299360256 token_bytes=529855624 weight_movement_bytes=34603008 total_nvlink_bytes=564458632 weight_to_token_ratio=0.065306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126468543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=345288 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267412680 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534480072 weight_movement_bytes=34603008 total_nvlink_bytes=569083080 weight_to_token_ratio=0.064741 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344608 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267403808 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463008 weight_movement_bytes=34603008 total_nvlink_bytes=569066016 weight_to_token_ratio=0.064744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267567104 dispatch_meta_bytes=344328 combine_payload_bytes=267567104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267911432 ingress_bytes=371376128 link_bytes=371376128 token_bytes=535478536 weight_movement_bytes=103809024 total_nvlink_bytes=639287560 weight_to_token_ratio=0.193862 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397458471e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343552 combine_payload_bytes=266289152 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266632704 ingress_bytes=404701184 link_bytes=404701184 token_bytes=532921856 weight_movement_bytes=138412032 total_nvlink_bytes=671333888 weight_to_token_ratio=0.259723 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522857974e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=343928 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460472 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577016 weight_movement_bytes=34603008 total_nvlink_bytes=569180024 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344304 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707184 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070064 weight_movement_bytes=34603008 total_nvlink_bytes=567673072 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64810 dup_rows=24048 dup_rows_remote=21056 dedup_ratio=0.244629 router_tokens_per_expert=[776, 776, 731, 763, 743, 799, 766, 743, 810, 795, 781, 753, 718, 794, 782, 761, 786, 752, 769, 771, 768, 724, 770, 755, 755, 755, 812, 806, 758, 768, 763, 802, 733, 794, 766, 770, 759, 748, 781, 787, 781, 750, 776, 751, 785, 764, 816, 734, 841, 761, 784, 805, 787, 783, 770, 765, 739, 784, 739, 733, 730, 776, 720, 735] payload_rows_per_dst=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] dup_rows_per_dst=[2940, 3042, 3024, 2992, 3024, 2994, 3096, 2936] combine_rows_per_src=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343792 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419952 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496112 weight_movement_bytes=34603008 total_nvlink_bytes=567099120 weight_to_token_ratio=0.064983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64786 dup_rows=23952 dup_rows_remote=20792 dedup_ratio=0.243652 router_tokens_per_expert=[755, 749, 714, 769, 788, 739, 759, 756, 806, 723, 779, 766, 774, 782, 819, 786, 728, 736, 797, 789, 746, 749, 713, 805, 779, 769, 788, 735, 778, 774, 763, 775, 806, 808, 776, 774, 706, 749, 820, 725, 785, 815, 742, 770, 775, 771, 801, 758, 795, 794, 770, 773, 745, 831, 729, 762, 769, 712, 742, 800, 749, 778, 790, 744] payload_rows_per_dst=[9126, 9460, 9060, 9174, 9310, 9510, 9146, 9566] dup_rows_per_dst=[2932, 3008, 2750, 2924, 3066, 3226, 2886, 3160] combine_rows_per_src=[9126, 9460, 9060, 9174, 9310, 9510, 9146, 9566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65016 dup_rows=23994 dup_rows_remote=21008 dedup_ratio=0.244080 router_tokens_per_expert=[758, 794, 810, 756, 737, 755, 725, 759, 735, 768, 738, 840, 749, 763, 708, 790, 729, 777, 796, 788, 738, 817, 786, 775, 806, 786, 795, 755, 771, 805, 778, 792, 824, 746, 762, 778, 780, 733, 793, 724, 692, 797, 798, 760, 779, 771, 730, 780, 761, 779, 758, 795, 748, 734, 783, 763, 794, 726, 792, 783, 762, 758, 746, 744] payload_rows_per_dst=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] dup_rows_per_dst=[2914, 2950, 3098, 3178, 2986, 3010, 2910, 2948] combine_rows_per_src=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343464 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265805224 ingress_bytes=369270784 link_bytes=369270784 token_bytes=531266984 weight_movement_bytes=103809024 total_nvlink_bytes=635076008 weight_to_token_ratio=0.195399 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389536231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64932 dup_rows=23982 dup_rows_remote=20940 dedup_ratio=0.243958 router_tokens_per_expert=[749, 796, 780, 751, 784, 801, 771, 777, 732, 756, 748, 767, 763, 788, 825, 751, 760, 793, 780, 757, 737, 772, 782, 762, 799, 771, 748, 755, 787, 718, 749, 775, 746, 775, 771, 774, 818, 780, 755, 810, 769, 774, 750, 765, 759, 842, 801, 711, 770, 789, 784, 709, 772, 733, 731, 799, 758, 798, 757, 771, 766, 724, 801, 706] payload_rows_per_dst=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] dup_rows_per_dst=[3042, 2978, 3020, 2998, 3110, 2950, 2984, 2900] combine_rows_per_src=[9390, 9282, 9266, 9206, 9334, 9440, 9190, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65200 dup_rows=23756 dup_rows_remote=20952 dedup_ratio=0.241659 router_tokens_per_expert=[820, 796, 754, 784, 777, 737, 738, 756, 753, 824, 728, 753, 773, 762, 749, 779, 791, 755, 766, 762, 741, 773, 797, 747, 756, 817, 771, 775, 818, 815, 791, 765, 755, 776, 741, 757, 790, 789, 732, 752, 782, 784, 761, 769, 789, 768, 736, 767, 788, 764, 774, 772, 744, 719, 728, 787, 774, 786, 768, 770, 722, 761, 763, 761] payload_rows_per_dst=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] dup_rows_per_dst=[3012, 3044, 2954, 3110, 2924, 2994, 2804, 2914] combine_rows_per_src=[9312, 9198, 9310, 9506, 9260, 9318, 9348, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342312 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265705768 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531069224 weight_movement_bytes=34603008 total_nvlink_bytes=565672232 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64706 dup_rows=24258 dup_rows_remote=21134 dedup_ratio=0.246765 router_tokens_per_expert=[741, 768, 830, 789, 772, 786, 790, 775, 791, 758, 806, 780, 779, 758, 790, 770, 768, 799, 768, 767, 729, 770, 784, 771, 711, 756, 779, 823, 772, 772, 762, 805, 776, 767, 814, 761, 722, 799, 737, 759, 749, 759, 707, 761, 793, 769, 760, 736, 759, 793, 755, 809, 807, 759, 723, 769, 732, 716, 753, 752, 708, 744, 775, 810] payload_rows_per_dst=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] dup_rows_per_dst=[3070, 3124, 2994, 3084, 2972, 2960, 3032, 3022] combine_rows_per_src=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64918 dup_rows=24106 dup_rows_remote=21180 dedup_ratio=0.245219 router_tokens_per_expert=[795, 780, 804, 713, 853, 732, 702, 731, 766, 826, 780, 780, 732, 722, 755, 782, 773, 742, 784, 749, 800, 757, 747, 765, 769, 779, 804, 768, 738, 810, 722, 745, 779, 765, 792, 772, 785, 749, 782, 737, 743, 776, 729, 744, 778, 784, 776, 790, 787, 788, 735, 730, 801, 787, 762, 713, 756, 805, 828, 740, 801, 821, 727, 785] payload_rows_per_dst=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] dup_rows_per_dst=[2966, 2944, 2978, 3048, 3064, 2990, 2926, 3190] combine_rows_per_src=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64726 dup_rows=24204 dup_rows_remote=21164 dedup_ratio=0.246216 router_tokens_per_expert=[774, 770, 813, 735, 795, 746, 789, 785, 750, 753, 760, 775, 771, 762, 750, 748, 784, 745, 739, 764, 751, 802, 762, 767, 784, 754, 773, 769, 745, 744, 810, 757, 751, 744, 777, 738, 747, 760, 776, 765, 810, 780, 827, 766, 761, 746, 817, 794, 813, 756, 760, 761, 712, 767, 761, 759, 768, 750, 774, 776, 791, 777, 774, 768] payload_rows_per_dst=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] dup_rows_per_dst=[3040, 2962, 3160, 2910, 3132, 2908, 3054, 3038] combine_rows_per_src=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65016 dup_rows=23994 dup_rows_remote=21008 dedup_ratio=0.244080 router_tokens_per_expert=[758, 794, 810, 756, 737, 755, 725, 759, 735, 768, 738, 840, 749, 763, 708, 790, 729, 777, 796, 788, 738, 817, 786, 775, 806, 786, 795, 755, 771, 805, 778, 792, 824, 746, 762, 778, 780, 733, 793, 724, 692, 797, 798, 760, 779, 771, 730, 780, 761, 779, 758, 795, 748, 734, 783, 763, 794, 726, 792, 783, 762, 758, 746, 744] payload_rows_per_dst=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] dup_rows_per_dst=[2914, 2950, 3098, 3178, 2986, 3010, 2910, 2948] combine_rows_per_src=[9274, 9232, 9314, 9398, 9294, 9204, 9332, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65086 dup_rows=23884 dup_rows_remote=20794 dedup_ratio=0.242961 router_tokens_per_expert=[763, 808, 823, 745, 755, 803, 825, 775, 701, 705, 763, 745, 737, 744, 754, 789, 743, 780, 830, 778, 756, 753, 790, 782, 795, 703, 804, 776, 744, 807, 720, 768, 761, 787, 748, 759, 812, 764, 760, 761, 756, 744, 812, 726, 804, 770, 778, 730, 700, 771, 741, 785, 752, 743, 738, 796, 784, 775, 805, 753, 802, 798, 795, 778] payload_rows_per_dst=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] dup_rows_per_dst=[3044, 2848, 3090, 2858, 3010, 3018, 2962, 3054] combine_rows_per_src=[9550, 9028, 9334, 9376, 9294, 9222, 9090, 9526] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344096 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649632 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955168 weight_movement_bytes=34603008 total_nvlink_bytes=567558176 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64810 dup_rows=24048 dup_rows_remote=21056 dedup_ratio=0.244629 router_tokens_per_expert=[776, 776, 731, 763, 743, 799, 766, 743, 810, 795, 781, 753, 718, 794, 782, 761, 786, 752, 769, 771, 768, 724, 770, 755, 755, 755, 812, 806, 758, 768, 763, 802, 733, 794, 766, 770, 759, 748, 781, 787, 781, 750, 776, 751, 785, 764, 816, 734, 841, 761, 784, 805, 787, 783, 770, 765, 739, 784, 739, 733, 730, 776, 720, 735] payload_rows_per_dst=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] dup_rows_per_dst=[2940, 3042, 3024, 2992, 3024, 2994, 3096, 2936] combine_rows_per_src=[9254, 9346, 9166, 9446, 9252, 9320, 9496, 8976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64960 dup_rows=24024 dup_rows_remote=20988 dedup_ratio=0.244385 router_tokens_per_expert=[753, 732, 746, 792, 753, 748, 767, 797, 831, 778, 799, 767, 762, 738, 754, 746, 799, 754, 781, 820, 807, 765, 742, 719, 766, 749, 771, 781, 768, 745, 791, 760, 762, 774, 679, 733, 744, 763, 803, 775, 803, 801, 750, 734, 759, 754, 781, 796, 812, 782, 823, 781, 768, 791, 751, 756, 722, 739, 746, 792, 809, 753, 786, 749] payload_rows_per_dst=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] dup_rows_per_dst=[2986, 3042, 3038, 3010, 2870, 3036, 3094, 2948] combine_rows_per_src=[9190, 9308, 9336, 9252, 9196, 9320, 9434, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343488 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266304960 ingress_bytes=369770496 link_bytes=369770496 token_bytes=532266432 weight_movement_bytes=103809024 total_nvlink_bytes=636075456 weight_to_token_ratio=0.195032 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391416607e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64706 dup_rows=24258 dup_rows_remote=21134 dedup_ratio=0.246765 router_tokens_per_expert=[741, 768, 830, 789, 772, 786, 790, 775, 791, 758, 806, 780, 779, 758, 790, 770, 768, 799, 768, 767, 729, 770, 784, 771, 711, 756, 779, 823, 772, 772, 762, 805, 776, 767, 814, 761, 722, 799, 737, 759, 749, 759, 707, 761, 793, 769, 760, 736, 759, 793, 755, 809, 807, 759, 723, 769, 732, 716, 753, 752, 708, 744, 775, 810] payload_rows_per_dst=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] dup_rows_per_dst=[3070, 3124, 2994, 3084, 2972, 2960, 3032, 3022] combine_rows_per_src=[9432, 9340, 9318, 9276, 9298, 9108, 9316, 8958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64918 dup_rows=23896 dup_rows_remote=20820 dedup_ratio=0.243083 router_tokens_per_expert=[761, 818, 778, 794, 796, 737, 777, 781, 732, 811, 747, 781, 746, 725, 755, 772, 772, 795, 762, 754, 806, 757, 769, 784, 788, 760, 798, 779, 729, 761, 756, 779, 751, 785, 830, 762, 740, 798, 740, 749, 785, 795, 805, 805, 773, 772, 756, 792, 760, 714, 791, 756, 758, 734, 695, 793, 727, 777, 789, 762, 732, 727, 778, 761] payload_rows_per_dst=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] dup_rows_per_dst=[3090, 2920, 3070, 2990, 2966, 3076, 2906, 2878] combine_rows_per_src=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344608 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267403808 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463008 weight_movement_bytes=34603008 total_nvlink_bytes=569066016 weight_to_token_ratio=0.064744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=343360 combine_payload_bytes=265035776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265379136 ingress_bytes=334241792 link_bytes=334241792 token_bytes=530414912 weight_movement_bytes=69206016 total_nvlink_bytes=599620928 weight_to_token_ratio=0.130475 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257724955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344392 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248520 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152648 weight_movement_bytes=69206016 total_nvlink_bytes=601358664 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65118 dup_rows=23966 dup_rows_remote=21008 dedup_ratio=0.243795 router_tokens_per_expert=[787, 799, 782, 735, 770, 745, 768, 809, 734, 796, 757, 771, 743, 792, 763, 777, 795, 766, 748, 745, 725, 821, 767, 777, 748, 766, 766, 742, 767, 763, 726, 811, 727, 758, 791, 786, 767, 787, 785, 788, 806, 768, 769, 773, 773, 767, 759, 757, 747, 766, 751, 799, 767, 776, 817, 779, 783, 735, 747, 760, 771, 778, 707, 747] payload_rows_per_dst=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] dup_rows_per_dst=[3146, 2936, 2976, 2958, 3080, 2982, 2978, 2910] combine_rows_per_src=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343560 combine_payload_bytes=265117696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265461256 ingress_bytes=265117696 link_bytes=265461256 token_bytes=530578952 weight_movement_bytes=0 total_nvlink_bytes=530578952 weight_to_token_ratio=0.000000 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.989093349e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344096 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649632 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955168 weight_movement_bytes=34603008 total_nvlink_bytes=567558176 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343520 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266935776 ingress_bytes=370401280 link_bytes=370401280 token_bytes=533528032 weight_movement_bytes=103809024 total_nvlink_bytes=637337056 weight_to_token_ratio=0.194571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393790197e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343464 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265805224 ingress_bytes=369270784 link_bytes=369270784 token_bytes=531266984 weight_movement_bytes=103809024 total_nvlink_bytes=635076008 weight_to_token_ratio=0.195399 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389536231e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343792 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419952 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496112 weight_movement_bytes=34603008 total_nvlink_bytes=567099120 weight_to_token_ratio=0.064983 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=343360 combine_payload_bytes=265035776 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265379136 ingress_bytes=334241792 link_bytes=334241792 token_bytes=530414912 weight_movement_bytes=69206016 total_nvlink_bytes=599620928 weight_to_token_ratio=0.130475 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257724955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73996 payload_rows_remote=64426 dup_rows=24308 dup_rows_remote=20874 dedup_ratio=0.247274 router_tokens_per_expert=[749, 780, 793, 787, 760, 774, 776, 778, 765, 783, 733, 794, 745, 772, 774, 779, 764, 747, 766, 780, 759, 753, 790, 732, 758, 743, 788, 752, 766, 798, 774, 747, 788, 763, 792, 752, 768, 778, 789, 794, 805, 748, 767, 766, 741, 790, 755, 773, 749, 764, 758, 742, 795, 767, 839, 715, 755, 749, 752, 764, 827, 730, 761, 757] payload_rows_per_dst=[8812, 9206, 9262, 9302, 9010, 9346, 9488, 9570] dup_rows_per_dst=[2562, 3084, 2920, 3042, 2994, 2944, 3328, 3434] combine_rows_per_src=[8812, 9206, 9262, 9302, 9010, 9346, 9488, 9570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64748 dup_rows=24148 dup_rows_remote=21168 dedup_ratio=0.245646 router_tokens_per_expert=[789, 794, 762, 799, 794, 807, 744, 786, 796, 725, 777, 770, 797, 730, 770, 738, 748, 790, 826, 751, 755, 756, 727, 746, 855, 768, 787, 815, 694, 743, 798, 770, 762, 781, 765, 778, 796, 775, 743, 794, 811, 706, 735, 766, 803, 747, 784, 704, 750, 735, 755, 726, 823, 761, 774, 766, 787, 799, 780, 734, 761, 776, 708, 760] payload_rows_per_dst=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] dup_rows_per_dst=[3166, 3002, 2982, 3078, 2980, 2952, 3018, 2970] combine_rows_per_src=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342952 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266247080 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151208 weight_movement_bytes=69206016 total_nvlink_bytes=601357224 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344504 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067832 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791160 weight_movement_bytes=34603008 total_nvlink_bytes=568394168 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=263888896 dispatch_meta_bytes=341200 combine_payload_bytes=263888896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264230096 ingress_bytes=298491904 link_bytes=298491904 token_bytes=528118992 weight_movement_bytes=34603008 total_nvlink_bytes=562722000 weight_to_token_ratio=0.065521 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123201004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73940 payload_rows_remote=64594 dup_rows=24364 dup_rows_remote=21242 dedup_ratio=0.247843 router_tokens_per_expert=[799, 734, 782, 777, 762, 783, 787, 791, 802, 809, 771, 773, 777, 758, 831, 800, 771, 776, 755, 731, 752, 772, 740, 743, 752, 765, 775, 772, 723, 764, 808, 816, 781, 709, 761, 805, 778, 760, 771, 763, 758, 745, 719, 773, 747, 769, 763, 778, 783, 771, 790, 765, 805, 816, 747, 764, 744, 760, 732, 797, 709, 798, 729, 711] payload_rows_per_dst=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] dup_rows_per_dst=[3122, 3400, 2928, 2970, 3070, 2920, 3090, 2864] combine_rows_per_src=[9346, 9672, 9152, 9092, 9314, 9056, 9212, 9096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343664 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265551472 ingress_bytes=369016832 link_bytes=369016832 token_bytes=530759280 weight_movement_bytes=103809024 total_nvlink_bytes=634568304 weight_to_token_ratio=0.195586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388580631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65026 dup_rows=24006 dup_rows_remote=21018 dedup_ratio=0.244202 router_tokens_per_expert=[761, 762, 744, 778, 784, 804, 829, 743, 746, 765, 780, 766, 776, 738, 788, 757, 816, 776, 765, 757, 733, 802, 778, 741, 772, 762, 731, 714, 792, 801, 750, 758, 763, 700, 762, 796, 772, 749, 832, 791, 726, 719, 774, 728, 712, 812, 788, 825, 809, 785, 735, 736, 787, 782, 746, 750, 767, 756, 789, 782, 756, 797, 794, 763] payload_rows_per_dst=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] dup_rows_per_dst=[3028, 2984, 3028, 2948, 3058, 2992, 2988, 2980] combine_rows_per_src=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65028 dup_rows=23894 dup_rows_remote=20972 dedup_ratio=0.243062 router_tokens_per_expert=[761, 733, 701, 797, 770, 767, 783, 750, 728, 781, 767, 759, 784, 758, 792, 783, 783, 787, 745, 784, 715, 732, 771, 751, 782, 782, 757, 793, 749, 777, 764, 757, 797, 771, 771, 790, 792, 757, 784, 746, 827, 762, 795, 771, 763, 797, 775, 770, 759, 767, 790, 753, 739, 770, 774, 782, 770, 767, 764, 791, 782, 748, 715, 770] payload_rows_per_dst=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] dup_rows_per_dst=[2920, 2922, 2886, 3048, 3040, 3116, 3000, 2962] combine_rows_per_src=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65118 dup_rows=23966 dup_rows_remote=21008 dedup_ratio=0.243795 router_tokens_per_expert=[787, 799, 782, 735, 770, 745, 768, 809, 734, 796, 757, 771, 743, 792, 763, 777, 795, 766, 748, 745, 725, 821, 767, 777, 748, 766, 766, 742, 767, 763, 726, 811, 727, 758, 791, 786, 767, 787, 785, 788, 806, 768, 769, 773, 773, 767, 759, 757, 747, 766, 751, 799, 767, 776, 817, 779, 783, 735, 747, 760, 771, 778, 707, 747] payload_rows_per_dst=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] dup_rows_per_dst=[3146, 2936, 2976, 2958, 3080, 2982, 2978, 2910] combine_rows_per_src=[9244, 9330, 9312, 9220, 9298, 9362, 9426, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64918 dup_rows=24106 dup_rows_remote=21180 dedup_ratio=0.245219 router_tokens_per_expert=[795, 780, 804, 713, 853, 732, 702, 731, 766, 826, 780, 780, 732, 722, 755, 782, 773, 742, 784, 749, 800, 757, 747, 765, 769, 779, 804, 768, 738, 810, 722, 745, 779, 765, 792, 772, 785, 749, 782, 737, 743, 776, 729, 744, 778, 784, 776, 790, 787, 788, 735, 730, 801, 787, 762, 713, 756, 805, 828, 740, 801, 821, 727, 785] payload_rows_per_dst=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] dup_rows_per_dst=[2966, 2944, 2978, 3048, 3064, 2990, 2926, 3190] combine_rows_per_src=[9254, 9342, 9256, 9222, 9258, 9250, 9280, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65116 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[729, 761, 765, 768, 787, 755, 808, 753, 799, 820, 801, 807, 730, 736, 739, 773, 820, 813, 775, 775, 760, 786, 776, 769, 783, 798, 747, 769, 732, 737, 736, 731, 731, 814, 778, 762, 807, 755, 777, 781, 764, 730, 755, 773, 734, 794, 778, 740, 758, 759, 773, 781, 762, 766, 765, 788, 728, 751, 770, 774, 772, 766, 731, 797] payload_rows_per_dst=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] dup_rows_per_dst=[3052, 2984, 3180, 2934, 2964, 2822, 3014, 2918] combine_rows_per_src=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64748 dup_rows=24148 dup_rows_remote=21168 dedup_ratio=0.245646 router_tokens_per_expert=[789, 794, 762, 799, 794, 807, 744, 786, 796, 725, 777, 770, 797, 730, 770, 738, 748, 790, 826, 751, 755, 756, 727, 746, 855, 768, 787, 815, 694, 743, 798, 770, 762, 781, 765, 778, 796, 775, 743, 794, 811, 706, 735, 766, 803, 747, 784, 704, 750, 735, 755, 726, 823, 761, 774, 766, 787, 799, 780, 734, 761, 776, 708, 760] payload_rows_per_dst=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] dup_rows_per_dst=[3166, 3002, 2982, 3078, 2980, 2952, 3018, 2970] combine_rows_per_src=[9384, 9204, 9216, 9382, 9408, 9160, 9162, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64788 dup_rows=24136 dup_rows_remote=21162 dedup_ratio=0.245524 router_tokens_per_expert=[760, 781, 796, 832, 809, 786, 753, 799, 765, 809, 704, 753, 795, 777, 750, 744, 772, 746, 751, 780, 782, 777, 785, 784, 783, 777, 743, 800, 771, 779, 780, 757, 786, 746, 766, 762, 775, 756, 823, 772, 760, 731, 773, 735, 793, 745, 723, 804, 747, 755, 767, 744, 756, 776, 748, 777, 681, 721, 761, 787, 776, 769, 777, 780] payload_rows_per_dst=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] dup_rows_per_dst=[3232, 2952, 2974, 3068, 3046, 2900, 2944, 3020] combine_rows_per_src=[9400, 9242, 9380, 9312, 9326, 9228, 9196, 9084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64918 dup_rows=23896 dup_rows_remote=20820 dedup_ratio=0.243083 router_tokens_per_expert=[761, 818, 778, 794, 796, 737, 777, 781, 732, 811, 747, 781, 746, 725, 755, 772, 772, 795, 762, 754, 806, 757, 769, 784, 788, 760, 798, 779, 729, 761, 756, 779, 751, 785, 830, 762, 740, 798, 740, 749, 785, 795, 805, 805, 773, 772, 756, 792, 760, 714, 791, 756, 758, 734, 695, 793, 727, 777, 789, 762, 732, 727, 778, 761] payload_rows_per_dst=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] dup_rows_per_dst=[3090, 2920, 3070, 2990, 2966, 3076, 2906, 2878] combine_rows_per_src=[9394, 9218, 9328, 9310, 9344, 9490, 9096, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65028 dup_rows=23894 dup_rows_remote=20972 dedup_ratio=0.243062 router_tokens_per_expert=[761, 733, 701, 797, 770, 767, 783, 750, 728, 781, 767, 759, 784, 758, 792, 783, 783, 787, 745, 784, 715, 732, 771, 751, 782, 782, 757, 793, 749, 777, 764, 757, 797, 771, 771, 790, 792, 757, 784, 746, 827, 762, 795, 771, 763, 797, 775, 770, 759, 767, 790, 753, 739, 770, 774, 782, 770, 767, 764, 791, 782, 748, 715, 770] payload_rows_per_dst=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] dup_rows_per_dst=[2920, 2922, 2886, 3048, 3040, 3116, 3000, 2962] combine_rows_per_src=[9204, 9382, 9250, 9274, 9376, 9404, 9268, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65258 dup_rows=23796 dup_rows_remote=20846 dedup_ratio=0.242065 router_tokens_per_expert=[810, 768, 752, 753, 716, 814, 772, 763, 793, 785, 768, 792, 759, 790, 753, 802, 779, 721, 748, 757, 731, 808, 779, 754, 727, 736, 783, 755, 783, 766, 773, 749, 789, 728, 803, 751, 777, 778, 767, 777, 782, 756, 809, 761, 761, 762, 703, 766, 775, 772, 737, 789, 778, 782, 778, 754, 791, 748, 770, 808, 777, 753, 776, 755] payload_rows_per_dst=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] dup_rows_per_dst=[2954, 3066, 2944, 2956, 2954, 2950, 3014, 2958] combine_rows_per_src=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344392 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248520 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152648 weight_movement_bytes=69206016 total_nvlink_bytes=601358664 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=343344 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264920368 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529497392 weight_movement_bytes=34603008 total_nvlink_bytes=564100400 weight_to_token_ratio=0.065351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344176 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690672 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037168 weight_movement_bytes=34603008 total_nvlink_bytes=567640176 weight_to_token_ratio=0.064917 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344000 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698688 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053376 weight_movement_bytes=34603008 total_nvlink_bytes=567656384 weight_to_token_ratio=0.064915 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344504 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067832 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791160 weight_movement_bytes=34603008 total_nvlink_bytes=568394168 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=342952 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266247080 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151208 weight_movement_bytes=69206016 total_nvlink_bytes=601357224 weight_to_token_ratio=0.130050 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21076 dedup_ratio=0.245524 router_tokens_per_expert=[787, 742, 797, 818, 726, 775, 724, 766, 756, 834, 773, 751, 738, 758, 766, 737, 772, 714, 742, 777, 765, 760, 779, 759, 797, 747, 795, 765, 762, 808, 738, 759, 736, 773, 779, 758, 773, 823, 762, 749, 790, 757, 791, 764, 768, 789, 801, 785, 795, 755, 720, 711, 739, 789, 740, 819, 755, 775, 776, 802, 735, 771, 785, 800] payload_rows_per_dst=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] dup_rows_per_dst=[3006, 3020, 2878, 3060, 3092, 3110, 2906, 3064] combine_rows_per_src=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343728 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058864 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774000 weight_movement_bytes=34603008 total_nvlink_bytes=568377008 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343664 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265551472 ingress_bytes=369016832 link_bytes=369016832 token_bytes=530759280 weight_movement_bytes=103809024 total_nvlink_bytes=634568304 weight_to_token_ratio=0.195586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388580631e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=343800 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265715448 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087096 weight_movement_bytes=34603008 total_nvlink_bytes=565690104 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65104 dup_rows=23946 dup_rows_remote=21016 dedup_ratio=0.243591 router_tokens_per_expert=[774, 812, 778, 762, 765, 793, 770, 820, 775, 754, 737, 768, 807, 763, 790, 787, 780, 799, 793, 808, 771, 773, 732, 744, 770, 778, 776, 726, 749, 734, 813, 773, 727, 770, 815, 769, 744, 753, 802, 766, 755, 741, 775, 744, 748, 751, 734, 783, 777, 748, 807, 749, 718, 844, 732, 757, 749, 789, 750, 785, 741, 754, 712, 789] payload_rows_per_dst=[9522, 9260, 9422, 9190, 9216, 9240, 9254, 9254] dup_rows_per_dst=[3026, 3102, 2978, 3038, 3040, 2822, 3010, 2930] combine_rows_per_src=[9522, 9260, 9422, 9190, 9216, 9240, 9254, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64796 dup_rows=24010 dup_rows_remote=20938 dedup_ratio=0.244242 router_tokens_per_expert=[758, 780, 755, 731, 802, 738, 748, 769, 775, 753, 803, 736, 771, 773, 781, 736, 758, 714, 716, 803, 764, 786, 794, 766, 799, 784, 767, 731, 763, 793, 734, 789, 809, 763, 799, 777, 763, 782, 799, 793, 717, 789, 747, 794, 755, 744, 788, 751, 759, 739, 761, 834, 727, 763, 769, 835, 762, 799, 799, 785, 780, 744, 717, 739] payload_rows_per_dst=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] dup_rows_per_dst=[2944, 2976, 2910, 3030, 3072, 2954, 3112, 3012] combine_rows_per_src=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344000 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698688 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053376 weight_movement_bytes=34603008 total_nvlink_bytes=567656384 weight_to_token_ratio=0.064915 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344416 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267641184 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937952 weight_movement_bytes=34603008 total_nvlink_bytes=569540960 weight_to_token_ratio=0.064686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343848 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266116904 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889960 weight_movement_bytes=138412032 total_nvlink_bytes=670301992 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010464 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676448 weight_movement_bytes=69206016 total_nvlink_bytes=602882464 weight_to_token_ratio=0.129678 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=342936 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265747352 ingress_bytes=334610432 link_bytes=334610432 token_bytes=531151768 weight_movement_bytes=69206016 total_nvlink_bytes=600357784 weight_to_token_ratio=0.130294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259112117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64638 dup_rows=24056 dup_rows_remote=20644 dedup_ratio=0.244710 router_tokens_per_expert=[768, 717, 737, 781, 778, 711, 841, 797, 809, 783, 762, 768, 788, 758, 756, 785, 789, 750, 750, 739, 758, 723, 781, 747, 739, 814, 739, 790, 781, 777, 778, 771, 731, 809, 788, 761, 721, 738, 803, 742, 766, 783, 776, 754, 784, 781, 755, 751, 789, 758, 753, 751, 791, 786, 757, 747, 772, 773, 742, 789, 794, 762, 804, 776] payload_rows_per_dst=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] dup_rows_per_dst=[3412, 2700, 3432, 2622, 3032, 2970, 2990, 2898] combine_rows_per_src=[9610, 8960, 9774, 8952, 9274, 9254, 9270, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65136 dup_rows=23824 dup_rows_remote=20802 dedup_ratio=0.242350 router_tokens_per_expert=[751, 717, 739, 779, 760, 819, 765, 737, 762, 773, 773, 748, 740, 783, 787, 817, 761, 722, 756, 818, 802, 749, 760, 757, 754, 806, 759, 777, 762, 761, 797, 775, 761, 756, 804, 736, 743, 742, 809, 753, 742, 740, 792, 803, 766, 808, 770, 760, 755, 803, 739, 758, 765, 720, 816, 747, 779, 772, 748, 807, 741, 782, 782, 787] payload_rows_per_dst=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] dup_rows_per_dst=[2926, 3022, 2920, 3066, 2884, 2958, 2974, 3074] combine_rows_per_src=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64974 dup_rows=24110 dup_rows_remote=21048 dedup_ratio=0.245260 router_tokens_per_expert=[746, 786, 742, 711, 731, 735, 743, 762, 799, 786, 802, 743, 754, 749, 760, 741, 780, 734, 776, 768, 825, 749, 729, 759, 755, 808, 752, 796, 773, 782, 786, 813, 770, 767, 768, 793, 769, 737, 733, 759, 800, 761, 790, 762, 807, 814, 745, 819, 758, 762, 801, 823, 746, 713, 759, 779, 753, 786, 783, 784, 741, 761, 764, 770] payload_rows_per_dst=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] dup_rows_per_dst=[2856, 3066, 2978, 3090, 2934, 3136, 3062, 2988] combine_rows_per_src=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65258 dup_rows=23796 dup_rows_remote=20846 dedup_ratio=0.242065 router_tokens_per_expert=[810, 768, 752, 753, 716, 814, 772, 763, 793, 785, 768, 792, 759, 790, 753, 802, 779, 721, 748, 757, 731, 808, 779, 754, 727, 736, 783, 755, 783, 766, 773, 749, 789, 728, 803, 751, 777, 778, 767, 777, 782, 756, 809, 761, 761, 762, 703, 766, 775, 772, 737, 789, 778, 782, 778, 754, 791, 748, 770, 808, 777, 753, 776, 755] payload_rows_per_dst=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] dup_rows_per_dst=[2954, 3066, 2944, 2956, 2954, 2950, 3014, 2958] combine_rows_per_src=[9342, 9418, 9210, 9188, 9386, 9250, 9316, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21076 dedup_ratio=0.245524 router_tokens_per_expert=[787, 742, 797, 818, 726, 775, 724, 766, 756, 834, 773, 751, 738, 758, 766, 737, 772, 714, 742, 777, 765, 760, 779, 759, 797, 747, 795, 765, 762, 808, 738, 759, 736, 773, 779, 758, 773, 823, 762, 749, 790, 757, 791, 764, 768, 789, 801, 785, 795, 755, 720, 711, 739, 789, 740, 819, 755, 775, 776, 802, 735, 771, 785, 800] payload_rows_per_dst=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] dup_rows_per_dst=[3006, 3020, 2878, 3060, 3092, 3110, 2906, 3064] combine_rows_per_src=[9264, 9206, 9258, 9282, 9214, 9380, 9230, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65026 dup_rows=24006 dup_rows_remote=21018 dedup_ratio=0.244202 router_tokens_per_expert=[761, 762, 744, 778, 784, 804, 829, 743, 746, 765, 780, 766, 776, 738, 788, 757, 816, 776, 765, 757, 733, 802, 778, 741, 772, 762, 731, 714, 792, 801, 750, 758, 763, 700, 762, 796, 772, 749, 832, 791, 726, 719, 774, 728, 712, 812, 788, 825, 809, 785, 735, 736, 787, 782, 746, 750, 767, 756, 789, 782, 756, 797, 794, 763] payload_rows_per_dst=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] dup_rows_per_dst=[3028, 2984, 3028, 2948, 3058, 2992, 2988, 2980] combine_rows_per_src=[9382, 9248, 9308, 9212, 9272, 9176, 9272, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65036 dup_rows=24032 dup_rows_remote=21034 dedup_ratio=0.244466 router_tokens_per_expert=[739, 734, 783, 744, 784, 780, 776, 724, 763, 776, 724, 798, 801, 813, 772, 754, 764, 751, 791, 722, 774, 785, 780, 750, 797, 813, 810, 766, 789, 773, 733, 774, 761, 743, 783, 783, 704, 766, 771, 752, 756, 742, 832, 793, 755, 720, 746, 777, 743, 753, 783, 758, 783, 809, 800, 749, 746, 809, 758, 842, 751, 759, 768, 720] payload_rows_per_dst=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] dup_rows_per_dst=[2924, 3026, 2998, 3160, 2936, 2950, 3084, 2954] combine_rows_per_src=[9204, 9376, 9236, 9350, 9190, 9292, 9272, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64840 dup_rows=24192 dup_rows_remote=21124 dedup_ratio=0.246094 router_tokens_per_expert=[811, 732, 726, 732, 786, 800, 761, 812, 745, 787, 771, 823, 781, 749, 764, 714, 809, 779, 749, 757, 753, 766, 740, 740, 717, 779, 815, 777, 788, 704, 783, 717, 775, 787, 782, 753, 788, 774, 802, 795, 803, 748, 782, 791, 728, 791, 766, 802, 756, 742, 808, 773, 729, 752, 763, 794, 751, 760, 797, 804, 734, 761, 741, 753] payload_rows_per_dst=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] dup_rows_per_dst=[3068, 3162, 3130, 2930, 2976, 2962, 2962, 3002] combine_rows_per_src=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64796 dup_rows=24010 dup_rows_remote=20938 dedup_ratio=0.244242 router_tokens_per_expert=[758, 780, 755, 731, 802, 738, 748, 769, 775, 753, 803, 736, 771, 773, 781, 736, 758, 714, 716, 803, 764, 786, 794, 766, 799, 784, 767, 731, 763, 793, 734, 789, 809, 763, 799, 777, 763, 782, 799, 793, 717, 789, 747, 794, 755, 744, 788, 751, 759, 739, 761, 834, 727, 763, 769, 835, 762, 799, 799, 785, 780, 744, 717, 739] payload_rows_per_dst=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] dup_rows_per_dst=[2944, 2976, 2910, 3030, 3072, 2954, 3112, 3012] combine_rows_per_src=[9218, 9280, 9292, 9290, 9498, 9216, 9262, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65136 dup_rows=23824 dup_rows_remote=20802 dedup_ratio=0.242350 router_tokens_per_expert=[751, 717, 739, 779, 760, 819, 765, 737, 762, 773, 773, 748, 740, 783, 787, 817, 761, 722, 756, 818, 802, 749, 760, 757, 754, 806, 759, 777, 762, 761, 797, 775, 761, 756, 804, 736, 743, 742, 809, 753, 742, 740, 792, 803, 766, 808, 770, 760, 755, 803, 739, 758, 765, 720, 816, 747, 779, 772, 748, 807, 741, 782, 782, 787] payload_rows_per_dst=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] dup_rows_per_dst=[2926, 3022, 2920, 3066, 2884, 2958, 2974, 3074] combine_rows_per_src=[9208, 9344, 9330, 9316, 9324, 9404, 9232, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64818 dup_rows=24176 dup_rows_remote=21108 dedup_ratio=0.245931 router_tokens_per_expert=[792, 732, 776, 766, 764, 773, 772, 737, 769, 763, 796, 794, 734, 770, 755, 772, 792, 783, 778, 732, 777, 786, 783, 755, 767, 802, 732, 784, 736, 752, 801, 774, 783, 769, 796, 763, 727, 765, 790, 759, 791, 744, 800, 812, 760, 787, 744, 751, 768, 795, 719, 809, 786, 759, 806, 778, 737, 731, 781, 722, 740, 747, 778, 756] payload_rows_per_dst=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] dup_rows_per_dst=[2986, 3000, 3040, 3004, 3118, 3068, 3126, 2834] combine_rows_per_src=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264757248 dispatch_meta_bytes=341128 combine_payload_bytes=264757248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265098376 ingress_bytes=299360256 link_bytes=299360256 token_bytes=529855624 weight_movement_bytes=34603008 total_nvlink_bytes=564458632 weight_to_token_ratio=0.065306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126468543e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343752 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267140808 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533937864 weight_movement_bytes=69206016 total_nvlink_bytes=603143880 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344416 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267641184 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937952 weight_movement_bytes=34603008 total_nvlink_bytes=569540960 weight_to_token_ratio=0.064686 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344088 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477592 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611096 weight_movement_bytes=34603008 total_nvlink_bytes=567214104 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343848 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266116904 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889960 weight_movement_bytes=138412032 total_nvlink_bytes=670301992 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64998 dup_rows=24016 dup_rows_remote=21026 dedup_ratio=0.244303 router_tokens_per_expert=[763, 746, 766, 785, 745, 784, 747, 742, 836, 763, 799, 739, 775, 723, 793, 793, 771, 778, 754, 762, 785, 791, 796, 816, 781, 799, 787, 734, 796, 749, 745, 749, 753, 747, 765, 704, 783, 763, 733, 765, 778, 763, 723, 791, 786, 761, 744, 786, 769, 794, 754, 754, 767, 798, 787, 788, 721, 776, 705, 750, 784, 796, 760, 812] payload_rows_per_dst=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] dup_rows_per_dst=[3024, 3060, 3086, 2990, 2956, 2936, 3046, 2918] combine_rows_per_src=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344176 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690672 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037168 weight_movement_bytes=34603008 total_nvlink_bytes=567640176 weight_to_token_ratio=0.064917 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=342936 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265747352 ingress_bytes=334610432 link_bytes=334610432 token_bytes=531151768 weight_movement_bytes=69206016 total_nvlink_bytes=600357784 weight_to_token_ratio=0.130294 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259112117e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344280 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731736 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119192 weight_movement_bytes=34603008 total_nvlink_bytes=567722200 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64588 dup_rows=24128 dup_rows_remote=20816 dedup_ratio=0.245443 router_tokens_per_expert=[743, 808, 791, 790, 776, 791, 770, 757, 726, 818, 746, 716, 748, 755, 751, 772, 757, 768, 730, 772, 778, 771, 826, 789, 791, 768, 762, 759, 775, 759, 777, 757, 782, 810, 747, 745, 764, 785, 708, 774, 763, 763, 760, 775, 790, 782, 767, 798, 780, 748, 744, 754, 769, 802, 770, 766, 796, 751, 763, 785, 767, 762, 718, 767] payload_rows_per_dst=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] dup_rows_per_dst=[3090, 2800, 2720, 3048, 3108, 3096, 2954, 3312] combine_rows_per_src=[9362, 9264, 8980, 9248, 9122, 9300, 9312, 9588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65208 dup_rows=23854 dup_rows_remote=20904 dedup_ratio=0.242655 router_tokens_per_expert=[785, 731, 775, 780, 714, 803, 757, 803, 719, 771, 771, 790, 711, 781, 779, 748, 698, 790, 790, 734, 791, 770, 776, 714, 781, 770, 747, 772, 782, 749, 783, 831, 754, 776, 777, 781, 729, 747, 758, 774, 781, 786, 761, 789, 728, 786, 788, 804, 783, 772, 773, 782, 777, 771, 799, 760, 766, 778, 782, 761, 782, 808, 688, 755] payload_rows_per_dst=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] dup_rows_per_dst=[2920, 2872, 2956, 3056, 2950, 3050, 3088, 2962] combine_rows_per_src=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343856 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265928496 ingress_bytes=403996672 link_bytes=403996672 token_bytes=531513136 weight_movement_bytes=138412032 total_nvlink_bytes=669925168 weight_to_token_ratio=0.260411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520206952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343752 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267140808 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533937864 weight_movement_bytes=69206016 total_nvlink_bytes=603143880 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343704 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265838232 ingress_bytes=369303552 link_bytes=369303552 token_bytes=531332760 weight_movement_bytes=103809024 total_nvlink_bytes=635141784 weight_to_token_ratio=0.195375 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389659535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344096 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575904 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807712 weight_movement_bytes=34603008 total_nvlink_bytes=567410720 weight_to_token_ratio=0.064945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341616 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264894064 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446512 weight_movement_bytes=34603008 total_nvlink_bytes=564049520 weight_to_token_ratio=0.065357 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65160 dup_rows=23950 dup_rows_remote=21104 dedup_ratio=0.243632 router_tokens_per_expert=[766, 757, 742, 769, 823, 724, 740, 812, 803, 709, 790, 737, 771, 718, 727, 765, 750, 810, 803, 778, 784, 738, 748, 796, 748, 758, 810, 808, 791, 741, 767, 725, 736, 756, 747, 747, 764, 841, 780, 753, 821, 800, 754, 761, 768, 769, 766, 785, 764, 802, 781, 828, 735, 762, 763, 774, 759, 790, 744, 761, 776, 782, 718, 757] payload_rows_per_dst=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] dup_rows_per_dst=[3028, 2846, 3080, 3020, 2968, 3050, 3008, 2950] combine_rows_per_src=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64726 dup_rows=24204 dup_rows_remote=21164 dedup_ratio=0.246216 router_tokens_per_expert=[774, 770, 813, 735, 795, 746, 789, 785, 750, 753, 760, 775, 771, 762, 750, 748, 784, 745, 739, 764, 751, 802, 762, 767, 784, 754, 773, 769, 745, 744, 810, 757, 751, 744, 777, 738, 747, 760, 776, 765, 810, 780, 827, 766, 761, 746, 817, 794, 813, 756, 760, 761, 712, 767, 761, 759, 768, 750, 774, 776, 791, 777, 774, 768] payload_rows_per_dst=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] dup_rows_per_dst=[3040, 2962, 3160, 2910, 3132, 2908, 3054, 3038] combine_rows_per_src=[9374, 9342, 9394, 9078, 9344, 9126, 9124, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64818 dup_rows=24176 dup_rows_remote=21108 dedup_ratio=0.245931 router_tokens_per_expert=[792, 732, 776, 766, 764, 773, 772, 737, 769, 763, 796, 794, 734, 770, 755, 772, 792, 783, 778, 732, 777, 786, 783, 755, 767, 802, 732, 784, 736, 752, 801, 774, 783, 769, 796, 763, 727, 765, 790, 759, 791, 744, 800, 812, 760, 787, 744, 751, 768, 795, 719, 809, 786, 759, 806, 778, 737, 731, 781, 722, 740, 747, 778, 756] payload_rows_per_dst=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] dup_rows_per_dst=[2986, 3000, 3040, 3004, 3118, 3068, 3126, 2834] combine_rows_per_src=[9238, 9306, 9332, 9292, 9186, 9310, 9314, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64918 dup_rows=24008 dup_rows_remote=20996 dedup_ratio=0.244222 router_tokens_per_expert=[749, 756, 746, 736, 799, 757, 743, 767, 774, 842, 743, 757, 755, 770, 799, 781, 763, 798, 744, 803, 758, 844, 787, 784, 816, 736, 751, 747, 776, 717, 727, 763, 781, 766, 764, 765, 741, 746, 721, 785, 774, 796, 778, 785, 765, 777, 766, 776, 804, 745, 795, 774, 789, 740, 781, 767, 747, 773, 747, 749, 751, 769, 751, 796] payload_rows_per_dst=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] dup_rows_per_dst=[2930, 3162, 3130, 2852, 2968, 3050, 3012, 2904] combine_rows_per_src=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64998 dup_rows=24016 dup_rows_remote=21026 dedup_ratio=0.244303 router_tokens_per_expert=[763, 746, 766, 785, 745, 784, 747, 742, 836, 763, 799, 739, 775, 723, 793, 793, 771, 778, 754, 762, 785, 791, 796, 816, 781, 799, 787, 734, 796, 749, 745, 749, 753, 747, 765, 704, 783, 763, 733, 765, 778, 763, 723, 791, 786, 761, 744, 786, 769, 794, 754, 754, 767, 798, 787, 788, 721, 776, 705, 750, 784, 796, 760, 812] payload_rows_per_dst=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] dup_rows_per_dst=[3024, 3060, 3086, 2990, 2956, 2936, 3046, 2918] combine_rows_per_src=[9132, 9382, 9420, 9290, 9070, 9328, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64974 dup_rows=24110 dup_rows_remote=21048 dedup_ratio=0.245260 router_tokens_per_expert=[746, 786, 742, 711, 731, 735, 743, 762, 799, 786, 802, 743, 754, 749, 760, 741, 780, 734, 776, 768, 825, 749, 729, 759, 755, 808, 752, 796, 773, 782, 786, 813, 770, 767, 768, 793, 769, 737, 733, 759, 800, 761, 790, 762, 807, 814, 745, 819, 758, 762, 801, 823, 746, 713, 759, 779, 753, 786, 783, 784, 741, 761, 764, 770] payload_rows_per_dst=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] dup_rows_per_dst=[2856, 3066, 2978, 3090, 2934, 3136, 3062, 2988] combine_rows_per_src=[9056, 9202, 9262, 9440, 9258, 9460, 9220, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65208 dup_rows=23854 dup_rows_remote=20904 dedup_ratio=0.242655 router_tokens_per_expert=[785, 731, 775, 780, 714, 803, 757, 803, 719, 771, 771, 790, 711, 781, 779, 748, 698, 790, 790, 734, 791, 770, 776, 714, 781, 770, 747, 772, 782, 749, 783, 831, 754, 776, 777, 781, 729, 747, 758, 774, 781, 786, 761, 789, 728, 786, 788, 804, 783, 772, 773, 782, 777, 771, 799, 760, 766, 778, 782, 761, 782, 808, 688, 755] payload_rows_per_dst=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] dup_rows_per_dst=[2920, 2872, 2956, 3056, 2950, 3050, 3088, 2962] combine_rows_per_src=[9376, 9268, 9170, 9374, 9242, 9396, 9346, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65068 dup_rows=24052 dup_rows_remote=20990 dedup_ratio=0.244670 router_tokens_per_expert=[759, 748, 771, 773, 762, 708, 791, 760, 774, 770, 782, 745, 782, 817, 735, 770, 727, 805, 783, 788, 791, 772, 713, 744, 809, 796, 748, 771, 770, 731, 779, 766, 775, 740, 756, 767, 740, 779, 750, 744, 768, 824, 813, 806, 782, 749, 779, 755, 747, 770, 778, 754, 813, 779, 783, 765, 724, 820, 779, 790, 749, 766, 744, 724] payload_rows_per_dst=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] dup_rows_per_dst=[2902, 2984, 3062, 2996, 2942, 3142, 2980, 3044] combine_rows_per_src=[9242, 9366, 9184, 9344, 9160, 9410, 9398, 9148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=345056 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267240416 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135776 weight_movement_bytes=34603008 total_nvlink_bytes=568738784 weight_to_token_ratio=0.064783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64546 dup_rows=24216 dup_rows_remote=20904 dedup_ratio=0.246338 router_tokens_per_expert=[776, 747, 779, 757, 773, 772, 756, 752, 783, 748, 813, 768, 759, 752, 764, 797, 822, 765, 765, 791, 804, 782, 729, 799, 757, 723, 808, 740, 743, 795, 764, 780, 784, 735, 794, 764, 778, 788, 738, 772, 710, 704, 766, 796, 808, 770, 750, 789, 712, 750, 773, 779, 783, 758, 760, 791, 796, 792, 776, 772, 728, 753, 747, 773] payload_rows_per_dst=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] dup_rows_per_dst=[3312, 2804, 3228, 2804, 3018, 3008, 3042, 3000] combine_rows_per_src=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65308 dup_rows=23780 dup_rows_remote=20958 dedup_ratio=0.241903 router_tokens_per_expert=[795, 774, 750, 767, 787, 810, 774, 759, 804, 743, 740, 838, 744, 795, 813, 778, 729, 781, 767, 794, 755, 805, 761, 779, 776, 790, 754, 791, 773, 757, 727, 728, 824, 776, 795, 789, 737, 721, 766, 789, 752, 723, 731, 789, 759, 739, 777, 749, 791, 710, 751, 760, 743, 748, 772, 751, 783, 775, 797, 762, 775, 738, 746, 796] payload_rows_per_dst=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] dup_rows_per_dst=[3054, 3150, 2914, 2940, 2976, 2822, 2886, 3038] combine_rows_per_src=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=64998 dup_rows=23876 dup_rows_remote=20798 dedup_ratio=0.242879 router_tokens_per_expert=[669, 778, 762, 809, 782, 779, 734, 776, 742, 803, 720, 781, 763, 769, 750, 770, 738, 773, 741, 815, 791, 781, 744, 794, 740, 799, 778, 751, 839, 758, 781, 808, 776, 725, 780, 783, 780, 762, 772, 711, 775, 734, 761, 733, 751, 772, 765, 771, 729, 767, 775, 728, 781, 765, 786, 805, 746, 803, 717, 809, 739, 783, 847, 803] payload_rows_per_dst=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] dup_rows_per_dst=[2958, 2902, 2936, 3078, 2986, 2932, 2994, 3090] combine_rows_per_src=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343704 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265838232 ingress_bytes=369303552 link_bytes=369303552 token_bytes=531332760 weight_movement_bytes=103809024 total_nvlink_bytes=635141784 weight_to_token_ratio=0.195375 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389659535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343560 combine_payload_bytes=265117696 prefetch_bytes=0 reduce_bytes=0 egress_bytes=265461256 ingress_bytes=265117696 link_bytes=265461256 token_bytes=530578952 weight_movement_bytes=0 total_nvlink_bytes=530578952 weight_to_token_ratio=0.000000 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=9.989093349e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344096 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575904 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807712 weight_movement_bytes=34603008 total_nvlink_bytes=567410720 weight_to_token_ratio=0.064945 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343656 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266247784 ingress_bytes=404316160 link_bytes=404316160 token_bytes=532151912 weight_movement_bytes=138412032 total_nvlink_bytes=670563944 weight_to_token_ratio=0.260099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521409160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65160 dup_rows=23950 dup_rows_remote=21104 dedup_ratio=0.243632 router_tokens_per_expert=[766, 757, 742, 769, 823, 724, 740, 812, 803, 709, 790, 737, 771, 718, 727, 765, 750, 810, 803, 778, 784, 738, 748, 796, 748, 758, 810, 808, 791, 741, 767, 725, 736, 756, 747, 747, 764, 841, 780, 753, 821, 800, 754, 761, 768, 769, 766, 785, 764, 802, 781, 828, 735, 762, 763, 774, 759, 790, 744, 761, 776, 782, 718, 757] payload_rows_per_dst=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] dup_rows_per_dst=[3028, 2846, 3080, 3020, 2968, 3050, 3008, 2950] combine_rows_per_src=[9238, 9194, 9334, 9276, 9280, 9398, 9410, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344088 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477592 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611096 weight_movement_bytes=34603008 total_nvlink_bytes=567214104 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21122 dedup_ratio=0.246379 router_tokens_per_expert=[751, 719, 754, 716, 792, 731, 778, 799, 821, 762, 752, 785, 769, 785, 787, 784, 776, 784, 756, 781, 815, 728, 816, 732, 769, 739, 760, 757, 734, 827, 809, 736, 799, 749, 754, 804, 785, 813, 713, 746, 790, 788, 719, 794, 775, 765, 731, 767, 755, 775, 757, 736, 769, 786, 778, 756, 777, 740, 781, 778, 758, 759, 820, 731] payload_rows_per_dst=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] dup_rows_per_dst=[2916, 2944, 3040, 2928, 3024, 3148, 3122, 3098] combine_rows_per_src=[9164, 9308, 9336, 9160, 9292, 9348, 9276, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344448 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436416 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528384 weight_movement_bytes=34603008 total_nvlink_bytes=569131392 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344232 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266862760 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381288 weight_movement_bytes=69206016 total_nvlink_bytes=602587304 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65120 dup_rows=23928 dup_rows_remote=20924 dedup_ratio=0.243408 router_tokens_per_expert=[840, 766, 754, 699, 777, 794, 825, 794, 783, 790, 786, 725, 748, 793, 750, 804, 734, 754, 717, 793, 769, 769, 766, 734, 760, 760, 771, 763, 784, 765, 771, 792, 762, 743, 707, 830, 737, 792, 791, 768, 763, 716, 787, 734, 774, 831, 794, 765, 766, 770, 798, 762, 743, 792, 762, 766, 796, 761, 731, 742, 726, 748, 793, 772] payload_rows_per_dst=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] dup_rows_per_dst=[3154, 2956, 2866, 3006, 3004, 3012, 2904, 3026] combine_rows_per_src=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267501568 dispatch_meta_bytes=345064 combine_payload_bytes=267501568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267846632 ingress_bytes=302104576 link_bytes=302104576 token_bytes=535348200 weight_movement_bytes=34603008 total_nvlink_bytes=569951208 weight_to_token_ratio=0.064636 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136795198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343184 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266574992 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806800 weight_movement_bytes=103809024 total_nvlink_bytes=636615824 weight_to_token_ratio=0.194834 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=341800 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264722216 ingress_bytes=298983424 link_bytes=298983424 token_bytes=529102632 weight_movement_bytes=34603008 total_nvlink_bytes=563705640 weight_to_token_ratio=0.065399 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125050554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=345056 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267240416 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135776 weight_movement_bytes=34603008 total_nvlink_bytes=568738784 weight_to_token_ratio=0.064783 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344024 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108888 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873752 weight_movement_bytes=69206016 total_nvlink_bytes=601079768 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65044 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[739, 737, 804, 804, 787, 785, 764, 776, 736, 744, 756, 765, 785, 736, 772, 786, 746, 780, 731, 747, 748, 742, 761, 800, 753, 747, 779, 758, 771, 737, 776, 772, 737, 784, 739, 804, 763, 751, 789, 794, 760, 782, 773, 783, 788, 766, 811, 770, 754, 801, 770, 793, 740, 758, 752, 788, 769, 824, 799, 771, 752, 773, 775, 715] payload_rows_per_dst=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] dup_rows_per_dst=[2986, 2972, 2822, 2988, 3002, 3238, 3060, 3004] combine_rows_per_src=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64918 dup_rows=24008 dup_rows_remote=20996 dedup_ratio=0.244222 router_tokens_per_expert=[749, 756, 746, 736, 799, 757, 743, 767, 774, 842, 743, 757, 755, 770, 799, 781, 763, 798, 744, 803, 758, 844, 787, 784, 816, 736, 751, 747, 776, 717, 727, 763, 781, 766, 764, 765, 741, 746, 721, 785, 774, 796, 778, 785, 765, 777, 766, 776, 804, 745, 795, 774, 789, 740, 781, 767, 747, 773, 747, 749, 751, 769, 751, 796] payload_rows_per_dst=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] dup_rows_per_dst=[2930, 3162, 3130, 2852, 2968, 3050, 3012, 2904] combine_rows_per_src=[9176, 9280, 9432, 9214, 9170, 9384, 9378, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65120 dup_rows=23928 dup_rows_remote=20924 dedup_ratio=0.243408 router_tokens_per_expert=[840, 766, 754, 699, 777, 794, 825, 794, 783, 790, 786, 725, 748, 793, 750, 804, 734, 754, 717, 793, 769, 769, 766, 734, 760, 760, 771, 763, 784, 765, 771, 792, 762, 743, 707, 830, 737, 792, 791, 768, 763, 716, 787, 734, 774, 831, 794, 765, 766, 770, 798, 762, 743, 792, 762, 766, 796, 761, 731, 742, 726, 748, 793, 772] payload_rows_per_dst=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] dup_rows_per_dst=[3154, 2956, 2866, 3006, 3004, 3012, 2904, 3026] combine_rows_per_src=[9344, 9402, 9206, 9326, 9256, 9316, 9414, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64984 dup_rows=24112 dup_rows_remote=21112 dedup_ratio=0.245280 router_tokens_per_expert=[782, 723, 788, 767, 761, 732, 770, 730, 774, 791, 801, 738, 767, 823, 754, 768, 758, 798, 759, 760, 798, 766, 734, 731, 752, 758, 757, 826, 790, 754, 766, 824, 769, 809, 758, 773, 757, 785, 750, 751, 760, 775, 780, 745, 751, 728, 788, 790, 774, 786, 788, 771, 775, 781, 741, 713, 770, 784, 755, 771, 795, 778, 737, 764] payload_rows_per_dst=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] dup_rows_per_dst=[2928, 3040, 3000, 3068, 2974, 3010, 3016, 3076] combine_rows_per_src=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65096 dup_rows=23970 dup_rows_remote=20950 dedup_ratio=0.243835 router_tokens_per_expert=[803, 742, 764, 773, 766, 761, 741, 788, 767, 740, 742, 730, 830, 831, 821, 764, 747, 778, 794, 806, 794, 795, 714, 772, 748, 775, 738, 784, 720, 732, 819, 799, 769, 804, 782, 807, 717, 749, 784, 780, 749, 767, 759, 786, 783, 752, 784, 749, 773, 780, 753, 730, 764, 743, 770, 792, 745, 772, 719, 772, 747, 754, 766, 773] payload_rows_per_dst=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] dup_rows_per_dst=[2968, 3092, 3050, 2992, 2950, 3020, 2972, 2926] combine_rows_per_src=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65158 dup_rows=23816 dup_rows_remote=20960 dedup_ratio=0.242269 router_tokens_per_expert=[749, 769, 777, 733, 786, 785, 803, 805, 775, 703, 810, 788, 813, 782, 770, 779, 769, 776, 754, 765, 760, 789, 778, 774, 763, 748, 757, 758, 778, 759, 767, 763, 795, 829, 787, 787, 772, 767, 750, 751, 738, 705, 770, 772, 794, 779, 758, 790, 743, 762, 759, 772, 763, 805, 761, 737, 727, 741, 740, 734, 770, 831, 749, 729] payload_rows_per_dst=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] dup_rows_per_dst=[3064, 3112, 3008, 2856, 3058, 2944, 2910, 2864] combine_rows_per_src=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343656 combine_payload_bytes=265904128 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266247784 ingress_bytes=404316160 link_bytes=404316160 token_bytes=532151912 weight_movement_bytes=138412032 total_nvlink_bytes=670563944 weight_to_token_ratio=0.260099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521409160e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64594 dup_rows=24144 dup_rows_remote=21050 dedup_ratio=0.245605 router_tokens_per_expert=[837, 766, 797, 780, 750, 748, 794, 778, 723, 763, 799, 774, 788, 779, 786, 751, 737, 755, 791, 765, 766, 813, 792, 716, 804, 745, 758, 769, 789, 732, 775, 742, 760, 745, 849, 775, 778, 751, 776, 757, 716, 804, 761, 759, 784, 748, 745, 751, 735, 737, 783, 728, 770, 781, 728, 749, 743, 787, 808, 776, 787, 758, 788, 773] payload_rows_per_dst=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] dup_rows_per_dst=[3094, 3068, 3110, 3092, 3376, 2644, 2698, 3062] combine_rows_per_src=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344176 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075696 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807216 weight_movement_bytes=103809024 total_nvlink_bytes=637616240 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65308 dup_rows=23780 dup_rows_remote=20958 dedup_ratio=0.241903 router_tokens_per_expert=[795, 774, 750, 767, 787, 810, 774, 759, 804, 743, 740, 838, 744, 795, 813, 778, 729, 781, 767, 794, 755, 805, 761, 779, 776, 790, 754, 791, 773, 757, 727, 728, 824, 776, 795, 789, 737, 721, 766, 789, 752, 723, 731, 789, 759, 739, 777, 749, 791, 710, 751, 760, 743, 748, 772, 751, 783, 775, 797, 762, 775, 738, 746, 796] payload_rows_per_dst=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] dup_rows_per_dst=[3054, 3150, 2914, 2940, 2976, 2822, 2886, 3038] combine_rows_per_src=[9378, 9360, 9428, 9252, 9418, 9216, 9166, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65044 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[739, 737, 804, 804, 787, 785, 764, 776, 736, 744, 756, 765, 785, 736, 772, 786, 746, 780, 731, 747, 748, 742, 761, 800, 753, 747, 779, 758, 771, 737, 776, 772, 737, 784, 739, 804, 763, 751, 789, 794, 760, 782, 773, 783, 788, 766, 811, 770, 754, 801, 770, 793, 740, 758, 752, 788, 769, 824, 799, 771, 752, 773, 775, 715] payload_rows_per_dst=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] dup_rows_per_dst=[2986, 2972, 2822, 2988, 3002, 3238, 3060, 3004] combine_rows_per_src=[9406, 9188, 9288, 9198, 9320, 9228, 9252, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64576 dup_rows=23950 dup_rows_remote=20666 dedup_ratio=0.243632 router_tokens_per_expert=[741, 780, 733, 792, 711, 782, 777, 790, 745, 735, 775, 770, 777, 799, 786, 835, 749, 747, 767, 754, 744, 727, 750, 747, 771, 752, 776, 758, 791, 780, 763, 778, 774, 769, 773, 734, 774, 760, 792, 802, 762, 760, 762, 787, 691, 773, 712, 775, 807, 795, 783, 763, 785, 779, 791, 796, 781, 765, 812, 781, 738, 769, 768, 757] payload_rows_per_dst=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] dup_rows_per_dst=[2926, 2726, 2838, 3020, 2882, 3142, 3132, 3284] combine_rows_per_src=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344384 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518848 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693312 weight_movement_bytes=34603008 total_nvlink_bytes=567296320 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344184 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977400 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610616 weight_movement_bytes=34603008 total_nvlink_bytes=568213624 weight_to_token_ratio=0.064847 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=64998 dup_rows=23876 dup_rows_remote=20798 dedup_ratio=0.242879 router_tokens_per_expert=[669, 778, 762, 809, 782, 779, 734, 776, 742, 803, 720, 781, 763, 769, 750, 770, 738, 773, 741, 815, 791, 781, 744, 794, 740, 799, 778, 751, 839, 758, 781, 808, 776, 725, 780, 783, 780, 762, 772, 711, 775, 734, 761, 733, 751, 772, 765, 771, 729, 767, 775, 728, 781, 765, 786, 805, 746, 803, 717, 809, 739, 783, 847, 803] payload_rows_per_dst=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] dup_rows_per_dst=[2958, 2902, 2936, 3078, 2986, 2932, 2994, 3090] combine_rows_per_src=[9220, 9294, 9418, 9430, 9192, 9192, 9278, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64904 dup_rows=23998 dup_rows_remote=21052 dedup_ratio=0.244120 router_tokens_per_expert=[732, 773, 741, 773, 798, 790, 788, 726, 755, 743, 746, 738, 760, 738, 793, 809, 792, 749, 783, 747, 760, 748, 776, 803, 781, 717, 789, 782, 763, 751, 780, 786, 799, 736, 719, 802, 777, 797, 719, 759, 798, 795, 817, 787, 794, 749, 768, 769, 712, 812, 766, 818, 773, 788, 754, 751, 738, 743, 769, 752, 789, 771, 758, 763] payload_rows_per_dst=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] dup_rows_per_dst=[3008, 2902, 2990, 2940, 3014, 3204, 2946, 2994] combine_rows_per_src=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65116 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[729, 761, 765, 768, 787, 755, 808, 753, 799, 820, 801, 807, 730, 736, 739, 773, 820, 813, 775, 775, 760, 786, 776, 769, 783, 798, 747, 769, 732, 737, 736, 731, 731, 814, 778, 762, 807, 755, 777, 781, 764, 730, 755, 773, 734, 794, 778, 740, 758, 759, 773, 781, 762, 766, 765, 788, 728, 751, 770, 774, 772, 766, 731, 797] payload_rows_per_dst=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] dup_rows_per_dst=[3052, 2984, 3180, 2934, 2964, 2822, 3014, 2918] combine_rows_per_src=[9320, 9306, 9368, 9132, 9446, 9314, 9290, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344472 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267231640 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534118808 weight_movement_bytes=34603008 total_nvlink_bytes=568721816 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=342576 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264919600 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529496624 weight_movement_bytes=34603008 total_nvlink_bytes=564099632 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344576 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266764800 ingress_bytes=335626240 link_bytes=335626240 token_bytes=533185024 weight_movement_bytes=69206016 total_nvlink_bytes=602391040 weight_to_token_ratio=0.129797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262934521e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64984 dup_rows=24112 dup_rows_remote=21112 dedup_ratio=0.245280 router_tokens_per_expert=[782, 723, 788, 767, 761, 732, 770, 730, 774, 791, 801, 738, 767, 823, 754, 768, 758, 798, 759, 760, 798, 766, 734, 731, 752, 758, 757, 826, 790, 754, 766, 824, 769, 809, 758, 773, 757, 785, 750, 751, 760, 775, 780, 745, 751, 728, 788, 790, 774, 786, 788, 771, 775, 781, 741, 713, 770, 784, 755, 771, 795, 778, 737, 764] payload_rows_per_dst=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] dup_rows_per_dst=[2928, 3040, 3000, 3068, 2974, 3010, 3016, 3076] combine_rows_per_src=[9178, 9392, 9208, 9386, 9330, 9224, 9242, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264503296 dispatch_meta_bytes=340968 combine_payload_bytes=264503296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264844264 ingress_bytes=299106304 link_bytes=299106304 token_bytes=529347560 weight_movement_bytes=34603008 total_nvlink_bytes=563950568 weight_to_token_ratio=0.065369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125512942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64904 dup_rows=23998 dup_rows_remote=21052 dedup_ratio=0.244120 router_tokens_per_expert=[732, 773, 741, 773, 798, 790, 788, 726, 755, 743, 746, 738, 760, 738, 793, 809, 792, 749, 783, 747, 760, 748, 776, 803, 781, 717, 789, 782, 763, 751, 780, 786, 799, 736, 719, 802, 777, 797, 719, 759, 798, 795, 817, 787, 794, 749, 768, 769, 712, 812, 766, 818, 773, 788, 754, 751, 738, 743, 769, 752, 789, 771, 758, 763] payload_rows_per_dst=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] dup_rows_per_dst=[3008, 2902, 2990, 2940, 3014, 3204, 2946, 2994] combine_rows_per_src=[9234, 9262, 9326, 9358, 9202, 9350, 9402, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267501568 dispatch_meta_bytes=345064 combine_payload_bytes=267501568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267846632 ingress_bytes=302104576 link_bytes=302104576 token_bytes=535348200 weight_movement_bytes=34603008 total_nvlink_bytes=569951208 weight_to_token_ratio=0.064636 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136795198e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64576 dup_rows=23950 dup_rows_remote=20666 dedup_ratio=0.243632 router_tokens_per_expert=[741, 780, 733, 792, 711, 782, 777, 790, 745, 735, 775, 770, 777, 799, 786, 835, 749, 747, 767, 754, 744, 727, 750, 747, 771, 752, 776, 758, 791, 780, 763, 778, 774, 769, 773, 734, 774, 760, 792, 802, 762, 760, 762, 787, 691, 773, 712, 775, 807, 795, 783, 763, 785, 779, 791, 796, 781, 765, 812, 781, 738, 769, 768, 757] payload_rows_per_dst=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] dup_rows_per_dst=[2926, 2726, 2838, 3020, 2882, 3142, 3132, 3284] combine_rows_per_src=[9188, 8998, 9132, 9318, 9150, 9324, 9466, 9778] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343184 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266574992 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806800 weight_movement_bytes=103809024 total_nvlink_bytes=636615824 weight_to_token_ratio=0.194834 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343824 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190608 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037392 weight_movement_bytes=34603008 total_nvlink_bytes=566640400 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343728 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058864 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774000 weight_movement_bytes=34603008 total_nvlink_bytes=568377008 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65048 dup_rows=23854 dup_rows_remote=20720 dedup_ratio=0.242655 router_tokens_per_expert=[758, 734, 761, 771, 768, 756, 750, 765, 733, 719, 788, 771, 761, 782, 723, 758, 777, 819, 755, 754, 711, 761, 731, 768, 775, 788, 834, 729, 745, 760, 750, 792, 776, 785, 764, 753, 815, 792, 809, 774, 798, 775, 761, 762, 804, 769, 749, 789, 794, 765, 773, 770, 811, 759, 759, 752, 756, 791, 729, 798, 784, 746, 744, 799] payload_rows_per_dst=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] dup_rows_per_dst=[2930, 2814, 2852, 2950, 3134, 3092, 3054, 3028] combine_rows_per_src=[9196, 9256, 9300, 9396, 9402, 9322, 9312, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344384 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518848 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693312 weight_movement_bytes=34603008 total_nvlink_bytes=567296320 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343824 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190608 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037392 weight_movement_bytes=34603008 total_nvlink_bytes=566640400 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64872 dup_rows=24046 dup_rows_remote=21038 dedup_ratio=0.244609 router_tokens_per_expert=[778, 779, 763, 784, 747, 773, 731, 757, 744, 764, 762, 763, 738, 843, 737, 764, 741, 743, 751, 790, 758, 763, 770, 754, 751, 774, 797, 779, 767, 731, 783, 781, 786, 771, 787, 778, 805, 736, 789, 779, 777, 779, 799, 784, 790, 758, 732, 778, 729, 780, 771, 785, 734, 745, 761, 766, 786, 785, 755, 828, 727, 770, 765, 777] payload_rows_per_dst=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] dup_rows_per_dst=[2988, 2942, 2982, 3042, 3082, 3008, 3000, 3002] combine_rows_per_src=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264503296 dispatch_meta_bytes=340968 combine_payload_bytes=264503296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264844264 ingress_bytes=299106304 link_bytes=299106304 token_bytes=529347560 weight_movement_bytes=34603008 total_nvlink_bytes=563950568 weight_to_token_ratio=0.065369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125512942e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64888 dup_rows=24068 dup_rows_remote=20982 dedup_ratio=0.244832 router_tokens_per_expert=[763, 773, 738, 759, 806, 791, 818, 767, 772, 784, 752, 745, 808, 791, 806, 759, 763, 768, 783, 790, 776, 735, 809, 786, 762, 743, 763, 779, 741, 781, 792, 761, 717, 793, 757, 766, 814, 757, 764, 787, 779, 766, 727, 771, 723, 781, 725, 731, 775, 749, 796, 775, 768, 750, 753, 744, 714, 803, 747, 760, 794, 790, 766, 746] payload_rows_per_dst=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] dup_rows_per_dst=[3026, 3086, 3122, 3048, 3032, 2848, 2946, 2960] combine_rows_per_src=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64954 dup_rows=23944 dup_rows_remote=20968 dedup_ratio=0.243571 router_tokens_per_expert=[822, 819, 773, 785, 801, 786, 701, 787, 740, 777, 718, 746, 805, 785, 795, 740, 777, 773, 758, 734, 776, 787, 769, 817, 731, 764, 782, 767, 764, 731, 760, 794, 748, 773, 790, 774, 757, 802, 729, 777, 770, 759, 764, 753, 787, 781, 778, 737, 721, 820, 778, 740, 770, 767, 765, 769, 705, 753, 803, 748, 796, 781, 761, 732] payload_rows_per_dst=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] dup_rows_per_dst=[3116, 3036, 2976, 2930, 2872, 3106, 2976, 2932] combine_rows_per_src=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65032 dup_rows=24012 dup_rows_remote=21072 dedup_ratio=0.244263 router_tokens_per_expert=[794, 764, 788, 781, 783, 767, 729, 806, 690, 769, 723, 762, 730, 818, 746, 802, 753, 750, 759, 782, 796, 773, 824, 787, 741, 771, 800, 726, 700, 803, 776, 783, 812, 747, 707, 762, 800, 815, 756, 726, 767, 802, 735, 755, 770, 739, 793, 780, 757, 743, 773, 775, 776, 811, 774, 749, 784, 742, 784, 734, 767, 789, 781, 771] payload_rows_per_dst=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] dup_rows_per_dst=[2992, 2920, 2980, 2940, 3010, 3020, 3010, 3140] combine_rows_per_src=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65018 dup_rows=23948 dup_rows_remote=20910 dedup_ratio=0.243612 router_tokens_per_expert=[779, 804, 721, 839, 775, 698, 747, 752, 762, 741, 799, 760, 744, 746, 785, 773, 810, 772, 770, 766, 812, 753, 768, 762, 710, 753, 756, 753, 808, 805, 762, 802, 748, 762, 773, 784, 800, 808, 773, 788, 734, 749, 714, 778, 798, 757, 766, 736, 793, 794, 784, 740, 798, 757, 778, 729, 774, 794, 768, 739, 781, 768, 750, 750] payload_rows_per_dst=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] dup_rows_per_dst=[3038, 2972, 3066, 2982, 2926, 2922, 3044, 2998] combine_rows_per_src=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343072 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266779680 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533216288 weight_movement_bytes=103809024 total_nvlink_bytes=637025312 weight_to_token_ratio=0.194685 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65096 dup_rows=23970 dup_rows_remote=20950 dedup_ratio=0.243835 router_tokens_per_expert=[803, 742, 764, 773, 766, 761, 741, 788, 767, 740, 742, 730, 830, 831, 821, 764, 747, 778, 794, 806, 794, 795, 714, 772, 748, 775, 738, 784, 720, 732, 819, 799, 769, 804, 782, 807, 717, 749, 784, 780, 749, 767, 759, 786, 783, 752, 784, 749, 773, 780, 753, 730, 764, 743, 770, 792, 745, 772, 719, 772, 747, 754, 766, 773] payload_rows_per_dst=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] dup_rows_per_dst=[2968, 3092, 3050, 2992, 2950, 3020, 2972, 2926] combine_rows_per_src=[9308, 9358, 9350, 9238, 9434, 9238, 9238, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64778 dup_rows=24292 dup_rows_remote=21274 dedup_ratio=0.247111 router_tokens_per_expert=[811, 792, 804, 745, 753, 763, 786, 746, 764, 746, 763, 780, 784, 724, 757, 740, 744, 749, 764, 767, 748, 802, 750, 796, 764, 804, 794, 734, 775, 775, 780, 752, 776, 791, 714, 778, 783, 795, 778, 766, 739, 759, 788, 759, 773, 773, 786, 771, 789, 770, 787, 793, 728, 761, 760, 753, 787, 767, 796, 746, 751, 766, 749, 764] payload_rows_per_dst=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] dup_rows_per_dst=[3042, 3020, 2992, 2834, 3166, 3236, 2984, 3018] combine_rows_per_src=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64888 dup_rows=24068 dup_rows_remote=20982 dedup_ratio=0.244832 router_tokens_per_expert=[763, 773, 738, 759, 806, 791, 818, 767, 772, 784, 752, 745, 808, 791, 806, 759, 763, 768, 783, 790, 776, 735, 809, 786, 762, 743, 763, 779, 741, 781, 792, 761, 717, 793, 757, 766, 814, 757, 764, 787, 779, 766, 727, 771, 723, 781, 725, 731, 775, 749, 796, 775, 768, 750, 753, 744, 714, 803, 747, 760, 794, 790, 766, 746] payload_rows_per_dst=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] dup_rows_per_dst=[3026, 3086, 3122, 3048, 3032, 2848, 2946, 2960] combine_rows_per_src=[9404, 9348, 9298, 9196, 9278, 9158, 9274, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343640 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059352 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775064 weight_movement_bytes=34603008 total_nvlink_bytes=566378072 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343688 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395272 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446856 weight_movement_bytes=34603008 total_nvlink_bytes=567049864 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343712 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657440 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971168 weight_movement_bytes=34603008 total_nvlink_bytes=567574176 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73930 payload_rows_remote=64518 dup_rows=24374 dup_rows_remote=21240 dedup_ratio=0.247945 router_tokens_per_expert=[751, 754, 773, 816, 776, 769, 718, 746, 765, 760, 804, 792, 746, 801, 759, 806, 746, 737, 762, 801, 759, 773, 753, 751, 781, 744, 761, 772, 709, 715, 817, 786, 797, 808, 733, 778, 723, 740, 771, 748, 801, 768, 741, 733, 792, 805, 728, 792, 769, 822, 782, 791, 738, 812, 797, 762, 754, 769, 761, 762, 750, 776, 750, 796] payload_rows_per_dst=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] dup_rows_per_dst=[2992, 3118, 2988, 3058, 2980, 3096, 3134, 3008] combine_rows_per_src=[9214, 9348, 9176, 9112, 9216, 9224, 9412, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65158 dup_rows=23816 dup_rows_remote=20960 dedup_ratio=0.242269 router_tokens_per_expert=[749, 769, 777, 733, 786, 785, 803, 805, 775, 703, 810, 788, 813, 782, 770, 779, 769, 776, 754, 765, 760, 789, 778, 774, 763, 748, 757, 758, 778, 759, 767, 763, 795, 829, 787, 787, 772, 767, 750, 751, 738, 705, 770, 772, 794, 779, 758, 790, 743, 762, 759, 772, 763, 805, 761, 737, 727, 741, 740, 734, 770, 831, 749, 729] payload_rows_per_dst=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] dup_rows_per_dst=[3064, 3112, 3008, 2856, 3058, 2944, 2910, 2864] combine_rows_per_src=[9350, 9328, 9322, 9330, 9418, 9268, 9294, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344208 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265674896 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005584 weight_movement_bytes=103809024 total_nvlink_bytes=634814608 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343480 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124728 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905976 weight_movement_bytes=69206016 total_nvlink_bytes=601111992 weight_to_token_ratio=0.130109 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64840 dup_rows=24192 dup_rows_remote=21124 dedup_ratio=0.246094 router_tokens_per_expert=[811, 732, 726, 732, 786, 800, 761, 812, 745, 787, 771, 823, 781, 749, 764, 714, 809, 779, 749, 757, 753, 766, 740, 740, 717, 779, 815, 777, 788, 704, 783, 717, 775, 787, 782, 753, 788, 774, 802, 795, 803, 748, 782, 791, 728, 791, 766, 802, 756, 742, 808, 773, 729, 752, 763, 794, 751, 760, 797, 804, 734, 761, 741, 753] payload_rows_per_dst=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] dup_rows_per_dst=[3068, 3162, 3130, 2930, 2976, 2962, 2962, 3002] combine_rows_per_src=[9272, 9512, 9282, 9230, 9130, 9214, 9272, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344184 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977400 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610616 weight_movement_bytes=34603008 total_nvlink_bytes=568213624 weight_to_token_ratio=0.064847 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64954 dup_rows=23944 dup_rows_remote=20968 dedup_ratio=0.243571 router_tokens_per_expert=[822, 819, 773, 785, 801, 786, 701, 787, 740, 777, 718, 746, 805, 785, 795, 740, 777, 773, 758, 734, 776, 787, 769, 817, 731, 764, 782, 767, 764, 731, 760, 794, 748, 773, 790, 774, 757, 802, 729, 777, 770, 759, 764, 753, 787, 781, 778, 737, 721, 820, 778, 740, 770, 767, 765, 769, 705, 753, 803, 748, 796, 781, 761, 732] payload_rows_per_dst=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] dup_rows_per_dst=[3116, 3036, 2976, 2930, 2872, 3106, 2976, 2932] combine_rows_per_src=[9432, 9176, 9406, 9256, 9428, 9152, 9284, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64778 dup_rows=24292 dup_rows_remote=21274 dedup_ratio=0.247111 router_tokens_per_expert=[811, 792, 804, 745, 753, 763, 786, 746, 764, 746, 763, 780, 784, 724, 757, 740, 744, 749, 764, 767, 748, 802, 750, 796, 764, 804, 794, 734, 775, 775, 780, 752, 776, 791, 714, 778, 783, 795, 778, 766, 739, 759, 788, 759, 773, 773, 786, 771, 789, 770, 787, 793, 728, 761, 760, 753, 787, 767, 796, 746, 751, 766, 749, 764] payload_rows_per_dst=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] dup_rows_per_dst=[3042, 3020, 2992, 2834, 3166, 3236, 2984, 3018] combine_rows_per_src=[9290, 9138, 9248, 8992, 9376, 9410, 9324, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264265728 dispatch_meta_bytes=343032 combine_payload_bytes=264265728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=264608760 ingress_bytes=368074752 link_bytes=368074752 token_bytes=528874488 weight_movement_bytes=103809024 total_nvlink_bytes=632683512 weight_to_token_ratio=0.196283 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385035659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344472 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267231640 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534118808 weight_movement_bytes=34603008 total_nvlink_bytes=568721816 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73904 payload_rows_remote=64742 dup_rows=24400 dup_rows_remote=21306 dedup_ratio=0.248210 router_tokens_per_expert=[769, 760, 785, 798, 799, 784, 771, 757, 733, 774, 742, 792, 821, 756, 813, 760, 779, 773, 738, 757, 767, 775, 795, 749, 777, 750, 755, 801, 765, 767, 778, 732, 817, 784, 725, 753, 759, 743, 752, 795, 792, 779, 769, 760, 824, 766, 766, 764, 782, 770, 760, 734, 740, 755, 731, 726, 770, 781, 798, 730, 744, 779, 745, 787] payload_rows_per_dst=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] dup_rows_per_dst=[3104, 3106, 3018, 3026, 3094, 3092, 2982, 2978] combine_rows_per_src=[9342, 9276, 9248, 9224, 9162, 9348, 9014, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343856 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265928496 ingress_bytes=403996672 link_bytes=403996672 token_bytes=531513136 weight_movement_bytes=138412032 total_nvlink_bytes=669925168 weight_to_token_ratio=0.260411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520206952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343688 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395272 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446856 weight_movement_bytes=34603008 total_nvlink_bytes=567049864 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344208 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265674896 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005584 weight_movement_bytes=103809024 total_nvlink_bytes=634814608 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64854 dup_rows=24002 dup_rows_remote=20992 dedup_ratio=0.244161 router_tokens_per_expert=[777, 740, 789, 743, 756, 748, 754, 742, 758, 761, 763, 755, 736, 749, 731, 768, 813, 778, 746, 772, 760, 764, 755, 740, 809, 825, 777, 785, 794, 714, 783, 778, 752, 792, 779, 719, 750, 799, 757, 779, 802, 815, 788, 721, 781, 829, 732, 761, 744, 757, 767, 773, 760, 731, 793, 769, 781, 769, 798, 767, 782, 781, 777, 784] payload_rows_per_dst=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] dup_rows_per_dst=[2854, 2898, 2990, 3146, 3034, 3010, 2990, 3080] combine_rows_per_src=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64970 dup_rows=23924 dup_rows_remote=20830 dedup_ratio=0.243368 router_tokens_per_expert=[746, 765, 713, 813, 766, 736, 796, 768, 783, 770, 740, 742, 747, 785, 763, 734, 766, 796, 737, 760, 821, 749, 785, 778, 774, 738, 760, 754, 794, 793, 863, 776, 775, 737, 758, 780, 787, 732, 764, 792, 784, 758, 751, 760, 799, 770, 787, 780, 750, 747, 768, 784, 758, 763, 760, 768, 749, 767, 793, 787, 702, 742, 812, 777] payload_rows_per_dst=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] dup_rows_per_dst=[2944, 2914, 3074, 3094, 2946, 2994, 2946, 3012] combine_rows_per_src=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64876 dup_rows=24156 dup_rows_remote=21218 dedup_ratio=0.245728 router_tokens_per_expert=[758, 689, 751, 740, 772, 744, 774, 760, 841, 795, 757, 748, 787, 761, 733, 770, 744, 758, 790, 761, 754, 767, 750, 781, 768, 765, 711, 759, 758, 773, 792, 760, 786, 782, 784, 770, 696, 732, 768, 734, 719, 782, 754, 794, 730, 755, 784, 777, 770, 793, 821, 788, 837, 774, 787, 740, 778, 852, 827, 815, 788, 784, 746, 734] payload_rows_per_dst=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] dup_rows_per_dst=[2924, 3054, 2938, 2938, 2938, 2948, 3232, 3184] combine_rows_per_src=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65112 dup_rows=23832 dup_rows_remote=20804 dedup_ratio=0.242432 router_tokens_per_expert=[769, 779, 798, 771, 746, 758, 791, 751, 773, 808, 732, 733, 776, 809, 791, 772, 807, 763, 731, 706, 703, 748, 764, 824, 818, 760, 745, 771, 795, 756, 793, 800, 784, 746, 763, 770, 735, 770, 758, 745, 763, 779, 777, 742, 754, 763, 745, 767, 760, 777, 770, 751, 767, 754, 759, 755, 769, 798, 764, 752, 793, 796, 795, 790] payload_rows_per_dst=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] dup_rows_per_dst=[3040, 3028, 2854, 3076, 2844, 2966, 2920, 3104] combine_rows_per_src=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64872 dup_rows=24046 dup_rows_remote=21038 dedup_ratio=0.244609 router_tokens_per_expert=[778, 779, 763, 784, 747, 773, 731, 757, 744, 764, 762, 763, 738, 843, 737, 764, 741, 743, 751, 790, 758, 763, 770, 754, 751, 774, 797, 779, 767, 731, 783, 781, 786, 771, 787, 778, 805, 736, 789, 779, 777, 779, 799, 784, 790, 758, 732, 778, 729, 780, 771, 785, 734, 745, 761, 766, 786, 785, 755, 828, 727, 770, 765, 777] payload_rows_per_dst=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] dup_rows_per_dst=[2988, 2942, 2982, 3042, 3082, 3008, 3000, 3002] combine_rows_per_src=[9236, 9288, 9158, 9284, 9380, 9386, 9142, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65054 dup_rows=24074 dup_rows_remote=21060 dedup_ratio=0.244893 router_tokens_per_expert=[789, 780, 745, 765, 719, 762, 787, 736, 791, 772, 789, 801, 726, 781, 781, 805, 778, 728, 763, 762, 758, 726, 773, 786, 789, 799, 748, 749, 782, 742, 752, 813, 740, 788, 824, 769, 760, 751, 740, 791, 779, 779, 737, 739, 757, 765, 835, 794, 822, 780, 762, 788, 740, 744, 777, 726, 767, 779, 782, 786, 746, 714, 729, 785] payload_rows_per_dst=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] dup_rows_per_dst=[3014, 3066, 3168, 3224, 2834, 3070, 2716, 2982] combine_rows_per_src=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64920 dup_rows=24104 dup_rows_remote=21140 dedup_ratio=0.245199 router_tokens_per_expert=[750, 763, 766, 808, 761, 791, 771, 753, 741, 790, 776, 755, 760, 753, 787, 755, 746, 728, 769, 808, 789, 751, 763, 771, 774, 771, 737, 763, 754, 762, 780, 739, 749, 773, 754, 809, 750, 749, 745, 772, 762, 767, 817, 801, 763, 769, 800, 768, 780, 780, 797, 755, 788, 744, 834, 735, 749, 721, 768, 786, 778, 726, 756, 822] payload_rows_per_dst=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] dup_rows_per_dst=[3056, 2972, 3016, 2796, 2856, 3392, 3052, 2964] combine_rows_per_src=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=344192 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265527424 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530710656 weight_movement_bytes=34603008 total_nvlink_bytes=565313664 weight_to_token_ratio=0.065201 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343384 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265985368 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531627352 weight_movement_bytes=69206016 total_nvlink_bytes=600833368 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343200 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266460320 ingress_bytes=369926144 link_bytes=369926144 token_bytes=532577440 weight_movement_bytes=103809024 total_nvlink_bytes=636386464 weight_to_token_ratio=0.194918 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392002298e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344376 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266076472 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531808568 weight_movement_bytes=34603008 total_nvlink_bytes=566411576 weight_to_token_ratio=0.065067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65112 dup_rows=23832 dup_rows_remote=20804 dedup_ratio=0.242432 router_tokens_per_expert=[769, 779, 798, 771, 746, 758, 791, 751, 773, 808, 732, 733, 776, 809, 791, 772, 807, 763, 731, 706, 703, 748, 764, 824, 818, 760, 745, 771, 795, 756, 793, 800, 784, 746, 763, 770, 735, 770, 758, 745, 763, 779, 777, 742, 754, 763, 745, 767, 760, 777, 770, 751, 767, 754, 759, 755, 769, 798, 764, 752, 793, 796, 795, 790] payload_rows_per_dst=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] dup_rows_per_dst=[3040, 3028, 2854, 3076, 2844, 2966, 2920, 3104] combine_rows_per_src=[9286, 9360, 9238, 9400, 9298, 9214, 9266, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344456 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805640 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266824 weight_movement_bytes=69206016 total_nvlink_bytes=602472840 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65002 dup_rows=23946 dup_rows_remote=20750 dedup_ratio=0.243591 router_tokens_per_expert=[753, 765, 764, 779, 792, 750, 739, 825, 752, 788, 790, 780, 774, 816, 745, 776, 795, 745, 766, 778, 724, 810, 767, 775, 777, 800, 818, 795, 758, 821, 763, 770, 768, 749, 746, 754, 768, 707, 742, 755, 786, 729, 795, 739, 749, 748, 715, 736, 790, 773, 790, 807, 777, 749, 798, 792, 791, 763, 764, 761, 741, 722, 738, 760] payload_rows_per_dst=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] dup_rows_per_dst=[2912, 2980, 3044, 3196, 2834, 2852, 3196, 2932] combine_rows_per_src=[9422, 9462, 9276, 9408, 9144, 9142, 9356, 9148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65032 dup_rows=24012 dup_rows_remote=21072 dedup_ratio=0.244263 router_tokens_per_expert=[794, 764, 788, 781, 783, 767, 729, 806, 690, 769, 723, 762, 730, 818, 746, 802, 753, 750, 759, 782, 796, 773, 824, 787, 741, 771, 800, 726, 700, 803, 776, 783, 812, 747, 707, 762, 800, 815, 756, 726, 767, 802, 735, 755, 770, 739, 793, 780, 757, 743, 773, 775, 776, 811, 774, 749, 784, 742, 784, 734, 767, 789, 781, 771] payload_rows_per_dst=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] dup_rows_per_dst=[2992, 2920, 2980, 2940, 3010, 3020, 3010, 3140] combine_rows_per_src=[9432, 9160, 9468, 9260, 9240, 9262, 9306, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343640 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059352 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775064 weight_movement_bytes=34603008 total_nvlink_bytes=566378072 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344240 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256560 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532168880 weight_movement_bytes=34603008 total_nvlink_bytes=566771888 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343664 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042416 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741168 weight_movement_bytes=34603008 total_nvlink_bytes=568344176 weight_to_token_ratio=0.064831 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64546 dup_rows=24216 dup_rows_remote=20904 dedup_ratio=0.246338 router_tokens_per_expert=[776, 747, 779, 757, 773, 772, 756, 752, 783, 748, 813, 768, 759, 752, 764, 797, 822, 765, 765, 791, 804, 782, 729, 799, 757, 723, 808, 740, 743, 795, 764, 780, 784, 735, 794, 764, 778, 788, 738, 772, 710, 704, 766, 796, 808, 770, 750, 789, 712, 750, 773, 779, 783, 758, 760, 791, 796, 792, 776, 772, 728, 753, 747, 773] payload_rows_per_dst=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] dup_rows_per_dst=[3312, 2804, 3228, 2804, 3018, 3008, 3042, 3000] combine_rows_per_src=[9542, 9136, 9460, 9044, 9288, 9174, 9314, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64876 dup_rows=24156 dup_rows_remote=21218 dedup_ratio=0.245728 router_tokens_per_expert=[758, 689, 751, 740, 772, 744, 774, 760, 841, 795, 757, 748, 787, 761, 733, 770, 744, 758, 790, 761, 754, 767, 750, 781, 768, 765, 711, 759, 758, 773, 792, 760, 786, 782, 784, 770, 696, 732, 768, 734, 719, 782, 754, 794, 730, 755, 784, 777, 770, 793, 821, 788, 837, 774, 787, 740, 778, 852, 827, 815, 788, 784, 746, 734] payload_rows_per_dst=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] dup_rows_per_dst=[2924, 3054, 2938, 2938, 2938, 2948, 3232, 3184] combine_rows_per_src=[9052, 9330, 9272, 9234, 9166, 9242, 9388, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64920 dup_rows=24104 dup_rows_remote=21140 dedup_ratio=0.245199 router_tokens_per_expert=[750, 763, 766, 808, 761, 791, 771, 753, 741, 790, 776, 755, 760, 753, 787, 755, 746, 728, 769, 808, 789, 751, 763, 771, 774, 771, 737, 763, 754, 762, 780, 739, 749, 773, 754, 809, 750, 749, 745, 772, 762, 767, 817, 801, 763, 769, 800, 768, 780, 780, 797, 755, 788, 744, 834, 735, 749, 721, 768, 786, 778, 726, 756, 822] payload_rows_per_dst=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] dup_rows_per_dst=[3056, 2972, 3016, 2796, 2856, 3392, 3052, 2964] combine_rows_per_src=[9270, 9262, 9234, 9248, 9086, 9478, 9342, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343008 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591200 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839392 weight_movement_bytes=34603008 total_nvlink_bytes=567442400 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65082 dup_rows=23878 dup_rows_remote=20924 dedup_ratio=0.242900 router_tokens_per_expert=[793, 750, 811, 837, 825, 740, 756, 768, 710, 788, 760, 749, 755, 745, 790, 783, 771, 742, 778, 792, 748, 757, 796, 809, 803, 764, 750, 780, 773, 789, 750, 769, 757, 794, 784, 794, 729, 724, 801, 766, 771, 773, 743, 755, 741, 793, 791, 735, 717, 736, 764, 749, 779, 748, 784, 789, 781, 789, 777, 801, 721, 702, 752, 781] payload_rows_per_dst=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] dup_rows_per_dst=[3036, 2996, 3028, 3024, 2954, 2970, 2958, 2912] combine_rows_per_src=[9524, 9164, 9358, 9332, 9344, 9234, 9174, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264380416 dispatch_meta_bytes=341800 combine_payload_bytes=264380416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264722216 ingress_bytes=298983424 link_bytes=298983424 token_bytes=529102632 weight_movement_bytes=34603008 total_nvlink_bytes=563705640 weight_to_token_ratio=0.065399 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125050554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344376 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266076472 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531808568 weight_movement_bytes=34603008 total_nvlink_bytes=566411576 weight_to_token_ratio=0.065067 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65228 dup_rows=23926 dup_rows_remote=20896 dedup_ratio=0.243388 router_tokens_per_expert=[743, 738, 787, 778, 814, 804, 768, 791, 777, 800, 814, 781, 753, 737, 731, 753, 771, 756, 776, 713, 770, 758, 767, 763, 774, 784, 782, 776, 739, 753, 763, 743, 730, 803, 768, 771, 795, 726, 810, 740, 798, 745, 777, 757, 777, 733, 755, 748, 807, 784, 812, 755, 751, 835, 788, 759, 795, 725, 769, 788, 767, 730, 743, 754] payload_rows_per_dst=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] dup_rows_per_dst=[3110, 3022, 2902, 2900, 2920, 3030, 3092, 2950] combine_rows_per_src=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65086 dup_rows=23894 dup_rows_remote=20854 dedup_ratio=0.243062 router_tokens_per_expert=[779, 676, 746, 768, 810, 775, 786, 778, 767, 753, 735, 743, 750, 793, 834, 783, 747, 771, 774, 767, 800, 773, 777, 773, 774, 796, 764, 770, 745, 762, 780, 725, 776, 767, 792, 795, 754, 762, 771, 719, 798, 803, 770, 768, 754, 737, 790, 725, 745, 816, 812, 703, 780, 825, 782, 796, 787, 736, 704, 772, 750, 754, 763, 772] payload_rows_per_dst=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] dup_rows_per_dst=[2942, 2980, 3040, 2944, 3044, 3036, 3092, 2816] combine_rows_per_src=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65114 dup_rows=23856 dup_rows_remote=20952 dedup_ratio=0.242676 router_tokens_per_expert=[761, 765, 753, 759, 732, 749, 803, 745, 755, 755, 760, 733, 784, 831, 777, 758, 770, 753, 759, 816, 790, 766, 709, 757, 773, 785, 780, 783, 774, 735, 736, 753, 724, 782, 777, 771, 783, 764, 793, 783, 735, 764, 765, 769, 759, 789, 764, 786, 755, 764, 794, 794, 803, 752, 781, 795, 790, 761, 792, 756, 765, 787, 781, 715] payload_rows_per_dst=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] dup_rows_per_dst=[2880, 2970, 2904, 2904, 3082, 3060, 3068, 2988] combine_rows_per_src=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344240 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256560 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532168880 weight_movement_bytes=34603008 total_nvlink_bytes=566771888 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64854 dup_rows=24002 dup_rows_remote=20992 dedup_ratio=0.244161 router_tokens_per_expert=[777, 740, 789, 743, 756, 748, 754, 742, 758, 761, 763, 755, 736, 749, 731, 768, 813, 778, 746, 772, 760, 764, 755, 740, 809, 825, 777, 785, 794, 714, 783, 778, 752, 792, 779, 719, 750, 799, 757, 779, 802, 815, 788, 721, 781, 829, 732, 761, 744, 757, 767, 773, 760, 731, 793, 769, 781, 769, 798, 767, 782, 781, 777, 784] payload_rows_per_dst=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] dup_rows_per_dst=[2854, 2898, 2990, 3146, 3034, 3010, 2990, 3080] combine_rows_per_src=[9244, 9144, 9266, 9384, 9220, 9448, 9198, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65262 dup_rows=23866 dup_rows_remote=20906 dedup_ratio=0.242778 router_tokens_per_expert=[794, 798, 775, 772, 744, 748, 793, 729, 779, 784, 725, 746, 754, 750, 790, 740, 771, 757, 784, 794, 765, 748, 759, 737, 712, 773, 726, 747, 726, 779, 773, 742, 809, 816, 830, 750, 779, 794, 809, 773, 797, 723, 788, 780, 786, 762, 785, 759, 784, 736, 802, 772, 742, 834, 723, 737, 795, 782, 784, 742, 779, 743, 793, 750] payload_rows_per_dst=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] dup_rows_per_dst=[2978, 2960, 2970, 2852, 3152, 2950, 2990, 3014] combine_rows_per_src=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=65106 dup_rows=23888 dup_rows_remote=20956 dedup_ratio=0.243001 router_tokens_per_expert=[727, 841, 753, 737, 770, 753, 756, 783, 808, 748, 756, 746, 770, 771, 746, 775, 792, 708, 748, 737, 713, 765, 794, 786, 774, 773, 832, 753, 754, 776, 810, 737, 733, 785, 850, 777, 810, 742, 816, 772, 781, 763, 669, 836, 721, 753, 789, 777, 724, 767, 794, 733, 738, 750, 814, 772, 749, 796, 779, 801, 753, 733, 792, 791] payload_rows_per_dst=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] dup_rows_per_dst=[2932, 3010, 3130, 3092, 3058, 2860, 2930, 2876] combine_rows_per_src=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344024 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266919896 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533495768 weight_movement_bytes=69206016 total_nvlink_bytes=602701784 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344496 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518384 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692272 weight_movement_bytes=34603008 total_nvlink_bytes=569295280 weight_to_token_ratio=0.064716 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343760 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936016 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528272 weight_movement_bytes=34603008 total_nvlink_bytes=568131280 weight_to_token_ratio=0.064857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65120 dup_rows=23826 dup_rows_remote=20742 dedup_ratio=0.242371 router_tokens_per_expert=[757, 748, 785, 827, 719, 766, 794, 800, 738, 752, 749, 778, 783, 745, 755, 764, 754, 780, 798, 755, 767, 803, 738, 791, 790, 696, 758, 744, 752, 743, 757, 767, 756, 779, 768, 749, 797, 781, 766, 758, 777, 720, 760, 737, 778, 792, 825, 757, 765, 777, 755, 769, 824, 773, 759, 767, 744, 808, 808, 782, 758, 767, 769, 774] payload_rows_per_dst=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] dup_rows_per_dst=[2992, 2858, 3030, 2878, 2936, 2966, 3082, 3084] combine_rows_per_src=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344264 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267051208 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533758152 weight_movement_bytes=34603008 total_nvlink_bytes=568361160 weight_to_token_ratio=0.064829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65262 dup_rows=23866 dup_rows_remote=20906 dedup_ratio=0.242778 router_tokens_per_expert=[794, 798, 775, 772, 744, 748, 793, 729, 779, 784, 725, 746, 754, 750, 790, 740, 771, 757, 784, 794, 765, 748, 759, 737, 712, 773, 726, 747, 726, 779, 773, 742, 809, 816, 830, 750, 779, 794, 809, 773, 797, 723, 788, 780, 786, 762, 785, 759, 784, 736, 802, 772, 742, 834, 723, 737, 795, 782, 784, 742, 779, 743, 793, 750] payload_rows_per_dst=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] dup_rows_per_dst=[2978, 2960, 2970, 2852, 3152, 2950, 2990, 3014] combine_rows_per_src=[9328, 9176, 9260, 9104, 9568, 9410, 9270, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65066 dup_rows=23984 dup_rows_remote=21082 dedup_ratio=0.243978 router_tokens_per_expert=[778, 767, 759, 788, 752, 783, 788, 763, 723, 769, 763, 792, 762, 785, 794, 786, 713, 734, 769, 728, 760, 809, 745, 776, 748, 779, 833, 736, 768, 709, 789, 754, 760, 745, 792, 780, 749, 809, 800, 732, 741, 761, 794, 721, 713, 808, 817, 804, 773, 756, 769, 785, 757, 747, 744, 747, 844, 746, 749, 773, 805, 771, 782, 776] payload_rows_per_dst=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] dup_rows_per_dst=[2984, 3068, 2872, 2992, 3070, 3002, 2902, 3094] combine_rows_per_src=[9372, 9280, 9196, 9240, 9264, 9316, 9254, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64970 dup_rows=23924 dup_rows_remote=20830 dedup_ratio=0.243368 router_tokens_per_expert=[746, 765, 713, 813, 766, 736, 796, 768, 783, 770, 740, 742, 747, 785, 763, 734, 766, 796, 737, 760, 821, 749, 785, 778, 774, 738, 760, 754, 794, 793, 863, 776, 775, 737, 758, 780, 787, 732, 764, 792, 784, 758, 751, 760, 799, 770, 787, 780, 750, 747, 768, 784, 758, 763, 760, 768, 749, 767, 793, 787, 702, 742, 812, 777] payload_rows_per_dst=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] dup_rows_per_dst=[2944, 2914, 3074, 3094, 2946, 2994, 2946, 3012] combine_rows_per_src=[9262, 9214, 9310, 9410, 9304, 9384, 9250, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343384 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265985368 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531627352 weight_movement_bytes=69206016 total_nvlink_bytes=600833368 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344248 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267018424 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533692600 weight_movement_bytes=103809024 total_nvlink_bytes=637501624 weight_to_token_ratio=0.194511 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343448 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267074968 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533806488 weight_movement_bytes=34603008 total_nvlink_bytes=568409496 weight_to_token_ratio=0.064823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344672 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657824 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534970976 weight_movement_bytes=34603008 total_nvlink_bytes=569573984 weight_to_token_ratio=0.064682 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64594 dup_rows=24144 dup_rows_remote=21050 dedup_ratio=0.245605 router_tokens_per_expert=[837, 766, 797, 780, 750, 748, 794, 778, 723, 763, 799, 774, 788, 779, 786, 751, 737, 755, 791, 765, 766, 813, 792, 716, 804, 745, 758, 769, 789, 732, 775, 742, 760, 745, 849, 775, 778, 751, 776, 757, 716, 804, 761, 759, 784, 748, 745, 751, 735, 737, 783, 728, 770, 781, 728, 749, 743, 787, 808, 776, 787, 758, 788, 773] payload_rows_per_dst=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] dup_rows_per_dst=[3094, 3068, 3110, 3092, 3376, 2644, 2698, 3062] combine_rows_per_src=[9566, 9452, 9268, 9208, 9688, 8742, 8858, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65086 dup_rows=23894 dup_rows_remote=20854 dedup_ratio=0.243062 router_tokens_per_expert=[779, 676, 746, 768, 810, 775, 786, 778, 767, 753, 735, 743, 750, 793, 834, 783, 747, 771, 774, 767, 800, 773, 777, 773, 774, 796, 764, 770, 745, 762, 780, 725, 776, 767, 792, 795, 754, 762, 771, 719, 798, 803, 770, 768, 754, 737, 790, 725, 745, 816, 812, 703, 780, 825, 782, 796, 787, 736, 704, 772, 750, 754, 763, 772] payload_rows_per_dst=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] dup_rows_per_dst=[2942, 2980, 3040, 2944, 3044, 3036, 3092, 2816] combine_rows_per_src=[9294, 9336, 9324, 9288, 9228, 9254, 9426, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344592 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266854928 ingress_bytes=370319360 link_bytes=370319360 token_bytes=533365264 weight_movement_bytes=103809024 total_nvlink_bytes=637174288 weight_to_token_ratio=0.194630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393481939e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343200 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266460320 ingress_bytes=369926144 link_bytes=369926144 token_bytes=532577440 weight_movement_bytes=103809024 total_nvlink_bytes=636386464 weight_to_token_ratio=0.194918 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392002298e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65120 dup_rows=23826 dup_rows_remote=20742 dedup_ratio=0.242371 router_tokens_per_expert=[757, 748, 785, 827, 719, 766, 794, 800, 738, 752, 749, 778, 783, 745, 755, 764, 754, 780, 798, 755, 767, 803, 738, 791, 790, 696, 758, 744, 752, 743, 757, 767, 756, 779, 768, 749, 797, 781, 766, 758, 777, 720, 760, 737, 778, 792, 825, 757, 765, 777, 755, 769, 824, 773, 759, 767, 744, 808, 808, 782, 758, 767, 769, 774] payload_rows_per_dst=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] dup_rows_per_dst=[2992, 2858, 3030, 2878, 2936, 2966, 3082, 3084] combine_rows_per_src=[9228, 9302, 9278, 9136, 9436, 9326, 9414, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65144 dup_rows=23860 dup_rows_remote=20792 dedup_ratio=0.242716 router_tokens_per_expert=[768, 763, 756, 768, 774, 761, 786, 792, 716, 773, 739, 733, 783, 757, 784, 811, 807, 803, 776, 827, 781, 750, 771, 765, 756, 739, 737, 744, 763, 796, 773, 754, 771, 766, 784, 750, 772, 790, 782, 769, 773, 711, 766, 763, 785, 780, 786, 802, 764, 747, 763, 749, 783, 751, 748, 803, 745, 729, 786, 771, 761, 748, 775, 773] payload_rows_per_dst=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] dup_rows_per_dst=[2954, 2924, 3068, 2896, 3068, 3026, 3044, 2880] combine_rows_per_src=[9382, 9268, 9492, 9228, 9300, 9306, 9172, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343760 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936016 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528272 weight_movement_bytes=34603008 total_nvlink_bytes=568131280 weight_to_token_ratio=0.064857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=342576 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264919600 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529496624 weight_movement_bytes=34603008 total_nvlink_bytes=564099632 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64940 dup_rows=24118 dup_rows_remote=21184 dedup_ratio=0.245341 router_tokens_per_expert=[767, 749, 768, 775, 746, 765, 788, 736, 794, 801, 777, 803, 791, 776, 743, 767, 714, 724, 763, 845, 765, 744, 766, 775, 791, 839, 785, 787, 783, 762, 723, 738, 780, 733, 748, 783, 742, 814, 741, 780, 732, 747, 757, 767, 780, 754, 782, 771, 750, 784, 785, 770, 792, 798, 764, 800, 782, 761, 750, 767, 766, 737, 736, 749] payload_rows_per_dst=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] dup_rows_per_dst=[2990, 3094, 3016, 3104, 2932, 2934, 3200, 2848] combine_rows_per_src=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65084 dup_rows=23976 dup_rows_remote=20926 dedup_ratio=0.243896 router_tokens_per_expert=[805, 790, 781, 726, 693, 726, 781, 720, 759, 760, 807, 804, 755, 775, 780, 808, 740, 784, 738, 799, 818, 805, 737, 741, 782, 723, 709, 798, 762, 793, 810, 770, 788, 812, 778, 727, 794, 773, 770, 746, 771, 775, 804, 766, 797, 783, 751, 756, 735, 722, 771, 743, 794, 769, 774, 801, 820, 741, 742, 728, 769, 750, 755, 768] payload_rows_per_dst=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] dup_rows_per_dst=[2876, 3036, 3066, 3050, 3046, 3052, 2968, 2882] combine_rows_per_src=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64884 dup_rows=24182 dup_rows_remote=21210 dedup_ratio=0.245992 router_tokens_per_expert=[759, 771, 761, 771, 767, 778, 788, 759, 772, 754, 775, 802, 735, 749, 775, 802, 719, 743, 801, 786, 759, 734, 764, 799, 757, 789, 728, 764, 756, 793, 787, 741, 796, 769, 775, 746, 760, 736, 717, 768, 798, 710, 788, 770, 718, 788, 774, 761, 824, 810, 753, 768, 794, 770, 781, 735, 756, 765, 780, 775, 757, 790, 790, 792] payload_rows_per_dst=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] dup_rows_per_dst=[3106, 3008, 2972, 3008, 2900, 2988, 3082, 3118] combine_rows_per_src=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65228 dup_rows=23926 dup_rows_remote=20896 dedup_ratio=0.243388 router_tokens_per_expert=[743, 738, 787, 778, 814, 804, 768, 791, 777, 800, 814, 781, 753, 737, 731, 753, 771, 756, 776, 713, 770, 758, 767, 763, 774, 784, 782, 776, 739, 753, 763, 743, 730, 803, 768, 771, 795, 726, 810, 740, 798, 745, 777, 757, 777, 733, 755, 748, 807, 784, 812, 755, 751, 835, 788, 759, 795, 725, 769, 788, 767, 730, 743, 754] payload_rows_per_dst=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] dup_rows_per_dst=[3110, 3022, 2902, 2900, 2920, 3030, 3092, 2950] combine_rows_per_src=[9336, 9270, 9246, 9328, 9366, 9150, 9490, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343448 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267074968 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533806488 weight_movement_bytes=34603008 total_nvlink_bytes=568409496 weight_to_token_ratio=0.064823 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65180 dup_rows=23900 dup_rows_remote=20860 dedup_ratio=0.243123 router_tokens_per_expert=[763, 758, 735, 778, 734, 762, 753, 742, 766, 743, 753, 745, 786, 749, 806, 784, 796, 774, 757, 771, 775, 810, 733, 731, 745, 785, 779, 761, 739, 819, 810, 797, 771, 705, 786, 758, 780, 817, 763, 750, 753, 771, 749, 771, 765, 747, 734, 769, 820, 780, 786, 785, 798, 743, 781, 770, 769, 776, 767, 786, 757, 764, 762, 780] payload_rows_per_dst=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] dup_rows_per_dst=[2882, 3040, 2866, 3080, 3012, 2868, 3156, 2996] combine_rows_per_src=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64820 dup_rows=24266 dup_rows_remote=21268 dedup_ratio=0.246847 router_tokens_per_expert=[791, 791, 749, 734, 757, 795, 746, 734, 786, 740, 746, 767, 769, 776, 783, 748, 748, 800, 737, 757, 729, 797, 802, 802, 772, 784, 793, 767, 773, 789, 796, 757, 760, 750, 776, 741, 754, 754, 797, 785, 805, 734, 750, 800, 731, 728, 784, 736, 791, 824, 793, 783, 792, 755, 761, 747, 739, 785, 822, 726, 763, 780, 756, 735] payload_rows_per_dst=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] dup_rows_per_dst=[2998, 3330, 3088, 3206, 2948, 2568, 3104, 3024] combine_rows_per_src=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=343744 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267173568 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003392 weight_movement_bytes=34603008 total_nvlink_bytes=568606400 weight_to_token_ratio=0.064799 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65056 dup_rows=23920 dup_rows_remote=20958 dedup_ratio=0.243327 router_tokens_per_expert=[796, 802, 771, 800, 764, 811, 739, 766, 765, 779, 764, 782, 775, 747, 765, 754, 793, 825, 737, 734, 740, 778, 764, 812, 736, 754, 744, 768, 768, 782, 818, 746, 800, 816, 764, 740, 733, 776, 756, 795, 737, 740, 795, 780, 754, 755, 798, 793, 751, 764, 768, 805, 754, 780, 748, 775, 743, 751, 775, 710, 745, 784, 735, 753] payload_rows_per_dst=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] dup_rows_per_dst=[3028, 3014, 2966, 2930, 3034, 3004, 2962, 2982] combine_rows_per_src=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65114 dup_rows=23856 dup_rows_remote=20952 dedup_ratio=0.242676 router_tokens_per_expert=[761, 765, 753, 759, 732, 749, 803, 745, 755, 755, 760, 733, 784, 831, 777, 758, 770, 753, 759, 816, 790, 766, 709, 757, 773, 785, 780, 783, 774, 735, 736, 753, 724, 782, 777, 771, 783, 764, 793, 783, 735, 764, 765, 769, 759, 789, 764, 786, 755, 764, 794, 794, 803, 752, 781, 795, 790, 761, 792, 756, 765, 787, 781, 715] payload_rows_per_dst=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] dup_rows_per_dst=[2880, 2970, 2904, 2904, 3082, 3060, 3068, 2988] combine_rows_per_src=[9254, 9336, 9336, 9334, 9272, 9202, 9408, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344496 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518384 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692272 weight_movement_bytes=34603008 total_nvlink_bytes=569295280 weight_to_token_ratio=0.064716 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298720 weight_movement_bytes=34603008 total_nvlink_bytes=568901728 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344496 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338736 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332976 weight_movement_bytes=34603008 total_nvlink_bytes=566935984 weight_to_token_ratio=0.065003 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=64816 dup_rows=23868 dup_rows_remote=20564 dedup_ratio=0.242798 router_tokens_per_expert=[761, 797, 728, 771, 730, 736, 784, 775, 704, 766, 766, 768, 783, 756, 767, 772, 740, 804, 743, 744, 774, 809, 774, 798, 718, 788, 747, 752, 784, 768, 775, 735, 780, 748, 769, 733, 796, 767, 753, 792, 723, 780, 765, 743, 771, 751, 816, 766, 770, 783, 759, 780, 777, 810, 771, 801, 770, 738, 814, 741, 760, 805, 816, 787] payload_rows_per_dst=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] dup_rows_per_dst=[2996, 2568, 2974, 2894, 2976, 3000, 3156, 3304] combine_rows_per_src=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344040 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928104 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512168 weight_movement_bytes=34603008 total_nvlink_bytes=568115176 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344376 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109240 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874104 weight_movement_bytes=69206016 total_nvlink_bytes=601080120 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65180 dup_rows=23900 dup_rows_remote=20860 dedup_ratio=0.243123 router_tokens_per_expert=[763, 758, 735, 778, 734, 762, 753, 742, 766, 743, 753, 745, 786, 749, 806, 784, 796, 774, 757, 771, 775, 810, 733, 731, 745, 785, 779, 761, 739, 819, 810, 797, 771, 705, 786, 758, 780, 817, 763, 750, 753, 771, 749, 771, 765, 747, 734, 769, 820, 780, 786, 785, 798, 743, 781, 770, 769, 776, 767, 786, 757, 764, 762, 780] payload_rows_per_dst=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] dup_rows_per_dst=[2882, 3040, 2866, 3080, 3012, 2868, 3156, 2996] combine_rows_per_src=[9168, 9224, 9428, 9390, 9248, 9250, 9370, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65056 dup_rows=23920 dup_rows_remote=20958 dedup_ratio=0.243327 router_tokens_per_expert=[796, 802, 771, 800, 764, 811, 739, 766, 765, 779, 764, 782, 775, 747, 765, 754, 793, 825, 737, 734, 740, 778, 764, 812, 736, 754, 744, 768, 768, 782, 818, 746, 800, 816, 764, 740, 733, 776, 756, 795, 737, 740, 795, 780, 754, 755, 798, 793, 751, 764, 768, 805, 754, 780, 748, 775, 743, 751, 775, 710, 745, 784, 735, 753] payload_rows_per_dst=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] dup_rows_per_dst=[3028, 3014, 2966, 2930, 3034, 3004, 2962, 2982] combine_rows_per_src=[9470, 9248, 9400, 9302, 9326, 9300, 9328, 9010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344352 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847072 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349792 weight_movement_bytes=34603008 total_nvlink_bytes=565952800 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=341520 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265827856 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531314192 weight_movement_bytes=34603008 total_nvlink_bytes=565917200 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65018 dup_rows=23948 dup_rows_remote=20910 dedup_ratio=0.243612 router_tokens_per_expert=[779, 804, 721, 839, 775, 698, 747, 752, 762, 741, 799, 760, 744, 746, 785, 773, 810, 772, 770, 766, 812, 753, 768, 762, 710, 753, 756, 753, 808, 805, 762, 802, 748, 762, 773, 784, 800, 808, 773, 788, 734, 749, 714, 778, 798, 757, 766, 736, 793, 794, 784, 740, 798, 757, 778, 729, 774, 794, 768, 739, 781, 768, 750, 750] payload_rows_per_dst=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] dup_rows_per_dst=[3038, 2972, 3066, 2982, 2926, 2922, 3044, 2998] combine_rows_per_src=[9338, 9260, 9460, 9264, 9388, 9142, 9302, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298720 weight_movement_bytes=34603008 total_nvlink_bytes=568901728 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344056 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266813432 ingress_bytes=370278400 link_bytes=370278400 token_bytes=533282808 weight_movement_bytes=103809024 total_nvlink_bytes=637091832 weight_to_token_ratio=0.194660 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393327809e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64884 dup_rows=24182 dup_rows_remote=21210 dedup_ratio=0.245992 router_tokens_per_expert=[759, 771, 761, 771, 767, 778, 788, 759, 772, 754, 775, 802, 735, 749, 775, 802, 719, 743, 801, 786, 759, 734, 764, 799, 757, 789, 728, 764, 756, 793, 787, 741, 796, 769, 775, 746, 760, 736, 717, 768, 798, 710, 788, 770, 718, 788, 774, 761, 824, 810, 753, 768, 794, 770, 781, 735, 756, 765, 780, 775, 757, 790, 790, 792] payload_rows_per_dst=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] dup_rows_per_dst=[3106, 3008, 2972, 3008, 2900, 2988, 3082, 3118] combine_rows_per_src=[9202, 9320, 9238, 9222, 9234, 9226, 9388, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344264 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267051208 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533758152 weight_movement_bytes=34603008 total_nvlink_bytes=568361160 weight_to_token_ratio=0.064829 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=64816 dup_rows=23868 dup_rows_remote=20564 dedup_ratio=0.242798 router_tokens_per_expert=[761, 797, 728, 771, 730, 736, 784, 775, 704, 766, 766, 768, 783, 756, 767, 772, 740, 804, 743, 744, 774, 809, 774, 798, 718, 788, 747, 752, 784, 768, 775, 735, 780, 748, 769, 733, 796, 767, 753, 792, 723, 780, 765, 743, 771, 751, 816, 766, 770, 783, 759, 780, 777, 810, 771, 801, 770, 738, 814, 741, 760, 805, 816, 787] payload_rows_per_dst=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] dup_rows_per_dst=[2996, 2568, 2974, 2894, 2976, 3000, 3156, 3304] combine_rows_per_src=[9168, 9026, 9262, 9240, 9300, 9366, 9454, 9620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65050 dup_rows=23924 dup_rows_remote=20954 dedup_ratio=0.243368 router_tokens_per_expert=[740, 765, 742, 746, 786, 807, 752, 799, 760, 788, 773, 720, 755, 761, 769, 752, 848, 709, 738, 753, 744, 776, 744, 773, 755, 735, 761, 813, 792, 725, 806, 740, 720, 800, 764, 740, 803, 769, 776, 779, 767, 790, 741, 762, 769, 784, 744, 793, 756, 792, 735, 806, 795, 746, 798, 751, 784, 799, 759, 794, 768, 769, 774, 798] payload_rows_per_dst=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] dup_rows_per_dst=[2954, 2948, 3000, 3004, 2964, 2970, 3078, 3006] combine_rows_per_src=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64986 dup_rows=24098 dup_rows_remote=21054 dedup_ratio=0.245138 router_tokens_per_expert=[794, 790, 790, 782, 786, 753, 791, 758, 773, 698, 802, 797, 770, 764, 760, 726, 786, 783, 774, 788, 749, 739, 752, 803, 822, 752, 741, 812, 740, 745, 744, 748, 788, 777, 754, 756, 784, 808, 732, 733, 755, 733, 742, 740, 778, 746, 755, 810, 771, 718, 813, 735, 757, 820, 767, 772, 716, 807, 796, 781, 765, 727, 820, 784] payload_rows_per_dst=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] dup_rows_per_dst=[3128, 2836, 3066, 2980, 3044, 2964, 3024, 3056] combine_rows_per_src=[9360, 9344, 9282, 9228, 9220, 9154, 9282, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343712 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657440 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971168 weight_movement_bytes=34603008 total_nvlink_bytes=567574176 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65056 dup_rows=24032 dup_rows_remote=20966 dedup_ratio=0.244466 router_tokens_per_expert=[708, 757, 780, 753, 813, 755, 736, 802, 793, 756, 791, 815, 754, 767, 811, 829, 803, 741, 804, 791, 722, 761, 731, 788, 730, 788, 828, 788, 812, 775, 772, 785, 719, 755, 779, 785, 791, 763, 724, 779, 770, 747, 763, 813, 789, 752, 699, 760, 750, 764, 742, 799, 756, 792, 710, 745, 784, 751, 739, 792, 756, 726, 803, 716] payload_rows_per_dst=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] dup_rows_per_dst=[2982, 3112, 3066, 3196, 2932, 2940, 2866, 2938] combine_rows_per_src=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74052 payload_rows_remote=64554 dup_rows=24252 dup_rows_remote=21100 dedup_ratio=0.246704 router_tokens_per_expert=[749, 816, 765, 798, 789, 750, 780, 771, 768, 769, 749, 691, 738, 789, 795, 805, 756, 731, 713, 750, 741, 725, 800, 747, 818, 744, 851, 784, 794, 772, 795, 767, 772, 715, 777, 745, 797, 768, 759, 781, 758, 812, 795, 742, 785, 751, 765, 775, 787, 792, 771, 780, 752, 772, 761, 764, 772, 814, 746, 775, 785, 752, 718, 704] payload_rows_per_dst=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] dup_rows_per_dst=[3146, 2958, 2934, 3152, 2916, 3070, 3108, 2968] combine_rows_per_src=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344376 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109240 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874104 weight_movement_bytes=69206016 total_nvlink_bytes=601080120 weight_to_token_ratio=0.130117 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64940 dup_rows=24118 dup_rows_remote=21184 dedup_ratio=0.245341 router_tokens_per_expert=[767, 749, 768, 775, 746, 765, 788, 736, 794, 801, 777, 803, 791, 776, 743, 767, 714, 724, 763, 845, 765, 744, 766, 775, 791, 839, 785, 787, 783, 762, 723, 738, 780, 733, 748, 783, 742, 814, 741, 780, 732, 747, 757, 767, 780, 754, 782, 771, 750, 784, 785, 770, 792, 798, 764, 800, 782, 761, 750, 767, 766, 737, 736, 749] payload_rows_per_dst=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] dup_rows_per_dst=[2990, 3094, 3016, 3104, 2932, 2934, 3200, 2848] combine_rows_per_src=[9198, 9410, 9176, 9312, 9310, 9246, 9286, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=341520 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265827856 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531314192 weight_movement_bytes=34603008 total_nvlink_bytes=565917200 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64956 dup_rows=24016 dup_rows_remote=21020 dedup_ratio=0.244303 router_tokens_per_expert=[747, 840, 782, 749, 758, 761, 765, 748, 791, 695, 773, 777, 781, 799, 796, 752, 761, 779, 768, 760, 748, 786, 762, 738, 788, 810, 772, 741, 779, 749, 803, 791, 748, 753, 736, 779, 760, 784, 776, 773, 777, 774, 772, 778, 786, 763, 778, 722, 805, 750, 756, 779, 799, 754, 774, 730, 726, 713, 749, 798, 782, 792, 790, 747] payload_rows_per_dst=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] dup_rows_per_dst=[2984, 2996, 3016, 3090, 2914, 2936, 3060, 3020] combine_rows_per_src=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344016 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788816 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533233616 weight_movement_bytes=138412032 total_nvlink_bytes=671645648 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344160 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526816 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709472 weight_movement_bytes=34603008 total_nvlink_bytes=567312480 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344088 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266813464 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533282840 weight_movement_bytes=69206016 total_nvlink_bytes=602488856 weight_to_token_ratio=0.129774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=342616 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264755800 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529168984 weight_movement_bytes=34603008 total_nvlink_bytes=563771992 weight_to_token_ratio=0.065391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65030 dup_rows=23992 dup_rows_remote=20962 dedup_ratio=0.244059 router_tokens_per_expert=[817, 804, 737, 739, 775, 689, 819, 775, 820, 749, 801, 790, 801, 758, 793, 738, 807, 750, 759, 736, 759, 821, 721, 737, 776, 782, 752, 789, 775, 752, 761, 813, 775, 760, 782, 748, 772, 754, 760, 734, 793, 716, 759, 761, 712, 797, 760, 754, 807, 745, 727, 819, 759, 759, 724, 717, 788, 794, 783, 778, 819, 737, 777, 787] payload_rows_per_dst=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] dup_rows_per_dst=[3030, 3042, 2920, 3206, 2988, 3154, 2558, 3094] combine_rows_per_src=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65048 dup_rows=23890 dup_rows_remote=20912 dedup_ratio=0.243022 router_tokens_per_expert=[823, 777, 750, 768, 783, 795, 748, 743, 744, 745, 823, 773, 807, 748, 748, 812, 790, 802, 786, 782, 766, 778, 799, 785, 740, 772, 746, 809, 732, 768, 731, 750, 781, 764, 755, 817, 783, 799, 732, 775, 732, 794, 804, 779, 760, 737, 762, 698, 717, 745, 808, 754, 759, 742, 761, 764, 727, 770, 778, 765, 801, 771, 780, 715] payload_rows_per_dst=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] dup_rows_per_dst=[2762, 2994, 3214, 3018, 3116, 2782, 3026, 2978] combine_rows_per_src=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64956 dup_rows=24016 dup_rows_remote=21020 dedup_ratio=0.244303 router_tokens_per_expert=[747, 840, 782, 749, 758, 761, 765, 748, 791, 695, 773, 777, 781, 799, 796, 752, 761, 779, 768, 760, 748, 786, 762, 738, 788, 810, 772, 741, 779, 749, 803, 791, 748, 753, 736, 779, 760, 784, 776, 773, 777, 774, 772, 778, 786, 763, 778, 722, 805, 750, 756, 779, 799, 754, 774, 730, 726, 713, 749, 798, 782, 792, 790, 747] payload_rows_per_dst=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] dup_rows_per_dst=[2984, 2996, 3016, 3090, 2914, 2936, 3060, 3020] combine_rows_per_src=[9316, 9332, 9188, 9376, 9304, 9364, 9234, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=20886 dedup_ratio=0.242676 router_tokens_per_expert=[778, 749, 799, 789, 769, 758, 755, 760, 759, 819, 754, 793, 784, 700, 751, 799, 776, 799, 766, 760, 792, 787, 763, 778, 760, 783, 742, 785, 776, 805, 710, 713, 719, 784, 817, 806, 776, 745, 738, 775, 787, 790, 756, 774, 818, 776, 757, 750, 804, 773, 758, 739, 749, 753, 774, 764, 766, 808, 737, 747, 739, 758, 775, 729] payload_rows_per_dst=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] dup_rows_per_dst=[3024, 3024, 3112, 2792, 2968, 2952, 2970, 3014] combine_rows_per_src=[9290, 9294, 9330, 9356, 9352, 9464, 9258, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65084 dup_rows=23976 dup_rows_remote=20926 dedup_ratio=0.243896 router_tokens_per_expert=[805, 790, 781, 726, 693, 726, 781, 720, 759, 760, 807, 804, 755, 775, 780, 808, 740, 784, 738, 799, 818, 805, 737, 741, 782, 723, 709, 798, 762, 793, 810, 770, 788, 812, 778, 727, 794, 773, 770, 746, 771, 775, 804, 766, 797, 783, 751, 756, 735, 722, 771, 743, 794, 769, 774, 801, 820, 741, 742, 728, 769, 750, 755, 768] payload_rows_per_dst=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] dup_rows_per_dst=[2876, 3036, 3066, 3050, 3046, 3052, 2968, 2882] combine_rows_per_src=[9168, 9460, 9258, 9244, 9330, 9354, 9250, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344496 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338736 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332976 weight_movement_bytes=34603008 total_nvlink_bytes=566935984 weight_to_token_ratio=0.065003 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343968 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266706848 ingress_bytes=404774912 link_bytes=404774912 token_bytes=533069728 weight_movement_bytes=138412032 total_nvlink_bytes=671481760 weight_to_token_ratio=0.259651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523135407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343840 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266780448 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217056 weight_movement_bytes=34603008 total_nvlink_bytes=567820064 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344304 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267362544 ingress_bytes=336224256 link_bytes=336224256 token_bytes=534380784 weight_movement_bytes=69206016 total_nvlink_bytes=603586800 weight_to_token_ratio=0.129507 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265184807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65054 dup_rows=24074 dup_rows_remote=21060 dedup_ratio=0.244893 router_tokens_per_expert=[789, 780, 745, 765, 719, 762, 787, 736, 791, 772, 789, 801, 726, 781, 781, 805, 778, 728, 763, 762, 758, 726, 773, 786, 789, 799, 748, 749, 782, 742, 752, 813, 740, 788, 824, 769, 760, 751, 740, 791, 779, 779, 737, 739, 757, 765, 835, 794, 822, 780, 762, 788, 740, 744, 777, 726, 767, 779, 782, 786, 746, 714, 729, 785] payload_rows_per_dst=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] dup_rows_per_dst=[3014, 3066, 3168, 3224, 2834, 3070, 2716, 2982] combine_rows_per_src=[9176, 9442, 9318, 9380, 9236, 9300, 9184, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344040 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928104 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512168 weight_movement_bytes=34603008 total_nvlink_bytes=568115176 weight_to_token_ratio=0.064859 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65056 dup_rows=24032 dup_rows_remote=20966 dedup_ratio=0.244466 router_tokens_per_expert=[708, 757, 780, 753, 813, 755, 736, 802, 793, 756, 791, 815, 754, 767, 811, 829, 803, 741, 804, 791, 722, 761, 731, 788, 730, 788, 828, 788, 812, 775, 772, 785, 719, 755, 779, 785, 791, 763, 724, 779, 770, 747, 763, 813, 789, 752, 699, 760, 750, 764, 742, 799, 756, 792, 710, 745, 784, 751, 739, 792, 756, 726, 803, 716] payload_rows_per_dst=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] dup_rows_per_dst=[2982, 3112, 3066, 3196, 2932, 2940, 2866, 2938] combine_rows_per_src=[9226, 9520, 9216, 9360, 9258, 9246, 9250, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65048 dup_rows=23890 dup_rows_remote=20912 dedup_ratio=0.243022 router_tokens_per_expert=[823, 777, 750, 768, 783, 795, 748, 743, 744, 745, 823, 773, 807, 748, 748, 812, 790, 802, 786, 782, 766, 778, 799, 785, 740, 772, 746, 809, 732, 768, 731, 750, 781, 764, 755, 817, 783, 799, 732, 775, 732, 794, 804, 779, 760, 737, 762, 698, 717, 745, 808, 754, 759, 742, 761, 764, 727, 770, 778, 765, 801, 771, 780, 715] payload_rows_per_dst=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] dup_rows_per_dst=[2762, 2994, 3214, 3018, 3116, 2782, 3026, 2978] combine_rows_per_src=[9098, 9406, 9362, 9266, 9492, 9108, 9316, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65082 dup_rows=23788 dup_rows_remote=20722 dedup_ratio=0.241984 router_tokens_per_expert=[820, 763, 732, 790, 745, 751, 822, 741, 762, 755, 775, 799, 749, 708, 782, 765, 753, 785, 744, 798, 791, 740, 742, 764, 751, 775, 753, 804, 765, 778, 804, 735, 760, 831, 772, 785, 727, 741, 769, 782, 776, 798, 834, 777, 707, 784, 781, 793, 744, 800, 763, 735, 805, 769, 767, 778, 763, 742, 781, 762, 787, 714, 756, 728] payload_rows_per_dst=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] dup_rows_per_dst=[3044, 2954, 2924, 3018, 2934, 3066, 3022, 2826] combine_rows_per_src=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65002 dup_rows=23972 dup_rows_remote=20978 dedup_ratio=0.243856 router_tokens_per_expert=[781, 768, 740, 816, 757, 790, 802, 754, 720, 759, 728, 767, 802, 778, 749, 745, 771, 796, 780, 766, 766, 732, 775, 776, 737, 775, 808, 775, 762, 749, 776, 734, 748, 731, 754, 742, 767, 731, 735, 780, 775, 804, 768, 802, 773, 761, 782, 810, 765, 758, 756, 799, 804, 745, 717, 794, 791, 754, 814, 775, 764, 734, 802, 783] payload_rows_per_dst=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] dup_rows_per_dst=[3006, 2894, 2994, 2972, 2912, 3124, 3010, 3060] combine_rows_per_src=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74558 payload_rows_remote=65132 dup_rows=23746 dup_rows_remote=20618 dedup_ratio=0.241557 router_tokens_per_expert=[714, 763, 816, 749, 768, 781, 788, 806, 832, 753, 772, 754, 752, 782, 755, 777, 728, 763, 825, 773, 721, 767, 766, 710, 796, 769, 804, 725, 785, 768, 755, 770, 797, 780, 785, 785, 775, 832, 781, 742, 812, 786, 793, 752, 790, 755, 740, 819, 752, 770, 734, 728, 798, 766, 743, 783, 800, 763, 715, 743, 704, 731, 763, 748] payload_rows_per_dst=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] dup_rows_per_dst=[2974, 2970, 2920, 2962, 3128, 3032, 2894, 2866] combine_rows_per_src=[9396, 9384, 9186, 9382, 9426, 9462, 9254, 9068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65174 dup_rows=23724 dup_rows_remote=20658 dedup_ratio=0.241333 router_tokens_per_expert=[741, 806, 708, 764, 796, 776, 750, 761, 766, 736, 766, 771, 794, 783, 755, 794, 771, 775, 796, 770, 856, 802, 789, 736, 760, 784, 733, 765, 799, 802, 801, 792, 765, 745, 777, 774, 779, 756, 764, 794, 771, 746, 736, 788, 779, 736, 793, 754, 793, 751, 733, 763, 754, 763, 797, 775, 763, 711, 773, 703, 716, 758, 786, 758] payload_rows_per_dst=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] dup_rows_per_dst=[2934, 3006, 3078, 3066, 2980, 2944, 2912, 2804] combine_rows_per_src=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65050 dup_rows=23924 dup_rows_remote=20954 dedup_ratio=0.243368 router_tokens_per_expert=[740, 765, 742, 746, 786, 807, 752, 799, 760, 788, 773, 720, 755, 761, 769, 752, 848, 709, 738, 753, 744, 776, 744, 773, 755, 735, 761, 813, 792, 725, 806, 740, 720, 800, 764, 740, 803, 769, 776, 779, 767, 790, 741, 762, 769, 784, 744, 793, 756, 792, 735, 806, 795, 746, 798, 751, 784, 799, 759, 794, 768, 769, 774, 798] payload_rows_per_dst=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] dup_rows_per_dst=[2954, 2948, 3000, 3004, 2964, 2970, 3078, 3006] combine_rows_per_src=[9320, 9208, 9170, 9250, 9338, 9330, 9280, 9484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344456 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805640 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266824 weight_movement_bytes=69206016 total_nvlink_bytes=602472840 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344088 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266813464 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533282840 weight_movement_bytes=69206016 total_nvlink_bytes=602488856 weight_to_token_ratio=0.129774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64972 dup_rows=24078 dup_rows_remote=21170 dedup_ratio=0.244934 router_tokens_per_expert=[780, 803, 801, 774, 808, 771, 808, 717, 750, 774, 719, 774, 756, 786, 773, 749, 824, 788, 782, 790, 774, 748, 746, 801, 704, 736, 779, 774, 776, 717, 751, 762, 793, 765, 751, 778, 764, 743, 747, 828, 767, 728, 825, 732, 751, 841, 771, 795, 734, 747, 770, 757, 746, 761, 750, 797, 801, 760, 739, 730, 766, 759, 816, 745] payload_rows_per_dst=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] dup_rows_per_dst=[3154, 2908, 3112, 2804, 3086, 3098, 2910, 3006] combine_rows_per_src=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343840 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266780448 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217056 weight_movement_bytes=34603008 total_nvlink_bytes=567820064 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343216 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266919088 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494960 weight_movement_bytes=69206016 total_nvlink_bytes=602700976 weight_to_token_ratio=0.129722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343920 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592112 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840304 weight_movement_bytes=34603008 total_nvlink_bytes=567443312 weight_to_token_ratio=0.064941 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343000 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267123672 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533904344 weight_movement_bytes=34603008 total_nvlink_bytes=568507352 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343328 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267296032 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534248736 weight_movement_bytes=69206016 total_nvlink_bytes=603454752 weight_to_token_ratio=0.129539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64770 dup_rows=24096 dup_rows_remote=20936 dedup_ratio=0.245117 router_tokens_per_expert=[786, 782, 768, 802, 758, 762, 736, 811, 774, 778, 797, 773, 802, 723, 747, 711, 804, 761, 726, 786, 757, 765, 778, 782, 769, 745, 778, 764, 772, 798, 756, 753, 794, 739, 788, 720, 719, 796, 741, 747, 794, 781, 779, 757, 803, 754, 728, 770, 747, 784, 748, 777, 810, 793, 733, 796, 749, 811, 796, 769, 740, 745, 767, 773] payload_rows_per_dst=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] dup_rows_per_dst=[3160, 3026, 2920, 2910, 3060, 3044, 2966, 3010] combine_rows_per_src=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65030 dup_rows=23890 dup_rows_remote=20844 dedup_ratio=0.243022 router_tokens_per_expert=[729, 732, 731, 715, 761, 766, 744, 740, 745, 755, 754, 789, 761, 772, 781, 742, 797, 749, 765, 727, 717, 772, 757, 735, 798, 748, 815, 783, 807, 767, 799, 796, 720, 777, 741, 774, 762, 816, 831, 765, 732, 796, 764, 775, 741, 824, 810, 737, 812, 797, 797, 752, 783, 747, 794, 773, 789, 796, 763, 780, 771, 756, 750, 778] payload_rows_per_dst=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] dup_rows_per_dst=[2848, 2964, 2862, 3122, 3038, 2876, 3134, 3046] combine_rows_per_src=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64972 dup_rows=24078 dup_rows_remote=21170 dedup_ratio=0.244934 router_tokens_per_expert=[780, 803, 801, 774, 808, 771, 808, 717, 750, 774, 719, 774, 756, 786, 773, 749, 824, 788, 782, 790, 774, 748, 746, 801, 704, 736, 779, 774, 776, 717, 751, 762, 793, 765, 751, 778, 764, 743, 747, 828, 767, 728, 825, 732, 751, 841, 771, 795, 734, 747, 770, 757, 746, 761, 750, 797, 801, 760, 739, 730, 766, 759, 816, 745] payload_rows_per_dst=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] dup_rows_per_dst=[3154, 2908, 3112, 2804, 3086, 3098, 2910, 3006] combine_rows_per_src=[9370, 9254, 9394, 9194, 9252, 9322, 9214, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65020 dup_rows=24074 dup_rows_remote=20960 dedup_ratio=0.244893 router_tokens_per_expert=[773, 746, 797, 759, 757, 764, 767, 744, 756, 785, 724, 780, 795, 722, 798, 791, 755, 787, 786, 732, 773, 804, 787, 771, 805, 748, 810, 728, 745, 766, 783, 729, 737, 745, 731, 739, 788, 782, 777, 732, 768, 765, 800, 789, 778, 805, 776, 779, 757, 780, 765, 773, 783, 766, 804, 734, 770, 729, 750, 782, 751, 783, 750, 817] payload_rows_per_dst=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] dup_rows_per_dst=[2968, 3040, 3084, 2934, 2896, 3108, 3114, 2930] combine_rows_per_src=[9246, 9262, 9306, 9294, 9166, 9412, 9210, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74052 payload_rows_remote=64554 dup_rows=24252 dup_rows_remote=21100 dedup_ratio=0.246704 router_tokens_per_expert=[749, 816, 765, 798, 789, 750, 780, 771, 768, 769, 749, 691, 738, 789, 795, 805, 756, 731, 713, 750, 741, 725, 800, 747, 818, 744, 851, 784, 794, 772, 795, 767, 772, 715, 777, 745, 797, 768, 759, 781, 758, 812, 795, 742, 785, 751, 765, 775, 787, 792, 771, 780, 752, 772, 761, 764, 772, 814, 746, 775, 785, 752, 718, 704] payload_rows_per_dst=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] dup_rows_per_dst=[3146, 2958, 2934, 3152, 2916, 3070, 3108, 2968] combine_rows_per_src=[9290, 9250, 8992, 9498, 9312, 9296, 9250, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344016 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788816 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533233616 weight_movement_bytes=138412032 total_nvlink_bytes=671645648 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342824 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640744 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938664 weight_movement_bytes=34603008 total_nvlink_bytes=565541672 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343496 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706376 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069256 weight_movement_bytes=34603008 total_nvlink_bytes=567672264 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344568 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469880 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595192 weight_movement_bytes=34603008 total_nvlink_bytes=567198200 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65082 dup_rows=23916 dup_rows_remote=20910 dedup_ratio=0.243286 router_tokens_per_expert=[752, 754, 807, 812, 743, 758, 762, 746, 753, 760, 739, 766, 754, 764, 784, 780, 785, 758, 784, 764, 732, 787, 774, 772, 792, 772, 743, 733, 736, 769, 792, 774, 759, 745, 776, 771, 781, 786, 757, 778, 785, 755, 739, 752, 821, 786, 718, 770, 798, 730, 818, 758, 820, 751, 742, 790, 764, 787, 768, 778, 814, 768, 727, 759] payload_rows_per_dst=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] dup_rows_per_dst=[2884, 2920, 3006, 2978, 3020, 3024, 3018, 3066] combine_rows_per_src=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343920 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266665840 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532987760 weight_movement_bytes=34603008 total_nvlink_bytes=567590768 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264413184 dispatch_meta_bytes=342616 combine_payload_bytes=264413184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264755800 ingress_bytes=299016192 link_bytes=299016192 token_bytes=529168984 weight_movement_bytes=34603008 total_nvlink_bytes=563771992 weight_to_token_ratio=0.065391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125173858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=65106 dup_rows=23888 dup_rows_remote=20956 dedup_ratio=0.243001 router_tokens_per_expert=[727, 841, 753, 737, 770, 753, 756, 783, 808, 748, 756, 746, 770, 771, 746, 775, 792, 708, 748, 737, 713, 765, 794, 786, 774, 773, 832, 753, 754, 776, 810, 737, 733, 785, 850, 777, 810, 742, 816, 772, 781, 763, 669, 836, 721, 753, 789, 777, 724, 767, 794, 733, 738, 750, 814, 772, 749, 796, 779, 801, 753, 733, 792, 791] payload_rows_per_dst=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] dup_rows_per_dst=[2932, 3010, 3130, 3092, 3058, 2860, 2930, 2876] combine_rows_per_src=[9310, 9230, 9458, 9358, 9512, 9318, 9254, 8976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65002 dup_rows=23972 dup_rows_remote=20978 dedup_ratio=0.243856 router_tokens_per_expert=[781, 768, 740, 816, 757, 790, 802, 754, 720, 759, 728, 767, 802, 778, 749, 745, 771, 796, 780, 766, 766, 732, 775, 776, 737, 775, 808, 775, 762, 749, 776, 734, 748, 731, 754, 742, 767, 731, 735, 780, 775, 804, 768, 802, 773, 761, 782, 810, 765, 758, 756, 799, 804, 745, 717, 794, 791, 754, 814, 775, 764, 734, 802, 783] payload_rows_per_dst=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] dup_rows_per_dst=[3006, 2894, 2994, 2972, 2912, 3124, 3010, 3060] combine_rows_per_src=[9410, 9202, 9330, 9260, 9064, 9426, 9266, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65030 dup_rows=23890 dup_rows_remote=20844 dedup_ratio=0.243022 router_tokens_per_expert=[729, 732, 731, 715, 761, 766, 744, 740, 745, 755, 754, 789, 761, 772, 781, 742, 797, 749, 765, 727, 717, 772, 757, 735, 798, 748, 815, 783, 807, 767, 799, 796, 720, 777, 741, 774, 762, 816, 831, 765, 732, 796, 764, 775, 741, 824, 810, 737, 812, 797, 797, 752, 783, 747, 794, 773, 789, 796, 763, 780, 771, 756, 750, 778] payload_rows_per_dst=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] dup_rows_per_dst=[2848, 2964, 2862, 3122, 3038, 2876, 3134, 3046] combine_rows_per_src=[8988, 9234, 9176, 9504, 9276, 9418, 9434, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65028 dup_rows=24004 dup_rows_remote=21024 dedup_ratio=0.244181 router_tokens_per_expert=[761, 781, 764, 784, 772, 740, 804, 806, 741, 754, 776, 808, 744, 768, 746, 772, 735, 786, 864, 797, 768, 761, 736, 736, 788, 747, 771, 786, 780, 763, 746, 774, 751, 766, 762, 767, 784, 809, 739, 799, 777, 770, 783, 743, 788, 737, 763, 765, 745, 746, 725, 690, 791, 761, 782, 808, 767, 761, 739, 762, 787, 808, 776, 742] payload_rows_per_dst=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] dup_rows_per_dst=[3096, 3044, 3056, 2970, 2948, 2980, 2892, 3018] combine_rows_per_src=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64712 dup_rows=24194 dup_rows_remote=21104 dedup_ratio=0.246114 router_tokens_per_expert=[811, 790, 747, 803, 804, 766, 762, 749, 766, 740, 802, 758, 750, 776, 749, 774, 777, 744, 773, 796, 768, 722, 788, 733, 758, 820, 748, 750, 816, 766, 740, 715, 742, 786, 807, 750, 772, 786, 801, 800, 746, 803, 786, 741, 743, 787, 758, 738, 781, 728, 818, 723, 776, 745, 712, 784, 753, 746, 804, 763, 790, 769, 791, 762] payload_rows_per_dst=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] dup_rows_per_dst=[3052, 3066, 2942, 3010, 3090, 2990, 2992, 3052] combine_rows_per_src=[9412, 9164, 9260, 9216, 9398, 9214, 9142, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65082 dup_rows=23788 dup_rows_remote=20722 dedup_ratio=0.241984 router_tokens_per_expert=[820, 763, 732, 790, 745, 751, 822, 741, 762, 755, 775, 799, 749, 708, 782, 765, 753, 785, 744, 798, 791, 740, 742, 764, 751, 775, 753, 804, 765, 778, 804, 735, 760, 831, 772, 785, 727, 741, 769, 782, 776, 798, 834, 777, 707, 784, 781, 793, 744, 800, 763, 735, 805, 769, 767, 778, 763, 742, 781, 762, 787, 714, 756, 728] payload_rows_per_dst=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] dup_rows_per_dst=[3044, 2954, 2924, 3018, 2934, 3066, 3022, 2826] combine_rows_per_src=[9284, 9236, 9310, 9312, 9400, 9434, 9300, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65060 dup_rows=23858 dup_rows_remote=20890 dedup_ratio=0.242696 router_tokens_per_expert=[788, 778, 783, 785, 762, 758, 740, 792, 750, 754, 748, 751, 732, 755, 754, 735, 794, 750, 753, 750, 804, 729, 744, 814, 769, 758, 807, 757, 798, 750, 768, 770, 763, 786, 792, 759, 756, 785, 716, 777, 752, 745, 773, 741, 836, 809, 748, 784, 754, 786, 776, 759, 797, 751, 751, 706, 783, 775, 797, 842, 761, 751, 769, 792] payload_rows_per_dst=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] dup_rows_per_dst=[2994, 2886, 2960, 2968, 2960, 3066, 2864, 3160] combine_rows_per_src=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344248 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267018424 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533692600 weight_movement_bytes=103809024 total_nvlink_bytes=637501624 weight_to_token_ratio=0.194511 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343968 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919840 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495712 weight_movement_bytes=34603008 total_nvlink_bytes=568098720 weight_to_token_ratio=0.064861 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344208 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266698896 ingress_bytes=370163712 link_bytes=370163712 token_bytes=533053584 weight_movement_bytes=103809024 total_nvlink_bytes=636862608 weight_to_token_ratio=0.194744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392896248e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343264 combine_payload_bytes=265060352 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265403616 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530463968 weight_movement_bytes=138412032 total_nvlink_bytes=668876000 weight_to_token_ratio=0.260926 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=343800 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266829560 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533315320 weight_movement_bytes=34603008 total_nvlink_bytes=567918328 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64916 dup_rows=24108 dup_rows_remote=21178 dedup_ratio=0.245239 router_tokens_per_expert=[724, 794, 769, 720, 744, 748, 794, 780, 791, 738, 756, 741, 754, 761, 804, 755, 714, 777, 759, 781, 747, 730, 773, 790, 832, 763, 724, 729, 781, 785, 750, 793, 773, 825, 716, 797, 760, 752, 735, 760, 781, 783, 768, 792, 788, 769, 762, 766, 785, 728, 775, 781, 749, 782, 769, 818, 764, 777, 771, 821, 725, 822, 758, 799] payload_rows_per_dst=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] dup_rows_per_dst=[2930, 3176, 3016, 3014, 3068, 2718, 3044, 3142] combine_rows_per_src=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64764 dup_rows=24092 dup_rows_remote=20888 dedup_ratio=0.245076 router_tokens_per_expert=[763, 789, 779, 732, 783, 747, 769, 777, 789, 782, 733, 797, 724, 773, 731, 783, 763, 787, 752, 737, 782, 750, 758, 778, 786, 751, 751, 748, 757, 809, 719, 781, 773, 780, 785, 785, 706, 752, 786, 762, 777, 785, 793, 749, 760, 770, 796, 765, 786, 697, 791, 749, 863, 741, 760, 764, 768, 781, 765, 750, 826, 807, 742, 778] payload_rows_per_dst=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] dup_rows_per_dst=[2848, 2938, 2862, 3222, 2926, 3058, 3034, 3204] combine_rows_per_src=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64900 dup_rows=23938 dup_rows_remote=20884 dedup_ratio=0.243510 router_tokens_per_expert=[771, 788, 802, 780, 772, 784, 762, 764, 784, 769, 771, 803, 809, 772, 787, 765, 746, 754, 768, 794, 754, 758, 754, 690, 743, 787, 757, 752, 713, 753, 795, 781, 769, 782, 768, 736, 817, 754, 756, 780, 771, 757, 780, 711, 755, 807, 744, 747, 774, 755, 737, 780, 756, 795, 826, 738, 808, 754, 797, 781, 760, 727, 769, 779] payload_rows_per_dst=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] dup_rows_per_dst=[3012, 3054, 2882, 2976, 2936, 3022, 3058, 2998] combine_rows_per_src=[9434, 9466, 9154, 9186, 9388, 9122, 9264, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343920 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592112 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840304 weight_movement_bytes=34603008 total_nvlink_bytes=567443312 weight_to_token_ratio=0.064941 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343496 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706376 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069256 weight_movement_bytes=34603008 total_nvlink_bytes=567672264 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65186 dup_rows=23858 dup_rows_remote=20914 dedup_ratio=0.242696 router_tokens_per_expert=[673, 768, 824, 758, 710, 803, 816, 757, 777, 770, 784, 774, 780, 797, 723, 739, 799, 818, 774, 743, 762, 758, 763, 768, 774, 764, 788, 731, 793, 781, 726, 763, 809, 801, 764, 763, 775, 733, 690, 749, 764, 786, 742, 743, 764, 760, 768, 821, 767, 774, 770, 754, 725, 798, 730, 784, 744, 762, 782, 817, 792, 794, 796, 773] payload_rows_per_dst=[9298, 9274, 9400, 9226, 9324, 9294, 9260, 9370] dup_rows_per_dst=[2920, 3014, 2970, 3014, 2844, 3002, 2944, 3150] combine_rows_per_src=[9298, 9274, 9400, 9226, 9324, 9294, 9260, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343264 combine_payload_bytes=265060352 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265403616 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530463968 weight_movement_bytes=138412032 total_nvlink_bytes=668876000 weight_to_token_ratio=0.260926 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65174 dup_rows=23724 dup_rows_remote=20658 dedup_ratio=0.241333 router_tokens_per_expert=[741, 806, 708, 764, 796, 776, 750, 761, 766, 736, 766, 771, 794, 783, 755, 794, 771, 775, 796, 770, 856, 802, 789, 736, 760, 784, 733, 765, 799, 802, 801, 792, 765, 745, 777, 774, 779, 756, 764, 794, 771, 746, 736, 788, 779, 736, 793, 754, 793, 751, 733, 763, 754, 763, 797, 775, 763, 711, 773, 703, 716, 758, 786, 758] payload_rows_per_dst=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] dup_rows_per_dst=[2934, 3006, 3078, 3066, 2980, 2944, 2912, 2804] combine_rows_per_src=[9270, 9324, 9512, 9406, 9328, 9262, 9346, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=344376 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266240312 ingress_bytes=404307968 link_bytes=404307968 token_bytes=532136248 weight_movement_bytes=138412032 total_nvlink_bytes=670548280 weight_to_token_ratio=0.260106 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521378334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342608 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265615952 ingress_bytes=334479360 link_bytes=334479360 token_bytes=530889296 weight_movement_bytes=69206016 total_nvlink_bytes=600095312 weight_to_token_ratio=0.130359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258618904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343216 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266919088 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494960 weight_movement_bytes=69206016 total_nvlink_bytes=602700976 weight_to_token_ratio=0.129722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64882 dup_rows=24100 dup_rows_remote=21230 dedup_ratio=0.245158 router_tokens_per_expert=[736, 766, 749, 786, 771, 733, 759, 819, 766, 773, 775, 749, 745, 795, 778, 726, 778, 781, 705, 767, 750, 767, 803, 745, 763, 723, 796, 696, 745, 808, 762, 753, 808, 771, 769, 777, 750, 747, 778, 774, 791, 778, 780, 794, 770, 756, 784, 700, 740, 795, 753, 779, 813, 814, 767, 779, 772, 773, 732, 808, 778, 812, 780, 762] payload_rows_per_dst=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] dup_rows_per_dst=[2872, 3078, 2870, 2866, 2984, 3124, 3116, 3190] combine_rows_per_src=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343136 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173536 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532003936 weight_movement_bytes=34603008 total_nvlink_bytes=566606944 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65054 dup_rows=23950 dup_rows_remote=21006 dedup_ratio=0.243632 router_tokens_per_expert=[802, 804, 777, 791, 773, 763, 742, 722, 764, 739, 781, 775, 765, 781, 745, 790, 832, 708, 793, 792, 736, 764, 761, 737, 806, 766, 787, 733, 759, 785, 792, 763, 805, 742, 752, 838, 767, 763, 828, 755, 765, 734, 764, 822, 736, 768, 763, 770, 777, 731, 754, 754, 716, 782, 758, 797, 749, 791, 746, 766, 779, 745, 782, 725] payload_rows_per_dst=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] dup_rows_per_dst=[3024, 3004, 3002, 2974, 3132, 2944, 2956, 2914] combine_rows_per_src=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344400 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346256 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348112 weight_movement_bytes=69206016 total_nvlink_bytes=603554128 weight_to_token_ratio=0.129515 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343328 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267296032 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534248736 weight_movement_bytes=69206016 total_nvlink_bytes=603454752 weight_to_token_ratio=0.129539 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64820 dup_rows=24266 dup_rows_remote=21268 dedup_ratio=0.246847 router_tokens_per_expert=[791, 791, 749, 734, 757, 795, 746, 734, 786, 740, 746, 767, 769, 776, 783, 748, 748, 800, 737, 757, 729, 797, 802, 802, 772, 784, 793, 767, 773, 789, 796, 757, 760, 750, 776, 741, 754, 754, 797, 785, 805, 734, 750, 800, 731, 728, 784, 736, 791, 824, 793, 783, 792, 755, 761, 747, 739, 785, 822, 726, 763, 780, 756, 735] payload_rows_per_dst=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] dup_rows_per_dst=[2998, 3330, 3088, 3206, 2948, 2568, 3104, 3024] combine_rows_per_src=[9218, 9610, 9278, 9244, 9286, 8858, 9356, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64764 dup_rows=24092 dup_rows_remote=20888 dedup_ratio=0.245076 router_tokens_per_expert=[763, 789, 779, 732, 783, 747, 769, 777, 789, 782, 733, 797, 724, 773, 731, 783, 763, 787, 752, 737, 782, 750, 758, 778, 786, 751, 751, 748, 757, 809, 719, 781, 773, 780, 785, 785, 706, 752, 786, 762, 777, 785, 793, 749, 760, 770, 796, 765, 786, 697, 791, 749, 863, 741, 760, 764, 768, 781, 765, 750, 826, 807, 742, 778] payload_rows_per_dst=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] dup_rows_per_dst=[2848, 2938, 2862, 3222, 2926, 3058, 3034, 3204] combine_rows_per_src=[9002, 9286, 9130, 9332, 9332, 9336, 9346, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65082 dup_rows=23916 dup_rows_remote=20910 dedup_ratio=0.243286 router_tokens_per_expert=[752, 754, 807, 812, 743, 758, 762, 746, 753, 760, 739, 766, 754, 764, 784, 780, 785, 758, 784, 764, 732, 787, 774, 772, 792, 772, 743, 733, 736, 769, 792, 774, 759, 745, 776, 771, 781, 786, 757, 778, 785, 755, 739, 752, 821, 786, 718, 770, 798, 730, 818, 758, 820, 751, 742, 790, 764, 787, 768, 778, 814, 768, 727, 759] payload_rows_per_dst=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] dup_rows_per_dst=[2884, 2920, 3006, 2978, 3020, 3024, 3018, 3066] combine_rows_per_src=[9384, 9280, 9306, 9244, 9286, 9228, 9396, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65050 dup_rows=23986 dup_rows_remote=21018 dedup_ratio=0.243998 router_tokens_per_expert=[770, 747, 756, 805, 766, 745, 793, 777, 759, 798, 758, 766, 770, 792, 807, 777, 825, 782, 783, 765, 726, 776, 757, 780, 779, 776, 727, 725, 781, 752, 777, 736, 760, 771, 807, 801, 777, 753, 748, 701, 798, 744, 798, 721, 758, 806, 725, 757, 732, 796, 756, 784, 757, 798, 791, 787, 772, 739, 769, 750, 744, 742, 763, 814] payload_rows_per_dst=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] dup_rows_per_dst=[3052, 3092, 3020, 2866, 2968, 2952, 3074, 2962] combine_rows_per_src=[9266, 9362, 9368, 9240, 9268, 9262, 9328, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344352 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847072 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349792 weight_movement_bytes=34603008 total_nvlink_bytes=565952800 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65028 dup_rows=24004 dup_rows_remote=21024 dedup_ratio=0.244181 router_tokens_per_expert=[761, 781, 764, 784, 772, 740, 804, 806, 741, 754, 776, 808, 744, 768, 746, 772, 735, 786, 864, 797, 768, 761, 736, 736, 788, 747, 771, 786, 780, 763, 746, 774, 751, 766, 762, 767, 784, 809, 739, 799, 777, 770, 783, 743, 788, 737, 763, 765, 745, 746, 725, 690, 791, 761, 782, 808, 767, 761, 739, 762, 787, 808, 776, 742] payload_rows_per_dst=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] dup_rows_per_dst=[3096, 3044, 3056, 2970, 2948, 2980, 2892, 3018] combine_rows_per_src=[9328, 9174, 9310, 9340, 9406, 9272, 9204, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65110 dup_rows=23928 dup_rows_remote=20954 dedup_ratio=0.243408 router_tokens_per_expert=[799, 736, 720, 824, 751, 793, 758, 809, 772, 798, 769, 790, 787, 774, 781, 774, 779, 756, 775, 716, 773, 742, 753, 749, 726, 797, 776, 774, 752, 757, 792, 746, 767, 772, 765, 726, 752, 772, 811, 739, 740, 808, 797, 761, 788, 777, 805, 756, 771, 753, 757, 784, 782, 754, 773, 812, 746, 769, 734, 738, 752, 763, 751, 779] payload_rows_per_dst=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] dup_rows_per_dst=[3002, 3082, 2882, 2974, 3032, 3078, 2952, 2926] combine_rows_per_src=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344448 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266101120 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857792 weight_movement_bytes=69206016 total_nvlink_bytes=601063808 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342608 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265615952 ingress_bytes=334479360 link_bytes=334479360 token_bytes=530889296 weight_movement_bytes=69206016 total_nvlink_bytes=600095312 weight_to_token_ratio=0.130359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258618904e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344240 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805424 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266608 weight_movement_bytes=34603008 total_nvlink_bytes=567869616 weight_to_token_ratio=0.064889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344256 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267034816 ingress_bytes=370499584 link_bytes=370499584 token_bytes=533725376 weight_movement_bytes=103809024 total_nvlink_bytes=637534400 weight_to_token_ratio=0.194499 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394160107e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344272 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789072 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233872 weight_movement_bytes=34603008 total_nvlink_bytes=567836880 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64714 dup_rows=24276 dup_rows_remote=21050 dedup_ratio=0.246948 router_tokens_per_expert=[756, 771, 741, 759, 775, 760, 767, 813, 793, 783, 736, 742, 760, 745, 722, 737, 783, 764, 786, 791, 751, 741, 753, 797, 722, 823, 759, 778, 775, 723, 773, 751, 807, 756, 768, 778, 785, 788, 786, 753, 784, 734, 788, 769, 774, 762, 798, 798, 729, 795, 785, 740, 806, 721, 769, 791, 805, 756, 752, 787, 762, 783, 736, 777] payload_rows_per_dst=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] dup_rows_per_dst=[3226, 3080, 3134, 2874, 2928, 3034, 2906, 3094] combine_rows_per_src=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65020 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[779, 780, 780, 754, 733, 773, 739, 761, 797, 790, 779, 741, 780, 732, 802, 770, 722, 748, 766, 807, 774, 779, 759, 748, 742, 742, 795, 800, 767, 772, 776, 734, 822, 758, 749, 764, 775, 751, 799, 771, 792, 803, 744, 747, 749, 752, 737, 793, 805, 773, 750, 784, 789, 749, 774, 780, 747, 791, 794, 778, 762, 781, 730, 738] payload_rows_per_dst=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] dup_rows_per_dst=[2962, 2646, 3438, 2750, 3192, 2966, 3088, 2968] combine_rows_per_src=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65188 dup_rows=23826 dup_rows_remote=21086 dedup_ratio=0.242371 router_tokens_per_expert=[782, 757, 793, 762, 761, 808, 764, 787, 730, 768, 787, 780, 739, 740, 736, 735, 798, 760, 806, 736, 821, 779, 750, 824, 795, 717, 734, 790, 709, 812, 746, 802, 814, 796, 716, 797, 727, 751, 749, 748, 721, 811, 765, 777, 782, 759, 766, 769, 718, 753, 732, 766, 795, 812, 769, 786, 808, 692, 801, 778, 788, 748, 779, 771] payload_rows_per_dst=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] dup_rows_per_dst=[3066, 2740, 3076, 3042, 2908, 2970, 2964, 3060] combine_rows_per_src=[9362, 9290, 9472, 9168, 9288, 9330, 9298, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343968 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919840 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495712 weight_movement_bytes=34603008 total_nvlink_bytes=568098720 weight_to_token_ratio=0.064861 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[741, 740, 808, 774, 759, 761, 787, 786, 757, 760, 792, 770, 727, 727, 773, 748, 788, 767, 791, 739, 765, 772, 777, 735, 783, 776, 776, 764, 774, 765, 803, 756, 760, 789, 798, 796, 769, 749, 738, 744, 776, 759, 741, 817, 814, 725, 803, 760, 796, 745, 781, 787, 751, 770, 715, 798, 782, 810, 777, 739, 772, 756, 767, 727] payload_rows_per_dst=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] dup_rows_per_dst=[2972, 2858, 3012, 3090, 2986, 3078, 3024, 3064] combine_rows_per_src=[9340, 9250, 9256, 9304, 9300, 9312, 9262, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65060 dup_rows=23858 dup_rows_remote=20890 dedup_ratio=0.242696 router_tokens_per_expert=[788, 778, 783, 785, 762, 758, 740, 792, 750, 754, 748, 751, 732, 755, 754, 735, 794, 750, 753, 750, 804, 729, 744, 814, 769, 758, 807, 757, 798, 750, 768, 770, 763, 786, 792, 759, 756, 785, 716, 777, 752, 745, 773, 741, 836, 809, 748, 784, 754, 786, 776, 759, 797, 751, 751, 706, 783, 775, 797, 842, 761, 751, 769, 792] payload_rows_per_dst=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] dup_rows_per_dst=[2994, 2886, 2960, 2968, 2960, 3066, 2864, 3160] combine_rows_per_src=[9378, 9072, 9316, 9386, 9308, 9310, 9296, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344272 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789072 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233872 weight_movement_bytes=34603008 total_nvlink_bytes=567836880 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343056 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265411600 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530480144 weight_movement_bytes=34603008 total_nvlink_bytes=565083152 weight_to_token_ratio=0.065230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344248 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666168 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988088 weight_movement_bytes=69206016 total_nvlink_bytes=602194104 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=65046 dup_rows=24066 dup_rows_remote=21098 dedup_ratio=0.244812 router_tokens_per_expert=[769, 780, 780, 767, 769, 721, 713, 773, 719, 805, 787, 777, 784, 795, 769, 734, 783, 757, 769, 768, 758, 734, 748, 763, 720, 807, 765, 809, 700, 774, 813, 781, 764, 797, 711, 787, 771, 771, 810, 780, 814, 746, 789, 719, 740, 773, 741, 772, 778, 741, 807, 749, 717, 778, 819, 722, 740, 816, 817, 768, 741, 790, 818, 775] payload_rows_per_dst=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] dup_rows_per_dst=[2884, 3074, 2968, 3028, 3032, 3016, 2950, 3114] combine_rows_per_src=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344208 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266698896 ingress_bytes=370163712 link_bytes=370163712 token_bytes=533053584 weight_movement_bytes=103809024 total_nvlink_bytes=636862608 weight_to_token_ratio=0.194744 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392896248e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=345096 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267355144 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534365192 weight_movement_bytes=34603008 total_nvlink_bytes=568968200 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64876 dup_rows=24166 dup_rows_remote=21048 dedup_ratio=0.245829 router_tokens_per_expert=[770, 779, 738, 830, 761, 754, 771, 732, 817, 767, 833, 755, 765, 757, 742, 712, 735, 757, 766, 778, 745, 780, 767, 709, 752, 795, 768, 766, 773, 770, 804, 743, 735, 784, 771, 780, 779, 801, 740, 795, 809, 812, 739, 777, 749, 767, 742, 795, 803, 761, 763, 763, 753, 750, 778, 740, 772, 774, 761, 770, 768, 799, 783, 748] payload_rows_per_dst=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] dup_rows_per_dst=[2958, 3068, 2904, 3066, 2976, 3118, 3006, 3070] combine_rows_per_src=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64862 dup_rows=24122 dup_rows_remote=21080 dedup_ratio=0.245382 router_tokens_per_expert=[773, 737, 812, 729, 780, 764, 771, 768, 786, 774, 803, 750, 757, 759, 788, 771, 728, 723, 785, 760, 784, 768, 719, 737, 809, 756, 777, 772, 774, 789, 739, 765, 777, 751, 716, 772, 798, 790, 822, 790, 768, 779, 742, 767, 746, 731, 787, 811, 778, 786, 765, 737, 766, 767, 852, 778, 761, 772, 722, 773, 768, 741, 733, 799] payload_rows_per_dst=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] dup_rows_per_dst=[3024, 3076, 2828, 3042, 3130, 2972, 3054, 2996] combine_rows_per_src=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344072 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412040 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480008 weight_movement_bytes=34603008 total_nvlink_bytes=567083016 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=343800 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266829560 ingress_bytes=301088768 link_bytes=301088768 token_bytes=533315320 weight_movement_bytes=34603008 total_nvlink_bytes=567918328 weight_to_token_ratio=0.064883 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132972794e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65030 dup_rows=23992 dup_rows_remote=20962 dedup_ratio=0.244059 router_tokens_per_expert=[817, 804, 737, 739, 775, 689, 819, 775, 820, 749, 801, 790, 801, 758, 793, 738, 807, 750, 759, 736, 759, 821, 721, 737, 776, 782, 752, 789, 775, 752, 761, 813, 775, 760, 782, 748, 772, 754, 760, 734, 793, 716, 759, 761, 712, 797, 760, 754, 807, 745, 727, 819, 759, 759, 724, 717, 788, 794, 783, 778, 819, 737, 777, 787] payload_rows_per_dst=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] dup_rows_per_dst=[3030, 3042, 2920, 3206, 2988, 3154, 2558, 3094] combine_rows_per_src=[9282, 9458, 9258, 9562, 9182, 9448, 8690, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65020 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[779, 780, 780, 754, 733, 773, 739, 761, 797, 790, 779, 741, 780, 732, 802, 770, 722, 748, 766, 807, 774, 779, 759, 748, 742, 742, 795, 800, 767, 772, 776, 734, 822, 758, 749, 764, 775, 751, 799, 771, 792, 803, 744, 747, 749, 752, 737, 793, 805, 773, 750, 784, 789, 749, 774, 780, 747, 791, 794, 778, 762, 781, 730, 738] payload_rows_per_dst=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] dup_rows_per_dst=[2962, 2646, 3438, 2750, 3192, 2966, 3088, 2968] combine_rows_per_src=[9164, 8940, 9564, 9144, 9436, 9380, 9392, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64882 dup_rows=24100 dup_rows_remote=21230 dedup_ratio=0.245158 router_tokens_per_expert=[736, 766, 749, 786, 771, 733, 759, 819, 766, 773, 775, 749, 745, 795, 778, 726, 778, 781, 705, 767, 750, 767, 803, 745, 763, 723, 796, 696, 745, 808, 762, 753, 808, 771, 769, 777, 750, 747, 778, 774, 791, 778, 780, 794, 770, 756, 784, 700, 740, 795, 753, 779, 813, 814, 767, 779, 772, 773, 732, 808, 778, 812, 780, 762] payload_rows_per_dst=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] dup_rows_per_dst=[2872, 3078, 2870, 2866, 2984, 3124, 3116, 3190] combine_rows_per_src=[9366, 9136, 9322, 9226, 9364, 9182, 9364, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65234 dup_rows=23946 dup_rows_remote=21100 dedup_ratio=0.243591 router_tokens_per_expert=[785, 780, 795, 739, 748, 789, 744, 767, 747, 790, 812, 734, 771, 772, 779, 786, 800, 721, 792, 761, 783, 734, 754, 782, 801, 735, 754, 818, 782, 783, 755, 821, 772, 745, 724, 737, 783, 741, 748, 735, 757, 678, 792, 726, 786, 744, 756, 793, 776, 796, 785, 789, 831, 770, 724, 743, 756, 813, 834, 734, 746, 744, 805, 775] payload_rows_per_dst=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] dup_rows_per_dst=[2960, 2986, 3062, 3052, 2846, 2844, 3042, 3154] combine_rows_per_src=[9334, 9396, 9192, 9446, 9124, 9220, 9386, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344576 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772992 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201408 weight_movement_bytes=34603008 total_nvlink_bytes=567804416 weight_to_token_ratio=0.064897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266075792 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807888 weight_movement_bytes=69206016 total_nvlink_bytes=601013904 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343968 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266706848 ingress_bytes=404774912 link_bytes=404774912 token_bytes=533069728 weight_movement_bytes=138412032 total_nvlink_bytes=671481760 weight_to_token_ratio=0.259651 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523135407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65054 dup_rows=23950 dup_rows_remote=21006 dedup_ratio=0.243632 router_tokens_per_expert=[802, 804, 777, 791, 773, 763, 742, 722, 764, 739, 781, 775, 765, 781, 745, 790, 832, 708, 793, 792, 736, 764, 761, 737, 806, 766, 787, 733, 759, 785, 792, 763, 805, 742, 752, 838, 767, 763, 828, 755, 765, 734, 764, 822, 736, 768, 763, 770, 777, 731, 754, 754, 716, 782, 758, 797, 749, 791, 746, 766, 779, 745, 782, 725] payload_rows_per_dst=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] dup_rows_per_dst=[3024, 3004, 3002, 2974, 3132, 2944, 2956, 2914] combine_rows_per_src=[9324, 9276, 9244, 9408, 9368, 9300, 9182, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64926 dup_rows=24032 dup_rows_remote=20976 dedup_ratio=0.244466 router_tokens_per_expert=[805, 729, 795, 813, 779, 813, 791, 765, 802, 807, 771, 725, 795, 776, 801, 724, 727, 801, 741, 718, 764, 776, 748, 806, 803, 819, 748, 740, 757, 750, 745, 772, 774, 761, 737, 752, 747, 739, 803, 741, 775, 765, 777, 771, 760, 790, 759, 753, 776, 764, 788, 760, 762, 746, 763, 781, 803, 709, 715, 738, 788, 779, 768, 802] payload_rows_per_dst=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] dup_rows_per_dst=[3118, 3056, 2964, 2986, 2856, 2976, 3062, 3014] combine_rows_per_src=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344248 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666168 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988088 weight_movement_bytes=69206016 total_nvlink_bytes=602194104 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344448 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266101120 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857792 weight_movement_bytes=69206016 total_nvlink_bytes=601063808 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343768 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266018520 ingress_bytes=404086784 link_bytes=404086784 token_bytes=531693272 weight_movement_bytes=138412032 total_nvlink_bytes=670105304 weight_to_token_ratio=0.260323 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520546037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64854 dup_rows=24110 dup_rows_remote=21088 dedup_ratio=0.245260 router_tokens_per_expert=[774, 741, 735, 786, 801, 771, 791, 764, 792, 752, 786, 790, 765, 763, 797, 753, 770, 849, 720, 737, 792, 791, 717, 761, 720, 757, 767, 806, 764, 775, 773, 780, 788, 790, 779, 765, 746, 725, 757, 768, 819, 791, 738, 728, 795, 759, 725, 760, 750, 740, 775, 775, 783, 781, 789, 726, 729, 763, 794, 806, 734, 766, 773, 795] payload_rows_per_dst=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] dup_rows_per_dst=[3022, 3424, 2982, 3006, 3048, 2936, 3050, 2642] combine_rows_per_src=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=345336 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267543800 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534742264 weight_movement_bytes=34603008 total_nvlink_bytes=569345272 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64960 dup_rows=23982 dup_rows_remote=20978 dedup_ratio=0.243958 router_tokens_per_expert=[793, 746, 768, 729, 707, 767, 777, 760, 767, 722, 738, 766, 747, 822, 798, 780, 793, 798, 823, 739, 778, 772, 786, 778, 727, 734, 781, 764, 787, 737, 707, 778, 749, 740, 786, 778, 777, 793, 773, 749, 705, 776, 813, 750, 776, 788, 770, 791, 805, 747, 833, 743, 771, 749, 770, 768, 770, 799, 778, 767, 776, 772, 758, 763] payload_rows_per_dst=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] dup_rows_per_dst=[2950, 3040, 3172, 2730, 2956, 3078, 3052, 3004] combine_rows_per_src=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64926 dup_rows=24032 dup_rows_remote=20976 dedup_ratio=0.244466 router_tokens_per_expert=[805, 729, 795, 813, 779, 813, 791, 765, 802, 807, 771, 725, 795, 776, 801, 724, 727, 801, 741, 718, 764, 776, 748, 806, 803, 819, 748, 740, 757, 750, 745, 772, 774, 761, 737, 752, 747, 739, 803, 741, 775, 765, 777, 771, 760, 790, 759, 753, 776, 764, 788, 760, 762, 746, 763, 781, 803, 709, 715, 738, 788, 779, 768, 802] payload_rows_per_dst=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] dup_rows_per_dst=[3118, 3056, 2964, 2986, 2856, 2976, 3062, 3014] combine_rows_per_src=[9462, 9346, 9198, 9282, 9252, 9324, 9218, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65078 dup_rows=23900 dup_rows_remote=20936 dedup_ratio=0.243123 router_tokens_per_expert=[752, 767, 726, 826, 731, 780, 773, 750, 790, 737, 774, 754, 739, 747, 784, 767, 757, 796, 780, 733, 761, 790, 736, 738, 744, 754, 798, 794, 826, 772, 779, 766, 768, 750, 801, 736, 763, 784, 735, 761, 768, 761, 760, 763, 767, 771, 825, 743, 768, 782, 784, 769, 764, 787, 767, 724, 815, 775, 762, 768, 770, 810, 756, 774] payload_rows_per_dst=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] dup_rows_per_dst=[2940, 2850, 2982, 3032, 3016, 3024, 2964, 3092] combine_rows_per_src=[9270, 9334, 9200, 9434, 9180, 9292, 9326, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65110 dup_rows=23928 dup_rows_remote=20954 dedup_ratio=0.243408 router_tokens_per_expert=[799, 736, 720, 824, 751, 793, 758, 809, 772, 798, 769, 790, 787, 774, 781, 774, 779, 756, 775, 716, 773, 742, 753, 749, 726, 797, 776, 774, 752, 757, 792, 746, 767, 772, 765, 726, 752, 772, 811, 739, 740, 808, 797, 761, 788, 777, 805, 756, 771, 753, 757, 784, 782, 754, 773, 812, 746, 769, 734, 738, 752, 763, 751, 779] payload_rows_per_dst=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] dup_rows_per_dst=[3002, 3082, 2882, 2974, 3032, 3078, 2952, 2926] combine_rows_per_src=[9378, 9408, 9204, 9266, 9176, 9386, 9420, 9138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343768 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265985752 ingress_bytes=369451008 link_bytes=369451008 token_bytes=531627736 weight_movement_bytes=103809024 total_nvlink_bytes=635436760 weight_to_token_ratio=0.195266 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390214400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65222 dup_rows=23810 dup_rows_remote=20880 dedup_ratio=0.242208 router_tokens_per_expert=[767, 774, 780, 743, 771, 784, 797, 776, 692, 754, 742, 750, 754, 792, 758, 766, 778, 745, 711, 743, 785, 762, 768, 809, 739, 794, 808, 790, 761, 782, 718, 780, 796, 783, 770, 757, 745, 747, 765, 758, 739, 758, 792, 770, 755, 840, 752, 809, 761, 773, 751, 763, 800, 745, 779, 762, 821, 772, 757, 763, 771, 781, 783, 761] payload_rows_per_dst=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] dup_rows_per_dst=[3064, 2842, 2930, 3030, 2934, 3016, 2974, 3020] combine_rows_per_src=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343752 combine_payload_bytes=266076160 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266419912 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496072 weight_movement_bytes=69206016 total_nvlink_bytes=601702088 weight_to_token_ratio=0.129965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344240 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805424 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266608 weight_movement_bytes=34603008 total_nvlink_bytes=567869616 weight_to_token_ratio=0.064889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65132 dup_rows=23742 dup_rows_remote=20800 dedup_ratio=0.241516 router_tokens_per_expert=[785, 761, 767, 743, 746, 740, 782, 785, 770, 776, 779, 793, 802, 758, 768, 740, 820, 740, 762, 799, 777, 753, 750, 747, 771, 743, 783, 792, 746, 748, 802, 765, 799, 793, 745, 793, 754, 790, 744, 721, 808, 768, 748, 840, 757, 730, 768, 767, 726, 731, 768, 769, 776, 812, 741, 769, 798, 801, 746, 748, 753, 750, 789, 757] payload_rows_per_dst=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] dup_rows_per_dst=[3062, 3032, 3014, 2826, 2954, 2942, 2912, 3000] combine_rows_per_src=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343608 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280504 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217400 weight_movement_bytes=69206016 total_nvlink_bytes=601423416 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65070 dup_rows=23852 dup_rows_remote=20866 dedup_ratio=0.242635 router_tokens_per_expert=[787, 751, 734, 761, 775, 776, 746, 722, 735, 806, 717, 747, 778, 818, 797, 760, 752, 714, 801, 765, 802, 764, 742, 805, 779, 747, 752, 796, 815, 742, 784, 769, 767, 796, 753, 774, 763, 722, 774, 757, 780, 762, 781, 729, 773, 790, 771, 754, 788, 733, 805, 748, 753, 707, 812, 766, 795, 768, 770, 764, 817, 748, 769, 824] payload_rows_per_dst=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] dup_rows_per_dst=[2946, 2924, 3012, 2986, 2892, 2958, 3000, 3134] combine_rows_per_src=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344056 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266903544 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533463032 weight_movement_bytes=69206016 total_nvlink_bytes=602669048 weight_to_token_ratio=0.129730 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344256 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267034816 ingress_bytes=370499584 link_bytes=370499584 token_bytes=533725376 weight_movement_bytes=103809024 total_nvlink_bytes=637534400 weight_to_token_ratio=0.194499 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394160107e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64770 dup_rows=24096 dup_rows_remote=20936 dedup_ratio=0.245117 router_tokens_per_expert=[786, 782, 768, 802, 758, 762, 736, 811, 774, 778, 797, 773, 802, 723, 747, 711, 804, 761, 726, 786, 757, 765, 778, 782, 769, 745, 778, 764, 772, 798, 756, 753, 794, 739, 788, 720, 719, 796, 741, 747, 794, 781, 779, 757, 803, 754, 728, 770, 747, 784, 748, 777, 810, 793, 733, 796, 749, 811, 796, 769, 740, 745, 767, 773] payload_rows_per_dst=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] dup_rows_per_dst=[3160, 3026, 2920, 2910, 3060, 3044, 2966, 3010] combine_rows_per_src=[9438, 9226, 9356, 9172, 9358, 9288, 9080, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64960 dup_rows=23982 dup_rows_remote=20978 dedup_ratio=0.243958 router_tokens_per_expert=[793, 746, 768, 729, 707, 767, 777, 760, 767, 722, 738, 766, 747, 822, 798, 780, 793, 798, 823, 739, 778, 772, 786, 778, 727, 734, 781, 764, 787, 737, 707, 778, 749, 740, 786, 778, 777, 793, 773, 749, 705, 776, 813, 750, 776, 788, 770, 791, 805, 747, 833, 743, 771, 749, 770, 768, 770, 799, 778, 767, 776, 772, 758, 763] payload_rows_per_dst=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] dup_rows_per_dst=[2950, 3040, 3172, 2730, 2956, 3078, 3052, 3004] combine_rows_per_src=[9144, 9240, 9362, 9098, 9334, 9394, 9388, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344408 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267493720 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643032 weight_movement_bytes=34603008 total_nvlink_bytes=569246040 weight_to_token_ratio=0.064722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65022 dup_rows=23954 dup_rows_remote=20930 dedup_ratio=0.243673 router_tokens_per_expert=[717, 773, 769, 783, 757, 790, 788, 737, 767, 748, 777, 750, 796, 718, 786, 768, 790, 753, 762, 736, 762, 751, 740, 789, 749, 755, 756, 764, 811, 739, 797, 757, 731, 769, 791, 762, 770, 778, 787, 788, 739, 752, 787, 729, 805, 756, 744, 805, 826, 770, 710, 769, 795, 794, 782, 787, 757, 777, 773, 765, 770, 789, 773, 787] payload_rows_per_dst=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] dup_rows_per_dst=[2884, 2978, 2924, 2998, 3024, 3002, 3082, 3062] combine_rows_per_src=[9344, 9242, 9242, 9258, 9328, 9232, 9384, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=65046 dup_rows=24066 dup_rows_remote=21098 dedup_ratio=0.244812 router_tokens_per_expert=[769, 780, 780, 767, 769, 721, 713, 773, 719, 805, 787, 777, 784, 795, 769, 734, 783, 757, 769, 768, 758, 734, 748, 763, 720, 807, 765, 809, 700, 774, 813, 781, 764, 797, 711, 787, 771, 771, 810, 780, 814, 746, 789, 719, 740, 773, 741, 772, 778, 741, 807, 749, 717, 778, 819, 722, 740, 816, 817, 768, 741, 790, 818, 775] payload_rows_per_dst=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] dup_rows_per_dst=[2884, 3074, 2968, 3028, 3032, 3016, 2950, 3114] combine_rows_per_src=[9260, 9266, 9192, 9310, 9350, 9172, 9272, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342824 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640744 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938664 weight_movement_bytes=34603008 total_nvlink_bytes=565541672 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343728 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124400 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905072 weight_movement_bytes=69206016 total_nvlink_bytes=603111088 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343744 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870464 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397184 weight_movement_bytes=69206016 total_nvlink_bytes=602603200 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64876 dup_rows=24166 dup_rows_remote=21048 dedup_ratio=0.245829 router_tokens_per_expert=[770, 779, 738, 830, 761, 754, 771, 732, 817, 767, 833, 755, 765, 757, 742, 712, 735, 757, 766, 778, 745, 780, 767, 709, 752, 795, 768, 766, 773, 770, 804, 743, 735, 784, 771, 780, 779, 801, 740, 795, 809, 812, 739, 777, 749, 767, 742, 795, 803, 761, 763, 763, 753, 750, 778, 740, 772, 774, 761, 770, 768, 799, 783, 748] payload_rows_per_dst=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] dup_rows_per_dst=[2958, 3068, 2904, 3066, 2976, 3118, 3006, 3070] combine_rows_per_src=[9312, 9228, 9170, 9276, 9394, 9262, 9216, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343752 combine_payload_bytes=266076160 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266419912 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496072 weight_movement_bytes=69206016 total_nvlink_bytes=601702088 weight_to_token_ratio=0.129965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21062 dedup_ratio=0.244324 router_tokens_per_expert=[744, 758, 749, 797, 774, 786, 723, 778, 774, 784, 762, 797, 788, 779, 787, 730, 812, 751, 756, 797, 751, 724, 769, 746, 774, 772, 721, 760, 801, 741, 728, 811, 777, 770, 756, 763, 782, 792, 757, 759, 788, 750, 785, 749, 808, 773, 738, 800, 808, 774, 758, 734, 780, 750, 780, 782, 824, 730, 720, 779, 713, 796, 758, 795] payload_rows_per_dst=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] dup_rows_per_dst=[2956, 3154, 3046, 2938, 2958, 3050, 2918, 2998] combine_rows_per_src=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343808 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673920 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004032 weight_movement_bytes=34603008 total_nvlink_bytes=567607040 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65080 dup_rows=24010 dup_rows_remote=21088 dedup_ratio=0.244242 router_tokens_per_expert=[764, 758, 725, 767, 800, 758, 790, 777, 785, 742, 808, 794, 766, 773, 785, 806, 746, 747, 751, 727, 776, 782, 750, 748, 770, 808, 767, 783, 776, 773, 809, 718, 766, 790, 750, 778, 799, 759, 765, 792, 763, 784, 743, 737, 792, 784, 776, 807, 766, 769, 785, 781, 752, 773, 720, 734, 769, 768, 788, 746, 744, 713, 766, 764] payload_rows_per_dst=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] dup_rows_per_dst=[3028, 3024, 2860, 2952, 3124, 3016, 3084, 2922] combine_rows_per_src=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64878 dup_rows=24116 dup_rows_remote=21126 dedup_ratio=0.245321 router_tokens_per_expert=[759, 758, 790, 774, 749, 796, 782, 702, 779, 772, 782, 762, 755, 799, 736, 765, 696, 739, 770, 790, 783, 769, 764, 811, 812, 769, 783, 811, 724, 747, 766, 769, 734, 804, 742, 770, 785, 770, 774, 782, 734, 807, 773, 742, 749, 760, 808, 758, 780, 854, 771, 785, 729, 804, 744, 755, 799, 732, 743, 754, 755, 755, 786, 751] payload_rows_per_dst=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] dup_rows_per_dst=[3076, 2990, 2902, 3002, 3038, 2982, 3104, 3022] combine_rows_per_src=[9144, 9310, 9342, 9360, 9284, 9280, 9340, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65074 dup_rows=24032 dup_rows_remote=21216 dedup_ratio=0.244466 router_tokens_per_expert=[716, 770, 771, 761, 774, 745, 755, 809, 809, 740, 787, 754, 764, 821, 735, 765, 772, 762, 745, 762, 745, 764, 719, 738, 797, 776, 733, 749, 810, 773, 758, 769, 824, 775, 736, 782, 768, 756, 736, 759, 728, 773, 759, 782, 823, 795, 809, 763, 769, 765, 739, 788, 760, 743, 807, 804, 750, 805, 771, 797, 760, 713, 768, 797] payload_rows_per_dst=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] dup_rows_per_dst=[3038, 3122, 2816, 3100, 2940, 3128, 2932, 2956] combine_rows_per_src=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65052 dup_rows=23918 dup_rows_remote=21044 dedup_ratio=0.243306 router_tokens_per_expert=[746, 743, 725, 754, 776, 784, 806, 720, 763, 778, 768, 801, 755, 765, 767, 786, 791, 724, 775, 753, 736, 798, 833, 748, 768, 761, 786, 756, 826, 755, 791, 745, 770, 769, 741, 748, 766, 726, 786, 776, 829, 765, 778, 773, 748, 766, 784, 794, 756, 784, 741, 838, 740, 736, 757, 752, 790, 798, 726, 815, 769, 831, 712, 705] payload_rows_per_dst=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] dup_rows_per_dst=[2892, 3020, 2990, 3052, 3000, 3084, 2874, 3006] combine_rows_per_src=[9216, 9346, 9326, 9324, 9164, 9390, 9334, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343808 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673920 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004032 weight_movement_bytes=34603008 total_nvlink_bytes=567607040 weight_to_token_ratio=0.064921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344576 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772992 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201408 weight_movement_bytes=34603008 total_nvlink_bytes=567804416 weight_to_token_ratio=0.064897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64862 dup_rows=24122 dup_rows_remote=21080 dedup_ratio=0.245382 router_tokens_per_expert=[773, 737, 812, 729, 780, 764, 771, 768, 786, 774, 803, 750, 757, 759, 788, 771, 728, 723, 785, 760, 784, 768, 719, 737, 809, 756, 777, 772, 774, 789, 739, 765, 777, 751, 716, 772, 798, 790, 822, 790, 768, 779, 742, 767, 746, 731, 787, 811, 778, 786, 765, 737, 766, 767, 852, 778, 761, 772, 722, 773, 768, 741, 733, 799] payload_rows_per_dst=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] dup_rows_per_dst=[3024, 3076, 2828, 3042, 3130, 2972, 3054, 2996] combine_rows_per_src=[9244, 9300, 9180, 9320, 9302, 9290, 9404, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344328 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266666248 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988168 weight_movement_bytes=69206016 total_nvlink_bytes=602194184 weight_to_token_ratio=0.129845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65066 dup_rows=23930 dup_rows_remote=20842 dedup_ratio=0.243429 router_tokens_per_expert=[754, 735, 742, 764, 765, 738, 766, 772, 819, 705, 782, 776, 764, 787, 759, 756, 751, 748, 793, 764, 779, 773, 762, 786, 788, 779, 743, 764, 752, 801, 723, 796, 732, 827, 760, 776, 718, 800, 759, 817, 738, 773, 805, 751, 784, 786, 760, 801, 795, 755, 739, 705, 813, 748, 759, 742, 772, 763, 810, 806, 751, 757, 807, 757] payload_rows_per_dst=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] dup_rows_per_dst=[2924, 2922, 3072, 2900, 3026, 3088, 2936, 3062] combine_rows_per_src=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344672 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912352 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533480032 weight_movement_bytes=103809024 total_nvlink_bytes=637289056 weight_to_token_ratio=0.194588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266075792 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807888 weight_movement_bytes=69206016 total_nvlink_bytes=601013904 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344016 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084304 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824592 weight_movement_bytes=34603008 total_nvlink_bytes=566427600 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64830 dup_rows=24160 dup_rows_remote=21170 dedup_ratio=0.245768 router_tokens_per_expert=[793, 732, 743, 780, 757, 735, 762, 745, 785, 788, 783, 720, 759, 765, 793, 758, 831, 747, 733, 785, 747, 778, 776, 818, 732, 765, 755, 764, 789, 780, 797, 770, 763, 736, 734, 788, 754, 773, 835, 758, 742, 777, 797, 730, 770, 777, 789, 768, 747, 792, 794, 739, 813, 730, 750, 775, 783, 771, 751, 771, 775, 782, 771, 752] payload_rows_per_dst=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] dup_rows_per_dst=[2924, 3062, 3104, 2990, 3010, 3110, 2926, 3034] combine_rows_per_src=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345160 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888264 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431368 weight_movement_bytes=34603008 total_nvlink_bytes=568034376 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250368 weight_movement_bytes=69206016 total_nvlink_bytes=602456384 weight_to_token_ratio=0.129781 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343768 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266018520 ingress_bytes=404086784 link_bytes=404086784 token_bytes=531693272 weight_movement_bytes=138412032 total_nvlink_bytes=670105304 weight_to_token_ratio=0.260323 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520546037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64916 dup_rows=24108 dup_rows_remote=21178 dedup_ratio=0.245239 router_tokens_per_expert=[724, 794, 769, 720, 744, 748, 794, 780, 791, 738, 756, 741, 754, 761, 804, 755, 714, 777, 759, 781, 747, 730, 773, 790, 832, 763, 724, 729, 781, 785, 750, 793, 773, 825, 716, 797, 760, 752, 735, 760, 781, 783, 768, 792, 788, 769, 762, 766, 785, 728, 775, 781, 749, 782, 769, 818, 764, 777, 771, 821, 725, 822, 758, 799] payload_rows_per_dst=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] dup_rows_per_dst=[2930, 3176, 3016, 3014, 3068, 2718, 3044, 3142] combine_rows_per_src=[9280, 9450, 9290, 9300, 9168, 9046, 9330, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65080 dup_rows=24010 dup_rows_remote=21088 dedup_ratio=0.244242 router_tokens_per_expert=[764, 758, 725, 767, 800, 758, 790, 777, 785, 742, 808, 794, 766, 773, 785, 806, 746, 747, 751, 727, 776, 782, 750, 748, 770, 808, 767, 783, 776, 773, 809, 718, 766, 790, 750, 778, 799, 759, 765, 792, 763, 784, 743, 737, 792, 784, 776, 807, 766, 769, 785, 781, 752, 773, 720, 734, 769, 768, 788, 746, 744, 713, 766, 764] payload_rows_per_dst=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] dup_rows_per_dst=[3028, 3024, 2860, 2952, 3124, 3016, 3084, 2922] combine_rows_per_src=[9250, 9224, 9194, 9266, 9464, 9336, 9346, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65154 dup_rows=23824 dup_rows_remote=20942 dedup_ratio=0.242350 router_tokens_per_expert=[807, 811, 759, 775, 794, 766, 757, 755, 758, 810, 742, 776, 759, 792, 752, 785, 796, 801, 745, 755, 780, 775, 734, 792, 809, 712, 748, 795, 752, 719, 763, 767, 734, 771, 793, 788, 760, 743, 756, 759, 771, 804, 760, 771, 779, 761, 776, 788, 767, 776, 769, 762, 770, 739, 778, 744, 745, 757, 769, 760, 742, 752, 767, 800] payload_rows_per_dst=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] dup_rows_per_dst=[3100, 2988, 2972, 2942, 2882, 3040, 3014, 2886] combine_rows_per_src=[9348, 9360, 9384, 9188, 9326, 9380, 9196, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65222 dup_rows=23810 dup_rows_remote=20880 dedup_ratio=0.242208 router_tokens_per_expert=[767, 774, 780, 743, 771, 784, 797, 776, 692, 754, 742, 750, 754, 792, 758, 766, 778, 745, 711, 743, 785, 762, 768, 809, 739, 794, 808, 790, 761, 782, 718, 780, 796, 783, 770, 757, 745, 747, 765, 758, 739, 758, 792, 770, 755, 840, 752, 809, 761, 773, 751, 763, 800, 745, 779, 762, 821, 772, 757, 763, 771, 781, 783, 761] payload_rows_per_dst=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] dup_rows_per_dst=[3064, 2842, 2930, 3030, 2934, 3016, 2974, 3020] combine_rows_per_src=[9320, 9174, 9272, 9314, 9308, 9414, 9294, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=344376 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266240312 ingress_bytes=404307968 link_bytes=404307968 token_bytes=532136248 weight_movement_bytes=138412032 total_nvlink_bytes=670548280 weight_to_token_ratio=0.260106 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521378334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266853968 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364304 weight_movement_bytes=34603008 total_nvlink_bytes=567967312 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=344000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265887680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531431360 weight_movement_bytes=34603008 total_nvlink_bytes=566034368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65132 dup_rows=23742 dup_rows_remote=20800 dedup_ratio=0.241516 router_tokens_per_expert=[785, 761, 767, 743, 746, 740, 782, 785, 770, 776, 779, 793, 802, 758, 768, 740, 820, 740, 762, 799, 777, 753, 750, 747, 771, 743, 783, 792, 746, 748, 802, 765, 799, 793, 745, 793, 754, 790, 744, 721, 808, 768, 748, 840, 757, 730, 768, 767, 726, 731, 768, 769, 776, 812, 741, 769, 798, 801, 746, 748, 753, 750, 789, 757] payload_rows_per_dst=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] dup_rows_per_dst=[3062, 3032, 3014, 2826, 2954, 2942, 2912, 3000] combine_rows_per_src=[9156, 9340, 9282, 9474, 9324, 9430, 9272, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344672 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912352 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533480032 weight_movement_bytes=103809024 total_nvlink_bytes=637289056 weight_to_token_ratio=0.194588 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64834 dup_rows=23992 dup_rows_remote=20738 dedup_ratio=0.244059 router_tokens_per_expert=[749, 789, 806, 766, 722, 727, 757, 768, 749, 781, 751, 797, 771, 787, 799, 726, 751, 756, 772, 779, 775, 762, 783, 734, 791, 782, 783, 775, 784, 780, 785, 792, 725, 768, 791, 723, 739, 770, 767, 775, 805, 743, 782, 763, 756, 720, 803, 791, 787, 778, 819, 745, 732, 750, 770, 800, 765, 781, 765, 790, 735, 762, 805, 718] payload_rows_per_dst=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] dup_rows_per_dst=[3254, 3116, 3122, 3088, 2918, 2812, 2732, 2950] combine_rows_per_src=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344384 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215168 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085952 weight_movement_bytes=34603008 total_nvlink_bytes=568688960 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344408 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267493720 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643032 weight_movement_bytes=34603008 total_nvlink_bytes=569246040 weight_to_token_ratio=0.064722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262011640 prefetch_bytes=553648128 reduce_bytes=899678208 weight_movement_bytes=1453326336 egress_bytes=6134960888 ingress_bytes=7580377088 link_bytes=7580377088 total_nvlink_bytes=13715337976 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.073037 dedup_ratio=0.243660 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=26 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240189074e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64966 dup_rows=24026 dup_rows_remote=20810 dedup_ratio=0.244405 router_tokens_per_expert=[770, 767, 781, 743, 775, 747, 759, 747, 785, 731, 765, 708, 764, 755, 761, 743, 792, 753, 761, 800, 799, 742, 758, 771, 816, 780, 769, 758, 710, 777, 752, 759, 726, 815, 792, 795, 818, 733, 741, 769, 680, 788, 776, 772, 770, 778, 726, 742, 811, 762, 762, 797, 803, 741, 795, 807, 795, 780, 764, 771, 791, 780, 804, 770] payload_rows_per_dst=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] dup_rows_per_dst=[2826, 2876, 3104, 3006, 2994, 2826, 3178, 3216] combine_rows_per_src=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74740 payload_rows_remote=65304 dup_rows=23564 dup_rows_remote=20558 dedup_ratio=0.239705 router_tokens_per_expert=[766, 764, 747, 784, 717, 795, 748, 743, 795, 749, 797, 742, 777, 782, 793, 786, 755, 820, 765, 763, 710, 742, 766, 777, 763, 803, 789, 790, 783, 758, 747, 758, 718, 728, 772, 762, 754, 794, 730, 759, 760, 771, 761, 781, 733, 778, 728, 780, 776, 761, 810, 773, 798, 831, 784, 749, 772, 773, 804, 745, 801, 749, 766, 777] payload_rows_per_dst=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] dup_rows_per_dst=[2862, 3006, 2944, 3024, 2800, 2890, 3086, 2952] combine_rows_per_src=[9266, 9436, 9252, 9358, 9234, 9294, 9478, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64958 dup_rows=23966 dup_rows_remote=20872 dedup_ratio=0.243795 router_tokens_per_expert=[728, 760, 825, 777, 751, 752, 803, 701, 765, 774, 788, 781, 789, 743, 792, 741, 782, 767, 797, 829, 764, 828, 742, 728, 714, 786, 764, 706, 798, 766, 795, 754, 793, 772, 775, 801, 803, 750, 727, 793, 774, 776, 771, 726, 793, 739, 744, 749, 727, 748, 771, 799, 755, 819, 756, 786, 770, 773, 758, 781, 758, 769, 767, 739] payload_rows_per_dst=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] dup_rows_per_dst=[3002, 2948, 3094, 2996, 3032, 2950, 3056, 2888] combine_rows_per_src=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254326472 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=6131117768 ingress_bytes=7714947072 link_bytes=7715290728 total_nvlink_bytes=13846064840 weight_to_token_ratio=0.129892 prefetch_share_of_ingress=0.085219 dedup_ratio=0.243881 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262261647e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64978 dup_rows=24142 dup_rows_remote=21250 dedup_ratio=0.245585 router_tokens_per_expert=[738, 790, 717, 758, 777, 781, 798, 774, 808, 779, 734, 758, 750, 737, 773, 788, 768, 741, 770, 742, 752, 788, 826, 761, 738, 759, 788, 776, 746, 766, 781, 748, 746, 846, 743, 816, 814, 757, 803, 723, 762, 783, 793, 798, 762, 744, 767, 731, 751, 780, 722, 773, 809, 735, 735, 733, 814, 776, 752, 782, 720, 792, 791, 789] payload_rows_per_dst=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] dup_rows_per_dst=[2998, 3028, 3090, 2968, 3082, 3048, 2892, 3036] combine_rows_per_src=[9268, 9226, 9206, 9236, 9414, 9232, 9184, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65070 dup_rows=23852 dup_rows_remote=20866 dedup_ratio=0.242635 router_tokens_per_expert=[787, 751, 734, 761, 775, 776, 746, 722, 735, 806, 717, 747, 778, 818, 797, 760, 752, 714, 801, 765, 802, 764, 742, 805, 779, 747, 752, 796, 815, 742, 784, 769, 767, 796, 753, 774, 763, 722, 774, 757, 780, 762, 781, 729, 773, 790, 771, 754, 788, 733, 805, 748, 753, 707, 812, 766, 795, 768, 770, 764, 817, 748, 769, 824] payload_rows_per_dst=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] dup_rows_per_dst=[2946, 2924, 3012, 2986, 2892, 2958, 3000, 3134] combine_rows_per_src=[9158, 9392, 9278, 9382, 9320, 9322, 9224, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=343728 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124400 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905072 weight_movement_bytes=69206016 total_nvlink_bytes=603111088 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342288 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902352 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462416 weight_movement_bytes=34603008 total_nvlink_bytes=566065424 weight_to_token_ratio=0.065109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343104 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266443840 ingress_bytes=404512768 link_bytes=404512768 token_bytes=532544576 weight_movement_bytes=138412032 total_nvlink_bytes=670956608 weight_to_token_ratio=0.259907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522148980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343448 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267828632 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535313816 weight_movement_bytes=103809024 total_nvlink_bytes=639122840 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343744 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870464 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397184 weight_movement_bytes=69206016 total_nvlink_bytes=602603200 weight_to_token_ratio=0.129746 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343320 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411288 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479256 weight_movement_bytes=69206016 total_nvlink_bytes=601685272 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344912 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494800 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644688 weight_movement_bytes=34603008 total_nvlink_bytes=567247696 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64714 dup_rows=24276 dup_rows_remote=21050 dedup_ratio=0.246948 router_tokens_per_expert=[756, 771, 741, 759, 775, 760, 767, 813, 793, 783, 736, 742, 760, 745, 722, 737, 783, 764, 786, 791, 751, 741, 753, 797, 722, 823, 759, 778, 775, 723, 773, 751, 807, 756, 768, 778, 785, 788, 786, 753, 784, 734, 788, 769, 774, 762, 798, 798, 729, 795, 785, 740, 806, 721, 769, 791, 805, 756, 752, 787, 762, 783, 736, 777] payload_rows_per_dst=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] dup_rows_per_dst=[3226, 3080, 3134, 2874, 2928, 3034, 2906, 3094] combine_rows_per_src=[9314, 9268, 9388, 9152, 9134, 9374, 9176, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64966 dup_rows=24026 dup_rows_remote=20810 dedup_ratio=0.244405 router_tokens_per_expert=[770, 767, 781, 743, 775, 747, 759, 747, 785, 731, 765, 708, 764, 755, 761, 743, 792, 753, 761, 800, 799, 742, 758, 771, 816, 780, 769, 758, 710, 777, 752, 759, 726, 815, 792, 795, 818, 733, 741, 769, 680, 788, 776, 772, 770, 778, 726, 742, 811, 762, 762, 797, 803, 741, 795, 807, 795, 780, 764, 771, 791, 780, 804, 770] payload_rows_per_dst=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] dup_rows_per_dst=[2826, 2876, 3104, 3006, 2994, 2826, 3178, 3216] combine_rows_per_src=[9352, 9148, 9248, 9236, 9384, 9238, 9360, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74592 payload_rows_remote=65242 dup_rows=23712 dup_rows_remote=20832 dedup_ratio=0.241211 router_tokens_per_expert=[818, 792, 756, 776, 777, 753, 775, 759, 719, 744, 780, 789, 793, 767, 768, 728, 783, 794, 761, 795, 783, 751, 781, 704, 760, 741, 749, 760, 808, 778, 731, 739, 729, 781, 794, 813, 756, 744, 735, 763, 749, 751, 776, 788, 798, 793, 765, 775, 758, 777, 800, 706, 790, 824, 788, 767, 749, 778, 740, 783, 764, 768, 756, 782] payload_rows_per_dst=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] dup_rows_per_dst=[3078, 2944, 2916, 2884, 2880, 3038, 3014, 2958] combine_rows_per_src=[9334, 9232, 9388, 9248, 9350, 9352, 9406, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65074 dup_rows=24032 dup_rows_remote=21216 dedup_ratio=0.244466 router_tokens_per_expert=[716, 770, 771, 761, 774, 745, 755, 809, 809, 740, 787, 754, 764, 821, 735, 765, 772, 762, 745, 762, 745, 764, 719, 738, 797, 776, 733, 749, 810, 773, 758, 769, 824, 775, 736, 782, 768, 756, 736, 759, 728, 773, 759, 782, 823, 795, 809, 763, 769, 765, 739, 788, 760, 743, 807, 804, 750, 805, 771, 797, 760, 713, 768, 797] payload_rows_per_dst=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] dup_rows_per_dst=[3038, 3122, 2816, 3100, 2940, 3128, 2932, 2956] combine_rows_per_src=[9164, 9228, 9198, 9230, 9332, 9336, 9418, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343056 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265411600 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530480144 weight_movement_bytes=34603008 total_nvlink_bytes=565083152 weight_to_token_ratio=0.065230 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65066 dup_rows=23930 dup_rows_remote=20842 dedup_ratio=0.243429 router_tokens_per_expert=[754, 735, 742, 764, 765, 738, 766, 772, 819, 705, 782, 776, 764, 787, 759, 756, 751, 748, 793, 764, 779, 773, 762, 786, 788, 779, 743, 764, 752, 801, 723, 796, 732, 827, 760, 776, 718, 800, 759, 817, 738, 773, 805, 751, 784, 786, 760, 801, 795, 755, 739, 705, 813, 748, 759, 742, 772, 763, 810, 806, 751, 757, 807, 757] payload_rows_per_dst=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] dup_rows_per_dst=[2924, 2922, 3072, 2900, 3026, 3088, 2936, 3062] combine_rows_per_src=[9148, 9374, 9240, 9392, 9352, 9308, 9176, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233454992 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120674704 ingress_bytes=7427694592 link_bytes=7427694592 total_nvlink_bytes=13548369296 weight_to_token_ratio=0.107485 prefetch_share_of_ingress=0.093173 dedup_ratio=0.244450 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215209425e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258284272 prefetch_bytes=692060160 reduce_bytes=380633088 weight_movement_bytes=1072693248 egress_bytes=6133101296 ingress_bytes=7197876224 link_bytes=7197876224 total_nvlink_bytes=13330977520 weight_to_token_ratio=0.087508 prefetch_share_of_ingress=0.096148 dedup_ratio=0.244418 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=11 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.177609946e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343104 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266443840 ingress_bytes=404512768 link_bytes=404512768 token_bytes=532544576 weight_movement_bytes=138412032 total_nvlink_bytes=670956608 weight_to_token_ratio=0.259907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522148980e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64690 dup_rows=24094 dup_rows_remote=20954 dedup_ratio=0.245097 router_tokens_per_expert=[742, 756, 785, 734, 788, 791, 726, 782, 806, 771, 718, 773, 763, 757, 764, 820, 775, 748, 747, 806, 763, 801, 765, 774, 776, 755, 785, 757, 797, 774, 762, 759, 766, 804, 770, 753, 745, 723, 775, 772, 802, 731, 752, 779, 769, 834, 781, 750, 721, 805, 761, 761, 785, 749, 810, 741, 714, 778, 781, 758, 766, 722, 769, 805] payload_rows_per_dst=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] dup_rows_per_dst=[3140, 2864, 3300, 3026, 2968, 2900, 2978, 2918] combine_rows_per_src=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=344296 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267575528 ingress_bytes=336437248 link_bytes=336437248 token_bytes=534806760 weight_movement_bytes=69206016 total_nvlink_bytes=604012776 weight_to_token_ratio=0.129404 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265986279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64770 dup_rows=24112 dup_rows_remote=20910 dedup_ratio=0.245280 router_tokens_per_expert=[777, 772, 730, 766, 776, 765, 739, 787, 801, 807, 788, 773, 777, 783, 802, 781, 738, 722, 772, 785, 761, 803, 785, 749, 754, 744, 771, 766, 718, 767, 821, 777, 755, 731, 775, 795, 782, 781, 806, 714, 733, 784, 799, 781, 817, 719, 762, 742, 786, 764, 779, 713, 760, 755, 772, 744, 768, 788, 792, 778, 726, 774, 746, 774] payload_rows_per_dst=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] dup_rows_per_dst=[2938, 3202, 2962, 3006, 3032, 3006, 2988, 2978] combine_rows_per_src=[9286, 9422, 9268, 9230, 9246, 9268, 9158, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=345160 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888264 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431368 weight_movement_bytes=34603008 total_nvlink_bytes=568034376 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64938 dup_rows=23962 dup_rows_remote=20826 dedup_ratio=0.243754 router_tokens_per_expert=[740, 818, 770, 766, 719, 803, 756, 771, 753, 776, 720, 725, 725, 762, 755, 788, 747, 738, 809, 774, 796, 783, 757, 754, 746, 748, 698, 757, 782, 761, 748, 793, 753, 785, 790, 764, 791, 759, 799, 776, 812, 788, 763, 756, 716, 782, 766, 767, 771, 764, 817, 799, 773, 797, 780, 769, 759, 813, 721, 771, 777, 781, 768, 787] payload_rows_per_dst=[9274, 9208, 9354, 9154, 9368, 9278, 9404, 9302] dup_rows_per_dst=[3012, 2800, 2962, 2912, 3066, 3022, 3136, 3052] combine_rows_per_src=[9274, 9208, 9354, 9154, 9368, 9278, 9404, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64830 dup_rows=24160 dup_rows_remote=21170 dedup_ratio=0.245768 router_tokens_per_expert=[793, 732, 743, 780, 757, 735, 762, 745, 785, 788, 783, 720, 759, 765, 793, 758, 831, 747, 733, 785, 747, 778, 776, 818, 732, 765, 755, 764, 789, 780, 797, 770, 763, 736, 734, 788, 754, 773, 835, 758, 742, 777, 797, 730, 770, 777, 789, 768, 747, 792, 794, 739, 813, 730, 750, 775, 783, 771, 751, 771, 775, 782, 771, 752] payload_rows_per_dst=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] dup_rows_per_dst=[2924, 3062, 3104, 2990, 3010, 3110, 2926, 3034] combine_rows_per_src=[9170, 9240, 9326, 9314, 9272, 9190, 9354, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266853968 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364304 weight_movement_bytes=34603008 total_nvlink_bytes=567967312 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342576 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312816 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283056 weight_movement_bytes=34603008 total_nvlink_bytes=564886064 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233454992 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120674704 ingress_bytes=7427694592 link_bytes=7427694592 total_nvlink_bytes=13548369296 weight_to_token_ratio=0.107485 prefetch_share_of_ingress=0.093173 dedup_ratio=0.244450 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215209425e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262407376 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6135160016 ingress_bytes=7407558656 link_bytes=7407558656 total_nvlink_bytes=13542718672 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.098098 dedup_ratio=0.243728 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211915081e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=342720 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265640640 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530938560 weight_movement_bytes=34603008 total_nvlink_bytes=565541568 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64854 dup_rows=24110 dup_rows_remote=21088 dedup_ratio=0.245260 router_tokens_per_expert=[774, 741, 735, 786, 801, 771, 791, 764, 792, 752, 786, 790, 765, 763, 797, 753, 770, 849, 720, 737, 792, 791, 717, 761, 720, 757, 767, 806, 764, 775, 773, 780, 788, 790, 779, 765, 746, 725, 757, 768, 819, 791, 738, 728, 795, 759, 725, 760, 750, 740, 775, 775, 783, 781, 789, 726, 729, 763, 794, 806, 734, 766, 773, 795] payload_rows_per_dst=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] dup_rows_per_dst=[3022, 3424, 2982, 3006, 3048, 2936, 3050, 2642] combine_rows_per_src=[9340, 9674, 9292, 9278, 9310, 9136, 9188, 8976] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343056 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266329104 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315152 weight_movement_bytes=69206016 total_nvlink_bytes=601521168 weight_to_token_ratio=0.130009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=344000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265887680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531431360 weight_movement_bytes=34603008 total_nvlink_bytes=566034368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262011640 prefetch_bytes=553648128 reduce_bytes=899678208 weight_movement_bytes=1453326336 egress_bytes=6134960888 ingress_bytes=7580377088 link_bytes=7580377088 total_nvlink_bytes=13715337976 weight_to_token_ratio=0.118523 prefetch_share_of_ingress=0.073037 dedup_ratio=0.243660 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=26 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240189074e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258702816 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133306848 ingress_bytes=7371104256 link_bytes=7371104256 total_nvlink_bytes=13504411104 weight_to_token_ratio=0.101618 prefetch_share_of_ingress=0.103277 dedup_ratio=0.243937 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205950952e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=343768 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265985752 ingress_bytes=369451008 link_bytes=369451008 token_bytes=531627736 weight_movement_bytes=103809024 total_nvlink_bytes=635436760 weight_to_token_ratio=0.195266 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390214400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64958 dup_rows=23966 dup_rows_remote=20872 dedup_ratio=0.243795 router_tokens_per_expert=[728, 760, 825, 777, 751, 752, 803, 701, 765, 774, 788, 781, 789, 743, 792, 741, 782, 767, 797, 829, 764, 828, 742, 728, 714, 786, 764, 706, 798, 766, 795, 754, 793, 772, 775, 801, 803, 750, 727, 793, 774, 776, 771, 726, 793, 739, 744, 749, 727, 748, 771, 799, 755, 819, 756, 786, 770, 773, 758, 781, 758, 769, 767, 739] payload_rows_per_dst=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] dup_rows_per_dst=[3002, 2948, 3094, 2996, 3032, 2950, 3056, 2888] combine_rows_per_src=[9192, 9398, 9380, 9170, 9396, 9194, 9266, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255037232 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6131476272 ingress_bytes=7507681280 link_bytes=7507681280 total_nvlink_bytes=13639157552 weight_to_token_ratio=0.112943 prefetch_share_of_ingress=0.078353 dedup_ratio=0.244336 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228295663e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254326472 prefetch_bytes=657457152 reduce_bytes=934281216 weight_movement_bytes=1591738368 egress_bytes=6131117768 ingress_bytes=7714947072 link_bytes=7715290728 total_nvlink_bytes=13846064840 weight_to_token_ratio=0.129892 prefetch_share_of_ingress=0.085219 dedup_ratio=0.243881 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=27 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.262261647e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343320 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411288 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479256 weight_movement_bytes=69206016 total_nvlink_bytes=601685272 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258284272 prefetch_bytes=692060160 reduce_bytes=380633088 weight_movement_bytes=1072693248 egress_bytes=6133101296 ingress_bytes=7197876224 link_bytes=7197876224 total_nvlink_bytes=13330977520 weight_to_token_ratio=0.087508 prefetch_share_of_ingress=0.096148 dedup_ratio=0.244418 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=11 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.177609946e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21062 dedup_ratio=0.244324 router_tokens_per_expert=[744, 758, 749, 797, 774, 786, 723, 778, 774, 784, 762, 797, 788, 779, 787, 730, 812, 751, 756, 797, 751, 724, 769, 746, 774, 772, 721, 760, 801, 741, 728, 811, 777, 770, 756, 763, 782, 792, 757, 759, 788, 750, 785, 749, 808, 773, 738, 800, 808, 774, 758, 734, 780, 750, 780, 782, 824, 730, 720, 779, 713, 796, 758, 795] payload_rows_per_dst=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] dup_rows_per_dst=[2956, 3154, 3046, 2938, 2958, 3050, 2918, 2998] combine_rows_per_src=[9266, 9486, 9166, 9280, 9354, 9332, 9176, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344328 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266666248 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988168 weight_movement_bytes=69206016 total_nvlink_bytes=602194184 weight_to_token_ratio=0.129845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64834 dup_rows=23992 dup_rows_remote=20738 dedup_ratio=0.244059 router_tokens_per_expert=[749, 789, 806, 766, 722, 727, 757, 768, 749, 781, 751, 797, 771, 787, 799, 726, 751, 756, 772, 779, 775, 762, 783, 734, 791, 782, 783, 775, 784, 780, 785, 792, 725, 768, 791, 723, 739, 770, 767, 775, 805, 743, 782, 763, 756, 720, 803, 791, 787, 778, 819, 745, 732, 750, 770, 800, 765, 781, 765, 790, 735, 762, 805, 718] payload_rows_per_dst=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] dup_rows_per_dst=[3254, 3116, 3122, 3088, 2918, 2812, 2732, 2950] combine_rows_per_src=[9478, 9334, 9638, 9346, 9214, 8978, 9066, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342288 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902352 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462416 weight_movement_bytes=34603008 total_nvlink_bytes=566065424 weight_to_token_ratio=0.065109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64690 dup_rows=24094 dup_rows_remote=20954 dedup_ratio=0.245097 router_tokens_per_expert=[742, 756, 785, 734, 788, 791, 726, 782, 806, 771, 718, 773, 763, 757, 764, 820, 775, 748, 747, 806, 763, 801, 765, 774, 776, 755, 785, 757, 797, 774, 762, 759, 766, 804, 770, 753, 745, 723, 775, 772, 802, 731, 752, 779, 769, 834, 781, 750, 721, 805, 761, 761, 785, 749, 810, 741, 714, 778, 781, 758, 766, 722, 769, 805] payload_rows_per_dst=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] dup_rows_per_dst=[3140, 2864, 3300, 3026, 2968, 2900, 2978, 2918] combine_rows_per_src=[9520, 9110, 9502, 9346, 9248, 9052, 9288, 9144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342576 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312816 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283056 weight_movement_bytes=34603008 total_nvlink_bytes=564886064 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64598 dup_rows=24140 dup_rows_remote=20862 dedup_ratio=0.245565 router_tokens_per_expert=[742, 779, 781, 759, 751, 799, 787, 773, 793, 793, 745, 789, 743, 739, 749, 720, 729, 779, 809, 744, 770, 781, 756, 803, 769, 778, 730, 744, 759, 767, 765, 766, 735, 767, 818, 767, 794, 807, 749, 786, 783, 807, 762, 784, 732, 755, 772, 742, 771, 780, 752, 824, 735, 740, 760, 804, 741, 770, 754, 782, 798, 730, 794, 766] payload_rows_per_dst=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] dup_rows_per_dst=[3278, 3126, 2842, 3292, 2880, 3112, 2810, 2800] combine_rows_per_src=[9566, 9402, 9044, 9574, 9016, 9324, 9136, 9102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255037232 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6131476272 ingress_bytes=7507681280 link_bytes=7507681280 total_nvlink_bytes=13639157552 weight_to_token_ratio=0.112943 prefetch_share_of_ingress=0.078353 dedup_ratio=0.244336 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228295663e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264593408 dispatch_meta_bytes=341840 combine_payload_bytes=264593408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264935248 ingress_bytes=299196416 link_bytes=299196416 token_bytes=529528656 weight_movement_bytes=34603008 total_nvlink_bytes=564131664 weight_to_token_ratio=0.065347 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125852026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12226985568 prefetch_bytes=761266176 reduce_bytes=761266176 weight_movement_bytes=1522532352 egress_bytes=6117441120 ingress_bytes=7632076800 link_bytes=7632420360 total_nvlink_bytes=13749517920 weight_to_token_ratio=0.124522 prefetch_share_of_ingress=0.099746 dedup_ratio=0.244912 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=22 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.248703624e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none - [2026-08-26 16:56:35] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 795.5 | throughput per GPU (TFLOP/s/GPU): 160.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.947229E+00 | loss scale: 1.0 | grad norm: 42.080 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64944 dup_rows=23980 dup_rows_remote=21000 dedup_ratio=0.243937 router_tokens_per_expert=[778, 771, 764, 755, 767, 785, 818, 736, 800, 776, 747, 771, 729, 757, 764, 782, 771, 755, 726, 724, 745, 780, 762, 734, 758, 761, 817, 789, 757, 774, 765, 750, 730, 764, 806, 798, 783, 819, 778, 702, 776, 773, 804, 756, 780, 757, 808, 789, 712, 755, 814, 794, 769, 781, 774, 742, 754, 767, 745, 741, 794, 762, 802, 755] payload_rows_per_dst=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] dup_rows_per_dst=[3104, 3010, 2890, 3058, 2980, 3032, 2974, 2932] combine_rows_per_src=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] + [2026-08-26 18:08:41] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 789.7 | throughput per GPU (TFLOP/s/GPU): 161.5 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.947229E+00 | loss scale: 1.0 | grad norm: 42.080 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64880 dup_rows=24276 dup_rows_remote=21192 dedup_ratio=0.246948 router_tokens_per_expert=[744, 774, 806, 762, 817, 777, 743, 803, 743, 797, 749, 766, 739, 784, 789, 749, 793, 799, 732, 743, 714, 748, 790, 762, 741, 803, 751, 768, 760, 735, 748, 830, 794, 762, 768, 769, 764, 776, 719, 815, 743, 781, 805, 738, 728, 788, 742, 762, 750, 795, 726, 744, 775, 789, 778, 784, 769, 787, 781, 749, 746, 800, 784, 782] payload_rows_per_dst=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] dup_rows_per_dst=[3046, 3084, 2952, 3064, 2988, 2998, 3012, 3132] combine_rows_per_src=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64982 dup_rows=24022 dup_rows_remote=20980 dedup_ratio=0.244364 router_tokens_per_expert=[766, 771, 779, 769, 811, 733, 739, 763, 749, 805, 807, 775, 758, 778, 764, 776, 783, 763, 705, 724, 713, 781, 771, 785, 756, 811, 784, 764, 804, 755, 748, 749, 798, 739, 749, 734, 724, 785, 761, 757, 740, 744, 811, 752, 787, 799, 806, 791, 734, 762, 794, 790, 729, 724, 784, 796, 781, 770, 772, 764, 761, 752, 842, 781] payload_rows_per_dst=[9238, 9332, 9190, 9300, 9170, 9412, 9304, 9336] dup_rows_per_dst=[3024, 3092, 2860, 3042, 2924, 3048, 2922, 3110] combine_rows_per_src=[9238, 9332, 9190, 9300, 9170, 9412, 9304, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64918 dup_rows=24124 dup_rows_remote=21094 dedup_ratio=0.245402 router_tokens_per_expert=[820, 767, 730, 745, 803, 779, 774, 771, 751, 771, 768, 743, 784, 780, 776, 749, 786, 753, 786, 727, 757, 788, 794, 755, 724, 791, 768, 780, 782, 770, 767, 763, 809, 746, 780, 776, 798, 721, 738, 744, 713, 767, 756, 726, 764, 811, 828, 754, 739, 852, 755, 718, 750, 783, 773, 779, 790, 767, 766, 755, 777, 776, 761, 778] payload_rows_per_dst=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] dup_rows_per_dst=[3146, 2980, 3030, 2984, 2964, 2980, 3030, 3010] combine_rows_per_src=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64896 dup_rows=24234 dup_rows_remote=21328 dedup_ratio=0.246521 router_tokens_per_expert=[736, 768, 738, 786, 748, 768, 741, 770, 705, 785, 780, 787, 707, 751, 772, 744, 767, 789, 769, 748, 776, 756, 754, 769, 760, 739, 842, 801, 770, 731, 788, 735, 782, 828, 756, 801, 781, 781, 764, 790, 758, 736, 781, 730, 783, 762, 722, 768, 804, 792, 758, 804, 777, 803, 759, 793, 762, 748, 787, 729, 785, 775, 802, 771] payload_rows_per_dst=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] dup_rows_per_dst=[2910, 2984, 3060, 3036, 3134, 2906, 3158, 3046] combine_rows_per_src=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64944 dup_rows=23980 dup_rows_remote=21000 dedup_ratio=0.243937 router_tokens_per_expert=[778, 771, 764, 755, 767, 785, 818, 736, 800, 776, 747, 771, 729, 757, 764, 782, 771, 755, 726, 724, 745, 780, 762, 734, 758, 761, 817, 789, 757, 774, 765, 750, 730, 764, 806, 798, 783, 819, 778, 702, 776, 773, 804, 756, 780, 757, 808, 789, 712, 755, 814, 794, 769, 781, 774, 742, 754, 767, 745, 741, 794, 762, 802, 755] payload_rows_per_dst=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] dup_rows_per_dst=[3104, 3010, 2890, 3058, 2980, 3032, 2974, 2932] combine_rows_per_src=[9244, 9242, 9104, 9284, 9380, 9454, 9308, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64912 dup_rows=24124 dup_rows_remote=21076 dedup_ratio=0.245402 router_tokens_per_expert=[732, 794, 822, 742, 800, 768, 736, 722, 753, 746, 815, 770, 747, 790, 739, 756, 756, 754, 783, 788, 790, 763, 800, 754, 773, 737, 765, 789, 829, 835, 775, 755, 781, 727, 788, 786, 781, 768, 804, 718, 777, 780, 758, 807, 750, 745, 711, 744, 753, 804, 787, 780, 769, 719, 740, 757, 789, 759, 777, 768, 749, 749, 812, 737] payload_rows_per_dst=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] dup_rows_per_dst=[2894, 2938, 3076, 3146, 3142, 2918, 2962, 3048] combine_rows_per_src=[9122, 9294, 9300, 9442, 9278, 9190, 9286, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64924 dup_rows=24084 dup_rows_remote=20984 dedup_ratio=0.244995 router_tokens_per_expert=[753, 803, 772, 779, 788, 714, 781, 826, 740, 747, 825, 763, 777, 706, 795, 736, 797, 754, 764, 786, 771, 768, 784, 757, 706, 759, 781, 796, 780, 703, 746, 763, 787, 768, 755, 754, 756, 749, 800, 758, 861, 743, 751, 761, 810, 764, 821, 761, 754, 759, 763, 802, 791, 751, 769, 809, 725, 788, 724, 787, 730, 760, 749, 772] payload_rows_per_dst=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] dup_rows_per_dst=[3124, 2952, 2958, 2888, 3000, 3158, 3100, 2904] combine_rows_per_src=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64880 dup_rows=24276 dup_rows_remote=21192 dedup_ratio=0.246948 router_tokens_per_expert=[744, 774, 806, 762, 817, 777, 743, 803, 743, 797, 749, 766, 739, 784, 789, 749, 793, 799, 732, 743, 714, 748, 790, 762, 741, 803, 751, 768, 760, 735, 748, 830, 794, 762, 768, 769, 764, 776, 719, 815, 743, 781, 805, 738, 728, 788, 742, 762, 750, 795, 726, 744, 775, 789, 778, 784, 769, 787, 781, 749, 746, 800, 784, 782] payload_rows_per_dst=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] dup_rows_per_dst=[3046, 3084, 2952, 3064, 2988, 2998, 3012, 3132] combine_rows_per_src=[9406, 9148, 9210, 9208, 9346, 9176, 9270, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64896 dup_rows=24234 dup_rows_remote=21328 dedup_ratio=0.246521 router_tokens_per_expert=[736, 768, 738, 786, 748, 768, 741, 770, 705, 785, 780, 787, 707, 751, 772, 744, 767, 789, 769, 748, 776, 756, 754, 769, 760, 739, 842, 801, 770, 731, 788, 735, 782, 828, 756, 801, 781, 781, 764, 790, 758, 736, 781, 730, 783, 762, 722, 768, 804, 792, 758, 804, 777, 803, 759, 793, 762, 748, 787, 729, 785, 775, 802, 771] payload_rows_per_dst=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] dup_rows_per_dst=[2910, 2984, 3060, 3036, 3134, 2906, 3158, 3046] combine_rows_per_src=[9200, 9078, 9196, 9296, 9432, 9174, 9422, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64918 dup_rows=24124 dup_rows_remote=21094 dedup_ratio=0.245402 router_tokens_per_expert=[820, 767, 730, 745, 803, 779, 774, 771, 751, 771, 768, 743, 784, 780, 776, 749, 786, 753, 786, 727, 757, 788, 794, 755, 724, 791, 768, 780, 782, 770, 767, 763, 809, 746, 780, 776, 798, 721, 738, 744, 713, 767, 756, 726, 764, 811, 828, 754, 739, 852, 755, 718, 750, 783, 773, 779, 790, 767, 766, 755, 777, 776, 761, 778] payload_rows_per_dst=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] dup_rows_per_dst=[3146, 2980, 3030, 2984, 2964, 2980, 3030, 3010] combine_rows_per_src=[9232, 9264, 9262, 9306, 9260, 9258, 9268, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64950 dup_rows=23996 dup_rows_remote=21048 dedup_ratio=0.244100 router_tokens_per_expert=[758, 780, 775, 758, 797, 752, 743, 789, 764, 794, 799, 801, 713, 761, 774, 798, 807, 758, 830, 764, 794, 721, 750, 773, 746, 775, 774, 741, 709, 802, 721, 767, 764, 810, 762, 806, 795, 773, 742, 783, 761, 796, 783, 778, 784, 736, 742, 728, 775, 751, 742, 750, 800, 793, 718, 803, 761, 744, 761, 758, 764, 762, 776, 763] payload_rows_per_dst=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] dup_rows_per_dst=[2948, 3228, 3218, 2914, 2924, 2994, 2844, 2926] combine_rows_per_src=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343776 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354400 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365024 weight_movement_bytes=69206016 total_nvlink_bytes=601571040 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=344288 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266092768 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531841248 weight_movement_bytes=34603008 total_nvlink_bytes=566444256 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343848 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510120 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676392 weight_movement_bytes=34603008 total_nvlink_bytes=567279400 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344048 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266248176 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532152304 weight_movement_bytes=34603008 total_nvlink_bytes=566755312 weight_to_token_ratio=0.065025 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343776 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354400 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365024 weight_movement_bytes=69206016 total_nvlink_bytes=601571040 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343952 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266223504 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103056 weight_movement_bytes=34603008 total_nvlink_bytes=566706064 weight_to_token_ratio=0.065031 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344896 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266158912 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531972928 weight_movement_bytes=69206016 total_nvlink_bytes=601178944 weight_to_token_ratio=0.130093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343632 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272336 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201040 weight_movement_bytes=69206016 total_nvlink_bytes=601407056 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=344288 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266092768 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531841248 weight_movement_bytes=34603008 total_nvlink_bytes=566444256 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64950 dup_rows=23996 dup_rows_remote=21048 dedup_ratio=0.244100 router_tokens_per_expert=[758, 780, 775, 758, 797, 752, 743, 789, 764, 794, 799, 801, 713, 761, 774, 798, 807, 758, 830, 764, 794, 721, 750, 773, 746, 775, 774, 741, 709, 802, 721, 767, 764, 810, 762, 806, 795, 773, 742, 783, 761, 796, 783, 778, 784, 736, 742, 728, 775, 751, 742, 750, 800, 793, 718, 803, 761, 744, 761, 758, 764, 762, 776, 763] payload_rows_per_dst=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] dup_rows_per_dst=[2948, 3228, 3218, 2914, 2924, 2994, 2844, 2926] combine_rows_per_src=[9358, 9452, 9594, 9156, 9128, 9220, 9148, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65202 dup_rows=23822 dup_rows_remote=20844 dedup_ratio=0.242330 router_tokens_per_expert=[734, 769, 748, 679, 810, 746, 756, 758, 845, 724, 745, 800, 799, 766, 768, 797, 748, 799, 771, 789, 802, 764, 752, 779, 788, 759, 782, 775, 767, 790, 757, 758, 776, 755, 770, 779, 756, 794, 766, 733, 769, 744, 743, 825, 796, 781, 748, 774, 789, 763, 734, 813, 750, 802, 750, 758, 769, 717, 778, 793, 759, 732, 752, 760] payload_rows_per_dst=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] dup_rows_per_dst=[2834, 3140, 2970, 2964, 2978, 2982, 2988, 2966] combine_rows_per_src=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65048 dup_rows=23928 dup_rows_remote=20826 dedup_ratio=0.243408 router_tokens_per_expert=[758, 772, 734, 757, 748, 784, 757, 801, 775, 780, 781, 767, 783, 772, 731, 725, 771, 759, 779, 776, 769, 753, 775, 808, 792, 780, 792, 758, 749, 717, 813, 814, 768, 743, 749, 828, 727, 727, 736, 763, 826, 724, 744, 754, 755, 760, 763, 785, 771, 779, 771, 753, 752, 767, 784, 824, 817, 750, 784, 742, 757, 763, 798, 758] payload_rows_per_dst=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] dup_rows_per_dst=[2920, 2974, 3040, 3102, 2864, 2974, 3062, 2992] combine_rows_per_src=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65018 dup_rows=24074 dup_rows_remote=21082 dedup_ratio=0.244893 router_tokens_per_expert=[747, 734, 782, 835, 799, 818, 771, 744, 770, 781, 772, 779, 721, 736, 791, 804, 736, 767, 754, 788, 757, 775, 764, 761, 736, 815, 803, 736, 798, 806, 742, 761, 743, 737, 714, 807, 761, 784, 778, 791, 770, 768, 820, 782, 760, 755, 753, 752, 764, 758, 764, 724, 772, 778, 800, 747, 785, 766, 761, 714, 780, 770, 749, 762] payload_rows_per_dst=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] dup_rows_per_dst=[3108, 3000, 2992, 3050, 3042, 2998, 2944, 2940] combine_rows_per_src=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64826 dup_rows=24118 dup_rows_remote=20980 dedup_ratio=0.245341 router_tokens_per_expert=[824, 740, 768, 768, 796, 749, 706, 758, 745, 776, 800, 758, 749, 772, 751, 735, 780, 773, 780, 758, 757, 767, 805, 758, 700, 755, 791, 772, 754, 804, 733, 701, 790, 842, 796, 770, 748, 770, 785, 788, 814, 729, 798, 749, 782, 731, 757, 758, 749, 771, 754, 705, 783, 782, 816, 755, 793, 767, 802, 760, 817, 795, 773, 740] payload_rows_per_dst=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] dup_rows_per_dst=[2978, 2998, 3034, 2876, 3082, 3080, 2932, 3138] combine_rows_per_src=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64814 dup_rows=24072 dup_rows_remote=21088 dedup_ratio=0.244873 router_tokens_per_expert=[778, 795, 753, 762, 730, 763, 769, 746, 752, 799, 778, 837, 729, 792, 786, 804, 744, 803, 751, 742, 760, 825, 756, 755, 769, 754, 791, 761, 798, 746, 769, 745, 739, 738, 785, 767, 772, 754, 683, 775, 808, 741, 773, 758, 753, 767, 774, 727, 755, 778, 768, 751, 765, 797, 751, 836, 805, 743, 759, 808, 788, 751, 760, 781] payload_rows_per_dst=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] dup_rows_per_dst=[3002, 3112, 3100, 2910, 2832, 3040, 2984, 3092] combine_rows_per_src=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344896 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266158912 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531972928 weight_movement_bytes=69206016 total_nvlink_bytes=601178944 weight_to_token_ratio=0.130093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344048 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266248176 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532152304 weight_movement_bytes=34603008 total_nvlink_bytes=566755312 weight_to_token_ratio=0.065025 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343992 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266379192 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532414392 weight_movement_bytes=138412032 total_nvlink_bytes=670826424 weight_to_token_ratio=0.259970 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64924 dup_rows=24084 dup_rows_remote=20984 dedup_ratio=0.244995 router_tokens_per_expert=[753, 803, 772, 779, 788, 714, 781, 826, 740, 747, 825, 763, 777, 706, 795, 736, 797, 754, 764, 786, 771, 768, 784, 757, 706, 759, 781, 796, 780, 703, 746, 763, 787, 768, 755, 754, 756, 749, 800, 758, 861, 743, 751, 761, 810, 764, 821, 761, 754, 759, 763, 802, 791, 751, 769, 809, 725, 788, 724, 787, 730, 760, 749, 772] payload_rows_per_dst=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] dup_rows_per_dst=[3124, 2952, 2958, 2888, 3000, 3158, 3100, 2904] combine_rows_per_src=[9308, 9226, 9404, 9180, 9254, 9386, 9296, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20954 dedup_ratio=0.243184 router_tokens_per_expert=[796, 775, 762, 730, 701, 822, 764, 789, 764, 743, 737, 760, 766, 811, 750, 790, 748, 772, 814, 778, 754, 817, 733, 712, 780, 795, 789, 769, 744, 751, 788, 760, 781, 782, 750, 812, 764, 789, 769, 739, 739, 786, 775, 811, 777, 760, 763, 714, 754, 748, 759, 761, 788, 789, 761, 765, 798, 746, 782, 759, 755, 739, 782, 791] payload_rows_per_dst=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] dup_rows_per_dst=[2972, 2952, 3060, 2940, 3090, 2884, 3000, 3008] combine_rows_per_src=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65048 dup_rows=23928 dup_rows_remote=20826 dedup_ratio=0.243408 router_tokens_per_expert=[758, 772, 734, 757, 748, 784, 757, 801, 775, 780, 781, 767, 783, 772, 731, 725, 771, 759, 779, 776, 769, 753, 775, 808, 792, 780, 792, 758, 749, 717, 813, 814, 768, 743, 749, 828, 727, 727, 736, 763, 826, 724, 744, 754, 755, 760, 763, 785, 771, 779, 771, 753, 752, 767, 784, 824, 817, 750, 784, 742, 757, 763, 798, 758] payload_rows_per_dst=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] dup_rows_per_dst=[2920, 2974, 3040, 3102, 2864, 2974, 3062, 2992] combine_rows_per_src=[9302, 9254, 9340, 9328, 9218, 9248, 9340, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65202 dup_rows=23822 dup_rows_remote=20844 dedup_ratio=0.242330 router_tokens_per_expert=[734, 769, 748, 679, 810, 746, 756, 758, 845, 724, 745, 800, 799, 766, 768, 797, 748, 799, 771, 789, 802, 764, 752, 779, 788, 759, 782, 775, 767, 790, 757, 758, 776, 755, 770, 779, 756, 794, 766, 733, 769, 744, 743, 825, 796, 781, 748, 774, 789, 763, 734, 813, 750, 802, 750, 758, 769, 717, 778, 793, 759, 732, 752, 760] payload_rows_per_dst=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] dup_rows_per_dst=[2834, 3140, 2970, 2964, 2978, 2982, 2988, 2966] combine_rows_per_src=[9166, 9348, 9438, 9388, 9280, 9378, 9330, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64826 dup_rows=24118 dup_rows_remote=20980 dedup_ratio=0.245341 router_tokens_per_expert=[824, 740, 768, 768, 796, 749, 706, 758, 745, 776, 800, 758, 749, 772, 751, 735, 780, 773, 780, 758, 757, 767, 805, 758, 700, 755, 791, 772, 754, 804, 733, 701, 790, 842, 796, 770, 748, 770, 785, 788, 814, 729, 798, 749, 782, 731, 757, 758, 749, 771, 754, 705, 783, 782, 816, 755, 793, 767, 802, 760, 817, 795, 773, 740] payload_rows_per_dst=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] dup_rows_per_dst=[2978, 2998, 3034, 2876, 3082, 3080, 2932, 3138] combine_rows_per_src=[9240, 9174, 9126, 9144, 9496, 9348, 9298, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64970 dup_rows=24062 dup_rows_remote=21124 dedup_ratio=0.244771 router_tokens_per_expert=[766, 752, 700, 752, 760, 726, 759, 770, 763, 746, 767, 757, 754, 732, 763, 755, 787, 790, 771, 787, 744, 725, 791, 758, 801, 772, 754, 793, 789, 780, 841, 789, 734, 763, 738, 741, 768, 805, 789, 782, 789, 751, 775, 769, 789, 735, 737, 760, 786, 771, 752, 806, 781, 773, 769, 794, 761, 793, 749, 796, 754, 754, 770, 824] payload_rows_per_dst=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] dup_rows_per_dst=[2830, 2952, 3020, 3142, 3040, 2938, 3110, 3030] combine_rows_per_src=[9140, 9122, 9286, 9496, 9200, 9272, 9354, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20954 dedup_ratio=0.243184 router_tokens_per_expert=[796, 775, 762, 730, 701, 822, 764, 789, 764, 743, 737, 760, 766, 811, 750, 790, 748, 772, 814, 778, 754, 817, 733, 712, 780, 795, 789, 769, 744, 751, 788, 760, 781, 782, 750, 812, 764, 789, 769, 739, 739, 786, 775, 811, 777, 760, 763, 714, 754, 748, 759, 761, 788, 789, 761, 765, 798, 746, 782, 759, 755, 739, 782, 791] payload_rows_per_dst=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] dup_rows_per_dst=[2972, 2952, 3060, 2940, 3090, 2884, 3000, 3008] combine_rows_per_src=[9306, 9290, 9196, 9412, 9282, 9366, 9250, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344184 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411576 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478968 weight_movement_bytes=34603008 total_nvlink_bytes=569081976 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343632 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272336 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201040 weight_movement_bytes=69206016 total_nvlink_bytes=601407056 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65018 dup_rows=24074 dup_rows_remote=21082 dedup_ratio=0.244893 router_tokens_per_expert=[747, 734, 782, 835, 799, 818, 771, 744, 770, 781, 772, 779, 721, 736, 791, 804, 736, 767, 754, 788, 757, 775, 764, 761, 736, 815, 803, 736, 798, 806, 742, 761, 743, 737, 714, 807, 761, 784, 778, 791, 770, 768, 820, 782, 760, 755, 753, 752, 764, 758, 764, 724, 772, 778, 800, 747, 785, 766, 761, 714, 780, 770, 749, 762] payload_rows_per_dst=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] dup_rows_per_dst=[3108, 3000, 2992, 3050, 3042, 2998, 2944, 2940] combine_rows_per_src=[9352, 9308, 9212, 9344, 9188, 9322, 9270, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64634 dup_rows=24058 dup_rows_remote=20706 dedup_ratio=0.244731 router_tokens_per_expert=[788, 745, 782, 789, 767, 791, 742, 760, 718, 688, 755, 751, 833, 795, 786, 785, 761, 805, 743, 740, 789, 798, 757, 774, 809, 755, 740, 744, 751, 769, 767, 778, 744, 790, 703, 809, 754, 781, 824, 780, 763, 755, 789, 777, 776, 793, 743, 760, 819, 732, 796, 765, 764, 762, 747, 745, 783, 751, 770, 758, 822, 785, 742, 715] payload_rows_per_dst=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] dup_rows_per_dst=[3352, 3006, 3016, 2806, 3090, 2992, 2870, 2926] combine_rows_per_src=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344248 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026616 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708984 weight_movement_bytes=34603008 total_nvlink_bytes=568311992 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343496 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780104 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216712 weight_movement_bytes=69206016 total_nvlink_bytes=602422728 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344400 combine_payload_bytes=266313728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266658128 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971856 weight_movement_bytes=103809024 total_nvlink_bytes=636780880 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344184 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411576 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478968 weight_movement_bytes=34603008 total_nvlink_bytes=569081976 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343224 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870520 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397816 weight_movement_bytes=103809024 total_nvlink_bytes=635206840 weight_to_token_ratio=0.195351 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343608 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265821752 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299896 weight_movement_bytes=69206016 total_nvlink_bytes=600505912 weight_to_token_ratio=0.130258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344376 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461496 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578616 weight_movement_bytes=34603008 total_nvlink_bytes=567181624 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344248 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026616 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708984 weight_movement_bytes=34603008 total_nvlink_bytes=568311992 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64634 dup_rows=24058 dup_rows_remote=20706 dedup_ratio=0.244731 router_tokens_per_expert=[788, 745, 782, 789, 767, 791, 742, 760, 718, 688, 755, 751, 833, 795, 786, 785, 761, 805, 743, 740, 789, 798, 757, 774, 809, 755, 740, 744, 751, 769, 767, 778, 744, 790, 703, 809, 754, 781, 824, 780, 763, 755, 789, 777, 776, 793, 743, 760, 819, 732, 796, 765, 764, 762, 747, 745, 783, 751, 770, 758, 822, 785, 742, 715] payload_rows_per_dst=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] dup_rows_per_dst=[3352, 3006, 3016, 2806, 3090, 2992, 2870, 2926] combine_rows_per_src=[9612, 9232, 9302, 9046, 9428, 9320, 9154, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344400 combine_payload_bytes=266313728 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266658128 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971856 weight_movement_bytes=103809024 total_nvlink_bytes=636780880 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=341360 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265082224 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529823088 weight_movement_bytes=34603008 total_nvlink_bytes=564426096 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65142 dup_rows=23914 dup_rows_remote=21042 dedup_ratio=0.243266 router_tokens_per_expert=[759, 783, 746, 775, 796, 747, 768, 750, 755, 742, 754, 807, 756, 746, 748, 752, 765, 797, 777, 742, 758, 779, 741, 762, 753, 694, 744, 754, 787, 789, 818, 789, 769, 811, 774, 775, 778, 778, 774, 773, 760, 730, 726, 775, 768, 804, 741, 794, 790, 748, 799, 766, 795, 780, 770, 769, 801, 808, 725, 740, 777, 783, 790, 748] payload_rows_per_dst=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] dup_rows_per_dst=[3016, 2872, 2866, 3000, 3046, 2936, 3160, 3018] combine_rows_per_src=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65296 dup_rows=23766 dup_rows_remote=20912 dedup_ratio=0.241760 router_tokens_per_expert=[823, 727, 784, 785, 831, 743, 783, 798, 769, 755, 754, 771, 777, 775, 767, 763, 719, 799, 756, 736, 749, 784, 724, 769, 788, 806, 733, 749, 739, 708, 762, 763, 792, 780, 796, 740, 779, 736, 777, 747, 786, 760, 791, 752, 791, 760, 793, 787, 736, 736, 795, 745, 790, 763, 803, 808, 709, 774, 772, 803, 793, 745, 775, 749] payload_rows_per_dst=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] dup_rows_per_dst=[3040, 2934, 2890, 2854, 3056, 3068, 3044, 2880] combine_rows_per_src=[9508, 9328, 9182, 9242, 9238, 9372, 9308, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64814 dup_rows=24072 dup_rows_remote=21088 dedup_ratio=0.244873 router_tokens_per_expert=[778, 795, 753, 762, 730, 763, 769, 746, 752, 799, 778, 837, 729, 792, 786, 804, 744, 803, 751, 742, 760, 825, 756, 755, 769, 754, 791, 761, 798, 746, 769, 745, 739, 738, 785, 767, 772, 754, 683, 775, 808, 741, 773, 758, 753, 767, 774, 727, 755, 778, 768, 751, 765, 797, 751, 836, 805, 743, 759, 808, 788, 751, 760, 781] payload_rows_per_dst=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] dup_rows_per_dst=[3002, 3112, 3100, 2910, 2832, 3040, 2984, 3092] combine_rows_per_src=[9190, 9442, 9172, 9356, 9194, 9162, 9418, 9298] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64964 dup_rows=24144 dup_rows_remote=21178 dedup_ratio=0.245605 router_tokens_per_expert=[772, 728, 755, 807, 809, 780, 751, 763, 786, 782, 751, 734, 760, 771, 715, 791, 742, 810, 784, 739, 799, 724, 801, 738, 797, 734, 749, 817, 797, 808, 771, 764, 743, 756, 745, 751, 785, 753, 821, 727, 790, 707, 748, 779, 724, 826, 741, 741, 777, 749, 807, 788, 798, 845, 825, 775, 725, 738, 773, 754, 744, 760, 766, 762] payload_rows_per_dst=[9314, 9206, 9296, 9374, 9196, 9184, 9500, 9090] dup_rows_per_dst=[3016, 2974, 2978, 3100, 2966, 2928, 3228, 2954] combine_rows_per_src=[9314, 9206, 9296, 9374, 9196, 9184, 9500, 9090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64990 dup_rows=23918 dup_rows_remote=20864 dedup_ratio=0.243306 router_tokens_per_expert=[755, 760, 724, 790, 802, 767, 752, 762, 741, 767, 782, 753, 754, 739, 769, 724, 775, 767, 760, 776, 771, 812, 786, 778, 713, 798, 756, 777, 801, 741, 743, 780, 761, 803, 795, 752, 749, 768, 780, 801, 763, 772, 798, 828, 807, 737, 777, 736, 807, 718, 763, 722, 740, 774, 787, 763, 790, 791, 795, 727, 758, 751, 802, 762] payload_rows_per_dst=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] dup_rows_per_dst=[2992, 2862, 3054, 2942, 3094, 3076, 2874, 3024] combine_rows_per_src=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64898 dup_rows=24076 dup_rows_remote=21030 dedup_ratio=0.244914 router_tokens_per_expert=[796, 795, 717, 747, 817, 747, 774, 773, 783, 773, 767, 755, 766, 780, 809, 805, 790, 749, 739, 739, 766, 752, 715, 807, 768, 784, 812, 773, 746, 775, 785, 815, 755, 774, 767, 753, 776, 746, 775, 784, 753, 753, 782, 776, 737, 784, 789, 784, 749, 782, 766, 761, 809, 744, 784, 740, 740, 726, 765, 746, 762, 762, 776, 733] payload_rows_per_dst=[9172, 9468, 9244, 9406, 9172, 9338, 9098, 9330] dup_rows_per_dst=[2946, 3212, 2870, 3076, 2990, 3080, 2856, 3046] combine_rows_per_src=[9172, 9468, 9244, 9406, 9172, 9338, 9098, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64998 dup_rows=24098 dup_rows_remote=21094 dedup_ratio=0.245138 router_tokens_per_expert=[814, 738, 806, 797, 744, 741, 791, 773, 770, 782, 776, 790, 717, 754, 759, 781, 764, 772, 759, 736, 769, 771, 753, 783, 821, 770, 790, 777, 761, 807, 738, 755, 768, 768, 798, 793, 779, 750, 716, 767, 767, 738, 760, 744, 759, 727, 773, 774, 740, 778, 764, 754, 735, 812, 758, 765, 759, 830, 762, 736, 785, 753, 771, 810] payload_rows_per_dst=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] dup_rows_per_dst=[3088, 3044, 3028, 3006, 2938, 2984, 3004, 3006] combine_rows_per_src=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64938 dup_rows=24048 dup_rows_remote=21092 dedup_ratio=0.244629 router_tokens_per_expert=[754, 753, 784, 776, 819, 770, 776, 786, 784, 782, 748, 741, 825, 783, 747, 808, 740, 766, 797, 817, 765, 767, 725, 771, 750, 783, 754, 735, 762, 727, 749, 781, 745, 779, 777, 800, 746, 774, 775, 757, 758, 736, 784, 747, 762, 826, 755, 769, 753, 738, 736, 772, 778, 756, 768, 772, 760, 739, 756, 737, 785, 781, 790, 816] payload_rows_per_dst=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] dup_rows_per_dst=[3132, 3094, 3046, 2924, 2962, 2956, 2914, 3020] combine_rows_per_src=[9304, 9342, 9250, 9158, 9344, 9318, 9232, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344736 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166368 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533988000 weight_movement_bytes=34603008 total_nvlink_bytes=568591008 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64990 dup_rows=23918 dup_rows_remote=20864 dedup_ratio=0.243306 router_tokens_per_expert=[755, 760, 724, 790, 802, 767, 752, 762, 741, 767, 782, 753, 754, 739, 769, 724, 775, 767, 760, 776, 771, 812, 786, 778, 713, 798, 756, 777, 801, 741, 743, 780, 761, 803, 795, 752, 749, 768, 780, 801, 763, 772, 798, 828, 807, 737, 777, 736, 807, 718, 763, 722, 740, 774, 787, 763, 790, 791, 795, 727, 758, 751, 802, 762] payload_rows_per_dst=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] dup_rows_per_dst=[2992, 2862, 3054, 2942, 3094, 3076, 2874, 3024] combine_rows_per_src=[9232, 9196, 9396, 9276, 9324, 9360, 9274, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344832 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267797248 ingress_bytes=336658432 link_bytes=336658432 token_bytes=535249664 weight_movement_bytes=69206016 total_nvlink_bytes=604455680 weight_to_token_ratio=0.129297 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266818576e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65142 dup_rows=23914 dup_rows_remote=21042 dedup_ratio=0.243266 router_tokens_per_expert=[759, 783, 746, 775, 796, 747, 768, 750, 755, 742, 754, 807, 756, 746, 748, 752, 765, 797, 777, 742, 758, 779, 741, 762, 753, 694, 744, 754, 787, 789, 818, 789, 769, 811, 774, 775, 778, 778, 774, 773, 760, 730, 726, 775, 768, 804, 741, 794, 790, 748, 799, 766, 795, 780, 770, 769, 801, 808, 725, 740, 777, 783, 790, 748] payload_rows_per_dst=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] dup_rows_per_dst=[3016, 2872, 2866, 3000, 3046, 2936, 3160, 3018] combine_rows_per_src=[9232, 9248, 9376, 9256, 9418, 9260, 9274, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65218 dup_rows=23874 dup_rows_remote=21094 dedup_ratio=0.242859 router_tokens_per_expert=[763, 762, 746, 722, 793, 704, 758, 748, 781, 766, 760, 777, 757, 798, 767, 749, 757, 778, 803, 813, 830, 764, 781, 775, 758, 813, 759, 752, 756, 797, 722, 772, 762, 752, 740, 771, 778, 789, 725, 813, 750, 791, 806, 782, 758, 751, 793, 741, 772, 791, 742, 786, 772, 752, 757, 793, 787, 761, 752, 744, 746, 756, 804, 754] payload_rows_per_dst=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] dup_rows_per_dst=[2780, 2994, 3174, 3026, 2936, 3018, 2970, 2976] combine_rows_per_src=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343608 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265821752 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299896 weight_movement_bytes=69206016 total_nvlink_bytes=600505912 weight_to_token_ratio=0.130258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344568 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266437112 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529656 weight_movement_bytes=103809024 total_nvlink_bytes=636338680 weight_to_token_ratio=0.194936 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=341360 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265082224 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529823088 weight_movement_bytes=34603008 total_nvlink_bytes=564426096 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343416 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542456 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741496 weight_movement_bytes=34603008 total_nvlink_bytes=567344504 weight_to_token_ratio=0.064953 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=344120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266330168 ingress_bytes=300589056 link_bytes=300589056 token_bytes=532316216 weight_movement_bytes=34603008 total_nvlink_bytes=566919224 weight_to_token_ratio=0.065005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131092418e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344368 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266576176 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807984 weight_movement_bytes=69206016 total_nvlink_bytes=602014000 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344736 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166368 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533988000 weight_movement_bytes=34603008 total_nvlink_bytes=568591008 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343416 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542456 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741496 weight_movement_bytes=34603008 total_nvlink_bytes=567344504 weight_to_token_ratio=0.064953 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=345248 combine_payload_bytes=267132928 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267478176 ingress_bytes=370941952 link_bytes=370941952 token_bytes=534611104 weight_movement_bytes=103809024 total_nvlink_bytes=638420128 weight_to_token_ratio=0.194177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395824702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65270 dup_rows=23660 dup_rows_remote=20752 dedup_ratio=0.240682 router_tokens_per_expert=[789, 764, 767, 765, 738, 772, 792, 733, 709, 799, 822, 760, 739, 774, 756, 782, 766, 756, 770, 770, 726, 859, 777, 759, 776, 735, 773, 807, 767, 732, 778, 825, 773, 774, 732, 795, 784, 775, 767, 751, 771, 735, 742, 726, 764, 754, 765, 795, 799, 746, 774, 742, 759, 766, 757, 796, 775, 787, 758, 745, 766, 777, 773, 792] payload_rows_per_dst=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] dup_rows_per_dst=[2990, 2908, 3012, 2958, 2994, 2870, 2986, 2942] combine_rows_per_src=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65068 dup_rows=23994 dup_rows_remote=20962 dedup_ratio=0.244080 router_tokens_per_expert=[814, 761, 777, 769, 788, 775, 784, 773, 768, 768, 808, 726, 778, 775, 791, 750, 759, 765, 755, 794, 745, 707, 797, 751, 775, 774, 750, 773, 771, 798, 749, 747, 759, 751, 797, 756, 746, 829, 750, 738, 802, 769, 778, 775, 758, 787, 735, 749, 785, 717, 710, 789, 755, 737, 789, 820, 743, 722, 761, 788, 722, 806, 802, 812] payload_rows_per_dst=[9382, 9324, 9274, 9242, 9344, 9274, 9202, 9268] dup_rows_per_dst=[3100, 3004, 2872, 3032, 2908, 3032, 3002, 3044] combine_rows_per_src=[9382, 9324, 9274, 9242, 9344, 9274, 9202, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64998 dup_rows=24098 dup_rows_remote=21094 dedup_ratio=0.245138 router_tokens_per_expert=[814, 738, 806, 797, 744, 741, 791, 773, 770, 782, 776, 790, 717, 754, 759, 781, 764, 772, 759, 736, 769, 771, 753, 783, 821, 770, 790, 777, 761, 807, 738, 755, 768, 768, 798, 793, 779, 750, 716, 767, 767, 738, 760, 744, 759, 727, 773, 774, 740, 778, 764, 754, 735, 812, 758, 765, 759, 830, 762, 736, 785, 753, 771, 810] payload_rows_per_dst=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] dup_rows_per_dst=[3088, 3044, 3028, 3006, 2938, 2984, 3004, 3006] combine_rows_per_src=[9320, 9214, 9186, 9432, 9340, 9100, 9208, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64872 dup_rows=24166 dup_rows_remote=21184 dedup_ratio=0.245829 router_tokens_per_expert=[800, 717, 769, 737, 778, 782, 828, 803, 740, 757, 768, 797, 766, 746, 731, 760, 744, 758, 762, 755, 731, 816, 745, 756, 804, 757, 778, 841, 772, 798, 783, 736, 814, 737, 793, 752, 756, 749, 779, 744, 767, 752, 780, 782, 787, 735, 703, 762, 806, 792, 754, 758, 743, 798, 782, 752, 796, 766, 753, 778, 746, 764, 826, 731] payload_rows_per_dst=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] dup_rows_per_dst=[3026, 2924, 2976, 3154, 2982, 2910, 3108, 3086] combine_rows_per_src=[9402, 9206, 9158, 9384, 9266, 9226, 9262, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65218 dup_rows=23874 dup_rows_remote=21094 dedup_ratio=0.242859 router_tokens_per_expert=[763, 762, 746, 722, 793, 704, 758, 748, 781, 766, 760, 777, 757, 798, 767, 749, 757, 778, 803, 813, 830, 764, 781, 775, 758, 813, 759, 752, 756, 797, 722, 772, 762, 752, 740, 771, 778, 789, 725, 813, 750, 791, 806, 782, 758, 751, 793, 741, 772, 791, 742, 786, 772, 752, 757, 793, 787, 761, 752, 744, 746, 756, 804, 754] payload_rows_per_dst=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] dup_rows_per_dst=[2780, 2994, 3174, 3026, 2936, 3018, 2970, 2976] combine_rows_per_src=[9212, 9316, 9468, 9232, 9284, 9326, 9360, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=65020 dup_rows=24180 dup_rows_remote=21278 dedup_ratio=0.245972 router_tokens_per_expert=[769, 741, 777, 782, 776, 748, 777, 775, 800, 771, 802, 815, 807, 795, 769, 765, 707, 803, 769, 774, 753, 682, 764, 751, 760, 827, 820, 778, 746, 740, 757, 790, 804, 734, 804, 755, 740, 757, 778, 780, 713, 746, 757, 763, 811, 765, 770, 773, 728, 711, 768, 751, 787, 794, 743, 732, 726, 774, 777, 770, 808, 798, 785, 760] payload_rows_per_dst=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] dup_rows_per_dst=[3082, 3162, 2902, 3104, 3014, 3058, 2846, 3012] combine_rows_per_src=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=64872 dup_rows=24120 dup_rows_remote=21238 dedup_ratio=0.245361 router_tokens_per_expert=[801, 760, 719, 774, 750, 762, 764, 792, 802, 781, 763, 767, 804, 765, 748, 779, 766, 734, 762, 753, 753, 749, 765, 747, 792, 745, 772, 762, 745, 708, 807, 790, 738, 808, 796, 773, 724, 779, 773, 785, 782, 790, 759, 767, 761, 772, 777, 745, 784, 794, 775, 806, 742, 797, 794, 760, 789, 762, 791, 753, 750, 751, 765, 729] payload_rows_per_dst=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] dup_rows_per_dst=[3012, 3050, 2854, 2892, 3068, 3306, 3056, 2882] combine_rows_per_src=[9232, 9368, 9204, 8918, 9284, 9432, 9434, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21078 dedup_ratio=0.243958 router_tokens_per_expert=[771, 791, 749, 771, 764, 745, 786, 733, 791, 772, 765, 783, 772, 794, 796, 725, 777, 767, 759, 770, 803, 759, 803, 749, 771, 747, 793, 793, 781, 786, 794, 720, 771, 779, 738, 783, 748, 786, 747, 799, 781, 798, 737, 754, 728, 794, 721, 778, 772, 739, 719, 769, 796, 770, 790, 751, 771, 737, 774, 732, 793, 823, 752, 742] payload_rows_per_dst=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] dup_rows_per_dst=[2916, 2970, 3068, 3176, 3048, 2904, 2892, 3008] combine_rows_per_src=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344088 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690008 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535035928 weight_movement_bytes=69206016 total_nvlink_bytes=604241944 weight_to_token_ratio=0.129348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=65020 dup_rows=24180 dup_rows_remote=21278 dedup_ratio=0.245972 router_tokens_per_expert=[769, 741, 777, 782, 776, 748, 777, 775, 800, 771, 802, 815, 807, 795, 769, 765, 707, 803, 769, 774, 753, 682, 764, 751, 760, 827, 820, 778, 746, 740, 757, 790, 804, 734, 804, 755, 740, 757, 778, 780, 713, 746, 757, 763, 811, 765, 770, 773, 728, 711, 768, 751, 787, 794, 743, 732, 726, 774, 777, 770, 808, 798, 785, 760] payload_rows_per_dst=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] dup_rows_per_dst=[3082, 3162, 2902, 3104, 3014, 3058, 2846, 3012] combine_rows_per_src=[9208, 9486, 9104, 9332, 9290, 9138, 9182, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=20996 dedup_ratio=0.243754 router_tokens_per_expert=[747, 754, 759, 756, 706, 790, 817, 729, 747, 735, 756, 765, 720, 766, 738, 755, 813, 807, 763, 727, 794, 759, 791, 787, 776, 834, 763, 760, 800, 800, 760, 740, 726, 812, 755, 767, 708, 790, 773, 775, 770, 790, 784, 768, 785, 786, 725, 788, 794, 775, 765, 722, 798, 820, 751, 791, 766, 749, 757, 785, 774, 757, 744, 788] payload_rows_per_dst=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] dup_rows_per_dst=[2966, 2812, 3294, 2942, 2996, 2990, 3040, 2922] combine_rows_per_src=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344120 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862648 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533381176 weight_movement_bytes=103809024 total_nvlink_bytes=637190200 weight_to_token_ratio=0.194624 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21078 dedup_ratio=0.243958 router_tokens_per_expert=[771, 791, 749, 771, 764, 745, 786, 733, 791, 772, 765, 783, 772, 794, 796, 725, 777, 767, 759, 770, 803, 759, 803, 749, 771, 747, 793, 793, 781, 786, 794, 720, 771, 779, 738, 783, 748, 786, 747, 799, 781, 798, 737, 754, 728, 794, 721, 778, 772, 739, 719, 769, 796, 770, 790, 751, 771, 737, 774, 732, 793, 823, 752, 742] payload_rows_per_dst=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] dup_rows_per_dst=[2916, 2970, 3068, 3176, 3048, 2904, 2892, 3008] combine_rows_per_src=[9304, 9426, 9306, 9194, 9254, 9278, 9320, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344224 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059936 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775648 weight_movement_bytes=34603008 total_nvlink_bytes=566378656 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65270 dup_rows=23660 dup_rows_remote=20752 dedup_ratio=0.240682 router_tokens_per_expert=[789, 764, 767, 765, 738, 772, 792, 733, 709, 799, 822, 760, 739, 774, 756, 782, 766, 756, 770, 770, 726, 859, 777, 759, 776, 735, 773, 807, 767, 732, 778, 825, 773, 774, 732, 795, 784, 775, 767, 751, 771, 735, 742, 726, 764, 754, 765, 795, 799, 746, 774, 742, 759, 766, 757, 796, 775, 787, 758, 745, 766, 777, 773, 792] payload_rows_per_dst=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] dup_rows_per_dst=[2990, 2908, 3012, 2958, 2994, 2870, 2986, 2942] combine_rows_per_src=[9250, 9374, 9354, 9428, 9308, 9234, 9292, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=345248 combine_payload_bytes=267132928 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267478176 ingress_bytes=370941952 link_bytes=370941952 token_bytes=534611104 weight_movement_bytes=103809024 total_nvlink_bytes=638420128 weight_to_token_ratio=0.194177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395824702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65038 dup_rows=23950 dup_rows_remote=20966 dedup_ratio=0.243632 router_tokens_per_expert=[746, 772, 771, 752, 738, 771, 742, 782, 781, 760, 771, 735, 790, 741, 782, 772, 760, 771, 725, 784, 800, 798, 745, 753, 757, 758, 758, 751, 743, 813, 773, 805, 736, 779, 741, 775, 774, 757, 808, 761, 791, 764, 750, 787, 725, 783, 718, 807, 735, 778, 768, 810, 798, 721, 795, 745, 748, 827, 797, 772, 735, 790, 804, 773] payload_rows_per_dst=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] dup_rows_per_dst=[2944, 2988, 2968, 2958, 2996, 3016, 2984, 3096] combine_rows_per_src=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=345192 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667112 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532989032 weight_movement_bytes=34603008 total_nvlink_bytes=567592040 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344368 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266576176 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807984 weight_movement_bytes=69206016 total_nvlink_bytes=602014000 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344224 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059936 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775648 weight_movement_bytes=34603008 total_nvlink_bytes=566378656 weight_to_token_ratio=0.065071 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344440 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266060152 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775864 weight_movement_bytes=103809024 total_nvlink_bytes=635584888 weight_to_token_ratio=0.195212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344488 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764712 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184936 weight_movement_bytes=34603008 total_nvlink_bytes=567787944 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344088 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690008 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535035928 weight_movement_bytes=69206016 total_nvlink_bytes=604241944 weight_to_token_ratio=0.129348 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344016 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739664 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135312 weight_movement_bytes=69206016 total_nvlink_bytes=602341328 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=345192 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266667112 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532989032 weight_movement_bytes=34603008 total_nvlink_bytes=567592040 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344208 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813584 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282960 weight_movement_bytes=34603008 total_nvlink_bytes=567885968 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65016 dup_rows=24120 dup_rows_remote=21102 dedup_ratio=0.245361 router_tokens_per_expert=[808, 778, 822, 780, 772, 761, 766, 770, 796, 764, 764, 773, 739, 765, 745, 747, 802, 780, 747, 788, 750, 750, 777, 733, 744, 741, 793, 789, 726, 775, 740, 722, 770, 747, 761, 770, 803, 772, 795, 780, 761, 780, 768, 748, 794, 767, 794, 792, 760, 764, 748, 805, 778, 796, 793, 739, 795, 733, 746, 763, 726, 732, 795, 770] payload_rows_per_dst=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] dup_rows_per_dst=[3052, 3018, 3048, 2914, 3040, 3062, 3040, 2946] combine_rows_per_src=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64932 dup_rows=24078 dup_rows_remote=20960 dedup_ratio=0.244934 router_tokens_per_expert=[758, 779, 788, 809, 732, 787, 817, 779, 791, 728, 756, 777, 784, 730, 762, 805, 744, 817, 799, 783, 806, 767, 758, 765, 786, 794, 831, 759, 777, 782, 751, 726, 793, 744, 747, 725, 739, 769, 758, 743, 799, 779, 735, 774, 746, 740, 770, 755, 766, 719, 768, 735, 757, 779, 712, 788, 758, 754, 732, 756, 805, 785, 774, 821] payload_rows_per_dst=[9378, 9284, 9390, 9294, 9176, 9150, 9244, 9310] dup_rows_per_dst=[3120, 2982, 3088, 3118, 2860, 3046, 2804, 3060] combine_rows_per_src=[9378, 9284, 9390, 9294, 9176, 9150, 9244, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64672 dup_rows=24300 dup_rows_remote=21186 dedup_ratio=0.247192 router_tokens_per_expert=[746, 804, 750, 715, 757, 759, 767, 801, 809, 750, 768, 779, 819, 760, 773, 787, 765, 712, 805, 761, 775, 759, 773, 748, 749, 773, 762, 806, 744, 756, 761, 727, 759, 741, 791, 758, 836, 752, 786, 800, 740, 786, 781, 800, 804, 766, 741, 788, 761, 785, 789, 767, 755, 720, 744, 752, 792, 739, 745, 762, 775, 762, 786, 769] payload_rows_per_dst=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] dup_rows_per_dst=[2998, 3230, 3000, 2984, 3114, 3024, 2856, 3094] combine_rows_per_src=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=20996 dedup_ratio=0.243754 router_tokens_per_expert=[747, 754, 759, 756, 706, 790, 817, 729, 747, 735, 756, 765, 720, 766, 738, 755, 813, 807, 763, 727, 794, 759, 791, 787, 776, 834, 763, 760, 800, 800, 760, 740, 726, 812, 755, 767, 708, 790, 773, 775, 770, 790, 784, 768, 785, 786, 725, 788, 794, 775, 765, 722, 798, 820, 751, 791, 766, 749, 757, 785, 774, 757, 744, 788] payload_rows_per_dst=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] dup_rows_per_dst=[2966, 2812, 3294, 2942, 2996, 2990, 3040, 2922] combine_rows_per_src=[9286, 9026, 9654, 9232, 9286, 9402, 9382, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64672 dup_rows=24300 dup_rows_remote=21186 dedup_ratio=0.247192 router_tokens_per_expert=[746, 804, 750, 715, 757, 759, 767, 801, 809, 750, 768, 779, 819, 760, 773, 787, 765, 712, 805, 761, 775, 759, 773, 748, 749, 773, 762, 806, 744, 756, 761, 727, 759, 741, 791, 758, 836, 752, 786, 800, 740, 786, 781, 800, 804, 766, 741, 788, 761, 785, 789, 767, 755, 720, 744, 752, 792, 739, 745, 762, 775, 762, 786, 769] payload_rows_per_dst=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] dup_rows_per_dst=[2998, 3230, 3000, 2984, 3114, 3024, 2856, 3094] combine_rows_per_src=[9200, 9260, 9196, 9172, 9332, 9388, 9290, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65038 dup_rows=23950 dup_rows_remote=20966 dedup_ratio=0.243632 router_tokens_per_expert=[746, 772, 771, 752, 738, 771, 742, 782, 781, 760, 771, 735, 790, 741, 782, 772, 760, 771, 725, 784, 800, 798, 745, 753, 757, 758, 758, 751, 743, 813, 773, 805, 736, 779, 741, 775, 774, 757, 808, 761, 791, 764, 750, 787, 725, 783, 718, 807, 735, 778, 768, 810, 798, 721, 795, 745, 748, 827, 797, 772, 735, 790, 804, 773] payload_rows_per_dst=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] dup_rows_per_dst=[2944, 2988, 2968, 2958, 2996, 3016, 2984, 3096] combine_rows_per_src=[9204, 9276, 9304, 9358, 9266, 9234, 9316, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65134 dup_rows=23730 dup_rows_remote=20696 dedup_ratio=0.241394 router_tokens_per_expert=[785, 796, 782, 755, 763, 795, 772, 742, 711, 731, 761, 736, 790, 750, 767, 767, 774, 795, 814, 787, 755, 770, 799, 792, 767, 721, 763, 754, 770, 734, 806, 810, 753, 757, 778, 804, 756, 769, 732, 773, 767, 794, 757, 773, 746, 746, 763, 792, 728, 802, 773, 768, 752, 783, 765, 744, 761, 754, 769, 769, 753, 751, 794, 812] payload_rows_per_dst=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] dup_rows_per_dst=[2946, 2836, 2654, 3142, 3020, 3022, 3076, 3034] combine_rows_per_src=[9198, 9152, 9118, 9592, 9226, 9342, 9506, 9440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64702 dup_rows=24188 dup_rows_remote=21198 dedup_ratio=0.246053 router_tokens_per_expert=[757, 752, 736, 791, 797, 771, 776, 733, 712, 760, 778, 787, 772, 766, 748, 818, 784, 764, 769, 804, 764, 788, 724, 805, 738, 787, 739, 820, 809, 730, 805, 740, 755, 791, 749, 754, 715, 792, 735, 805, 757, 792, 815, 739, 749, 727, 760, 774, 780, 766, 792, 777, 760, 775, 783, 769, 731, 747, 783, 750, 759, 806, 769, 772] payload_rows_per_dst=[9234, 9252, 9414, 9344, 9224, 9136, 9330, 9182] dup_rows_per_dst=[2992, 3030, 2990, 2992, 2968, 3090, 3074, 3052] combine_rows_per_src=[9234, 9252, 9414, 9344, 9224, 9136, 9330, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343568 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305040 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266512 weight_movement_bytes=34603008 total_nvlink_bytes=566869520 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344472 combine_payload_bytes=266305536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266650008 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955544 weight_movement_bytes=69206016 total_nvlink_bytes=602161560 weight_to_token_ratio=0.129853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65040 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[752, 777, 817, 735, 760, 791, 739, 788, 760, 733, 795, 776, 788, 803, 778, 756, 774, 766, 750, 817, 768, 773, 779, 769, 761, 782, 720, 768, 782, 737, 783, 746, 783, 783, 751, 727, 734, 801, 780, 773, 781, 789, 764, 726, 749, 757, 787, 770, 744, 736, 768, 739, 785, 753, 775, 738, 758, 826, 764, 812, 810, 761, 740, 765] payload_rows_per_dst=[9352, 9310, 9442, 9154, 9260, 9252, 9128, 9394] dup_rows_per_dst=[2966, 3068, 2950, 3004, 3004, 2994, 2948, 3078] combine_rows_per_src=[9352, 9310, 9442, 9154, 9260, 9252, 9128, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64888 dup_rows=24040 dup_rows_remote=20990 dedup_ratio=0.244548 router_tokens_per_expert=[792, 736, 790, 777, 797, 779, 785, 757, 729, 749, 799, 749, 759, 764, 802, 807, 791, 787, 717, 742, 802, 721, 767, 795, 704, 748, 771, 738, 765, 756, 765, 769, 790, 805, 781, 802, 809, 713, 735, 818, 773, 762, 760, 767, 779, 790, 756, 710, 758, 801, 812, 728, 790, 775, 729, 781, 744, 739, 735, 809, 782, 746, 765, 799] payload_rows_per_dst=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] dup_rows_per_dst=[3050, 3024, 2978, 2930, 3130, 2924, 3054, 2950] combine_rows_per_src=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343568 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305040 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266512 weight_movement_bytes=34603008 total_nvlink_bytes=566869520 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65016 dup_rows=24120 dup_rows_remote=21102 dedup_ratio=0.245361 router_tokens_per_expert=[808, 778, 822, 780, 772, 761, 766, 770, 796, 764, 764, 773, 739, 765, 745, 747, 802, 780, 747, 788, 750, 750, 777, 733, 744, 741, 793, 789, 726, 775, 740, 722, 770, 747, 761, 770, 803, 772, 795, 780, 761, 780, 768, 748, 794, 767, 794, 792, 760, 764, 748, 805, 778, 796, 793, 739, 795, 733, 746, 763, 726, 732, 795, 770] payload_rows_per_dst=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] dup_rows_per_dst=[3052, 3018, 3048, 2914, 3040, 3062, 3040, 2946] combine_rows_per_src=[9462, 9168, 9206, 9146, 9356, 9346, 9326, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343432 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239944 ingress_bytes=299499520 link_bytes=299499520 token_bytes=530136456 weight_movement_bytes=34603008 total_nvlink_bytes=564739464 weight_to_token_ratio=0.065272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126992582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344208 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813584 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282960 weight_movement_bytes=34603008 total_nvlink_bytes=567885968 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65016 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[765, 791, 734, 711, 755, 780, 742, 712, 749, 797, 771, 798, 759, 777, 767, 781, 766, 829, 768, 779, 743, 799, 810, 788, 805, 780, 828, 781, 760, 779, 761, 714, 768, 792, 749, 804, 711, 755, 758, 772, 787, 802, 726, 755, 731, 743, 781, 804, 768, 780, 835, 745, 770, 781, 741, 747, 783, 732, 754, 771, 744, 700, 780, 804] payload_rows_per_dst=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] dup_rows_per_dst=[2802, 3086, 3184, 3070, 2932, 3004, 3002, 2876] combine_rows_per_src=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343432 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265239944 ingress_bytes=299499520 link_bytes=299499520 token_bytes=530136456 weight_movement_bytes=34603008 total_nvlink_bytes=564739464 weight_to_token_ratio=0.065272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126992582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344016 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739664 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135312 weight_movement_bytes=69206016 total_nvlink_bytes=602341328 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343320 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132184 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921048 weight_movement_bytes=34603008 total_nvlink_bytes=568524056 weight_to_token_ratio=0.064809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=343600 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265362992 ingress_bytes=368828416 link_bytes=368828416 token_bytes=530382384 weight_movement_bytes=103809024 total_nvlink_bytes=634191408 weight_to_token_ratio=0.195725 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387871636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344232 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748072 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151912 weight_movement_bytes=34603008 total_nvlink_bytes=567754920 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344472 combine_payload_bytes=266305536 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266650008 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955544 weight_movement_bytes=69206016 total_nvlink_bytes=602161560 weight_to_token_ratio=0.129853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343512 combine_payload_bytes=265781248 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266124760 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906008 weight_movement_bytes=69206016 total_nvlink_bytes=601112024 weight_to_token_ratio=0.130109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64990 dup_rows=24116 dup_rows_remote=21010 dedup_ratio=0.245321 router_tokens_per_expert=[794, 709, 763, 743, 794, 743, 753, 765, 793, 760, 758, 811, 790, 756, 772, 712, 771, 757, 771, 755, 777, 826, 785, 774, 745, 742, 706, 808, 781, 798, 763, 758, 702, 792, 764, 803, 766, 754, 778, 753, 732, 740, 794, 775, 741, 792, 772, 786, 770, 770, 838, 771, 744, 775, 772, 805, 762, 742, 796, 760, 753, 765, 748, 804] payload_rows_per_dst=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] dup_rows_per_dst=[2942, 3106, 3134, 2984, 2928, 3000, 3090, 2932] combine_rows_per_src=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=65002 dup_rows=24190 dup_rows_remote=21258 dedup_ratio=0.246073 router_tokens_per_expert=[770, 719, 773, 798, 763, 766, 762, 747, 759, 793, 761, 736, 728, 720, 756, 790, 819, 789, 762, 778, 813, 812, 808, 734, 770, 679, 801, 784, 701, 745, 733, 809, 811, 812, 785, 773, 796, 739, 796, 771, 788, 814, 790, 777, 802, 775, 761, 791, 805, 731, 761, 754, 782, 767, 767, 751, 742, 755, 750, 762, 775, 737, 733, 721] payload_rows_per_dst=[9206, 9166, 9432, 9112, 9398, 9398, 9294, 9108] dup_rows_per_dst=[2990, 2920, 3198, 2932, 3168, 3198, 2942, 2842] combine_rows_per_src=[9206, 9166, 9432, 9112, 9398, 9398, 9294, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64890 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[775, 791, 722, 741, 722, 763, 775, 826, 740, 805, 773, 784, 743, 763, 779, 775, 753, 797, 758, 820, 777, 801, 732, 807, 767, 767, 775, 790, 767, 726, 749, 764, 764, 781, 745, 712, 739, 808, 775, 787, 726, 772, 771, 772, 822, 750, 777, 788, 774, 744, 760, 741, 800, 777, 718, 782, 772, 753, 754, 776, 795, 770, 753, 767] payload_rows_per_dst=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] dup_rows_per_dst=[2996, 3022, 3152, 3024, 2956, 3116, 2912, 2970] combine_rows_per_src=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64888 dup_rows=24040 dup_rows_remote=20990 dedup_ratio=0.244548 router_tokens_per_expert=[792, 736, 790, 777, 797, 779, 785, 757, 729, 749, 799, 749, 759, 764, 802, 807, 791, 787, 717, 742, 802, 721, 767, 795, 704, 748, 771, 738, 765, 756, 765, 769, 790, 805, 781, 802, 809, 713, 735, 818, 773, 762, 760, 767, 779, 790, 756, 710, 758, 801, 812, 728, 790, 775, 729, 781, 744, 739, 735, 809, 782, 746, 765, 799] payload_rows_per_dst=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] dup_rows_per_dst=[3050, 3024, 2978, 2930, 3130, 2924, 3054, 2950] combine_rows_per_src=[9376, 9292, 9266, 9102, 9376, 9270, 9294, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=345040 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266593232 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841424 weight_movement_bytes=34603008 total_nvlink_bytes=567444432 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65104 dup_rows=24018 dup_rows_remote=20974 dedup_ratio=0.244324 router_tokens_per_expert=[763, 776, 752, 788, 713, 789, 794, 760, 748, 737, 782, 795, 754, 755, 767, 793, 749, 802, 777, 801, 777, 785, 793, 785, 749, 768, 772, 753, 768, 760, 737, 766, 733, 774, 766, 799, 741, 740, 764, 789, 768, 785, 785, 795, 698, 765, 789, 779, 786, 774, 802, 771, 752, 760, 755, 761, 775, 752, 775, 758, 761, 758, 747, 787] payload_rows_per_dst=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] dup_rows_per_dst=[2988, 3076, 2874, 2990, 2992, 3062, 2992, 3044] combine_rows_per_src=[9282, 9186, 9352, 9156, 9220, 9578, 9330, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65250 dup_rows=23726 dup_rows_remote=20834 dedup_ratio=0.241353 router_tokens_per_expert=[787, 754, 769, 760, 760, 786, 783, 780, 838, 748, 744, 747, 737, 767, 807, 791, 761, 709, 798, 760, 806, 727, 794, 755, 750, 762, 737, 748, 808, 836, 735, 791, 731, 730, 724, 798, 760, 787, 756, 816, 760, 790, 742, 790, 795, 739, 749, 754, 783, 751, 785, 726, 764, 750, 816, 732, 807, 775, 802, 771, 784, 752, 712, 786] payload_rows_per_dst=[9348, 9398, 9328, 9358, 9272, 9238, 9330, 9306] dup_rows_per_dst=[3010, 2960, 2892, 2976, 2932, 3000, 2884, 3072] combine_rows_per_src=[9348, 9398, 9328, 9358, 9272, 9238, 9330, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344000 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543040 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742080 weight_movement_bytes=34603008 total_nvlink_bytes=567345088 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64890 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[775, 791, 722, 741, 722, 763, 775, 826, 740, 805, 773, 784, 743, 763, 779, 775, 753, 797, 758, 820, 777, 801, 732, 807, 767, 767, 775, 790, 767, 726, 749, 764, 764, 781, 745, 712, 739, 808, 775, 787, 726, 772, 771, 772, 822, 750, 777, 788, 774, 744, 760, 741, 800, 777, 718, 782, 772, 753, 754, 776, 795, 770, 753, 767] payload_rows_per_dst=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] dup_rows_per_dst=[2996, 3022, 3152, 3024, 2956, 3116, 2912, 2970] combine_rows_per_src=[9234, 9302, 9338, 9186, 9266, 9240, 9280, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65014 dup_rows=24096 dup_rows_remote=21134 dedup_ratio=0.245117 router_tokens_per_expert=[762, 777, 687, 758, 778, 757, 782, 781, 784, 772, 765, 745, 760, 776, 779, 790, 786, 789, 821, 774, 748, 760, 790, 781, 768, 778, 771, 765, 756, 756, 783, 767, 761, 805, 748, 790, 766, 740, 756, 771, 776, 743, 775, 777, 729, 768, 786, 724, 757, 758, 732, 730, 770, 767, 752, 763, 825, 796, 766, 771, 776, 777, 744, 807] payload_rows_per_dst=[9192, 9310, 9382, 9342, 9262, 9194, 9142, 9384] dup_rows_per_dst=[2972, 3032, 3116, 2946, 3012, 2962, 2916, 3140] combine_rows_per_src=[9192, 9310, 9382, 9342, 9262, 9194, 9142, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65016 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[765, 791, 734, 711, 755, 780, 742, 712, 749, 797, 771, 798, 759, 777, 767, 781, 766, 829, 768, 779, 743, 799, 810, 788, 805, 780, 828, 781, 760, 779, 761, 714, 768, 792, 749, 804, 711, 755, 758, 772, 787, 802, 726, 755, 731, 743, 781, 804, 768, 780, 835, 745, 770, 781, 741, 747, 783, 732, 754, 771, 744, 700, 780, 804] payload_rows_per_dst=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] dup_rows_per_dst=[2802, 3086, 3184, 3070, 2932, 3004, 3002, 2876] combine_rows_per_src=[9178, 9312, 9380, 9346, 9286, 9254, 9332, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64586 dup_rows=24272 dup_rows_remote=21004 dedup_ratio=0.246908 router_tokens_per_expert=[728, 728, 799, 717, 835, 714, 758, 726, 782, 793, 815, 759, 776, 791, 779, 768, 780, 801, 798, 740, 765, 742, 723, 735, 747, 776, 759, 755, 801, 771, 764, 736, 729, 773, 758, 779, 730, 752, 737, 793, 755, 768, 784, 773, 808, 765, 756, 764, 764, 837, 749, 793, 795, 800, 729, 823, 773, 751, 789, 754, 799, 753, 748, 810] payload_rows_per_dst=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] dup_rows_per_dst=[3268, 3222, 2570, 3194, 3064, 2826, 3132, 2996] combine_rows_per_src=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=345040 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266593232 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841424 weight_movement_bytes=34603008 total_nvlink_bytes=567444432 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64990 dup_rows=24116 dup_rows_remote=21010 dedup_ratio=0.245321 router_tokens_per_expert=[794, 709, 763, 743, 794, 743, 753, 765, 793, 760, 758, 811, 790, 756, 772, 712, 771, 757, 771, 755, 777, 826, 785, 774, 745, 742, 706, 808, 781, 798, 763, 758, 702, 792, 764, 803, 766, 754, 778, 753, 732, 740, 794, 775, 741, 792, 772, 786, 770, 770, 838, 771, 744, 775, 772, 805, 762, 742, 796, 760, 753, 765, 748, 804] payload_rows_per_dst=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] dup_rows_per_dst=[2942, 3106, 3134, 2984, 2928, 3000, 3090, 2932] combine_rows_per_src=[9186, 9198, 9298, 9218, 9296, 9264, 9400, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344328 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133768 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531923208 weight_movement_bytes=34603008 total_nvlink_bytes=566526216 weight_to_token_ratio=0.065053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343512 combine_payload_bytes=265781248 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266124760 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531906008 weight_movement_bytes=69206016 total_nvlink_bytes=601112024 weight_to_token_ratio=0.130109 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65232 dup_rows=23760 dup_rows_remote=20876 dedup_ratio=0.241699 router_tokens_per_expert=[770, 766, 757, 787, 764, 728, 764, 786, 724, 713, 756, 750, 810, 752, 772, 769, 743, 785, 796, 752, 810, 774, 783, 729, 767, 739, 789, 745, 722, 841, 763, 742, 794, 801, 782, 706, 777, 771, 751, 773, 798, 727, 780, 758, 760, 805, 757, 792, 726, 760, 777, 760, 761, 760, 791, 763, 816, 793, 780, 770, 790, 778, 757, 790] payload_rows_per_dst=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] dup_rows_per_dst=[2998, 2820, 3090, 2926, 3066, 2956, 2884, 3020] combine_rows_per_src=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344312 combine_payload_bytes=266665984 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267010296 ingress_bytes=370475008 link_bytes=370475008 token_bytes=533676280 weight_movement_bytes=103809024 total_nvlink_bytes=637485304 weight_to_token_ratio=0.194517 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394067629e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344336 combine_payload_bytes=267264000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267608336 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872336 weight_movement_bytes=69206016 total_nvlink_bytes=604078352 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=344592 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266641936 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532939280 weight_movement_bytes=69206016 total_nvlink_bytes=602145296 weight_to_token_ratio=0.129857 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344000 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543040 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742080 weight_movement_bytes=34603008 total_nvlink_bytes=567345088 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344328 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133768 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531923208 weight_movement_bytes=34603008 total_nvlink_bytes=566526216 weight_to_token_ratio=0.065053 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=342360 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886616 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430872 weight_movement_bytes=34603008 total_nvlink_bytes=564033880 weight_to_token_ratio=0.065359 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=21102 dedup_ratio=0.244649 router_tokens_per_expert=[753, 771, 807, 758, 735, 767, 737, 804, 760, 792, 786, 780, 747, 770, 766, 746, 776, 758, 754, 740, 735, 763, 798, 742, 785, 743, 774, 800, 753, 755, 714, 746, 763, 766, 742, 817, 791, 756, 783, 777, 777, 756, 760, 763, 770, 745, 756, 756, 789, 767, 796, 749, 736, 775, 784, 781, 783, 797, 742, 777, 830, 803, 744, 806] payload_rows_per_dst=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] dup_rows_per_dst=[3086, 2948, 2924, 2952, 3098, 2918, 3058, 3066] combine_rows_per_src=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65112 dup_rows=23906 dup_rows_remote=20754 dedup_ratio=0.243184 router_tokens_per_expert=[826, 792, 736, 787, 816, 814, 749, 792, 772, 833, 778, 758, 774, 746, 767, 729, 808, 744, 761, 756, 750, 730, 777, 797, 775, 793, 780, 778, 827, 724, 746, 796, 713, 750, 758, 783, 752, 774, 807, 772, 791, 742, 745, 742, 750, 769, 765, 772, 809, 749, 789, 782, 733, 775, 738, 754, 733, 761, 764, 748, 792, 755, 689, 785] payload_rows_per_dst=[9588, 9212, 9414, 9286, 9230, 9206, 9280, 9182] dup_rows_per_dst=[3036, 3102, 2832, 3152, 2988, 2946, 2978, 2872] combine_rows_per_src=[9588, 9212, 9414, 9286, 9230, 9206, 9280, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344432 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534704 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724976 weight_movement_bytes=34603008 total_nvlink_bytes=569327984 weight_to_token_ratio=0.064712 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65162 dup_rows=23784 dup_rows_remote=20680 dedup_ratio=0.241943 router_tokens_per_expert=[761, 782, 746, 736, 811, 813, 771, 764, 731, 758, 736, 776, 773, 718, 758, 785, 761, 827, 771, 814, 763, 714, 756, 826, 749, 755, 760, 760, 757, 753, 720, 719, 733, 777, 767, 775, 757, 794, 783, 845, 777, 738, 784, 761, 824, 752, 780, 763, 792, 801, 752, 806, 753, 761, 801, 776, 775, 747, 748, 724, 784, 749, 792, 757] payload_rows_per_dst=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] dup_rows_per_dst=[3028, 2910, 3014, 2812, 3104, 3070, 3014, 2832] combine_rows_per_src=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343464 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042216 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533740968 weight_movement_bytes=34603008 total_nvlink_bytes=568343976 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64586 dup_rows=24272 dup_rows_remote=21004 dedup_ratio=0.246908 router_tokens_per_expert=[728, 728, 799, 717, 835, 714, 758, 726, 782, 793, 815, 759, 776, 791, 779, 768, 780, 801, 798, 740, 765, 742, 723, 735, 747, 776, 759, 755, 801, 771, 764, 736, 729, 773, 758, 779, 730, 752, 737, 793, 755, 768, 784, 773, 808, 765, 756, 764, 764, 837, 749, 793, 795, 800, 729, 823, 773, 751, 789, 754, 799, 753, 748, 810] payload_rows_per_dst=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] dup_rows_per_dst=[3268, 3222, 2570, 3194, 3064, 2826, 3132, 2996] combine_rows_per_src=[9446, 9314, 8894, 9670, 9228, 8874, 9464, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21130 dedup_ratio=0.245524 router_tokens_per_expert=[771, 748, 765, 740, 763, 736, 761, 773, 798, 824, 770, 795, 784, 761, 725, 799, 767, 790, 804, 768, 754, 783, 780, 745, 739, 785, 767, 725, 775, 781, 767, 802, 750, 791, 738, 730, 813, 749, 783, 784, 814, 827, 803, 761, 759, 746, 758, 768, 739, 675, 765, 749, 788, 739, 737, 762, 740, 801, 813, 736, 739, 796, 809, 745] payload_rows_per_dst=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] dup_rows_per_dst=[2746, 2906, 3020, 3384, 2972, 3156, 2946, 3006] combine_rows_per_src=[8950, 9152, 9362, 9770, 9144, 9290, 9122, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=65012 dup_rows=24176 dup_rows_remote=21192 dedup_ratio=0.245931 router_tokens_per_expert=[738, 809, 780, 794, 791, 809, 787, 794, 788, 771, 791, 716, 745, 808, 824, 720, 788, 734, 741, 766, 771, 730, 769, 751, 730, 731, 761, 806, 772, 793, 774, 737, 766, 755, 747, 806, 784, 766, 797, 771, 786, 746, 794, 675, 789, 791, 803, 751, 779, 775, 837, 749, 711, 723, 758, 783, 792, 773, 715, 809, 796, 752, 709, 745] payload_rows_per_dst=[9332, 9348, 9116, 9302, 9278, 9238, 9206, 9308] dup_rows_per_dst=[3272, 2978, 2984, 2906, 3106, 3032, 3024, 2874] combine_rows_per_src=[9332, 9348, 9116, 9302, 9278, 9238, 9206, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343368 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267246920 ingress_bytes=405315584 link_bytes=405315584 token_bytes=534150472 weight_movement_bytes=138412032 total_nvlink_bytes=672562504 weight_to_token_ratio=0.259126 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525169912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344040 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207208 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070376 weight_movement_bytes=69206016 total_nvlink_bytes=601276392 weight_to_token_ratio=0.130069 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64876 dup_rows=24018 dup_rows_remote=20934 dedup_ratio=0.244324 router_tokens_per_expert=[784, 791, 748, 747, 769, 749, 787, 771, 754, 765, 823, 791, 713, 737, 824, 774, 704, 761, 810, 756, 757, 829, 754, 753, 727, 735, 773, 777, 747, 752, 737, 779, 779, 782, 734, 790, 734, 824, 809, 720, 765, 802, 766, 784, 793, 788, 788, 761, 784, 736, 780, 771, 753, 737, 793, 759, 781, 736, 710, 791, 732, 797, 817, 778] payload_rows_per_dst=[9342, 9366, 9250, 9126, 9294, 9410, 9142, 9356] dup_rows_per_dst=[2950, 2996, 2998, 2928, 3050, 3084, 3084, 2928] combine_rows_per_src=[9342, 9366, 9250, 9126, 9294, 9410, 9142, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=21102 dedup_ratio=0.244649 router_tokens_per_expert=[753, 771, 807, 758, 735, 767, 737, 804, 760, 792, 786, 780, 747, 770, 766, 746, 776, 758, 754, 740, 735, 763, 798, 742, 785, 743, 774, 800, 753, 755, 714, 746, 763, 766, 742, 817, 791, 756, 783, 777, 777, 756, 760, 763, 770, 745, 756, 756, 789, 767, 796, 749, 736, 775, 784, 781, 783, 797, 742, 777, 830, 803, 744, 806] payload_rows_per_dst=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] dup_rows_per_dst=[3086, 2948, 2924, 2952, 3098, 2918, 3058, 3066] combine_rows_per_src=[9178, 9346, 9208, 9188, 9292, 9248, 9296, 9498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65010 dup_rows=24040 dup_rows_remote=21066 dedup_ratio=0.244548 router_tokens_per_expert=[777, 754, 768, 775, 773, 717, 780, 764, 724, 797, 806, 708, 774, 791, 780, 834, 739, 786, 719, 768, 809, 763, 754, 781, 725, 792, 760, 745, 738, 779, 727, 757, 726, 764, 757, 748, 764, 786, 745, 792, 775, 786, 826, 780, 773, 729, 737, 723, 783, 799, 807, 773, 747, 822, 755, 801, 768, 813, 764, 791, 767, 750, 756, 781] payload_rows_per_dst=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] dup_rows_per_dst=[2974, 3002, 3068, 2928, 2920, 3012, 3114, 3022] combine_rows_per_src=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65162 dup_rows=23784 dup_rows_remote=20680 dedup_ratio=0.241943 router_tokens_per_expert=[761, 782, 746, 736, 811, 813, 771, 764, 731, 758, 736, 776, 773, 718, 758, 785, 761, 827, 771, 814, 763, 714, 756, 826, 749, 755, 760, 760, 757, 753, 720, 719, 733, 777, 767, 775, 757, 794, 783, 845, 777, 738, 784, 761, 824, 752, 780, 763, 792, 801, 752, 806, 753, 761, 801, 776, 775, 747, 748, 724, 784, 749, 792, 757] payload_rows_per_dst=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] dup_rows_per_dst=[3028, 2910, 3014, 2812, 3104, 3070, 3014, 2832] combine_rows_per_src=[9340, 9160, 9450, 9134, 9358, 9288, 9470, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=343464 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042216 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533740968 weight_movement_bytes=34603008 total_nvlink_bytes=568343976 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=342360 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886616 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430872 weight_movement_bytes=34603008 total_nvlink_bytes=564033880 weight_to_token_ratio=0.065359 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73902 payload_rows_remote=64770 dup_rows=24402 dup_rows_remote=21424 dedup_ratio=0.248230 router_tokens_per_expert=[759, 735, 780, 785, 811, 714, 756, 774, 792, 780, 771, 811, 799, 755, 768, 750, 799, 807, 774, 815, 747, 786, 773, 720, 790, 734, 774, 818, 730, 806, 729, 778, 802, 768, 748, 753, 774, 733, 779, 787, 768, 737, 772, 787, 750, 744, 757, 766, 771, 745, 755, 755, 736, 779, 730, 784, 748, 826, 797, 777, 734, 763, 739, 768] payload_rows_per_dst=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] dup_rows_per_dst=[2986, 3132, 3166, 3064, 3080, 2948, 2978, 3048] combine_rows_per_src=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65232 dup_rows=23760 dup_rows_remote=20876 dedup_ratio=0.241699 router_tokens_per_expert=[770, 766, 757, 787, 764, 728, 764, 786, 724, 713, 756, 750, 810, 752, 772, 769, 743, 785, 796, 752, 810, 774, 783, 729, 767, 739, 789, 745, 722, 841, 763, 742, 794, 801, 782, 706, 777, 771, 751, 773, 798, 727, 780, 758, 760, 805, 757, 792, 726, 760, 777, 760, 761, 760, 791, 763, 816, 793, 780, 770, 790, 778, 757, 790] payload_rows_per_dst=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] dup_rows_per_dst=[2998, 2820, 3090, 2926, 3066, 2956, 2884, 3020] combine_rows_per_src=[9246, 9272, 9254, 9290, 9244, 9398, 9312, 9528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343680 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723520 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103360 weight_movement_bytes=34603008 total_nvlink_bytes=565706368 weight_to_token_ratio=0.065153 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344816 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633968 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532923120 weight_movement_bytes=34603008 total_nvlink_bytes=567526128 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343680 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723520 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103360 weight_movement_bytes=34603008 total_nvlink_bytes=565706368 weight_to_token_ratio=0.065153 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343240 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075336 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807432 weight_movement_bytes=69206016 total_nvlink_bytes=601013448 weight_to_token_ratio=0.130134 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344040 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207208 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070376 weight_movement_bytes=69206016 total_nvlink_bytes=601276392 weight_to_token_ratio=0.130069 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344304 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266625264 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532906224 weight_movement_bytes=69206016 total_nvlink_bytes=602112240 weight_to_token_ratio=0.129865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343368 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267246920 ingress_bytes=405315584 link_bytes=405315584 token_bytes=534150472 weight_movement_bytes=138412032 total_nvlink_bytes=672562504 weight_to_token_ratio=0.259126 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525169912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344432 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534704 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724976 weight_movement_bytes=34603008 total_nvlink_bytes=569327984 weight_to_token_ratio=0.064712 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65150 dup_rows=23842 dup_rows_remote=20886 dedup_ratio=0.242533 router_tokens_per_expert=[757, 795, 802, 717, 759, 783, 710, 749, 727, 745, 748, 786, 737, 805, 792, 794, 800, 797, 711, 723, 765, 779, 763, 778, 779, 808, 781, 739, 758, 745, 843, 781, 811, 786, 772, 762, 777, 748, 804, 748, 771, 726, 767, 799, 769, 776, 776, 764, 765, 775, 756, 757, 738, 727, 776, 772, 760, 788, 747, 754, 763, 811, 777, 774] payload_rows_per_dst=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] dup_rows_per_dst=[2950, 2956, 2932, 3078, 3014, 2974, 2936, 3002] combine_rows_per_src=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64928 dup_rows=23902 dup_rows_remote=20828 dedup_ratio=0.243144 router_tokens_per_expert=[756, 727, 742, 732, 790, 716, 773, 792, 732, 749, 768, 752, 762, 738, 769, 777, 773, 746, 753, 764, 785, 779, 777, 753, 761, 771, 806, 776, 809, 821, 732, 798, 787, 752, 769, 770, 800, 762, 767, 805, 714, 726, 795, 759, 781, 806, 780, 778, 759, 797, 758, 784, 783, 780, 762, 791, 781, 742, 747, 770, 801, 786, 692, 789] payload_rows_per_dst=[9236, 9240, 9272, 9474, 9324, 9232, 9360, 9264] dup_rows_per_dst=[2820, 2854, 2988, 3074, 3100, 3046, 3068, 2952] combine_rows_per_src=[9236, 9240, 9272, 9474, 9324, 9232, 9360, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344776 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265642696 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530940616 weight_movement_bytes=34603008 total_nvlink_bytes=565543624 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65078 dup_rows=23976 dup_rows_remote=20960 dedup_ratio=0.243896 router_tokens_per_expert=[734, 759, 762, 753, 762, 787, 752, 767, 759, 758, 748, 763, 816, 775, 735, 739, 787, 817, 757, 748, 770, 739, 742, 773, 800, 774, 758, 801, 808, 779, 743, 787, 749, 730, 777, 764, 753, 764, 803, 760, 807, 790, 729, 782, 740, 774, 780, 788, 772, 719, 775, 789, 771, 839, 757, 744, 781, 776, 774, 757, 739, 760, 787, 770] payload_rows_per_dst=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] dup_rows_per_dst=[2920, 2936, 3016, 3116, 2914, 3030, 3044, 3000] combine_rows_per_src=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65220 dup_rows=23794 dup_rows_remote=20860 dedup_ratio=0.242045 router_tokens_per_expert=[742, 769, 783, 795, 817, 795, 777, 793, 746, 765, 757, 744, 782, 790, 791, 754, 790, 755, 806, 740, 767, 762, 720, 727, 775, 744, 787, 729, 767, 781, 710, 793, 745, 777, 776, 762, 820, 729, 736, 767, 761, 781, 757, 811, 771, 784, 766, 778, 769, 767, 801, 771, 769, 811, 783, 782, 763, 774, 724, 783, 762, 750, 757, 712] payload_rows_per_dst=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] dup_rows_per_dst=[3124, 2932, 2950, 2882, 2934, 2996, 3092, 2884] combine_rows_per_src=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344144 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267198544 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534052944 weight_movement_bytes=34603008 total_nvlink_bytes=568655952 weight_to_token_ratio=0.064793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64792 dup_rows=24218 dup_rows_remote=21114 dedup_ratio=0.246358 router_tokens_per_expert=[732, 788, 754, 789, 777, 763, 757, 768, 721, 758, 764, 782, 818, 832, 776, 771, 794, 766, 762, 735, 733, 774, 751, 745, 722, 761, 775, 809, 757, 772, 761, 737, 798, 813, 733, 747, 746, 798, 807, 786, 762, 778, 757, 817, 730, 806, 804, 745, 758, 738, 792, 730, 810, 747, 831, 773, 736, 752, 743, 737, 755, 711, 779, 829] payload_rows_per_dst=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] dup_rows_per_dst=[2998, 3048, 2990, 3072, 3108, 3104, 2982, 2916] combine_rows_per_src=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343024 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288112 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233200 weight_movement_bytes=69206016 total_nvlink_bytes=601439216 weight_to_token_ratio=0.130029 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65010 dup_rows=24040 dup_rows_remote=21066 dedup_ratio=0.244548 router_tokens_per_expert=[777, 754, 768, 775, 773, 717, 780, 764, 724, 797, 806, 708, 774, 791, 780, 834, 739, 786, 719, 768, 809, 763, 754, 781, 725, 792, 760, 745, 738, 779, 727, 757, 726, 764, 757, 748, 764, 786, 745, 792, 775, 786, 826, 780, 773, 729, 737, 723, 783, 799, 807, 773, 747, 822, 755, 801, 768, 813, 764, 791, 767, 750, 756, 781] payload_rows_per_dst=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] dup_rows_per_dst=[2974, 3002, 3068, 2928, 2920, 3012, 3114, 3022] combine_rows_per_src=[9254, 9426, 9222, 9118, 9182, 9244, 9460, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64750 dup_rows=24058 dup_rows_remote=20918 dedup_ratio=0.244731 router_tokens_per_expert=[748, 734, 819, 781, 732, 763, 773, 771, 744, 817, 735, 760, 768, 774, 769, 772, 750, 784, 737, 721, 781, 760, 771, 740, 795, 771, 765, 772, 748, 748, 775, 754, 771, 761, 762, 794, 746, 814, 770, 763, 785, 782, 763, 729, 774, 752, 758, 826, 801, 788, 769, 743, 743, 780, 759, 758, 824, 785, 748, 831, 781, 732, 761, 767] payload_rows_per_dst=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] dup_rows_per_dst=[2948, 3000, 2892, 2920, 3052, 3110, 2996, 3140] combine_rows_per_src=[9294, 9278, 9196, 9336, 9132, 9228, 9286, 9496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344320 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267485440 ingress_bytes=336347136 link_bytes=336347136 token_bytes=534626560 weight_movement_bytes=69206016 total_nvlink_bytes=603832576 weight_to_token_ratio=0.129447 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265647195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64792 dup_rows=24218 dup_rows_remote=21114 dedup_ratio=0.246358 router_tokens_per_expert=[732, 788, 754, 789, 777, 763, 757, 768, 721, 758, 764, 782, 818, 832, 776, 771, 794, 766, 762, 735, 733, 774, 751, 745, 722, 761, 775, 809, 757, 772, 761, 737, 798, 813, 733, 747, 746, 798, 807, 786, 762, 778, 757, 817, 730, 806, 804, 745, 758, 738, 792, 730, 810, 747, 831, 773, 736, 752, 743, 737, 755, 711, 779, 829] payload_rows_per_dst=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] dup_rows_per_dst=[2998, 3048, 2990, 3072, 3108, 3104, 2982, 2916] combine_rows_per_src=[9258, 9396, 9130, 9116, 9348, 9294, 9376, 9168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65078 dup_rows=23976 dup_rows_remote=20960 dedup_ratio=0.243896 router_tokens_per_expert=[734, 759, 762, 753, 762, 787, 752, 767, 759, 758, 748, 763, 816, 775, 735, 739, 787, 817, 757, 748, 770, 739, 742, 773, 800, 774, 758, 801, 808, 779, 743, 787, 749, 730, 777, 764, 753, 764, 803, 760, 807, 790, 729, 782, 740, 774, 780, 788, 772, 719, 775, 789, 771, 839, 757, 744, 781, 776, 774, 757, 739, 760, 787, 770] payload_rows_per_dst=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] dup_rows_per_dst=[2920, 2936, 3016, 3116, 2914, 3030, 3044, 3000] combine_rows_per_src=[9232, 9250, 9250, 9384, 9286, 9350, 9288, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65150 dup_rows=23842 dup_rows_remote=20886 dedup_ratio=0.242533 router_tokens_per_expert=[757, 795, 802, 717, 759, 783, 710, 749, 727, 745, 748, 786, 737, 805, 792, 794, 800, 797, 711, 723, 765, 779, 763, 778, 779, 808, 781, 739, 758, 745, 843, 781, 811, 786, 772, 762, 777, 748, 804, 748, 771, 726, 767, 799, 769, 776, 776, 764, 765, 775, 756, 757, 738, 727, 776, 772, 760, 788, 747, 754, 763, 811, 777, 774] payload_rows_per_dst=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] dup_rows_per_dst=[2950, 2956, 2932, 3078, 3014, 2974, 2936, 3002] combine_rows_per_src=[9194, 9312, 9300, 9390, 9402, 9322, 9196, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65182 dup_rows=23796 dup_rows_remote=20850 dedup_ratio=0.242065 router_tokens_per_expert=[806, 796, 815, 758, 773, 779, 716, 740, 780, 756, 809, 783, 743, 799, 726, 822, 778, 802, 759, 779, 778, 672, 776, 744, 808, 753, 764, 778, 775, 764, 758, 764, 768, 737, 768, 752, 719, 764, 764, 762, 787, 787, 725, 756, 794, 786, 768, 803, 779, 749, 762, 729, 743, 794, 814, 766, 793, 788, 732, 787, 767, 771, 730, 755] payload_rows_per_dst=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] dup_rows_per_dst=[2976, 3094, 2982, 2964, 2838, 3022, 2946, 2974] combine_rows_per_src=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344304 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266625264 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532906224 weight_movement_bytes=69206016 total_nvlink_bytes=602112240 weight_to_token_ratio=0.129865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64818 dup_rows=24060 dup_rows_remote=21078 dedup_ratio=0.244751 router_tokens_per_expert=[736, 776, 763, 799, 757, 747, 760, 775, 748, 808, 807, 764, 776, 769, 745, 812, 740, 732, 821, 741, 815, 757, 786, 763, 807, 787, 771, 802, 746, 732, 825, 734, 747, 760, 761, 725, 791, 700, 792, 820, 777, 719, 692, 783, 759, 788, 759, 809, 750, 762, 766, 784, 710, 785, 759, 765, 764, 774, 793, 746, 793, 751, 781, 786] payload_rows_per_dst=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] dup_rows_per_dst=[3046, 3144, 3054, 2982, 2948, 2926, 2944, 3016] combine_rows_per_src=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=342672 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265558672 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530774672 weight_movement_bytes=34603008 total_nvlink_bytes=565377680 weight_to_token_ratio=0.065193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343624 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265731656 ingress_bytes=334594048 link_bytes=334594048 token_bytes=531119688 weight_movement_bytes=69206016 total_nvlink_bytes=600325704 weight_to_token_ratio=0.130302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259050466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64810 dup_rows=24110 dup_rows_remote=20948 dedup_ratio=0.245260 router_tokens_per_expert=[779, 787, 739, 780, 743, 748, 771, 710, 745, 722, 769, 804, 740, 772, 809, 780, 787, 758, 789, 783, 753, 810, 800, 802, 765, 758, 797, 751, 763, 760, 760, 760, 803, 788, 751, 768, 764, 775, 745, 766, 775, 789, 775, 793, 753, 742, 765, 713, 827, 721, 775, 722, 723, 786, 755, 766, 799, 787, 795, 788, 748, 748, 756, 797] payload_rows_per_dst=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] dup_rows_per_dst=[3162, 3102, 3310, 2770, 2942, 2826, 2892, 3106] combine_rows_per_src=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65220 dup_rows=23794 dup_rows_remote=20860 dedup_ratio=0.242045 router_tokens_per_expert=[742, 769, 783, 795, 817, 795, 777, 793, 746, 765, 757, 744, 782, 790, 791, 754, 790, 755, 806, 740, 767, 762, 720, 727, 775, 744, 787, 729, 767, 781, 710, 793, 745, 777, 776, 762, 820, 729, 736, 767, 761, 781, 757, 811, 771, 784, 766, 778, 769, 767, 801, 771, 769, 811, 783, 782, 763, 774, 724, 783, 762, 750, 757, 712] payload_rows_per_dst=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] dup_rows_per_dst=[3124, 2932, 2950, 2882, 2934, 2996, 3092, 2884] combine_rows_per_src=[9418, 9326, 9184, 9290, 9290, 9422, 9414, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73902 payload_rows_remote=64770 dup_rows=24402 dup_rows_remote=21424 dedup_ratio=0.248230 router_tokens_per_expert=[759, 735, 780, 785, 811, 714, 756, 774, 792, 780, 771, 811, 799, 755, 768, 750, 799, 807, 774, 815, 747, 786, 773, 720, 790, 734, 774, 818, 730, 806, 729, 778, 802, 768, 748, 753, 774, 733, 779, 787, 768, 737, 772, 787, 750, 744, 757, 766, 771, 745, 755, 755, 736, 779, 730, 784, 748, 826, 797, 777, 734, 763, 739, 768] payload_rows_per_dst=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] dup_rows_per_dst=[2986, 3132, 3166, 3064, 3080, 2948, 2978, 3048] combine_rows_per_src=[9242, 9320, 9276, 9254, 9208, 9214, 9132, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344152 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266903640 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463128 weight_movement_bytes=34603008 total_nvlink_bytes=568066136 weight_to_token_ratio=0.064865 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344144 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267198544 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534052944 weight_movement_bytes=34603008 total_nvlink_bytes=568655952 weight_to_token_ratio=0.064793 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344128 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267329600 ingress_bytes=405397504 link_bytes=405397504 token_bytes=534315072 weight_movement_bytes=138412032 total_nvlink_bytes=672727104 weight_to_token_ratio=0.259046 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525478170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64974 dup_rows=24166 dup_rows_remote=21244 dedup_ratio=0.245829 router_tokens_per_expert=[750, 760, 790, 783, 779, 746, 772, 798, 771, 786, 745, 767, 784, 788, 787, 759, 717, 766, 760, 777, 750, 778, 755, 771, 781, 753, 771, 748, 797, 723, 780, 752, 711, 749, 744, 806, 749, 771, 749, 764, 766, 786, 780, 767, 753, 791, 748, 768, 762, 772, 782, 783, 789, 776, 790, 790, 798, 747, 800, 760, 746, 782, 780, 749] payload_rows_per_dst=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] dup_rows_per_dst=[3088, 3046, 2948, 2988, 2922, 3014, 3138, 3022] combine_rows_per_src=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343584 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838112 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332640 weight_movement_bytes=69206016 total_nvlink_bytes=600538656 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64810 dup_rows=24110 dup_rows_remote=20948 dedup_ratio=0.245260 router_tokens_per_expert=[779, 787, 739, 780, 743, 748, 771, 710, 745, 722, 769, 804, 740, 772, 809, 780, 787, 758, 789, 783, 753, 810, 800, 802, 765, 758, 797, 751, 763, 760, 760, 760, 803, 788, 751, 768, 764, 775, 745, 766, 775, 789, 775, 793, 753, 742, 765, 713, 827, 721, 775, 722, 723, 786, 755, 766, 799, 787, 795, 788, 748, 748, 756, 797] payload_rows_per_dst=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] dup_rows_per_dst=[3162, 3102, 3310, 2770, 2942, 2826, 2892, 3106] combine_rows_per_src=[9384, 9238, 9718, 9074, 9390, 8958, 9102, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344872 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478376 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611880 weight_movement_bytes=34603008 total_nvlink_bytes=567214888 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65032 dup_rows=24006 dup_rows_remote=20846 dedup_ratio=0.244202 router_tokens_per_expert=[773, 812, 740, 771, 736, 737, 815, 787, 795, 705, 805, 729, 750, 774, 758, 766, 776, 767, 739, 779, 744, 751, 746, 805, 753, 752, 825, 788, 737, 755, 749, 776, 722, 778, 738, 759, 770, 787, 804, 761, 761, 769, 747, 745, 814, 808, 801, 800, 733, 767, 755, 786, 788, 760, 777, 736, 751, 771, 757, 771, 778, 738, 829, 796] payload_rows_per_dst=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] dup_rows_per_dst=[2686, 2890, 2972, 2690, 2940, 3230, 3438, 3160] combine_rows_per_src=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65040 dup_rows=24014 dup_rows_remote=21078 dedup_ratio=0.244283 router_tokens_per_expert=[811, 768, 763, 812, 830, 773, 771, 751, 752, 740, 742, 791, 757, 772, 711, 792, 765, 780, 768, 789, 720, 722, 795, 788, 790, 763, 747, 807, 783, 770, 849, 798, 704, 825, 760, 795, 776, 773, 758, 792, 785, 734, 725, 746, 825, 765, 746, 767, 803, 716, 720, 761, 738, 790, 731, 748, 798, 774, 750, 747, 736, 766, 739, 789] payload_rows_per_dst=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] dup_rows_per_dst=[3100, 2930, 2992, 3196, 2990, 2936, 2940, 2930] combine_rows_per_src=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64766 dup_rows=24218 dup_rows_remote=21112 dedup_ratio=0.246358 router_tokens_per_expert=[774, 763, 774, 762, 779, 782, 747, 799, 786, 762, 769, 818, 791, 778, 724, 770, 780, 766, 791, 762, 804, 796, 761, 753, 754, 783, 774, 767, 773, 807, 779, 722, 792, 775, 726, 756, 723, 824, 778, 777, 766, 778, 741, 792, 789, 728, 734, 746, 738, 778, 762, 821, 768, 750, 757, 769, 763, 745, 796, 748, 782, 711, 706, 783] payload_rows_per_dst=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] dup_rows_per_dst=[3074, 3118, 3106, 2972, 2994, 2962, 3058, 2934] combine_rows_per_src=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21004 dedup_ratio=0.244629 router_tokens_per_expert=[766, 755, 768, 776, 748, 815, 788, 814, 764, 802, 797, 773, 749, 773, 799, 739, 729, 783, 744, 782, 768, 784, 714, 734, 744, 816, 775, 774, 786, 716, 746, 767, 829, 809, 701, 760, 776, 781, 784, 793, 779, 749, 757, 812, 786, 781, 759, 753, 718, 715, 708, 732, 773, 765, 788, 747, 788, 774, 784, 777, 811, 796, 729, 750] payload_rows_per_dst=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] dup_rows_per_dst=[3100, 3044, 2862, 2936, 3128, 3026, 2826, 3126] combine_rows_per_src=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343624 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265731656 ingress_bytes=334594048 link_bytes=334594048 token_bytes=531119688 weight_movement_bytes=69206016 total_nvlink_bytes=600325704 weight_to_token_ratio=0.130302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259050466e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=342672 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265558672 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530774672 weight_movement_bytes=34603008 total_nvlink_bytes=565377680 weight_to_token_ratio=0.065193 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343032 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265804792 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531266552 weight_movement_bytes=34603008 total_nvlink_bytes=565869560 weight_to_token_ratio=0.065133 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20968 dedup_ratio=0.243184 router_tokens_per_expert=[754, 752, 769, 756, 772, 732, 755, 795, 734, 716, 721, 717, 739, 753, 772, 791, 804, 759, 772, 805, 753, 814, 768, 722, 775, 742, 786, 809, 808, 819, 792, 778, 789, 736, 766, 787, 815, 805, 745, 802, 729, 787, 761, 750, 744, 784, 758, 792, 755, 792, 795, 752, 743, 726, 786, 765, 772, 752, 780, 770, 796, 777, 746, 761] payload_rows_per_dst=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] dup_rows_per_dst=[2934, 2798, 3002, 3110, 3178, 2948, 2938, 2998] combine_rows_per_src=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65110 dup_rows=23848 dup_rows_remote=20938 dedup_ratio=0.242594 router_tokens_per_expert=[760, 714, 759, 737, 778, 770, 790, 771, 700, 769, 765, 787, 741, 776, 800, 729, 763, 777, 762, 750, 769, 772, 774, 792, 804, 756, 755, 741, 783, 754, 728, 807, 824, 768, 755, 764, 780, 770, 787, 740, 783, 765, 802, 807, 786, 760, 775, 802, 733, 781, 804, 804, 763, 789, 775, 724, 779, 770, 752, 760, 733, 798, 758, 728] payload_rows_per_dst=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] dup_rows_per_dst=[2876, 2944, 2994, 2910, 3108, 3118, 2966, 2932] combine_rows_per_src=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344320 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267485440 ingress_bytes=336347136 link_bytes=336347136 token_bytes=534626560 weight_movement_bytes=69206016 total_nvlink_bytes=603832576 weight_to_token_ratio=0.129447 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265647195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344776 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265642696 ingress_bytes=299900928 link_bytes=299900928 token_bytes=530940616 weight_movement_bytes=34603008 total_nvlink_bytes=565543624 weight_to_token_ratio=0.065173 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128503048e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21004 dedup_ratio=0.244629 router_tokens_per_expert=[766, 755, 768, 776, 748, 815, 788, 814, 764, 802, 797, 773, 749, 773, 799, 739, 729, 783, 744, 782, 768, 784, 714, 734, 744, 816, 775, 774, 786, 716, 746, 767, 829, 809, 701, 760, 776, 781, 784, 793, 779, 749, 757, 812, 786, 781, 759, 753, 718, 715, 708, 732, 773, 765, 788, 747, 788, 774, 784, 777, 811, 796, 729, 750] payload_rows_per_dst=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] dup_rows_per_dst=[3100, 3044, 2862, 2936, 3128, 3026, 2826, 3126] combine_rows_per_src=[9360, 9348, 9214, 9312, 9338, 9326, 9066, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64818 dup_rows=24060 dup_rows_remote=21078 dedup_ratio=0.244751 router_tokens_per_expert=[736, 776, 763, 799, 757, 747, 760, 775, 748, 808, 807, 764, 776, 769, 745, 812, 740, 732, 821, 741, 815, 757, 786, 763, 807, 787, 771, 802, 746, 732, 825, 734, 747, 760, 761, 725, 791, 700, 792, 820, 777, 719, 692, 783, 759, 788, 759, 809, 750, 762, 766, 784, 710, 785, 759, 765, 764, 774, 793, 746, 793, 751, 781, 786] payload_rows_per_dst=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] dup_rows_per_dst=[3046, 3144, 3054, 2982, 2948, 2926, 2944, 3016] combine_rows_per_src=[9180, 9314, 9256, 9426, 9244, 9246, 9218, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343512 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714584 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085656 weight_movement_bytes=103809024 total_nvlink_bytes=636894680 weight_to_token_ratio=0.194732 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344472 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748312 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533152152 weight_movement_bytes=34603008 total_nvlink_bytes=567755160 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343512 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265625048 ingress_bytes=299884544 link_bytes=299884544 token_bytes=530906584 weight_movement_bytes=34603008 total_nvlink_bytes=565509592 weight_to_token_ratio=0.065177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128441396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64946 dup_rows=24118 dup_rows_remote=21190 dedup_ratio=0.245341 router_tokens_per_expert=[782, 778, 753, 814, 750, 789, 789, 765, 753, 776, 745, 762, 767, 774, 760, 801, 772, 761, 750, 776, 774, 755, 785, 724, 779, 764, 772, 742, 758, 762, 740, 739, 771, 750, 785, 793, 712, 724, 780, 769, 809, 808, 765, 785, 756, 816, 696, 722, 784, 780, 767, 806, 792, 783, 781, 790, 718, 729, 735, 787, 776, 783, 771, 818] payload_rows_per_dst=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] dup_rows_per_dst=[3096, 3080, 2904, 2910, 2928, 3126, 3104, 2970] combine_rows_per_src=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64766 dup_rows=24218 dup_rows_remote=21112 dedup_ratio=0.246358 router_tokens_per_expert=[774, 763, 774, 762, 779, 782, 747, 799, 786, 762, 769, 818, 791, 778, 724, 770, 780, 766, 791, 762, 804, 796, 761, 753, 754, 783, 774, 767, 773, 807, 779, 722, 792, 775, 726, 756, 723, 824, 778, 777, 766, 778, 741, 792, 789, 728, 734, 746, 738, 778, 762, 821, 768, 750, 757, 769, 763, 745, 796, 748, 782, 711, 706, 783] payload_rows_per_dst=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] dup_rows_per_dst=[3074, 3118, 3106, 2972, 2994, 2962, 3058, 2934] combine_rows_per_src=[9286, 9278, 9320, 9346, 9308, 9186, 9228, 9134] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206816 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069984 weight_movement_bytes=34603008 total_nvlink_bytes=566672992 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344304 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026672 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533709040 weight_movement_bytes=34603008 total_nvlink_bytes=568312048 weight_to_token_ratio=0.064835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344192 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267034752 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725312 weight_movement_bytes=138412032 total_nvlink_bytes=672137344 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344544 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266363360 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532382176 weight_movement_bytes=34603008 total_nvlink_bytes=566985184 weight_to_token_ratio=0.064997 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343584 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838112 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332640 weight_movement_bytes=69206016 total_nvlink_bytes=600538656 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65040 dup_rows=24014 dup_rows_remote=21078 dedup_ratio=0.244283 router_tokens_per_expert=[811, 768, 763, 812, 830, 773, 771, 751, 752, 740, 742, 791, 757, 772, 711, 792, 765, 780, 768, 789, 720, 722, 795, 788, 790, 763, 747, 807, 783, 770, 849, 798, 704, 825, 760, 795, 776, 773, 758, 792, 785, 734, 725, 746, 825, 765, 746, 767, 803, 716, 720, 761, 738, 790, 731, 748, 798, 774, 750, 747, 736, 766, 739, 789] payload_rows_per_dst=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] dup_rows_per_dst=[3100, 2930, 2992, 3196, 2990, 2936, 2940, 2930] combine_rows_per_src=[9458, 9184, 9262, 9418, 9376, 9250, 9074, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65032 dup_rows=24006 dup_rows_remote=20846 dedup_ratio=0.244202 router_tokens_per_expert=[773, 812, 740, 771, 736, 737, 815, 787, 795, 705, 805, 729, 750, 774, 758, 766, 776, 767, 739, 779, 744, 751, 746, 805, 753, 752, 825, 788, 737, 755, 749, 776, 722, 778, 738, 759, 770, 787, 804, 761, 761, 769, 747, 745, 814, 808, 801, 800, 733, 767, 755, 786, 788, 760, 777, 736, 751, 771, 757, 771, 778, 738, 829, 796] payload_rows_per_dst=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] dup_rows_per_dst=[2686, 2890, 2972, 2690, 2940, 3230, 3438, 3160] combine_rows_per_src=[9040, 9274, 9242, 8860, 9324, 9576, 9716, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64958 dup_rows=24056 dup_rows_remote=20898 dedup_ratio=0.244710 router_tokens_per_expert=[772, 753, 762, 796, 761, 800, 774, 806, 764, 798, 762, 800, 750, 783, 755, 786, 762, 755, 759, 767, 750, 760, 739, 801, 810, 786, 759, 772, 811, 734, 751, 746, 744, 740, 804, 762, 723, 761, 751, 784, 746, 748, 712, 731, 795, 790, 766, 768, 791, 735, 745, 801, 762, 734, 733, 739, 780, 781, 781, 815, 762, 790, 795, 799] payload_rows_per_dst=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] dup_rows_per_dst=[3158, 3114, 3118, 2982, 2988, 2834, 2940, 2922] combine_rows_per_src=[9290, 9406, 9364, 9356, 9150, 9278, 9140, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64974 dup_rows=24166 dup_rows_remote=21244 dedup_ratio=0.245829 router_tokens_per_expert=[750, 760, 790, 783, 779, 746, 772, 798, 771, 786, 745, 767, 784, 788, 787, 759, 717, 766, 760, 777, 750, 778, 755, 771, 781, 753, 771, 748, 797, 723, 780, 752, 711, 749, 744, 806, 749, 771, 749, 764, 766, 786, 780, 767, 753, 791, 748, 768, 762, 772, 782, 783, 789, 776, 790, 790, 798, 747, 800, 760, 746, 782, 780, 749] payload_rows_per_dst=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] dup_rows_per_dst=[3088, 3046, 2948, 2988, 2922, 3014, 3138, 3022] combine_rows_per_src=[9268, 9328, 9200, 9222, 9164, 9304, 9350, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343512 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265625048 ingress_bytes=299884544 link_bytes=299884544 token_bytes=530906584 weight_movement_bytes=34603008 total_nvlink_bytes=565509592 weight_to_token_ratio=0.065177 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128441396e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65182 dup_rows=23796 dup_rows_remote=20850 dedup_ratio=0.242065 router_tokens_per_expert=[806, 796, 815, 758, 773, 779, 716, 740, 780, 756, 809, 783, 743, 799, 726, 822, 778, 802, 759, 779, 778, 672, 776, 744, 808, 753, 764, 778, 775, 764, 758, 764, 768, 737, 768, 752, 719, 764, 764, 762, 787, 787, 725, 756, 794, 786, 768, 803, 779, 749, 762, 729, 743, 794, 814, 766, 793, 788, 732, 787, 767, 771, 730, 755] payload_rows_per_dst=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] dup_rows_per_dst=[2976, 3094, 2982, 2964, 2838, 3022, 2946, 2974] combine_rows_per_src=[9390, 9342, 9194, 9364, 9230, 9390, 9326, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=64664 dup_rows=23850 dup_rows_remote=20458 dedup_ratio=0.242615 router_tokens_per_expert=[755, 736, 770, 806, 778, 700, 747, 811, 784, 748, 822, 784, 707, 758, 753, 763, 756, 823, 780, 783, 760, 733, 738, 740, 773, 748, 799, 796, 778, 719, 799, 794, 771, 757, 732, 744, 790, 745, 746, 754, 819, 790, 821, 786, 771, 746, 765, 744, 795, 756, 730, 781, 792, 782, 792, 751, 751, 776, 754, 777, 784, 780, 761, 768] payload_rows_per_dst=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] dup_rows_per_dst=[2904, 2982, 2986, 2996, 2870, 2678, 3042, 3392] combine_rows_per_src=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65210 dup_rows=23920 dup_rows_remote=20886 dedup_ratio=0.243327 router_tokens_per_expert=[813, 793, 777, 756, 750, 776, 765, 814, 730, 723, 764, 769, 775, 773, 728, 736, 762, 774, 770, 750, 811, 778, 779, 769, 773, 774, 770, 767, 775, 806, 764, 759, 786, 759, 785, 767, 738, 787, 756, 755, 795, 770, 736, 764, 767, 757, 797, 718, 813, 802, 840, 752, 750, 733, 731, 829, 763, 763, 742, 734, 774, 785, 713, 768] payload_rows_per_dst=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] dup_rows_per_dst=[3064, 2856, 3010, 3016, 2916, 3034, 3090, 2934] combine_rows_per_src=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65104 dup_rows=23920 dup_rows_remote=20986 dedup_ratio=0.243327 router_tokens_per_expert=[748, 781, 775, 730, 787, 732, 750, 757, 759, 767, 798, 806, 751, 796, 759, 771, 728, 755, 782, 819, 766, 773, 748, 736, 764, 762, 755, 758, 835, 767, 758, 763, 794, 760, 732, 691, 829, 826, 798, 783, 812, 788, 777, 769, 766, 806, 733, 805, 748, 794, 771, 721, 799, 712, 774, 763, 768, 717, 780, 794, 746, 789, 729, 742] payload_rows_per_dst=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] dup_rows_per_dst=[2988, 3074, 2934, 3066, 3070, 3058, 2918, 2812] combine_rows_per_src=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344472 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266748312 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533152152 weight_movement_bytes=34603008 total_nvlink_bytes=567755160 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343512 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714584 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085656 weight_movement_bytes=103809024 total_nvlink_bytes=636894680 weight_to_token_ratio=0.194732 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343424 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266411392 ingress_bytes=404480000 link_bytes=404480000 token_bytes=532479360 weight_movement_bytes=138412032 total_nvlink_bytes=670891392 weight_to_token_ratio=0.259939 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522025677e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344872 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478376 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611880 weight_movement_bytes=34603008 total_nvlink_bytes=567214888 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344128 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267329600 ingress_bytes=405397504 link_bytes=405397504 token_bytes=534315072 weight_movement_bytes=138412032 total_nvlink_bytes=672727104 weight_to_token_ratio=0.259046 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525478170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64828 dup_rows=24272 dup_rows_remote=21378 dedup_ratio=0.246908 router_tokens_per_expert=[751, 794, 792, 779, 782, 800, 751, 721, 787, 730, 746, 726, 773, 745, 768, 774, 798, 821, 763, 755, 747, 794, 745, 728, 808, 766, 814, 804, 862, 793, 792, 773, 745, 776, 747, 720, 787, 760, 778, 783, 785, 772, 748, 743, 790, 767, 731, 754, 808, 734, 775, 732, 761, 804, 773, 782, 763, 757, 774, 790, 730, 752, 723, 726] payload_rows_per_dst=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] dup_rows_per_dst=[3084, 2894, 3044, 3300, 3048, 3014, 3072, 2816] combine_rows_per_src=[9256, 9204, 9258, 9524, 9144, 9166, 9266, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65082 dup_rows=23894 dup_rows_remote=21030 dedup_ratio=0.243062 router_tokens_per_expert=[759, 771, 743, 757, 749, 769, 772, 764, 766, 781, 768, 753, 768, 771, 746, 771, 800, 771, 731, 720, 767, 748, 733, 753, 770, 763, 730, 791, 786, 810, 782, 774, 771, 780, 784, 776, 806, 782, 743, 756, 817, 747, 768, 733, 755, 724, 817, 764, 762, 763, 736, 777, 767, 788, 743, 760, 764, 777, 728, 841, 789, 791, 786, 820] payload_rows_per_dst=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] dup_rows_per_dst=[2972, 3008, 2924, 2968, 3034, 2954, 2864, 3170] combine_rows_per_src=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65210 dup_rows=23870 dup_rows_remote=20974 dedup_ratio=0.242818 router_tokens_per_expert=[749, 775, 792, 753, 791, 792, 788, 765, 766, 789, 789, 765, 811, 788, 784, 773, 760, 752, 806, 782, 773, 749, 762, 731, 775, 757, 745, 782, 757, 753, 751, 740, 779, 747, 764, 769, 710, 794, 721, 772, 766, 769, 798, 769, 749, 760, 732, 729, 788, 741, 733, 804, 790, 812, 768, 803, 768, 755, 764, 797, 755, 768, 745, 788] payload_rows_per_dst=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] dup_rows_per_dst=[3044, 3094, 2972, 2896, 2980, 2812, 3036, 3036] combine_rows_per_src=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264863744 dispatch_meta_bytes=340488 combine_payload_bytes=264863744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265204232 ingress_bytes=299466752 link_bytes=299466752 token_bytes=530067976 weight_movement_bytes=34603008 total_nvlink_bytes=564670984 weight_to_token_ratio=0.065280 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126869279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344384 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444544 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544704 weight_movement_bytes=69206016 total_nvlink_bytes=603750720 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65110 dup_rows=23848 dup_rows_remote=20938 dedup_ratio=0.242594 router_tokens_per_expert=[760, 714, 759, 737, 778, 770, 790, 771, 700, 769, 765, 787, 741, 776, 800, 729, 763, 777, 762, 750, 769, 772, 774, 792, 804, 756, 755, 741, 783, 754, 728, 807, 824, 768, 755, 764, 780, 770, 787, 740, 783, 765, 802, 807, 786, 760, 775, 802, 733, 781, 804, 804, 763, 789, 775, 724, 779, 770, 752, 760, 733, 798, 758, 728] payload_rows_per_dst=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] dup_rows_per_dst=[2876, 2944, 2994, 2910, 3108, 3118, 2966, 2932] combine_rows_per_src=[9282, 9190, 9324, 9346, 9268, 9442, 9380, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=65014 dup_rows=24038 dup_rows_remote=20862 dedup_ratio=0.244527 router_tokens_per_expert=[746, 800, 779, 759, 745, 790, 764, 744, 785, 780, 737, 788, 742, 770, 756, 782, 774, 785, 782, 768, 773, 756, 770, 797, 788, 728, 717, 747, 707, 774, 743, 785, 724, 785, 805, 776, 771, 762, 786, 805, 729, 781, 716, 750, 791, 792, 820, 743, 732, 783, 721, 762, 822, 772, 832, 805, 766, 772, 774, 756, 770, 735, 783, 770] payload_rows_per_dst=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] dup_rows_per_dst=[2922, 2994, 3054, 2818, 3176, 2908, 3076, 3090] combine_rows_per_src=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344360 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010344 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676328 weight_movement_bytes=34603008 total_nvlink_bytes=568279336 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65104 dup_rows=23920 dup_rows_remote=20986 dedup_ratio=0.243327 router_tokens_per_expert=[748, 781, 775, 730, 787, 732, 750, 757, 759, 767, 798, 806, 751, 796, 759, 771, 728, 755, 782, 819, 766, 773, 748, 736, 764, 762, 755, 758, 835, 767, 758, 763, 794, 760, 732, 691, 829, 826, 798, 783, 812, 788, 777, 769, 766, 806, 733, 805, 748, 794, 771, 721, 799, 712, 774, 763, 768, 717, 780, 794, 746, 789, 729, 742] payload_rows_per_dst=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] dup_rows_per_dst=[2988, 3074, 2934, 3066, 3070, 3058, 2918, 2812] combine_rows_per_src=[9132, 9340, 9280, 9258, 9356, 9454, 9246, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344824 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880312 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415800 weight_movement_bytes=34603008 total_nvlink_bytes=566018808 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344448 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920320 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496192 weight_movement_bytes=69206016 total_nvlink_bytes=602702208 weight_to_token_ratio=0.129722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344736 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267444896 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534545056 weight_movement_bytes=34603008 total_nvlink_bytes=569148064 weight_to_token_ratio=0.064734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343504 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266640848 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938192 weight_movement_bytes=34603008 total_nvlink_bytes=567541200 weight_to_token_ratio=0.064929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344192 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267034752 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725312 weight_movement_bytes=138412032 total_nvlink_bytes=672137344 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65210 dup_rows=23920 dup_rows_remote=20886 dedup_ratio=0.243327 router_tokens_per_expert=[813, 793, 777, 756, 750, 776, 765, 814, 730, 723, 764, 769, 775, 773, 728, 736, 762, 774, 770, 750, 811, 778, 779, 769, 773, 774, 770, 767, 775, 806, 764, 759, 786, 759, 785, 767, 738, 787, 756, 755, 795, 770, 736, 764, 767, 757, 797, 718, 813, 802, 840, 752, 750, 733, 731, 829, 763, 763, 742, 734, 774, 785, 713, 768] payload_rows_per_dst=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] dup_rows_per_dst=[3064, 2856, 3010, 3016, 2916, 3034, 3090, 2934] combine_rows_per_src=[9424, 9140, 9376, 9360, 9350, 9174, 9410, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=64664 dup_rows=23850 dup_rows_remote=20458 dedup_ratio=0.242615 router_tokens_per_expert=[755, 736, 770, 806, 778, 700, 747, 811, 784, 748, 822, 784, 707, 758, 753, 763, 756, 823, 780, 783, 760, 733, 738, 740, 773, 748, 799, 796, 778, 719, 799, 794, 771, 757, 732, 744, 790, 745, 746, 754, 819, 790, 821, 786, 771, 746, 765, 744, 795, 756, 730, 781, 792, 782, 792, 751, 751, 776, 754, 777, 784, 780, 761, 768] payload_rows_per_dst=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] dup_rows_per_dst=[2904, 2982, 2986, 2996, 2870, 2678, 3042, 3392] combine_rows_per_src=[9278, 9256, 9240, 9194, 9338, 9026, 9332, 9790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344360 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010344 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676328 weight_movement_bytes=34603008 total_nvlink_bytes=568279336 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64950 dup_rows=23990 dup_rows_remote=20946 dedup_ratio=0.244039 router_tokens_per_expert=[781, 749, 792, 775, 786, 732, 793, 796, 782, 763, 773, 834, 731, 771, 739, 748, 775, 785, 781, 782, 743, 732, 769, 789, 761, 787, 710, 800, 783, 710, 746, 761, 813, 719, 787, 763, 747, 730, 770, 807, 753, 788, 808, 763, 785, 786, 829, 776, 795, 793, 750, 790, 800, 735, 790, 768, 700, 709, 762, 720, 759, 762, 798, 738] payload_rows_per_dst=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] dup_rows_per_dst=[3044, 3212, 3200, 2830, 2958, 3092, 2814, 2840] combine_rows_per_src=[9364, 9688, 9418, 8980, 9314, 9484, 9010, 9056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64946 dup_rows=24118 dup_rows_remote=21190 dedup_ratio=0.245341 router_tokens_per_expert=[782, 778, 753, 814, 750, 789, 789, 765, 753, 776, 745, 762, 767, 774, 760, 801, 772, 761, 750, 776, 774, 755, 785, 724, 779, 764, 772, 742, 758, 762, 740, 739, 771, 750, 785, 793, 712, 724, 780, 769, 809, 808, 765, 785, 756, 816, 696, 722, 784, 780, 767, 806, 792, 783, 781, 790, 718, 729, 735, 787, 776, 783, 771, 818] payload_rows_per_dst=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] dup_rows_per_dst=[3096, 3080, 2904, 2910, 2928, 3126, 3104, 2970] combine_rows_per_src=[9344, 9196, 9290, 9202, 9240, 9188, 9462, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65108 dup_rows=23906 dup_rows_remote=20968 dedup_ratio=0.243184 router_tokens_per_expert=[754, 752, 769, 756, 772, 732, 755, 795, 734, 716, 721, 717, 739, 753, 772, 791, 804, 759, 772, 805, 753, 814, 768, 722, 775, 742, 786, 809, 808, 819, 792, 778, 789, 736, 766, 787, 815, 805, 745, 802, 729, 787, 761, 750, 744, 784, 758, 792, 755, 792, 795, 752, 743, 726, 786, 765, 772, 752, 780, 770, 796, 777, 746, 761] payload_rows_per_dst=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] dup_rows_per_dst=[2934, 2798, 3002, 3110, 3178, 2948, 2938, 2998] combine_rows_per_src=[9236, 9088, 9392, 9508, 9312, 9262, 9290, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344384 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444544 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544704 weight_movement_bytes=69206016 total_nvlink_bytes=603750720 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=64952 dup_rows=23976 dup_rows_remote=20970 dedup_ratio=0.243896 router_tokens_per_expert=[826, 676, 763, 799, 759, 757, 784, 754, 764, 754, 787, 776, 747, 751, 782, 768, 744, 751, 733, 793, 746, 800, 782, 731, 747, 760, 734, 737, 781, 778, 764, 767, 754, 823, 740, 740, 779, 780, 785, 812, 799, 713, 804, 759, 809, 762, 787, 799, 762, 763, 773, 757, 759, 798, 773, 737, 750, 813, 754, 795, 786, 790, 749, 753] payload_rows_per_dst=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] dup_rows_per_dst=[3006, 2788, 2994, 2914, 3230, 3028, 3010, 3006] combine_rows_per_src=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65014 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[777, 751, 767, 743, 717, 747, 727, 758, 740, 756, 743, 774, 818, 780, 743, 771, 757, 777, 798, 741, 787, 754, 740, 741, 766, 779, 766, 761, 744, 780, 736, 807, 790, 741, 763, 780, 763, 757, 756, 812, 802, 783, 748, 778, 751, 780, 788, 740, 793, 814, 775, 743, 799, 784, 798, 771, 738, 778, 828, 721, 763, 788, 780, 801] payload_rows_per_dst=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] dup_rows_per_dst=[2852, 3030, 2972, 2912, 2980, 2994, 3192, 3012] combine_rows_per_src=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264863744 dispatch_meta_bytes=340488 combine_payload_bytes=264863744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265204232 ingress_bytes=299466752 link_bytes=299466752 token_bytes=530067976 weight_movement_bytes=34603008 total_nvlink_bytes=564670984 weight_to_token_ratio=0.065280 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126869279e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343584 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378784 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413984 weight_movement_bytes=69206016 total_nvlink_bytes=601620000 weight_to_token_ratio=0.129985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65312 dup_rows=23880 dup_rows_remote=20934 dedup_ratio=0.242920 router_tokens_per_expert=[779, 752, 756, 754, 745, 784, 766, 762, 815, 807, 790, 743, 795, 743, 777, 807, 734, 755, 786, 733, 747, 752, 768, 754, 776, 750, 768, 753, 746, 790, 802, 820, 764, 752, 761, 766, 734, 812, 757, 770, 787, 767, 743, 714, 781, 744, 751, 726, 775, 750, 794, 807, 786, 796, 800, 774, 763, 767, 745, 803, 774, 736, 794, 750] payload_rows_per_dst=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] dup_rows_per_dst=[2884, 3092, 2946, 2998, 2998, 2864, 3098, 3000] combine_rows_per_src=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344544 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266363360 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532382176 weight_movement_bytes=34603008 total_nvlink_bytes=566985184 weight_to_token_ratio=0.064997 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344304 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026672 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533709040 weight_movement_bytes=34603008 total_nvlink_bytes=568312048 weight_to_token_ratio=0.064835 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64934 dup_rows=24102 dup_rows_remote=21108 dedup_ratio=0.245178 router_tokens_per_expert=[716, 738, 778, 796, 771, 807, 803, 815, 775, 757, 791, 772, 760, 737, 813, 726, 749, 776, 774, 786, 764, 770, 791, 781, 766, 784, 768, 802, 761, 777, 761, 743, 759, 749, 733, 772, 730, 786, 738, 741, 755, 760, 799, 754, 732, 764, 793, 793, 830, 794, 752, 778, 753, 708, 737, 785, 764, 784, 762, 765, 802, 742, 755, 775] payload_rows_per_dst=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] dup_rows_per_dst=[3102, 2994, 3070, 3012, 2920, 2954, 3010, 3040] combine_rows_per_src=[9346, 9268, 9312, 9312, 9096, 9346, 9264, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65074 dup_rows=23876 dup_rows_remote=20900 dedup_ratio=0.242879 router_tokens_per_expert=[779, 738, 736, 770, 741, 790, 715, 769, 782, 791, 768, 746, 739, 799, 761, 744, 796, 780, 800, 780, 780, 731, 783, 810, 759, 767, 747, 807, 757, 774, 799, 795, 753, 776, 802, 726, 774, 761, 720, 798, 759, 739, 771, 744, 701, 790, 775, 713, 766, 794, 746, 774, 752, 757, 789, 787, 813, 793, 787, 728, 760, 756, 824, 791] payload_rows_per_dst=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] dup_rows_per_dst=[2950, 2980, 2988, 3018, 2962, 2882, 2976, 3120] combine_rows_per_src=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65214 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[742, 785, 763, 789, 724, 741, 761, 750, 809, 793, 769, 777, 734, 781, 776, 763, 743, 800, 752, 787, 788, 782, 750, 832, 774, 794, 782, 754, 723, 800, 764, 716, 784, 784, 758, 785, 765, 743, 761, 753, 779, 741, 700, 747, 728, 724, 745, 797, 781, 777, 816, 762, 762, 782, 763, 781, 827, 808, 779, 762, 755, 770, 793, 742] payload_rows_per_dst=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] dup_rows_per_dst=[2888, 3084, 3018, 2944, 3050, 2736, 3048, 3052] combine_rows_per_src=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343688 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266387080 ingress_bytes=369852416 link_bytes=369852416 token_bytes=532430472 weight_movement_bytes=103809024 total_nvlink_bytes=636239496 weight_to_token_ratio=0.194972 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391724866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64992 dup_rows=23998 dup_rows_remote=20908 dedup_ratio=0.244120 router_tokens_per_expert=[714, 778, 716, 762, 757, 789, 774, 781, 749, 715, 713, 756, 778, 780, 812, 766, 749, 787, 765, 739, 814, 737, 786, 808, 735, 826, 772, 781, 768, 778, 787, 781, 794, 749, 774, 784, 789, 774, 783, 755, 773, 753, 783, 767, 774, 755, 763, 748, 746, 781, 754, 772, 748, 761, 754, 756, 813, 809, 754, 811, 767, 740, 773, 742] payload_rows_per_dst=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] dup_rows_per_dst=[2938, 2898, 3106, 3018, 3090, 2976, 2878, 3094] combine_rows_per_src=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65210 dup_rows=23870 dup_rows_remote=20974 dedup_ratio=0.242818 router_tokens_per_expert=[749, 775, 792, 753, 791, 792, 788, 765, 766, 789, 789, 765, 811, 788, 784, 773, 760, 752, 806, 782, 773, 749, 762, 731, 775, 757, 745, 782, 757, 753, 751, 740, 779, 747, 764, 769, 710, 794, 721, 772, 766, 769, 798, 769, 749, 760, 732, 729, 788, 741, 733, 804, 790, 812, 768, 803, 768, 755, 764, 797, 755, 768, 745, 788] payload_rows_per_dst=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] dup_rows_per_dst=[3044, 3094, 2972, 2896, 2980, 2812, 3036, 3036] combine_rows_per_src=[9366, 9436, 9258, 9224, 9132, 9332, 9442, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65312 dup_rows=23880 dup_rows_remote=20934 dedup_ratio=0.242920 router_tokens_per_expert=[779, 752, 756, 754, 745, 784, 766, 762, 815, 807, 790, 743, 795, 743, 777, 807, 734, 755, 786, 733, 747, 752, 768, 754, 776, 750, 768, 753, 746, 790, 802, 820, 764, 752, 761, 766, 734, 812, 757, 770, 787, 767, 743, 714, 781, 744, 751, 726, 775, 750, 794, 807, 786, 796, 800, 774, 763, 767, 745, 803, 774, 736, 794, 750] payload_rows_per_dst=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] dup_rows_per_dst=[2884, 3092, 2946, 2998, 2998, 2864, 3098, 3000] combine_rows_per_src=[9312, 9462, 9112, 9412, 9234, 9162, 9466, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343856 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641200 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938544 weight_movement_bytes=34603008 total_nvlink_bytes=567541552 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267517952 dispatch_meta_bytes=344984 combine_payload_bytes=267517952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267862936 ingress_bytes=302120960 link_bytes=302120960 token_bytes=535380888 weight_movement_bytes=34603008 total_nvlink_bytes=569983896 weight_to_token_ratio=0.064633 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136856850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65014 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[777, 751, 767, 743, 717, 747, 727, 758, 740, 756, 743, 774, 818, 780, 743, 771, 757, 777, 798, 741, 787, 754, 740, 741, 766, 779, 766, 761, 744, 780, 736, 807, 790, 741, 763, 780, 763, 757, 756, 812, 802, 783, 748, 778, 751, 780, 788, 740, 793, 814, 775, 743, 799, 784, 798, 771, 738, 778, 828, 721, 763, 788, 780, 801] payload_rows_per_dst=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] dup_rows_per_dst=[2852, 3030, 2972, 2912, 2980, 2994, 3192, 3012] combine_rows_per_src=[9122, 9220, 9218, 9366, 9344, 9346, 9362, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344168 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313832 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283496 weight_movement_bytes=103809024 total_nvlink_bytes=636092520 weight_to_token_ratio=0.195026 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343896 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266887000 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430104 weight_movement_bytes=103809024 total_nvlink_bytes=637239128 weight_to_token_ratio=0.194607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344360 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460904 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577448 weight_movement_bytes=34603008 total_nvlink_bytes=569180456 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343600 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550832 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758064 weight_movement_bytes=69206016 total_nvlink_bytes=601964080 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344736 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267444896 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534545056 weight_movement_bytes=34603008 total_nvlink_bytes=569148064 weight_to_token_ratio=0.064734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=64952 dup_rows=23976 dup_rows_remote=20970 dedup_ratio=0.243896 router_tokens_per_expert=[826, 676, 763, 799, 759, 757, 784, 754, 764, 754, 787, 776, 747, 751, 782, 768, 744, 751, 733, 793, 746, 800, 782, 731, 747, 760, 734, 737, 781, 778, 764, 767, 754, 823, 740, 740, 779, 780, 785, 812, 799, 713, 804, 759, 809, 762, 787, 799, 762, 763, 773, 757, 759, 798, 773, 737, 750, 813, 754, 795, 786, 790, 749, 753] payload_rows_per_dst=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] dup_rows_per_dst=[3006, 2788, 2994, 2914, 3230, 3028, 3010, 3006] combine_rows_per_src=[9230, 9148, 9268, 9222, 9416, 9436, 9232, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267517952 dispatch_meta_bytes=344984 combine_payload_bytes=267517952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267862936 ingress_bytes=302120960 link_bytes=302120960 token_bytes=535380888 weight_movement_bytes=34603008 total_nvlink_bytes=569983896 weight_to_token_ratio=0.064633 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136856850e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=64974 dup_rows=23762 dup_rows_remote=20620 dedup_ratio=0.241720 router_tokens_per_expert=[748, 787, 781, 755, 755, 790, 795, 759, 748, 785, 794, 752, 747, 761, 726, 786, 772, 755, 787, 774, 785, 735, 764, 754, 733, 745, 784, 780, 813, 732, 747, 816, 774, 757, 802, 772, 756, 779, 775, 748, 783, 800, 756, 763, 783, 761, 739, 782, 739, 765, 747, 781, 751, 822, 699, 806, 764, 767, 775, 770, 780, 751, 805, 755] payload_rows_per_dst=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] dup_rows_per_dst=[3142, 2890, 2924, 3136, 3028, 3208, 2568, 2866] combine_rows_per_src=[9568, 9342, 9328, 9512, 9298, 9594, 8836, 9064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=65014 dup_rows=24038 dup_rows_remote=20862 dedup_ratio=0.244527 router_tokens_per_expert=[746, 800, 779, 759, 745, 790, 764, 744, 785, 780, 737, 788, 742, 770, 756, 782, 774, 785, 782, 768, 773, 756, 770, 797, 788, 728, 717, 747, 707, 774, 743, 785, 724, 785, 805, 776, 771, 762, 786, 805, 729, 781, 716, 750, 791, 792, 820, 743, 732, 783, 721, 762, 822, 772, 832, 805, 766, 772, 774, 756, 770, 735, 783, 770] payload_rows_per_dst=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] dup_rows_per_dst=[2922, 2994, 3054, 2818, 3176, 2908, 3076, 3090] combine_rows_per_src=[9332, 9286, 9356, 9160, 9252, 9336, 9382, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65082 dup_rows=23894 dup_rows_remote=21030 dedup_ratio=0.243062 router_tokens_per_expert=[759, 771, 743, 757, 749, 769, 772, 764, 766, 781, 768, 753, 768, 771, 746, 771, 800, 771, 731, 720, 767, 748, 733, 753, 770, 763, 730, 791, 786, 810, 782, 774, 771, 780, 784, 776, 806, 782, 743, 756, 817, 747, 768, 733, 755, 724, 817, 764, 762, 763, 736, 777, 767, 788, 743, 760, 764, 777, 728, 841, 789, 791, 786, 820] payload_rows_per_dst=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] dup_rows_per_dst=[2972, 3008, 2924, 2968, 3034, 2954, 2864, 3170] combine_rows_per_src=[9196, 9240, 9122, 9444, 9362, 9296, 9328, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343856 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641200 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938544 weight_movement_bytes=34603008 total_nvlink_bytes=567541552 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64800 dup_rows=24132 dup_rows_remote=21100 dedup_ratio=0.245483 router_tokens_per_expert=[783, 792, 754, 769, 748, 733, 791, 758, 820, 769, 719, 743, 768, 786, 810, 755, 790, 786, 812, 755, 769, 772, 798, 734, 789, 754, 756, 753, 783, 688, 775, 747, 756, 720, 743, 772, 758, 811, 737, 806, 738, 784, 732, 752, 780, 721, 773, 806, 788, 758, 747, 780, 783, 778, 786, 785, 751, 786, 716, 806, 783, 785, 777, 795] payload_rows_per_dst=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] dup_rows_per_dst=[2972, 2576, 3054, 2912, 2962, 3466, 3158, 3032] combine_rows_per_src=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343688 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266387080 ingress_bytes=369852416 link_bytes=369852416 token_bytes=532430472 weight_movement_bytes=103809024 total_nvlink_bytes=636239496 weight_to_token_ratio=0.194972 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391724866e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=342376 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266475880 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532609384 weight_movement_bytes=34603008 total_nvlink_bytes=567212392 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65328 dup_rows=23830 dup_rows_remote=21096 dedup_ratio=0.242411 router_tokens_per_expert=[784, 764, 786, 784, 750, 776, 765, 805, 743, 815, 823, 761, 809, 746, 753, 805, 759, 753, 801, 713, 753, 790, 755, 746, 767, 730, 778, 770, 774, 785, 788, 821, 781, 765, 755, 757, 784, 769, 817, 740, 756, 729, 765, 743, 731, 744, 711, 761, 755, 745, 765, 736, 785, 796, 728, 740, 756, 804, 734, 766, 820, 788, 803, 771] payload_rows_per_dst=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] dup_rows_per_dst=[3070, 3114, 2906, 3044, 3016, 2734, 2936, 3010] combine_rows_per_src=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64940 dup_rows=24096 dup_rows_remote=21042 dedup_ratio=0.245117 router_tokens_per_expert=[790, 754, 779, 738, 775, 746, 744, 755, 853, 720, 774, 751, 763, 694, 802, 771, 747, 789, 763, 755, 759, 776, 802, 770, 746, 781, 765, 807, 791, 805, 721, 756, 739, 775, 722, 814, 782, 722, 767, 821, 776, 752, 786, 759, 763, 805, 776, 801, 748, 796, 773, 751, 771, 785, 740, 735, 796, 784, 729, 759, 760, 782, 734, 807] payload_rows_per_dst=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] dup_rows_per_dst=[3042, 2964, 3054, 3028, 2996, 3134, 2928, 2950] combine_rows_per_src=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343504 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266640848 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938192 weight_movement_bytes=34603008 total_nvlink_bytes=567541200 weight_to_token_ratio=0.064929 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344448 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920320 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496192 weight_movement_bytes=69206016 total_nvlink_bytes=602702208 weight_to_token_ratio=0.129722 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65052 dup_rows=23926 dup_rows_remote=20838 dedup_ratio=0.243388 router_tokens_per_expert=[824, 752, 754, 768, 754, 721, 784, 784, 782, 749, 796, 771, 792, 802, 779, 736, 735, 816, 816, 769, 739, 736, 730, 773, 768, 802, 754, 737, 800, 762, 752, 760, 785, 778, 800, 780, 826, 738, 758, 759, 749, 749, 748, 787, 800, 745, 761, 765, 747, 801, 755, 744, 806, 796, 763, 733, 769, 764, 782, 771, 741, 759, 749, 747] payload_rows_per_dst=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] dup_rows_per_dst=[3002, 3088, 3000, 2940, 3130, 2864, 2962, 2940] combine_rows_per_src=[9280, 9326, 9228, 9330, 9318, 9344, 9328, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64978 dup_rows=23954 dup_rows_remote=20946 dedup_ratio=0.243673 router_tokens_per_expert=[788, 785, 745, 769, 798, 774, 758, 709, 770, 758, 779, 749, 777, 783, 815, 798, 738, 787, 736, 805, 759, 741, 764, 796, 689, 774, 777, 737, 761, 778, 774, 778, 738, 753, 746, 777, 793, 768, 750, 777, 756, 764, 776, 807, 784, 702, 754, 774, 810, 761, 829, 791, 746, 740, 757, 756, 736, 807, 768, 798, 779, 741, 747, 818] payload_rows_per_dst=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] dup_rows_per_dst=[2976, 3126, 2926, 2948, 2952, 3012, 3008, 3006] combine_rows_per_src=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64924 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[753, 803, 789, 719, 765, 770, 774, 829, 794, 748, 741, 768, 771, 757, 802, 754, 730, 755, 757, 780, 769, 727, 735, 790, 770, 784, 777, 735, 713, 785, 809, 771, 783, 781, 766, 806, 745, 806, 759, 753, 782, 780, 793, 756, 719, 771, 744, 781, 752, 773, 760, 808, 782, 773, 755, 725, 785, 787, 784, 754, 773, 752, 756, 784] payload_rows_per_dst=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] dup_rows_per_dst=[3100, 3014, 2964, 3046, 3114, 2920, 2940, 3040] combine_rows_per_src=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21118 dedup_ratio=0.244995 router_tokens_per_expert=[778, 741, 814, 736, 768, 746, 793, 787, 771, 738, 789, 770, 758, 748, 760, 766, 733, 793, 803, 809, 770, 757, 802, 748, 770, 813, 755, 766, 788, 775, 806, 758, 791, 796, 810, 728, 750, 764, 732, 743, 744, 752, 762, 804, 763, 769, 763, 747, 801, 766, 771, 799, 710, 811, 726, 745, 782, 738, 750, 795, 757, 742, 723, 809] payload_rows_per_dst=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] dup_rows_per_dst=[3088, 3008, 3050, 3108, 2966, 2944, 2954, 2966] combine_rows_per_src=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65214 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[742, 785, 763, 789, 724, 741, 761, 750, 809, 793, 769, 777, 734, 781, 776, 763, 743, 800, 752, 787, 788, 782, 750, 832, 774, 794, 782, 754, 723, 800, 764, 716, 784, 784, 758, 785, 765, 743, 761, 753, 779, 741, 700, 747, 728, 724, 745, 797, 781, 777, 816, 762, 762, 782, 763, 781, 827, 808, 779, 762, 755, 770, 793, 742] payload_rows_per_dst=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] dup_rows_per_dst=[2888, 3084, 3018, 2944, 3050, 2736, 3048, 3052] combine_rows_per_src=[9222, 9320, 9450, 9270, 9216, 9186, 9400, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64940 dup_rows=24096 dup_rows_remote=21042 dedup_ratio=0.245117 router_tokens_per_expert=[790, 754, 779, 738, 775, 746, 744, 755, 853, 720, 774, 751, 763, 694, 802, 771, 747, 789, 763, 755, 759, 776, 802, 770, 746, 781, 765, 807, 791, 805, 721, 756, 739, 775, 722, 814, 782, 722, 767, 821, 776, 752, 786, 759, 763, 805, 776, 801, 748, 796, 773, 751, 771, 785, 740, 735, 796, 784, 729, 759, 760, 782, 734, 807] payload_rows_per_dst=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] dup_rows_per_dst=[3042, 2964, 3054, 3028, 2996, 3134, 2928, 2950] combine_rows_per_src=[9120, 9292, 9268, 9316, 9288, 9302, 9270, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343600 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764400 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185200 weight_movement_bytes=103809024 total_nvlink_bytes=634994224 weight_to_token_ratio=0.195429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=345696 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267929184 ingress_bytes=302186496 link_bytes=302186496 token_bytes=535512672 weight_movement_bytes=34603008 total_nvlink_bytes=570115680 weight_to_token_ratio=0.064617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137103456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343928 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338168 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332408 weight_movement_bytes=69206016 total_nvlink_bytes=601538424 weight_to_token_ratio=0.130005 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65328 dup_rows=23830 dup_rows_remote=21096 dedup_ratio=0.242411 router_tokens_per_expert=[784, 764, 786, 784, 750, 776, 765, 805, 743, 815, 823, 761, 809, 746, 753, 805, 759, 753, 801, 713, 753, 790, 755, 746, 767, 730, 778, 770, 774, 785, 788, 821, 781, 765, 755, 757, 784, 769, 817, 740, 756, 729, 765, 743, 731, 744, 711, 761, 755, 745, 765, 736, 785, 796, 728, 740, 756, 804, 734, 766, 820, 788, 803, 771] payload_rows_per_dst=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] dup_rows_per_dst=[3070, 3114, 2906, 3044, 3016, 2734, 2936, 3010] combine_rows_per_src=[9358, 9396, 9234, 9382, 9320, 9146, 9164, 9474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64800 dup_rows=24132 dup_rows_remote=21100 dedup_ratio=0.245483 router_tokens_per_expert=[783, 792, 754, 769, 748, 733, 791, 758, 820, 769, 719, 743, 768, 786, 810, 755, 790, 786, 812, 755, 769, 772, 798, 734, 789, 754, 756, 753, 783, 688, 775, 747, 756, 720, 743, 772, 758, 811, 737, 806, 738, 784, 732, 752, 780, 721, 773, 806, 788, 758, 747, 780, 783, 778, 786, 785, 751, 786, 716, 806, 783, 785, 777, 795] payload_rows_per_dst=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] dup_rows_per_dst=[2972, 2576, 3054, 2912, 2962, 3466, 3158, 3032] combine_rows_per_src=[9284, 8540, 9378, 9178, 9244, 9930, 9246, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343560 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796552 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249544 weight_movement_bytes=69206016 total_nvlink_bytes=602455560 weight_to_token_ratio=0.129782 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344064 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272768 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201472 weight_movement_bytes=69206016 total_nvlink_bytes=601407488 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343696 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266493584 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643472 weight_movement_bytes=69206016 total_nvlink_bytes=601849488 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344304 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266412272 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480240 weight_movement_bytes=69206016 total_nvlink_bytes=601686256 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344360 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267460904 ingress_bytes=301719552 link_bytes=301719552 token_bytes=534577448 weight_movement_bytes=34603008 total_nvlink_bytes=569180456 weight_to_token_ratio=0.064730 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135346384e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343928 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338168 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332408 weight_movement_bytes=69206016 total_nvlink_bytes=601538424 weight_to_token_ratio=0.130005 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64842 dup_rows=24008 dup_rows_remote=20790 dedup_ratio=0.244222 router_tokens_per_expert=[795, 783, 749, 787, 793, 757, 736, 769, 750, 783, 778, 778, 772, 756, 805, 812, 812, 772, 790, 781, 752, 769, 756, 744, 752, 743, 714, 784, 769, 765, 772, 809, 785, 726, 762, 766, 796, 760, 735, 765, 774, 708, 797, 714, 733, 771, 731, 770, 771, 775, 786, 747, 775, 807, 759, 826, 772, 745, 765, 755, 785, 765, 785, 754] payload_rows_per_dst=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] dup_rows_per_dst=[3218, 3340, 2880, 3054, 2992, 2834, 2990, 2700] combine_rows_per_src=[9454, 9668, 9212, 9314, 9164, 9162, 9350, 8972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64992 dup_rows=23998 dup_rows_remote=20908 dedup_ratio=0.244120 router_tokens_per_expert=[714, 778, 716, 762, 757, 789, 774, 781, 749, 715, 713, 756, 778, 780, 812, 766, 749, 787, 765, 739, 814, 737, 786, 808, 735, 826, 772, 781, 768, 778, 787, 781, 794, 749, 774, 784, 789, 774, 783, 755, 773, 753, 783, 767, 774, 755, 763, 748, 746, 781, 754, 772, 748, 761, 754, 756, 813, 809, 754, 811, 767, 740, 773, 742] payload_rows_per_dst=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] dup_rows_per_dst=[2938, 2898, 3106, 3018, 3090, 2976, 2878, 3094] combine_rows_per_src=[9204, 9240, 9264, 9438, 9314, 9256, 9266, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65074 dup_rows=23876 dup_rows_remote=20900 dedup_ratio=0.242879 router_tokens_per_expert=[779, 738, 736, 770, 741, 790, 715, 769, 782, 791, 768, 746, 739, 799, 761, 744, 796, 780, 800, 780, 780, 731, 783, 810, 759, 767, 747, 807, 757, 774, 799, 795, 753, 776, 802, 726, 774, 761, 720, 798, 759, 739, 771, 744, 701, 790, 775, 713, 766, 794, 746, 774, 752, 757, 789, 787, 813, 793, 787, 728, 760, 756, 824, 791] payload_rows_per_dst=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] dup_rows_per_dst=[2950, 2980, 2988, 3018, 2962, 2882, 2976, 3120] combine_rows_per_src=[9126, 9280, 9532, 9392, 9258, 9102, 9354, 9384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=345696 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267929184 ingress_bytes=302186496 link_bytes=302186496 token_bytes=535512672 weight_movement_bytes=34603008 total_nvlink_bytes=570115680 weight_to_token_ratio=0.064617 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137103456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343600 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764400 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185200 weight_movement_bytes=103809024 total_nvlink_bytes=634994224 weight_to_token_ratio=0.195429 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=342528 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265935360 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528192 weight_movement_bytes=34603008 total_nvlink_bytes=566131200 weight_to_token_ratio=0.065101 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20886 dedup_ratio=0.244364 router_tokens_per_expert=[778, 779, 811, 740, 774, 748, 797, 764, 770, 739, 734, 726, 721, 797, 723, 758, 776, 784, 799, 784, 754, 719, 772, 760, 760, 784, 794, 772, 799, 747, 763, 743, 788, 763, 808, 822, 775, 784, 766, 767, 722, 755, 743, 717, 775, 719, 754, 768, 781, 800, 850, 732, 782, 804, 746, 797, 790, 747, 768, 756, 828, 754, 770, 752] payload_rows_per_dst=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] dup_rows_per_dst=[3046, 2852, 2958, 3078, 2980, 2820, 3152, 3136] combine_rows_per_src=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65174 dup_rows=23820 dup_rows_remote=20798 dedup_ratio=0.242310 router_tokens_per_expert=[751, 779, 752, 814, 739, 788, 838, 752, 749, 753, 800, 807, 756, 766, 733, 761, 769, 816, 738, 725, 804, 749, 784, 785, 767, 805, 784, 764, 763, 741, 764, 747, 778, 747, 767, 768, 791, 783, 743, 767, 796, 793, 759, 782, 752, 754, 774, 756, 730, 767, 721, 726, 744, 750, 719, 790, 751, 771, 766, 766, 807, 790, 774, 827] payload_rows_per_dst=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] dup_rows_per_dst=[3110, 2888, 2956, 2962, 3002, 3022, 2840, 3040] combine_rows_per_src=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65092 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[803, 752, 735, 760, 772, 796, 752, 763, 751, 780, 811, 792, 707, 787, 790, 735, 733, 764, 791, 783, 753, 773, 755, 758, 762, 752, 776, 756, 771, 739, 769, 761, 777, 774, 763, 757, 756, 808, 811, 760, 721, 775, 790, 732, 800, 812, 800, 763, 759, 775, 798, 746, 801, 738, 785, 792, 740, 719, 780, 763, 778, 760, 793, 744] payload_rows_per_dst=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] dup_rows_per_dst=[2992, 2942, 2954, 2868, 3074, 3090, 3052, 2974] combine_rows_per_src=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343896 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266887000 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430104 weight_movement_bytes=103809024 total_nvlink_bytes=637239128 weight_to_token_ratio=0.194607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343600 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550832 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758064 weight_movement_bytes=69206016 total_nvlink_bytes=601964080 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65106 dup_rows=23924 dup_rows_remote=20992 dedup_ratio=0.243368 router_tokens_per_expert=[740, 771, 767, 767, 789, 787, 771, 768, 764, 771, 756, 754, 775, 770, 762, 751, 798, 844, 757, 814, 793, 760, 729, 761, 736, 776, 784, 750, 767, 763, 735, 787, 744, 733, 762, 753, 768, 763, 751, 751, 748, 747, 807, 786, 753, 795, 747, 766, 783, 746, 744, 806, 749, 771, 770, 776, 818, 735, 809, 780, 778, 774, 772, 750] payload_rows_per_dst=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] dup_rows_per_dst=[2960, 2932, 3042, 2908, 2946, 3036, 3054, 3046] combine_rows_per_src=[9360, 9274, 9470, 9288, 9104, 9262, 9236, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65094 dup_rows=23844 dup_rows_remote=20776 dedup_ratio=0.242554 router_tokens_per_expert=[770, 765, 751, 740, 759, 796, 739, 762, 759, 756, 788, 754, 800, 788, 737, 801, 764, 806, 772, 786, 755, 763, 767, 763, 817, 738, 792, 786, 755, 760, 809, 760, 741, 748, 783, 755, 741, 786, 793, 759, 717, 728, 797, 681, 756, 752, 759, 781, 770, 765, 796, 799, 774, 815, 762, 776, 775, 761, 756, 765, 786, 786, 774, 757] payload_rows_per_dst=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] dup_rows_per_dst=[2906, 2890, 3016, 3068, 2980, 2798, 3112, 3074] combine_rows_per_src=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64924 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[753, 803, 789, 719, 765, 770, 774, 829, 794, 748, 741, 768, 771, 757, 802, 754, 730, 755, 757, 780, 769, 727, 735, 790, 770, 784, 777, 735, 713, 785, 809, 771, 783, 781, 766, 806, 745, 806, 759, 753, 782, 780, 793, 756, 719, 771, 744, 781, 752, 773, 760, 808, 782, 773, 755, 725, 785, 787, 784, 754, 773, 752, 756, 784] payload_rows_per_dst=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] dup_rows_per_dst=[3100, 3014, 2964, 3046, 3114, 2920, 2940, 3040] combine_rows_per_src=[9304, 9256, 9122, 9242, 9284, 9332, 9316, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65092 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[803, 752, 735, 760, 772, 796, 752, 763, 751, 780, 811, 792, 707, 787, 790, 735, 733, 764, 791, 783, 753, 773, 755, 758, 762, 752, 776, 756, 771, 739, 769, 761, 777, 774, 763, 757, 756, 808, 811, 760, 721, 775, 790, 732, 800, 812, 800, 763, 759, 775, 798, 746, 801, 738, 785, 792, 740, 719, 780, 763, 778, 760, 793, 744] payload_rows_per_dst=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] dup_rows_per_dst=[2992, 2942, 2954, 2868, 3074, 3090, 3052, 2974] combine_rows_per_src=[9274, 9364, 9266, 9304, 9338, 9296, 9336, 9180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21000 dedup_ratio=0.244283 router_tokens_per_expert=[802, 723, 761, 769, 776, 755, 781, 714, 771, 784, 791, 793, 752, 827, 715, 800, 799, 736, 752, 782, 744, 787, 737, 756, 784, 779, 758, 768, 794, 821, 744, 745, 774, 708, 768, 773, 765, 777, 750, 756, 835, 769, 740, 740, 757, 801, 755, 822, 768, 766, 777, 781, 776, 778, 781, 777, 761, 773, 782, 799, 760, 736, 707, 740] payload_rows_per_dst=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] dup_rows_per_dst=[2916, 3062, 2972, 3082, 2894, 3146, 3014, 2928] combine_rows_per_src=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65042 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[747, 769, 780, 794, 734, 786, 789, 768, 746, 829, 752, 751, 780, 749, 752, 754, 713, 798, 777, 725, 766, 800, 781, 771, 756, 811, 824, 780, 753, 744, 774, 804, 744, 760, 775, 749, 755, 780, 789, 743, 780, 744, 780, 798, 798, 809, 766, 766, 767, 758, 750, 766, 716, 757, 715, 806, 763, 751, 745, 748, 811, 753, 793, 760] payload_rows_per_dst=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] dup_rows_per_dst=[2984, 2986, 3030, 3042, 2984, 3100, 2928, 3002] combine_rows_per_src=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343200 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266230944 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532118688 weight_movement_bytes=69206016 total_nvlink_bytes=601324704 weight_to_token_ratio=0.130057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343888 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267296592 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534249296 weight_movement_bytes=69206016 total_nvlink_bytes=603455312 weight_to_token_ratio=0.129539 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344336 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961168 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578000 weight_movement_bytes=69206016 total_nvlink_bytes=602784016 weight_to_token_ratio=0.129702 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65174 dup_rows=23820 dup_rows_remote=20798 dedup_ratio=0.242310 router_tokens_per_expert=[751, 779, 752, 814, 739, 788, 838, 752, 749, 753, 800, 807, 756, 766, 733, 761, 769, 816, 738, 725, 804, 749, 784, 785, 767, 805, 784, 764, 763, 741, 764, 747, 778, 747, 767, 768, 791, 783, 743, 767, 796, 793, 759, 782, 752, 754, 774, 756, 730, 767, 721, 726, 744, 750, 719, 790, 751, 771, 766, 766, 807, 790, 774, 827] payload_rows_per_dst=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] dup_rows_per_dst=[3110, 2888, 2956, 2962, 3002, 3022, 2840, 3040] combine_rows_per_src=[9316, 9362, 9384, 9308, 9286, 9310, 9054, 9464] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20886 dedup_ratio=0.244364 router_tokens_per_expert=[778, 779, 811, 740, 774, 748, 797, 764, 770, 739, 734, 726, 721, 797, 723, 758, 776, 784, 799, 784, 754, 719, 772, 760, 760, 784, 794, 772, 799, 747, 763, 743, 788, 763, 808, 822, 775, 784, 766, 767, 722, 755, 743, 717, 775, 719, 754, 768, 781, 800, 850, 732, 782, 804, 746, 797, 790, 747, 768, 756, 828, 754, 770, 752] payload_rows_per_dst=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] dup_rows_per_dst=[3046, 2852, 2958, 3078, 2980, 2820, 3152, 3136] combine_rows_per_src=[9336, 9084, 9308, 9246, 9392, 9086, 9462, 9368] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=344392 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267018568 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533692744 weight_movement_bytes=34603008 total_nvlink_bytes=568295752 weight_to_token_ratio=0.064837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343480 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968504 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593528 weight_movement_bytes=69206016 total_nvlink_bytes=602799544 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344064 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272768 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201472 weight_movement_bytes=69206016 total_nvlink_bytes=601407488 weight_to_token_ratio=0.130037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344336 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961168 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578000 weight_movement_bytes=69206016 total_nvlink_bytes=602784016 weight_to_token_ratio=0.129702 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65244 dup_rows=23884 dup_rows_remote=21042 dedup_ratio=0.242961 router_tokens_per_expert=[725, 757, 739, 784, 722, 788, 735, 735, 717, 737, 769, 776, 758, 786, 777, 766, 798, 741, 787, 796, 801, 711, 734, 761, 795, 755, 790, 778, 752, 764, 814, 763, 791, 756, 787, 780, 758, 735, 789, 760, 786, 718, 762, 798, 793, 813, 787, 812, 769, 741, 782, 762, 780, 779, 749, 787, 726, 765, 759, 763, 816, 803, 754, 781] payload_rows_per_dst=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] dup_rows_per_dst=[2842, 3198, 2996, 2900, 3014, 3042, 2996, 2896] combine_rows_per_src=[9176, 9472, 9268, 9306, 9400, 9442, 9200, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343584 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266157600 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971616 weight_movement_bytes=69206016 total_nvlink_bytes=601177632 weight_to_token_ratio=0.130093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344456 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756488 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168520 weight_movement_bytes=34603008 total_nvlink_bytes=567771528 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343888 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267296592 ingress_bytes=336158720 link_bytes=336158720 token_bytes=534249296 weight_movement_bytes=69206016 total_nvlink_bytes=603455312 weight_to_token_ratio=0.129539 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264938200e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64978 dup_rows=23954 dup_rows_remote=20946 dedup_ratio=0.243673 router_tokens_per_expert=[788, 785, 745, 769, 798, 774, 758, 709, 770, 758, 779, 749, 777, 783, 815, 798, 738, 787, 736, 805, 759, 741, 764, 796, 689, 774, 777, 737, 761, 778, 774, 778, 738, 753, 746, 777, 793, 768, 750, 777, 756, 764, 776, 807, 784, 702, 754, 774, 810, 761, 829, 791, 746, 740, 757, 756, 736, 807, 768, 798, 779, 741, 747, 818] payload_rows_per_dst=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] dup_rows_per_dst=[2976, 3126, 2926, 2948, 2952, 3012, 3008, 3006] combine_rows_per_src=[9276, 9332, 9326, 9188, 9252, 9222, 9372, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64958 dup_rows=24084 dup_rows_remote=21118 dedup_ratio=0.244995 router_tokens_per_expert=[778, 741, 814, 736, 768, 746, 793, 787, 771, 738, 789, 770, 758, 748, 760, 766, 733, 793, 803, 809, 770, 757, 802, 748, 770, 813, 755, 766, 788, 775, 806, 758, 791, 796, 810, 728, 750, 764, 732, 743, 744, 752, 762, 804, 763, 769, 763, 747, 801, 766, 771, 799, 710, 811, 726, 745, 782, 738, 750, 795, 757, 742, 723, 809] payload_rows_per_dst=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] dup_rows_per_dst=[3088, 3008, 3050, 3108, 2966, 2944, 2954, 2966] combine_rows_per_src=[9238, 9192, 9380, 9354, 9262, 9264, 9304, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343200 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266230944 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532118688 weight_movement_bytes=69206016 total_nvlink_bytes=601324704 weight_to_token_ratio=0.130057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=345144 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584568 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823992 weight_movement_bytes=34603008 total_nvlink_bytes=569427000 weight_to_token_ratio=0.064700 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64870 dup_rows=24196 dup_rows_remote=21168 dedup_ratio=0.246134 router_tokens_per_expert=[748, 786, 739, 757, 797, 823, 788, 753, 774, 763, 722, 732, 740, 792, 735, 739, 776, 789, 833, 766, 790, 684, 753, 756, 798, 773, 776, 794, 764, 748, 805, 839, 755, 798, 773, 718, 772, 770, 746, 779, 792, 777, 791, 780, 758, 756, 746, 746, 768, 775, 689, 769, 770, 758, 761, 847, 770, 784, 765, 761, 764, 756, 773, 753] payload_rows_per_dst=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] dup_rows_per_dst=[3118, 2858, 2694, 3182, 3374, 3004, 2938, 3028] combine_rows_per_src=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64990 dup_rows=23958 dup_rows_remote=20858 dedup_ratio=0.243713 router_tokens_per_expert=[765, 786, 719, 768, 778, 748, 758, 756, 736, 818, 756, 753, 724, 792, 757, 747, 782, 750, 797, 746, 812, 776, 780, 784, 760, 797, 768, 738, 783, 787, 732, 773, 766, 764, 775, 755, 756, 737, 753, 841, 751, 757, 762, 807, 792, 762, 762, 835, 764, 791, 762, 790, 749, 752, 735, 757, 798, 816, 740, 718, 782, 795, 767, 735] payload_rows_per_dst=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] dup_rows_per_dst=[3048, 2880, 3054, 3012, 2902, 3100, 2982, 2980] combine_rows_per_src=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64888 dup_rows=23990 dup_rows_remote=20954 dedup_ratio=0.244039 router_tokens_per_expert=[768, 835, 691, 761, 779, 746, 776, 785, 760, 776, 809, 735, 754, 752, 765, 769, 779, 805, 752, 744, 816, 765, 780, 790, 738, 747, 723, 833, 745, 752, 767, 790, 763, 754, 765, 793, 728, 763, 753, 779, 806, 769, 784, 825, 770, 782, 762, 756, 774, 741, 800, 767, 734, 799, 741, 778, 775, 744, 739, 717, 755, 830, 758, 761] payload_rows_per_dst=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] dup_rows_per_dst=[2996, 2852, 3036, 2978, 2940, 3166, 3060, 2962] combine_rows_per_src=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343696 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266493584 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643472 weight_movement_bytes=69206016 total_nvlink_bytes=601849488 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344304 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266412272 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480240 weight_movement_bytes=69206016 total_nvlink_bytes=601686256 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64840 dup_rows=24238 dup_rows_remote=21234 dedup_ratio=0.246562 router_tokens_per_expert=[744, 776, 782, 803, 723, 763, 742, 743, 750, 783, 776, 768, 767, 765, 750, 756, 739, 789, 732, 797, 818, 770, 709, 786, 773, 771, 778, 801, 796, 753, 792, 770, 716, 751, 745, 768, 785, 791, 729, 704, 792, 805, 743, 773, 825, 753, 835, 763, 780, 778, 762, 781, 760, 741, 781, 788, 757, 746, 757, 745, 780, 768, 769, 816] payload_rows_per_dst=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] dup_rows_per_dst=[2948, 3004, 3010, 3108, 2854, 3188, 3080, 3046] combine_rows_per_src=[9204, 9226, 9270, 9360, 9124, 9390, 9262, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64880 dup_rows=23942 dup_rows_remote=20900 dedup_ratio=0.243551 router_tokens_per_expert=[720, 798, 786, 788, 708, 764, 777, 746, 815, 817, 737, 775, 746, 776, 757, 780, 751, 778, 814, 748, 746, 820, 792, 763, 709, 812, 704, 762, 736, 757, 765, 755, 768, 719, 739, 818, 774, 760, 775, 730, 759, 794, 820, 780, 732, 738, 745, 730, 791, 787, 737, 782, 798, 786, 797, 784, 761, 780, 802, 774, 796, 762, 772, 760] payload_rows_per_dst=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] dup_rows_per_dst=[3042, 2990, 3076, 2742, 3020, 2986, 3042, 3044] combine_rows_per_src=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65036 dup_rows=23904 dup_rows_remote=20940 dedup_ratio=0.243164 router_tokens_per_expert=[781, 742, 797, 790, 782, 765, 743, 726, 757, 760, 768, 730, 784, 727, 760, 826, 748, 732, 809, 795, 737, 800, 822, 765, 765, 830, 716, 746, 808, 739, 794, 766, 779, 739, 818, 765, 785, 785, 794, 769, 708, 739, 778, 739, 799, 752, 786, 754, 768, 777, 778, 734, 740, 727, 776, 786, 758, 766, 785, 722, 764, 792, 802, 778] payload_rows_per_dst=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] dup_rows_per_dst=[2952, 2962, 3020, 2964, 3064, 2934, 2982, 3026] combine_rows_per_src=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65094 dup_rows=23844 dup_rows_remote=20776 dedup_ratio=0.242554 router_tokens_per_expert=[770, 765, 751, 740, 759, 796, 739, 762, 759, 756, 788, 754, 800, 788, 737, 801, 764, 806, 772, 786, 755, 763, 767, 763, 817, 738, 792, 786, 755, 760, 809, 760, 741, 748, 783, 755, 741, 786, 793, 759, 717, 728, 797, 681, 756, 752, 759, 781, 770, 765, 796, 799, 774, 815, 762, 776, 775, 761, 756, 765, 786, 786, 774, 757] payload_rows_per_dst=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] dup_rows_per_dst=[2906, 2890, 3016, 3068, 2980, 2798, 3112, 3074] combine_rows_per_src=[9258, 9476, 9336, 9366, 9232, 9144, 9402, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64888 dup_rows=23990 dup_rows_remote=20954 dedup_ratio=0.244039 router_tokens_per_expert=[768, 835, 691, 761, 779, 746, 776, 785, 760, 776, 809, 735, 754, 752, 765, 769, 779, 805, 752, 744, 816, 765, 780, 790, 738, 747, 723, 833, 745, 752, 767, 790, 763, 754, 765, 793, 728, 763, 753, 779, 806, 769, 784, 825, 770, 782, 762, 756, 774, 741, 800, 767, 734, 799, 741, 778, 775, 744, 739, 717, 755, 830, 758, 761] payload_rows_per_dst=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] dup_rows_per_dst=[2996, 2852, 3036, 2978, 2940, 3166, 3060, 2962] combine_rows_per_src=[9286, 9388, 9426, 9212, 9256, 9342, 9208, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=344152 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266051672 ingress_bytes=300310528 link_bytes=300310528 token_bytes=531759192 weight_movement_bytes=34603008 total_nvlink_bytes=566362200 weight_to_token_ratio=0.065073 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130044340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343392 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542432 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741472 weight_movement_bytes=69206016 total_nvlink_bytes=601947488 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65330 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[785, 740, 762, 814, 761, 708, 722, 776, 773, 789, 787, 751, 777, 790, 750, 764, 778, 795, 803, 754, 770, 744, 757, 762, 807, 760, 709, 768, 815, 758, 807, 759, 767, 756, 759, 743, 738, 766, 763, 787, 721, 761, 772, 798, 771, 760, 733, 791, 730, 787, 818, 833, 782, 805, 772, 780, 725, 745, 784, 764, 752, 746, 789, 759] payload_rows_per_dst=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] dup_rows_per_dst=[2900, 2996, 2974, 2998, 3052, 3002, 3082, 2864] combine_rows_per_src=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343368 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124616 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905864 weight_movement_bytes=69206016 total_nvlink_bytes=601111880 weight_to_token_ratio=0.130110 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64990 dup_rows=23958 dup_rows_remote=20858 dedup_ratio=0.243713 router_tokens_per_expert=[765, 786, 719, 768, 778, 748, 758, 756, 736, 818, 756, 753, 724, 792, 757, 747, 782, 750, 797, 746, 812, 776, 780, 784, 760, 797, 768, 738, 783, 787, 732, 773, 766, 764, 775, 755, 756, 737, 753, 841, 751, 757, 762, 807, 792, 762, 762, 835, 764, 791, 762, 790, 749, 752, 735, 757, 798, 816, 740, 718, 782, 795, 767, 735] payload_rows_per_dst=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] dup_rows_per_dst=[3048, 2880, 3054, 3012, 2902, 3100, 2982, 2980] combine_rows_per_src=[9108, 9286, 9400, 9264, 9392, 9356, 9218, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64870 dup_rows=24196 dup_rows_remote=21168 dedup_ratio=0.246134 router_tokens_per_expert=[748, 786, 739, 757, 797, 823, 788, 753, 774, 763, 722, 732, 740, 792, 735, 739, 776, 789, 833, 766, 790, 684, 753, 756, 798, 773, 776, 794, 764, 748, 805, 839, 755, 798, 773, 718, 772, 770, 746, 779, 792, 777, 791, 780, 758, 756, 746, 746, 768, 775, 689, 769, 770, 758, 761, 847, 770, 784, 765, 761, 764, 756, 773, 753] payload_rows_per_dst=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] dup_rows_per_dst=[3118, 2858, 2694, 3182, 3374, 3004, 2938, 3028] combine_rows_per_src=[9264, 9136, 8954, 9412, 9494, 9288, 9322, 9238] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=344296 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928936 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513576 weight_movement_bytes=34603008 total_nvlink_bytes=566116584 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343480 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968504 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593528 weight_movement_bytes=69206016 total_nvlink_bytes=602799544 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265781248 dispatch_meta_bytes=343368 combine_payload_bytes=265781248 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266124616 ingress_bytes=334987264 link_bytes=334987264 token_bytes=531905864 weight_movement_bytes=69206016 total_nvlink_bytes=601111880 weight_to_token_ratio=0.130110 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260530106e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=64996 dup_rows=23796 dup_rows_remote=20648 dedup_ratio=0.242065 router_tokens_per_expert=[737, 763, 767, 769, 768, 722, 767, 799, 788, 740, 803, 770, 786, 749, 725, 808, 778, 756, 802, 804, 735, 774, 767, 749, 796, 772, 766, 754, 736, 764, 766, 732, 718, 775, 803, 785, 784, 793, 769, 773, 745, 807, 781, 743, 769, 756, 807, 728, 807, 756, 784, 745, 769, 744, 751, 788, 777, 812, 755, 790, 801, 741, 741, 743] payload_rows_per_dst=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] dup_rows_per_dst=[3148, 3248, 2980, 2770, 2994, 2810, 2880, 2966] combine_rows_per_src=[9512, 9428, 9442, 9080, 9406, 9040, 9246, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343120 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266091600 ingress_bytes=404160512 link_bytes=404160512 token_bytes=531840080 weight_movement_bytes=138412032 total_nvlink_bytes=670252112 weight_to_token_ratio=0.260251 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520823469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343904 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266731360 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533118816 weight_movement_bytes=69206016 total_nvlink_bytes=602324832 weight_to_token_ratio=0.129813 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343392 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542432 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741472 weight_movement_bytes=69206016 total_nvlink_bytes=601947488 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21000 dedup_ratio=0.244283 router_tokens_per_expert=[802, 723, 761, 769, 776, 755, 781, 714, 771, 784, 791, 793, 752, 827, 715, 800, 799, 736, 752, 782, 744, 787, 737, 756, 784, 779, 758, 768, 794, 821, 744, 745, 774, 708, 768, 773, 765, 777, 750, 756, 835, 769, 740, 740, 757, 801, 755, 822, 768, 766, 777, 781, 776, 778, 781, 777, 761, 773, 782, 799, 760, 736, 707, 740] payload_rows_per_dst=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] dup_rows_per_dst=[2916, 3062, 2972, 3082, 2894, 3146, 3014, 2928] combine_rows_per_src=[9246, 9404, 9214, 9304, 9248, 9292, 9394, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65042 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[747, 769, 780, 794, 734, 786, 789, 768, 746, 829, 752, 751, 780, 749, 752, 754, 713, 798, 777, 725, 766, 800, 781, 771, 756, 811, 824, 780, 753, 744, 774, 804, 744, 760, 775, 749, 755, 780, 789, 743, 780, 744, 780, 798, 798, 809, 766, 766, 767, 758, 750, 766, 716, 757, 715, 806, 763, 751, 745, 748, 811, 753, 793, 760] payload_rows_per_dst=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] dup_rows_per_dst=[2984, 2986, 3030, 3042, 2984, 3100, 2928, 3002] combine_rows_per_src=[9350, 9240, 9232, 9450, 9206, 9382, 9142, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=344152 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266051672 ingress_bytes=300310528 link_bytes=300310528 token_bytes=531759192 weight_movement_bytes=34603008 total_nvlink_bytes=566362200 weight_to_token_ratio=0.065073 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130044340e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=344584 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267936264 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535527944 weight_movement_bytes=34603008 total_nvlink_bytes=570130952 weight_to_token_ratio=0.064615 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=342576 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266566192 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532789808 weight_movement_bytes=34603008 total_nvlink_bytes=567392816 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343584 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266157600 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971616 weight_movement_bytes=69206016 total_nvlink_bytes=601177632 weight_to_token_ratio=0.130093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344456 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756488 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168520 weight_movement_bytes=34603008 total_nvlink_bytes=567771528 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65258 dup_rows=23902 dup_rows_remote=21004 dedup_ratio=0.243144 router_tokens_per_expert=[762, 779, 795, 799, 796, 758, 741, 795, 754, 734, 771, 715, 755, 750, 791, 751, 789, 722, 814, 772, 750, 776, 795, 768, 819, 800, 780, 777, 754, 766, 759, 767, 796, 731, 808, 766, 795, 732, 795, 758, 774, 740, 775, 735, 809, 764, 761, 734, 705, 770, 741, 746, 768, 768, 785, 755, 781, 808, 758, 770, 768, 800, 713, 789] payload_rows_per_dst=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] dup_rows_per_dst=[3122, 2898, 3024, 2994, 3060, 2942, 2890, 2972] combine_rows_per_src=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65018 dup_rows=24020 dup_rows_remote=21124 dedup_ratio=0.244344 router_tokens_per_expert=[787, 738, 803, 795, 764, 757, 822, 767, 788, 778, 693, 773, 777, 746, 738, 762, 722, 783, 756, 760, 800, 740, 756, 764, 764, 779, 767, 749, 783, 787, 776, 780, 773, 739, 757, 752, 787, 760, 767, 710, 794, 800, 838, 743, 770, 818, 791, 775, 736, 751, 765, 800, 736, 763, 772, 757, 781, 788, 797, 726, 732, 780, 760, 780] payload_rows_per_dst=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] dup_rows_per_dst=[3046, 2952, 2896, 3018, 2926, 3172, 2970, 3040] combine_rows_per_src=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65036 dup_rows=23904 dup_rows_remote=20940 dedup_ratio=0.243164 router_tokens_per_expert=[781, 742, 797, 790, 782, 765, 743, 726, 757, 760, 768, 730, 784, 727, 760, 826, 748, 732, 809, 795, 737, 800, 822, 765, 765, 830, 716, 746, 808, 739, 794, 766, 779, 739, 818, 765, 785, 785, 794, 769, 708, 739, 778, 739, 799, 752, 786, 754, 768, 777, 778, 734, 740, 727, 776, 786, 758, 766, 785, 722, 764, 792, 802, 778] payload_rows_per_dst=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] dup_rows_per_dst=[2952, 2962, 3020, 2964, 3064, 2934, 2982, 3026] combine_rows_per_src=[9300, 9262, 9396, 9364, 9404, 9176, 9190, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65038 dup_rows=24006 dup_rows_remote=21044 dedup_ratio=0.244202 router_tokens_per_expert=[752, 745, 789, 771, 806, 746, 727, 780, 771, 787, 775, 755, 739, 727, 777, 757, 794, 738, 791, 791, 780, 779, 832, 719, 768, 776, 810, 794, 755, 722, 741, 721, 789, 761, 731, 788, 767, 762, 774, 761, 775, 785, 746, 745, 709, 772, 804, 775, 782, 791, 805, 749, 759, 746, 749, 768, 798, 817, 798, 763, 791, 774, 720, 783] payload_rows_per_dst=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] dup_rows_per_dst=[2978, 2890, 3014, 3032, 3070, 2962, 3026, 3034] combine_rows_per_src=[9254, 9286, 9434, 9142, 9196, 9260, 9272, 9454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64876 dup_rows=24088 dup_rows_remote=21048 dedup_ratio=0.245036 router_tokens_per_expert=[776, 809, 785, 820, 737, 710, 777, 756, 751, 799, 789, 759, 779, 771, 781, 802, 808, 745, 757, 811, 732, 770, 832, 746, 703, 755, 793, 746, 778, 764, 808, 746, 761, 735, 770, 754, 745, 791, 780, 782, 773, 765, 784, 768, 769, 758, 773, 706, 795, 774, 774, 704, 738, 744, 779, 758, 755, 734, 766, 780, 785, 790, 773, 794] payload_rows_per_dst=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] dup_rows_per_dst=[2742, 3292, 2866, 2932, 3062, 2942, 3212, 3040] combine_rows_per_src=[9092, 9538, 9284, 9068, 9360, 9250, 9284, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65018 dup_rows=24020 dup_rows_remote=21124 dedup_ratio=0.244344 router_tokens_per_expert=[787, 738, 803, 795, 764, 757, 822, 767, 788, 778, 693, 773, 777, 746, 738, 762, 722, 783, 756, 760, 800, 740, 756, 764, 764, 779, 767, 749, 783, 787, 776, 780, 773, 739, 757, 752, 787, 760, 767, 710, 794, 800, 838, 743, 770, 818, 791, 775, 736, 751, 765, 800, 736, 763, 772, 757, 781, 788, 797, 726, 732, 780, 760, 780] payload_rows_per_dst=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] dup_rows_per_dst=[3046, 2952, 2896, 3018, 2926, 3172, 2970, 3040] combine_rows_per_src=[9420, 9158, 9266, 9352, 9164, 9486, 9190, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=345048 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641816 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534938584 weight_movement_bytes=34603008 total_nvlink_bytes=569541592 weight_to_token_ratio=0.064686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65088 dup_rows=24008 dup_rows_remote=20982 dedup_ratio=0.244222 router_tokens_per_expert=[769, 812, 777, 751, 785, 810, 784, 776, 741, 738, 789, 773, 777, 761, 783, 727, 796, 729, 762, 759, 784, 751, 790, 803, 744, 784, 739, 835, 838, 753, 772, 758, 809, 743, 785, 774, 787, 782, 752, 773, 739, 778, 737, 753, 750, 809, 791, 729, 739, 777, 782, 740, 770, 732, 803, 774, 737, 766, 752, 752, 714, 745, 740, 788] payload_rows_per_dst=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] dup_rows_per_dst=[3074, 2988, 2992, 2960, 3050, 3028, 3026, 2890] combine_rows_per_src=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65258 dup_rows=23902 dup_rows_remote=21004 dedup_ratio=0.243144 router_tokens_per_expert=[762, 779, 795, 799, 796, 758, 741, 795, 754, 734, 771, 715, 755, 750, 791, 751, 789, 722, 814, 772, 750, 776, 795, 768, 819, 800, 780, 777, 754, 766, 759, 767, 796, 731, 808, 766, 795, 732, 795, 758, 774, 740, 775, 735, 809, 764, 761, 734, 705, 770, 741, 746, 768, 768, 785, 755, 781, 808, 758, 770, 768, 800, 713, 789] payload_rows_per_dst=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] dup_rows_per_dst=[3122, 2898, 3024, 2994, 3060, 2942, 2890, 2972] combine_rows_per_src=[9328, 9144, 9348, 9450, 9302, 9242, 9186, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344568 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266658296 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532972024 weight_movement_bytes=69206016 total_nvlink_bytes=602178040 weight_to_token_ratio=0.129849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[778, 796, 771, 754, 804, 748, 769, 773, 804, 768, 743, 740, 793, 799, 759, 753, 754, 766, 783, 774, 710, 786, 774, 769, 801, 769, 739, 774, 784, 788, 732, 795, 770, 728, 761, 731, 769, 776, 731, 828, 775, 795, 779, 815, 741, 782, 719, 792, 768, 719, 745, 754, 726, 733, 808, 751, 759, 767, 760, 785, 812, 785, 799, 739] payload_rows_per_dst=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] dup_rows_per_dst=[3140, 3024, 2958, 3002, 2958, 3042, 2836, 2996] combine_rows_per_src=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343904 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266731360 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533118816 weight_movement_bytes=69206016 total_nvlink_bytes=602324832 weight_to_token_ratio=0.129813 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64880 dup_rows=23942 dup_rows_remote=20900 dedup_ratio=0.243551 router_tokens_per_expert=[720, 798, 786, 788, 708, 764, 777, 746, 815, 817, 737, 775, 746, 776, 757, 780, 751, 778, 814, 748, 746, 820, 792, 763, 709, 812, 704, 762, 736, 757, 765, 755, 768, 719, 739, 818, 774, 760, 775, 730, 759, 794, 820, 780, 732, 738, 745, 730, 791, 787, 737, 782, 798, 786, 797, 784, 761, 780, 802, 774, 796, 762, 772, 760] payload_rows_per_dst=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] dup_rows_per_dst=[3042, 2990, 3076, 2742, 3020, 2986, 3042, 3044] combine_rows_per_src=[9132, 9416, 9348, 9258, 9146, 9210, 9482, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64898 dup_rows=24102 dup_rows_remote=21000 dedup_ratio=0.245178 router_tokens_per_expert=[746, 714, 792, 762, 798, 776, 819, 794, 786, 728, 809, 782, 765, 782, 768, 771, 734, 781, 727, 740, 779, 767, 759, 754, 742, 761, 727, 746, 784, 765, 780, 763, 740, 769, 759, 746, 769, 747, 776, 758, 822, 803, 793, 766, 799, 767, 773, 775, 745, 794, 770, 776, 770, 770, 761, 741, 785, 806, 783, 740, 785, 760, 748, 755] payload_rows_per_dst=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] dup_rows_per_dst=[3102, 3020, 2930, 2976, 2926, 3180, 2992, 2976] combine_rows_per_src=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344328 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739976 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135624 weight_movement_bytes=69206016 total_nvlink_bytes=602341640 weight_to_token_ratio=0.129809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64886 dup_rows=24198 dup_rows_remote=21110 dedup_ratio=0.246155 router_tokens_per_expert=[747, 778, 836, 774, 774, 767, 753, 818, 771, 760, 731, 769, 786, 705, 721, 764, 762, 766, 774, 777, 830, 770, 777, 723, 807, 753, 782, 781, 749, 783, 753, 758, 780, 751, 792, 750, 800, 793, 763, 725, 806, 772, 742, 737, 781, 756, 794, 753, 775, 735, 785, 768, 769, 729, 830, 755, 743, 777, 768, 801, 740, 794, 726, 763] payload_rows_per_dst=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] dup_rows_per_dst=[3160, 2794, 3058, 3022, 3088, 3074, 2980, 3022] combine_rows_per_src=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344568 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266658296 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532972024 weight_movement_bytes=69206016 total_nvlink_bytes=602178040 weight_to_token_ratio=0.129849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65330 dup_rows=23868 dup_rows_remote=20816 dedup_ratio=0.242798 router_tokens_per_expert=[785, 740, 762, 814, 761, 708, 722, 776, 773, 789, 787, 751, 777, 790, 750, 764, 778, 795, 803, 754, 770, 744, 757, 762, 807, 760, 709, 768, 815, 758, 807, 759, 767, 756, 759, 743, 738, 766, 763, 787, 721, 761, 772, 798, 771, 760, 733, 791, 730, 787, 818, 833, 782, 805, 772, 780, 725, 745, 784, 764, 752, 746, 789, 759] payload_rows_per_dst=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] dup_rows_per_dst=[2900, 2996, 2974, 2998, 3052, 3002, 3082, 2864] combine_rows_per_src=[9236, 9366, 9352, 9368, 9106, 9212, 9532, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343696 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266075792 ingress_bytes=300335104 link_bytes=300335104 token_bytes=531807888 weight_movement_bytes=34603008 total_nvlink_bytes=566410896 weight_to_token_ratio=0.065067 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130136817e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64898 dup_rows=24102 dup_rows_remote=21000 dedup_ratio=0.245178 router_tokens_per_expert=[746, 714, 792, 762, 798, 776, 819, 794, 786, 728, 809, 782, 765, 782, 768, 771, 734, 781, 727, 740, 779, 767, 759, 754, 742, 761, 727, 746, 784, 765, 780, 763, 740, 769, 759, 746, 769, 747, 776, 758, 822, 803, 793, 766, 799, 767, 773, 775, 745, 794, 770, 776, 770, 770, 761, 741, 785, 806, 783, 740, 785, 760, 748, 755] payload_rows_per_dst=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] dup_rows_per_dst=[3102, 3020, 2930, 2976, 2926, 3180, 2992, 2976] combine_rows_per_src=[9304, 9362, 9152, 9200, 9202, 9372, 9262, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344280 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944728 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545176 weight_movement_bytes=34603008 total_nvlink_bytes=568148184 weight_to_token_ratio=0.064855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266526416 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709072 weight_movement_bytes=103809024 total_nvlink_bytes=636518096 weight_to_token_ratio=0.194870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=345048 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641816 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534938584 weight_movement_bytes=34603008 total_nvlink_bytes=569541592 weight_to_token_ratio=0.064686 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343984 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117040 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890096 weight_movement_bytes=34603008 total_nvlink_bytes=566493104 weight_to_token_ratio=0.065057 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343120 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266091600 ingress_bytes=404160512 link_bytes=404160512 token_bytes=531840080 weight_movement_bytes=138412032 total_nvlink_bytes=670252112 weight_to_token_ratio=0.260251 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520823469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267591680 dispatch_meta_bytes=344584 combine_payload_bytes=267591680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267936264 ingress_bytes=302194688 link_bytes=302194688 token_bytes=535527944 weight_movement_bytes=34603008 total_nvlink_bytes=570130952 weight_to_token_ratio=0.064615 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137134282e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343592 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266165800 ingress_bytes=404234240 link_bytes=404234240 token_bytes=531988008 weight_movement_bytes=138412032 total_nvlink_bytes=670400040 weight_to_token_ratio=0.260179 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521100902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20836 dedup_ratio=0.242716 router_tokens_per_expert=[743, 760, 775, 750, 720, 701, 808, 768, 750, 754, 803, 780, 742, 837, 762, 779, 766, 778, 790, 780, 756, 768, 782, 766, 834, 750, 750, 786, 761, 737, 783, 731, 757, 766, 811, 735, 754, 794, 766, 772, 801, 750, 793, 782, 784, 731, 763, 762, 733, 803, 752, 754, 787, 812, 800, 713, 751, 783, 768, 771, 779, 773, 772, 730] payload_rows_per_dst=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] dup_rows_per_dst=[2894, 3050, 3014, 2866, 3028, 3024, 2968, 3016] combine_rows_per_src=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65136 dup_rows=23870 dup_rows_remote=20900 dedup_ratio=0.242818 router_tokens_per_expert=[748, 750, 762, 709, 760, 753, 769, 772, 783, 836, 756, 705, 787, 734, 762, 771, 771, 772, 736, 751, 766, 749, 761, 834, 749, 771, 767, 720, 796, 786, 762, 822, 750, 706, 762, 810, 759, 776, 786, 787, 770, 796, 760, 765, 744, 788, 754, 761, 795, 749, 810, 823, 796, 736, 809, 768, 751, 771, 806, 785, 721, 756, 768, 764] payload_rows_per_dst=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] dup_rows_per_dst=[2856, 2970, 2978, 2964, 2964, 3020, 3108, 3010] combine_rows_per_src=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65082 dup_rows=23880 dup_rows_remote=20892 dedup_ratio=0.242920 router_tokens_per_expert=[776, 752, 799, 806, 733, 756, 777, 751, 748, 777, 729, 735, 750, 791, 754, 799, 770, 745, 761, 796, 775, 747, 806, 765, 793, 750, 787, 774, 715, 764, 806, 794, 786, 742, 807, 776, 737, 778, 745, 779, 727, 754, 744, 767, 778, 753, 782, 754, 767, 771, 786, 768, 763, 712, 793, 769, 796, 729, 837, 769, 782, 813, 709, 798] payload_rows_per_dst=[9290, 9232, 9342, 9320, 9310, 9190, 9260, 9480] dup_rows_per_dst=[3010, 2934, 2988, 3046, 2990, 2928, 2998, 2986] combine_rows_per_src=[9290, 9232, 9342, 9320, 9310, 9190, 9260, 9480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64892 dup_rows=24012 dup_rows_remote=20860 dedup_ratio=0.244263 router_tokens_per_expert=[739, 786, 772, 792, 768, 748, 766, 774, 788, 750, 756, 742, 785, 792, 717, 772, 754, 745, 757, 817, 791, 726, 800, 763, 744, 799, 777, 771, 705, 743, 776, 780, 747, 811, 781, 760, 767, 789, 817, 732, 774, 720, 745, 737, 783, 750, 784, 760, 822, 804, 759, 785, 749, 756, 748, 783, 817, 744, 768, 742, 794, 795, 806, 728] payload_rows_per_dst=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] dup_rows_per_dst=[3020, 2948, 2894, 2874, 3178, 2900, 3046, 3152] combine_rows_per_src=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64674 dup_rows=24258 dup_rows_remote=21084 dedup_ratio=0.246765 router_tokens_per_expert=[779, 765, 752, 719, 758, 827, 767, 760, 772, 760, 845, 743, 792, 747, 765, 763, 778, 739, 754, 760, 777, 755, 782, 780, 712, 758, 779, 767, 823, 739, 801, 797, 769, 753, 765, 736, 744, 762, 766, 783, 779, 802, 809, 801, 780, 708, 720, 771, 787, 783, 760, 820, 804, 788, 755, 776, 760, 721, 733, 781, 741, 747, 776, 757] payload_rows_per_dst=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] dup_rows_per_dst=[3008, 3134, 2996, 3116, 2950, 3020, 3174, 2860] combine_rows_per_src=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20954 dedup_ratio=0.243693 router_tokens_per_expert=[778, 796, 771, 754, 804, 748, 769, 773, 804, 768, 743, 740, 793, 799, 759, 753, 754, 766, 783, 774, 710, 786, 774, 769, 801, 769, 739, 774, 784, 788, 732, 795, 770, 728, 761, 731, 769, 776, 731, 828, 775, 795, 779, 815, 741, 782, 719, 792, 768, 719, 745, 754, 726, 733, 808, 751, 759, 767, 760, 785, 812, 785, 799, 739] payload_rows_per_dst=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] dup_rows_per_dst=[3140, 3024, 2958, 3002, 2958, 3042, 2836, 2996] combine_rows_per_src=[9246, 9294, 9274, 9362, 9230, 9354, 9172, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20836 dedup_ratio=0.242716 router_tokens_per_expert=[743, 760, 775, 750, 720, 701, 808, 768, 750, 754, 803, 780, 742, 837, 762, 779, 766, 778, 790, 780, 756, 768, 782, 766, 834, 750, 750, 786, 761, 737, 783, 731, 757, 766, 811, 735, 754, 794, 766, 772, 801, 750, 793, 782, 784, 731, 763, 762, 733, 803, 752, 754, 787, 812, 800, 713, 751, 783, 768, 771, 779, 773, 772, 730] payload_rows_per_dst=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] dup_rows_per_dst=[2894, 3050, 3014, 2866, 3028, 3024, 2968, 3016] combine_rows_per_src=[9156, 9364, 9358, 9398, 9282, 9308, 9340, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65054 dup_rows=24026 dup_rows_remote=21034 dedup_ratio=0.244405 router_tokens_per_expert=[769, 780, 768, 734, 787, 783, 758, 776, 763, 761, 722, 777, 751, 777, 824, 744, 745, 775, 761, 755, 779, 739, 754, 772, 800, 750, 742, 789, 727, 784, 728, 788, 762, 757, 771, 790, 775, 785, 798, 760, 762, 771, 771, 762, 795, 794, 832, 720, 756, 776, 792, 786, 770, 771, 768, 736, 789, 809, 773, 779, 743, 729, 749, 759] payload_rows_per_dst=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] dup_rows_per_dst=[2972, 2904, 2988, 2992, 3066, 3036, 3044, 3024] combine_rows_per_src=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65136 dup_rows=23870 dup_rows_remote=20900 dedup_ratio=0.242818 router_tokens_per_expert=[748, 750, 762, 709, 760, 753, 769, 772, 783, 836, 756, 705, 787, 734, 762, 771, 771, 772, 736, 751, 766, 749, 761, 834, 749, 771, 767, 720, 796, 786, 762, 822, 750, 706, 762, 810, 759, 776, 786, 787, 770, 796, 760, 765, 744, 788, 754, 761, 795, 749, 810, 823, 796, 736, 809, 768, 751, 771, 806, 785, 721, 756, 768, 764] payload_rows_per_dst=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] dup_rows_per_dst=[2856, 2970, 2978, 2964, 2964, 3020, 3108, 3010] combine_rows_per_src=[9190, 9298, 9302, 9382, 9308, 9256, 9464, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343888 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267140944 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938000 weight_movement_bytes=34603008 total_nvlink_bytes=568541008 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64844 dup_rows=24230 dup_rows_remote=21144 dedup_ratio=0.246480 router_tokens_per_expert=[763, 768, 799, 815, 781, 777, 819, 771, 788, 760, 813, 751, 741, 765, 763, 779, 746, 738, 776, 723, 799, 809, 716, 779, 764, 800, 793, 806, 818, 757, 722, 754, 725, 803, 805, 747, 777, 720, 798, 783, 739, 766, 714, 808, 721, 723, 775, 760, 744, 752, 802, 813, 768, 754, 731, 814, 770, 751, 767, 706, 710, 806, 814, 733] payload_rows_per_dst=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] dup_rows_per_dst=[3158, 3056, 2968, 3102, 3086, 2844, 3072, 2944] combine_rows_per_src=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64892 dup_rows=24012 dup_rows_remote=20860 dedup_ratio=0.244263 router_tokens_per_expert=[739, 786, 772, 792, 768, 748, 766, 774, 788, 750, 756, 742, 785, 792, 717, 772, 754, 745, 757, 817, 791, 726, 800, 763, 744, 799, 777, 771, 705, 743, 776, 780, 747, 811, 781, 760, 767, 789, 817, 732, 774, 720, 745, 737, 783, 750, 784, 760, 822, 804, 759, 785, 749, 756, 748, 783, 817, 744, 768, 742, 794, 795, 806, 728] payload_rows_per_dst=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] dup_rows_per_dst=[3020, 2948, 2894, 2874, 3178, 2900, 3046, 3152] combine_rows_per_src=[9270, 9256, 9042, 9144, 9472, 9328, 9380, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344144 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141200 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938256 weight_movement_bytes=34603008 total_nvlink_bytes=568541264 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343896 combine_payload_bytes=266575872 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266919768 ingress_bytes=370384896 link_bytes=370384896 token_bytes=533495640 weight_movement_bytes=103809024 total_nvlink_bytes=637304664 weight_to_token_ratio=0.194583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393728545e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266526416 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709072 weight_movement_bytes=103809024 total_nvlink_bytes=636518096 weight_to_token_ratio=0.194870 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65088 dup_rows=24008 dup_rows_remote=20982 dedup_ratio=0.244222 router_tokens_per_expert=[769, 812, 777, 751, 785, 810, 784, 776, 741, 738, 789, 773, 777, 761, 783, 727, 796, 729, 762, 759, 784, 751, 790, 803, 744, 784, 739, 835, 838, 753, 772, 758, 809, 743, 785, 774, 787, 782, 752, 773, 739, 778, 737, 753, 750, 809, 791, 729, 739, 777, 782, 740, 770, 732, 803, 774, 737, 766, 752, 752, 714, 745, 740, 788] payload_rows_per_dst=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] dup_rows_per_dst=[3074, 2988, 2992, 2960, 3050, 3028, 3026, 2890] combine_rows_per_src=[9454, 9190, 9356, 9486, 9360, 9144, 9208, 9098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=343888 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267140944 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938000 weight_movement_bytes=34603008 total_nvlink_bytes=568541008 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64886 dup_rows=24198 dup_rows_remote=21110 dedup_ratio=0.246155 router_tokens_per_expert=[747, 778, 836, 774, 774, 767, 753, 818, 771, 760, 731, 769, 786, 705, 721, 764, 762, 766, 774, 777, 830, 770, 777, 723, 807, 753, 782, 781, 749, 783, 753, 758, 780, 751, 792, 750, 800, 793, 763, 725, 806, 772, 742, 737, 781, 756, 794, 753, 775, 735, 785, 768, 769, 729, 830, 755, 743, 777, 768, 801, 740, 794, 726, 763] payload_rows_per_dst=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] dup_rows_per_dst=[3160, 2794, 3058, 3022, 3088, 3074, 2980, 3022] combine_rows_per_src=[9334, 9220, 9300, 9310, 9220, 9208, 9312, 9202] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65192 dup_rows=23834 dup_rows_remote=20628 dedup_ratio=0.242452 router_tokens_per_expert=[743, 719, 764, 770, 744, 769, 795, 775, 797, 787, 781, 774, 748, 797, 796, 744, 811, 803, 726, 796, 768, 797, 787, 785, 755, 773, 833, 750, 762, 721, 778, 759, 747, 745, 763, 742, 765, 795, 755, 748, 752, 778, 783, 751, 743, 822, 760, 777, 792, 818, 748, 773, 737, 815, 731, 745, 742, 781, 777, 717, 781, 801, 729, 732] payload_rows_per_dst=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] dup_rows_per_dst=[3206, 3014, 3070, 2912, 2932, 3094, 2778, 2828] combine_rows_per_src=[9278, 9532, 9476, 9350, 9188, 9238, 9116, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343008 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266140640 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531938272 weight_movement_bytes=34603008 total_nvlink_bytes=566541280 weight_to_token_ratio=0.065051 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=343032 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265247736 ingress_bytes=368713728 link_bytes=368713728 token_bytes=530152440 weight_movement_bytes=103809024 total_nvlink_bytes=633961464 weight_to_token_ratio=0.195810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387440075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344352 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805536 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266720 weight_movement_bytes=69206016 total_nvlink_bytes=602472736 weight_to_token_ratio=0.129777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344144 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141200 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938256 weight_movement_bytes=34603008 total_nvlink_bytes=568541264 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343952 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265944976 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546000 weight_movement_bytes=103809024 total_nvlink_bytes=635355024 weight_to_token_ratio=0.195296 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344280 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944728 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545176 weight_movement_bytes=34603008 total_nvlink_bytes=568148184 weight_to_token_ratio=0.064855 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343984 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117040 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890096 weight_movement_bytes=34603008 total_nvlink_bytes=566493104 weight_to_token_ratio=0.065057 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343280 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267369712 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534396144 weight_movement_bytes=34603008 total_nvlink_bytes=568999152 weight_to_token_ratio=0.064752 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65016 dup_rows=23982 dup_rows_remote=21004 dedup_ratio=0.243958 router_tokens_per_expert=[776, 777, 763, 781, 745, 769, 774, 739, 757, 719, 738, 738, 772, 780, 765, 809, 828, 793, 768, 785, 781, 744, 751, 762, 798, 786, 712, 820, 786, 725, 767, 758, 792, 762, 772, 718, 807, 736, 767, 770, 785, 804, 776, 724, 780, 744, 782, 747, 780, 796, 789, 764, 720, 770, 750, 746, 786, 772, 812, 766, 821, 744, 736, 768] payload_rows_per_dst=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] dup_rows_per_dst=[3042, 2932, 2962, 3030, 2988, 2978, 2996, 3054] combine_rows_per_src=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64686 dup_rows=24182 dup_rows_remote=21154 dedup_ratio=0.245992 router_tokens_per_expert=[788, 726, 761, 794, 767, 781, 772, 787, 775, 789, 792, 753, 847, 813, 740, 723, 743, 797, 749, 733, 760, 790, 759, 766, 779, 748, 754, 753, 741, 714, 862, 754, 762, 762, 780, 791, 813, 800, 816, 742, 728, 751, 781, 723, 739, 774, 772, 770, 766, 777, 770, 762, 801, 707, 775, 734, 791, 787, 765, 751, 771, 755, 776, 750] payload_rows_per_dst=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] dup_rows_per_dst=[2984, 3028, 2992, 3074, 3138, 2970, 2990, 3006] combine_rows_per_src=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64856 dup_rows=24008 dup_rows_remote=20944 dedup_ratio=0.244222 router_tokens_per_expert=[782, 719, 778, 735, 753, 773, 756, 797, 738, 768, 806, 769, 751, 775, 761, 740, 779, 788, 768, 758, 834, 791, 776, 758, 790, 747, 766, 767, 763, 746, 704, 797, 756, 736, 785, 745, 793, 763, 779, 744, 761, 745, 796, 723, 832, 813, 772, 706, 785, 765, 783, 745, 743, 755, 760, 805, 785, 757, 791, 816, 806, 810, 746, 718] payload_rows_per_dst=[9152, 9278, 9440, 9208, 9230, 9290, 9282, 9416] dup_rows_per_dst=[3034, 2938, 3064, 2952, 2972, 3006, 3000, 3042] combine_rows_per_src=[9152, 9278, 9440, 9208, 9230, 9290, 9282, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65144 dup_rows=23952 dup_rows_remote=20922 dedup_ratio=0.243652 router_tokens_per_expert=[754, 742, 777, 790, 767, 797, 780, 787, 786, 768, 755, 769, 742, 770, 719, 776, 819, 768, 797, 761, 735, 742, 789, 732, 771, 803, 732, 805, 800, 805, 778, 746, 747, 747, 725, 762, 758, 760, 764, 781, 778, 777, 719, 773, 771, 767, 805, 749, 802, 737, 773, 792, 773, 826, 764, 777, 787, 723, 764, 741, 709, 794, 738, 807] payload_rows_per_dst=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] dup_rows_per_dst=[2920, 2952, 2934, 2840, 2874, 3154, 3248, 3030] combine_rows_per_src=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65396 dup_rows=23908 dup_rows_remote=20960 dedup_ratio=0.243205 router_tokens_per_expert=[784, 746, 754, 774, 718, 785, 755, 744, 769, 760, 791, 746, 746, 771, 807, 711, 834, 760, 795, 742, 794, 806, 745, 789, 797, 828, 780, 754, 795, 768, 770, 800, 763, 714, 752, 717, 779, 795, 787, 801, 825, 761, 853, 725, 801, 743, 779, 738, 706, 717, 738, 775, 753, 733, 775, 777, 756, 713, 764, 775, 779, 791, 785, 764] payload_rows_per_dst=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] dup_rows_per_dst=[2890, 2946, 3080, 3164, 2916, 3040, 2948, 2924] combine_rows_per_src=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65054 dup_rows=24026 dup_rows_remote=21034 dedup_ratio=0.244405 router_tokens_per_expert=[769, 780, 768, 734, 787, 783, 758, 776, 763, 761, 722, 777, 751, 777, 824, 744, 745, 775, 761, 755, 779, 739, 754, 772, 800, 750, 742, 789, 727, 784, 728, 788, 762, 757, 771, 790, 775, 785, 798, 760, 762, 771, 771, 762, 795, 794, 832, 720, 756, 776, 792, 786, 770, 771, 768, 736, 789, 809, 773, 779, 743, 729, 749, 759] payload_rows_per_dst=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] dup_rows_per_dst=[2972, 2904, 2988, 2992, 3066, 3036, 3044, 3024] combine_rows_per_src=[9338, 9334, 9172, 9224, 9330, 9378, 9266, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65016 dup_rows=23982 dup_rows_remote=21004 dedup_ratio=0.243958 router_tokens_per_expert=[776, 777, 763, 781, 745, 769, 774, 739, 757, 719, 738, 738, 772, 780, 765, 809, 828, 793, 768, 785, 781, 744, 751, 762, 798, 786, 712, 820, 786, 725, 767, 758, 792, 762, 772, 718, 807, 736, 767, 770, 785, 804, 776, 724, 780, 744, 782, 747, 780, 796, 789, 764, 720, 770, 750, 746, 786, 772, 812, 766, 821, 744, 736, 768] payload_rows_per_dst=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] dup_rows_per_dst=[3042, 2932, 2962, 3030, 2988, 2978, 2996, 3054] combine_rows_per_src=[9206, 9224, 9462, 9274, 9260, 9306, 9234, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64992 dup_rows=24034 dup_rows_remote=21098 dedup_ratio=0.244486 router_tokens_per_expert=[695, 754, 769, 761, 745, 740, 726, 794, 819, 806, 751, 809, 736, 803, 774, 791, 746, 770, 767, 764, 749, 733, 784, 749, 754, 777, 756, 781, 753, 780, 822, 684, 750, 774, 744, 757, 778, 764, 786, 808, 753, 774, 782, 760, 786, 751, 797, 772, 768, 818, 780, 828, 716, 752, 745, 758, 785, 780, 794, 772, 777, 770, 775, 756] payload_rows_per_dst=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] dup_rows_per_dst=[2912, 3128, 2880, 2936, 3052, 3086, 3052, 2988] combine_rows_per_src=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64686 dup_rows=24182 dup_rows_remote=21154 dedup_ratio=0.245992 router_tokens_per_expert=[788, 726, 761, 794, 767, 781, 772, 787, 775, 789, 792, 753, 847, 813, 740, 723, 743, 797, 749, 733, 760, 790, 759, 766, 779, 748, 754, 753, 741, 714, 862, 754, 762, 762, 780, 791, 813, 800, 816, 742, 728, 751, 781, 723, 739, 774, 772, 770, 766, 777, 770, 762, 801, 707, 775, 734, 791, 787, 765, 751, 771, 755, 776, 750] payload_rows_per_dst=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] dup_rows_per_dst=[2984, 3028, 2992, 3074, 3138, 2970, 2990, 3006] combine_rows_per_src=[9368, 9436, 9202, 9136, 9394, 9106, 9194, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344080 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649616 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955152 weight_movement_bytes=34603008 total_nvlink_bytes=567558160 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65144 dup_rows=23952 dup_rows_remote=20922 dedup_ratio=0.243652 router_tokens_per_expert=[754, 742, 777, 790, 767, 797, 780, 787, 786, 768, 755, 769, 742, 770, 719, 776, 819, 768, 797, 761, 735, 742, 789, 732, 771, 803, 732, 805, 800, 805, 778, 746, 747, 747, 725, 762, 758, 760, 764, 781, 778, 777, 719, 773, 771, 767, 805, 749, 802, 737, 773, 792, 773, 826, 764, 777, 787, 723, 764, 741, 709, 794, 738, 807] payload_rows_per_dst=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] dup_rows_per_dst=[2920, 2952, 2934, 2840, 2874, 3154, 3248, 3030] combine_rows_per_src=[9116, 9218, 9240, 9314, 9214, 9476, 9566, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264953856 dispatch_meta_bytes=343360 combine_payload_bytes=264953856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265297216 ingress_bytes=299556864 link_bytes=299556864 token_bytes=530251072 weight_movement_bytes=34603008 total_nvlink_bytes=564854080 weight_to_token_ratio=0.065258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127208363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343200 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993376 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643552 weight_movement_bytes=103809024 total_nvlink_bytes=635452576 weight_to_token_ratio=0.195261 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344352 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805536 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266720 weight_movement_bytes=69206016 total_nvlink_bytes=602472736 weight_to_token_ratio=0.129777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64844 dup_rows=24230 dup_rows_remote=21144 dedup_ratio=0.246480 router_tokens_per_expert=[763, 768, 799, 815, 781, 777, 819, 771, 788, 760, 813, 751, 741, 765, 763, 779, 746, 738, 776, 723, 799, 809, 716, 779, 764, 800, 793, 806, 818, 757, 722, 754, 725, 803, 805, 747, 777, 720, 798, 783, 739, 766, 714, 808, 721, 723, 775, 760, 744, 752, 802, 813, 768, 754, 731, 814, 770, 751, 767, 706, 710, 806, 814, 733] payload_rows_per_dst=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] dup_rows_per_dst=[3158, 3056, 2968, 3102, 3086, 2844, 3072, 2944] combine_rows_per_src=[9428, 9264, 9204, 9326, 9230, 9168, 9284, 9170] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65202 dup_rows=23790 dup_rows_remote=20858 dedup_ratio=0.242004 router_tokens_per_expert=[717, 780, 750, 726, 727, 746, 810, 754, 774, 758, 804, 784, 784, 780, 784, 780, 761, 770, 753, 742, 779, 748, 757, 788, 798, 748, 739, 837, 780, 809, 740, 776, 743, 796, 728, 746, 772, 785, 796, 756, 762, 754, 727, 737, 722, 769, 780, 746, 778, 759, 786, 795, 769, 739, 753, 816, 779, 754, 808, 785, 799, 817, 771, 742] payload_rows_per_dst=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] dup_rows_per_dst=[2844, 3052, 2886, 3038, 2932, 2862, 3078, 3098] combine_rows_per_src=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74046 payload_rows_remote=64674 dup_rows=24258 dup_rows_remote=21084 dedup_ratio=0.246765 router_tokens_per_expert=[779, 765, 752, 719, 758, 827, 767, 760, 772, 760, 845, 743, 792, 747, 765, 763, 778, 739, 754, 760, 777, 755, 782, 780, 712, 758, 779, 767, 823, 739, 801, 797, 769, 753, 765, 736, 744, 762, 766, 783, 779, 802, 809, 801, 780, 708, 720, 771, 787, 783, 760, 820, 804, 788, 755, 776, 760, 721, 733, 781, 741, 747, 776, 757] payload_rows_per_dst=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] dup_rows_per_dst=[3008, 3134, 2996, 3116, 2950, 3020, 3174, 2860] combine_rows_per_src=[9246, 9240, 9254, 9236, 9206, 9320, 9372, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344080 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649616 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955152 weight_movement_bytes=34603008 total_nvlink_bytes=567558160 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64848 dup_rows=24114 dup_rows_remote=20990 dedup_ratio=0.245300 router_tokens_per_expert=[773, 811, 763, 747, 796, 759, 748, 783, 752, 788, 789, 845, 784, 768, 782, 750, 758, 764, 739, 796, 774, 759, 764, 751, 765, 767, 749, 732, 801, 755, 752, 749, 800, 829, 719, 797, 755, 760, 784, 795, 785, 760, 794, 736, 750, 728, 741, 777, 775, 755, 734, 751, 738, 766, 788, 783, 784, 740, 787, 766, 727, 760, 775, 800] payload_rows_per_dst=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] dup_rows_per_dst=[3124, 3070, 2918, 3014, 3000, 2940, 3032, 3016] combine_rows_per_src=[9342, 9468, 9228, 9234, 9306, 9202, 9148, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=345424 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268207440 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536069456 weight_movement_bytes=34603008 total_nvlink_bytes=570672464 weight_to_token_ratio=0.064549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344360 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551592 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758824 weight_movement_bytes=34603008 total_nvlink_bytes=567361832 weight_to_token_ratio=0.064951 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344264 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174088 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003912 weight_movement_bytes=34603008 total_nvlink_bytes=568606920 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264953856 dispatch_meta_bytes=343360 combine_payload_bytes=264953856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265297216 ingress_bytes=299556864 link_bytes=299556864 token_bytes=530251072 weight_movement_bytes=34603008 total_nvlink_bytes=564854080 weight_to_token_ratio=0.065258 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127208363e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344240 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411632 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534479024 weight_movement_bytes=34603008 total_nvlink_bytes=569082032 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343952 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265944976 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546000 weight_movement_bytes=103809024 total_nvlink_bytes=635355024 weight_to_token_ratio=0.195296 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264904704 dispatch_meta_bytes=343032 combine_payload_bytes=264904704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265247736 ingress_bytes=368713728 link_bytes=368713728 token_bytes=530152440 weight_movement_bytes=103809024 total_nvlink_bytes=633961464 weight_to_token_ratio=0.195810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387440075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265960760 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578168 weight_movement_bytes=69206016 total_nvlink_bytes=600784184 weight_to_token_ratio=0.130190 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65316 dup_rows=23940 dup_rows_remote=21066 dedup_ratio=0.243530 router_tokens_per_expert=[740, 733, 825, 755, 790, 825, 813, 752, 751, 805, 764, 802, 816, 792, 787, 812, 749, 859, 760, 781, 794, 776, 719, 730, 762, 766, 754, 793, 759, 779, 773, 795, 801, 740, 767, 734, 760, 764, 743, 752, 728, 729, 735, 729, 745, 756, 777, 762, 730, 758, 802, 741, 749, 782, 807, 759, 776, 782, 743, 787, 759, 772, 737, 735] payload_rows_per_dst=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] dup_rows_per_dst=[3070, 3132, 2934, 3092, 2936, 2874, 3024, 2878] combine_rows_per_src=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64938 dup_rows=24198 dup_rows_remote=21024 dedup_ratio=0.246155 router_tokens_per_expert=[761, 716, 755, 799, 757, 734, 830, 744, 773, 779, 750, 785, 747, 757, 785, 795, 739, 811, 737, 784, 768, 742, 821, 776, 726, 740, 769, 767, 780, 766, 757, 779, 772, 784, 721, 766, 729, 775, 782, 811, 801, 738, 763, 754, 820, 754, 705, 803, 707, 786, 769, 813, 791, 796, 749, 768, 798, 765, 781, 762, 777, 768, 762, 753] payload_rows_per_dst=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] dup_rows_per_dst=[2930, 3174, 3044, 2942, 3052, 3006, 3018, 3032] combine_rows_per_src=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65094 dup_rows=23858 dup_rows_remote=20806 dedup_ratio=0.242696 router_tokens_per_expert=[791, 805, 750, 719, 773, 787, 759, 751, 767, 805, 814, 682, 795, 778, 784, 767, 794, 752, 746, 761, 812, 732, 788, 817, 778, 809, 784, 704, 749, 781, 762, 715, 787, 801, 748, 763, 753, 800, 789, 753, 731, 768, 750, 790, 765, 745, 806, 761, 731, 753, 778, 778, 754, 748, 797, 738, 739, 787, 792, 800, 750, 761, 732, 793] payload_rows_per_dst=[9262, 9308, 9352, 9222, 9352, 9324, 9304, 9322] dup_rows_per_dst=[3008, 3076, 3052, 2942, 3036, 2908, 2850, 2986] combine_rows_per_src=[9262, 9308, 9352, 9222, 9352, 9324, 9304, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65192 dup_rows=23798 dup_rows_remote=20790 dedup_ratio=0.242086 router_tokens_per_expert=[807, 751, 795, 806, 708, 756, 740, 779, 751, 783, 783, 754, 692, 757, 719, 782, 771, 749, 763, 742, 800, 764, 769, 775, 792, 806, 753, 761, 757, 764, 772, 764, 753, 741, 792, 754, 801, 767, 752, 750, 732, 756, 765, 800, 755, 802, 807, 759, 784, 764, 749, 755, 767, 780, 780, 782, 749, 785, 778, 763, 856, 750, 787, 772] payload_rows_per_dst=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] dup_rows_per_dst=[2984, 2838, 2988, 2996, 2882, 3014, 3008, 3088] combine_rows_per_src=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64992 dup_rows=24034 dup_rows_remote=21098 dedup_ratio=0.244486 router_tokens_per_expert=[695, 754, 769, 761, 745, 740, 726, 794, 819, 806, 751, 809, 736, 803, 774, 791, 746, 770, 767, 764, 749, 733, 784, 749, 754, 777, 756, 781, 753, 780, 822, 684, 750, 774, 744, 757, 778, 764, 786, 808, 753, 774, 782, 760, 786, 751, 797, 772, 768, 818, 780, 828, 716, 752, 745, 758, 785, 780, 794, 772, 777, 770, 775, 756] payload_rows_per_dst=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] dup_rows_per_dst=[2912, 3128, 2880, 2936, 3052, 3086, 3052, 2988] combine_rows_per_src=[9056, 9450, 9244, 9278, 9270, 9264, 9278, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65316 dup_rows=23940 dup_rows_remote=21066 dedup_ratio=0.243530 router_tokens_per_expert=[740, 733, 825, 755, 790, 825, 813, 752, 751, 805, 764, 802, 816, 792, 787, 812, 749, 859, 760, 781, 794, 776, 719, 730, 762, 766, 754, 793, 759, 779, 773, 795, 801, 740, 767, 734, 760, 764, 743, 752, 728, 729, 735, 729, 745, 756, 777, 762, 730, 758, 802, 741, 749, 782, 807, 759, 776, 782, 743, 787, 759, 772, 737, 735] payload_rows_per_dst=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] dup_rows_per_dst=[3070, 3132, 2934, 3092, 2936, 2874, 3024, 2878] combine_rows_per_src=[9396, 9526, 9402, 9270, 9186, 9048, 9232, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65074 dup_rows=23892 dup_rows_remote=20974 dedup_ratio=0.243042 router_tokens_per_expert=[743, 798, 771, 735, 722, 755, 746, 756, 776, 754, 797, 832, 794, 787, 760, 787, 776, 788, 698, 705, 804, 766, 792, 763, 745, 772, 784, 796, 769, 728, 756, 798, 788, 736, 767, 759, 784, 753, 755, 738, 788, 812, 708, 767, 813, 741, 818, 796, 770, 741, 759, 769, 785, 807, 748, 784, 776, 760, 754, 752, 778, 764, 777, 752] payload_rows_per_dst=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] dup_rows_per_dst=[2868, 2854, 2918, 2960, 2986, 3314, 3074, 2918] combine_rows_per_src=[9184, 9102, 9266, 9372, 9174, 9680, 9296, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65204 dup_rows=23924 dup_rows_remote=21012 dedup_ratio=0.243368 router_tokens_per_expert=[829, 763, 729, 754, 771, 801, 689, 781, 770, 761, 777, 781, 748, 804, 820, 738, 761, 788, 771, 757, 750, 801, 778, 780, 736, 755, 777, 756, 727, 764, 782, 747, 720, 789, 816, 786, 758, 798, 758, 793, 780, 812, 803, 761, 746, 808, 740, 764, 744, 743, 778, 748, 748, 777, 750, 721, 784, 786, 760, 758, 747, 797, 732, 801] payload_rows_per_dst=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] dup_rows_per_dst=[2994, 2994, 3100, 2912, 2946, 3166, 2818, 2994] combine_rows_per_src=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64938 dup_rows=24198 dup_rows_remote=21024 dedup_ratio=0.246155 router_tokens_per_expert=[761, 716, 755, 799, 757, 734, 830, 744, 773, 779, 750, 785, 747, 757, 785, 795, 739, 811, 737, 784, 768, 742, 821, 776, 726, 740, 769, 767, 780, 766, 757, 779, 772, 784, 721, 766, 729, 775, 782, 811, 801, 738, 763, 754, 820, 754, 705, 803, 707, 786, 769, 813, 791, 796, 749, 768, 798, 765, 781, 762, 777, 768, 762, 753] payload_rows_per_dst=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] dup_rows_per_dst=[2930, 3174, 3044, 2942, 3052, 3006, 3018, 3032] combine_rows_per_src=[9262, 9168, 9312, 9226, 9228, 9270, 9340, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64742 dup_rows=23832 dup_rows_remote=20510 dedup_ratio=0.242432 router_tokens_per_expert=[764, 804, 823, 783, 802, 734, 798, 800, 750, 780, 726, 772, 764, 794, 755, 801, 722, 747, 738, 770, 792, 786, 774, 786, 759, 789, 765, 753, 838, 744, 768, 720, 768, 737, 744, 799, 775, 755, 775, 763, 732, 806, 795, 786, 734, 731, 767, 741, 763, 800, 770, 787, 796, 774, 767, 773, 777, 695, 724, 790, 767, 729, 790, 741] payload_rows_per_dst=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] dup_rows_per_dst=[3322, 2792, 3068, 3038, 3100, 2860, 2812, 2840] combine_rows_per_src=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64848 dup_rows=24158 dup_rows_remote=21114 dedup_ratio=0.245748 router_tokens_per_expert=[755, 775, 771, 792, 707, 749, 793, 758, 781, 784, 757, 780, 778, 744, 746, 786, 727, 771, 838, 757, 754, 777, 720, 770, 738, 796, 775, 761, 743, 798, 785, 750, 770, 788, 801, 766, 750, 747, 768, 781, 762, 749, 770, 754, 785, 777, 722, 748, 795, 810, 774, 749, 755, 803, 738, 781, 784, 780, 787, 744, 807, 785, 753, 753] payload_rows_per_dst=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] dup_rows_per_dst=[2982, 3154, 2944, 3038, 3044, 2876, 3030, 3090] combine_rows_per_src=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343600 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266968624 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533593648 weight_movement_bytes=34603008 total_nvlink_bytes=568196656 weight_to_token_ratio=0.064849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343848 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=266329896 ingress_bytes=439001088 link_bytes=439001088 token_bytes=532315944 weight_movement_bytes=173015040 total_nvlink_bytes=705330984 weight_to_token_ratio=0.325023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651925752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344360 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551592 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758824 weight_movement_bytes=34603008 total_nvlink_bytes=567361832 weight_to_token_ratio=0.064951 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343928 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370360 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396792 weight_movement_bytes=69206016 total_nvlink_bytes=603602808 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65202 dup_rows=23790 dup_rows_remote=20858 dedup_ratio=0.242004 router_tokens_per_expert=[717, 780, 750, 726, 727, 746, 810, 754, 774, 758, 804, 784, 784, 780, 784, 780, 761, 770, 753, 742, 779, 748, 757, 788, 798, 748, 739, 837, 780, 809, 740, 776, 743, 796, 728, 746, 772, 785, 796, 756, 762, 754, 727, 737, 722, 769, 780, 746, 778, 759, 786, 795, 769, 739, 753, 816, 779, 754, 808, 785, 799, 817, 771, 742] payload_rows_per_dst=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] dup_rows_per_dst=[2844, 3052, 2886, 3038, 2932, 2862, 3078, 3098] combine_rows_per_src=[9176, 9444, 9310, 9416, 9312, 9132, 9312, 9412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345528 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879864 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535414200 weight_movement_bytes=34603008 total_nvlink_bytes=570017208 weight_to_token_ratio=0.064628 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343600 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266968624 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533593648 weight_movement_bytes=34603008 total_nvlink_bytes=568196656 weight_to_token_ratio=0.064849 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64742 dup_rows=23832 dup_rows_remote=20510 dedup_ratio=0.242432 router_tokens_per_expert=[764, 804, 823, 783, 802, 734, 798, 800, 750, 780, 726, 772, 764, 794, 755, 801, 722, 747, 738, 770, 792, 786, 774, 786, 759, 789, 765, 753, 838, 744, 768, 720, 768, 737, 744, 799, 775, 755, 775, 763, 732, 806, 795, 786, 734, 731, 767, 741, 763, 800, 770, 787, 796, 774, 767, 773, 777, 695, 724, 790, 767, 729, 790, 741] payload_rows_per_dst=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] dup_rows_per_dst=[3322, 2792, 3068, 3038, 3100, 2860, 2812, 2840] combine_rows_per_src=[9730, 9056, 9370, 9234, 9468, 9324, 9312, 8978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65396 dup_rows=23908 dup_rows_remote=20960 dedup_ratio=0.243205 router_tokens_per_expert=[784, 746, 754, 774, 718, 785, 755, 744, 769, 760, 791, 746, 746, 771, 807, 711, 834, 760, 795, 742, 794, 806, 745, 789, 797, 828, 780, 754, 795, 768, 770, 800, 763, 714, 752, 717, 779, 795, 787, 801, 825, 761, 853, 725, 801, 743, 779, 738, 706, 717, 738, 775, 753, 733, 775, 777, 756, 713, 764, 775, 779, 791, 785, 764] payload_rows_per_dst=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] dup_rows_per_dst=[2890, 2946, 3080, 3164, 2916, 3040, 2948, 2924] combine_rows_per_src=[9230, 9256, 9450, 9420, 9300, 9410, 9000, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344192 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266887296 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430400 weight_movement_bytes=69206016 total_nvlink_bytes=602636416 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344864 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267420448 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534496032 weight_movement_bytes=69206016 total_nvlink_bytes=603702048 weight_to_token_ratio=0.129479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343848 combine_payload_bytes=265986048 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=266329896 ingress_bytes=439001088 link_bytes=439001088 token_bytes=532315944 weight_movement_bytes=173015040 total_nvlink_bytes=705330984 weight_to_token_ratio=0.325023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651925752e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344240 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267411632 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534479024 weight_movement_bytes=34603008 total_nvlink_bytes=569082032 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341008 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524240 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530707472 weight_movement_bytes=34603008 total_nvlink_bytes=565310480 weight_to_token_ratio=0.065202 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343848 combine_payload_bytes=265617408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265961256 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578664 weight_movement_bytes=69206016 total_nvlink_bytes=600784680 weight_to_token_ratio=0.130190 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65114 dup_rows=23920 dup_rows_remote=20858 dedup_ratio=0.243327 router_tokens_per_expert=[770, 735, 772, 759, 755, 752, 753, 743, 754, 808, 750, 769, 793, 756, 757, 722, 746, 729, 814, 797, 801, 768, 741, 811, 779, 758, 741, 818, 759, 759, 772, 818, 804, 714, 752, 773, 753, 810, 778, 740, 776, 800, 763, 790, 768, 718, 775, 763, 790, 783, 785, 775, 741, 760, 789, 743, 749, 800, 759, 827, 718, 812, 759, 726] payload_rows_per_dst=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] dup_rows_per_dst=[2932, 2992, 3084, 3076, 3004, 2906, 3062, 2864] combine_rows_per_src=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267862016 dispatch_meta_bytes=345424 combine_payload_bytes=267862016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268207440 ingress_bytes=302465024 link_bytes=302465024 token_bytes=536069456 weight_movement_bytes=34603008 total_nvlink_bytes=570672464 weight_to_token_ratio=0.064549 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138151535e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64660 dup_rows=24178 dup_rows_remote=21016 dedup_ratio=0.245951 router_tokens_per_expert=[808, 754, 710, 754, 725, 714, 744, 747, 783, 804, 785, 762, 794, 743, 783, 777, 745, 793, 792, 816, 774, 796, 800, 798, 778, 804, 732, 794, 760, 754, 739, 710, 760, 758, 754, 794, 776, 780, 810, 795, 751, 716, 774, 713, 747, 763, 794, 758, 752, 775, 764, 743, 790, 785, 741, 755, 814, 790, 785, 788, 778, 746, 765, 766] payload_rows_per_dst=[9052, 9320, 9466, 9168, 9400, 9074, 9246, 9400] dup_rows_per_dst=[2860, 3142, 3162, 2974, 3054, 2958, 2964, 3064] combine_rows_per_src=[9052, 9320, 9466, 9168, 9400, 9074, 9246, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20878 dedup_ratio=0.244059 router_tokens_per_expert=[754, 779, 782, 779, 741, 774, 763, 824, 716, 720, 821, 764, 711, 734, 793, 758, 748, 747, 803, 759, 791, 755, 731, 805, 712, 825, 723, 772, 777, 800, 770, 787, 765, 744, 790, 814, 783, 740, 786, 750, 798, 799, 783, 780, 800, 740, 757, 809, 778, 791, 771, 764, 778, 727, 770, 736, 764, 763, 773, 761, 745, 748, 826, 701] payload_rows_per_dst=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] dup_rows_per_dst=[2968, 2818, 3068, 3004, 3056, 3114, 3016, 2948] combine_rows_per_src=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65068 dup_rows=23986 dup_rows_remote=21060 dedup_ratio=0.243998 router_tokens_per_expert=[780, 772, 750, 765, 767, 748, 779, 792, 770, 770, 769, 796, 723, 816, 780, 722, 764, 741, 733, 803, 739, 729, 787, 797, 784, 773, 775, 813, 772, 801, 745, 784, 774, 760, 746, 756, 716, 749, 787, 722, 747, 766, 792, 760, 727, 774, 836, 782, 789, 762, 747, 762, 795, 802, 778, 796, 769, 760, 801, 774, 750, 733, 760, 741] payload_rows_per_dst=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] dup_rows_per_dst=[2992, 3008, 2968, 3078, 2912, 3050, 3052, 2926] combine_rows_per_src=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64892 dup_rows=24194 dup_rows_remote=21134 dedup_ratio=0.246114 router_tokens_per_expert=[763, 765, 798, 782, 765, 747, 766, 733, 782, 750, 753, 780, 775, 763, 790, 797, 809, 791, 754, 740, 763, 719, 767, 740, 769, 744, 792, 791, 747, 729, 774, 793, 772, 775, 761, 794, 763, 749, 760, 783, 774, 788, 802, 728, 739, 794, 764, 772, 755, 777, 737, 767, 712, 766, 758, 804, 785, 748, 791, 726, 769, 827, 788, 793] payload_rows_per_dst=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] dup_rows_per_dst=[3008, 3040, 2942, 3060, 3050, 3054, 2952, 3088] combine_rows_per_src=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64644 dup_rows=24060 dup_rows_remote=20728 dedup_ratio=0.244751 router_tokens_per_expert=[767, 801, 792, 788, 770, 762, 780, 797, 757, 747, 845, 762, 765, 777, 718, 775, 782, 775, 737, 792, 704, 809, 720, 803, 752, 759, 747, 792, 745, 763, 762, 728, 774, 820, 800, 788, 731, 780, 795, 735, 752, 771, 741, 765, 736, 756, 796, 733, 769, 726, 779, 785, 749, 724, 801, 740, 777, 766, 801, 787, 773, 807, 753, 769] payload_rows_per_dst=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] dup_rows_per_dst=[3332, 3028, 2926, 2948, 2890, 2970, 3092, 2874] combine_rows_per_src=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64816 dup_rows=24178 dup_rows_remote=21304 dedup_ratio=0.245951 router_tokens_per_expert=[794, 783, 772, 737, 776, 773, 809, 715, 758, 787, 815, 785, 809, 782, 810, 738, 786, 772, 754, 753, 751, 773, 765, 784, 749, 765, 772, 780, 761, 822, 724, 770, 752, 769, 797, 784, 733, 723, 742, 792, 778, 742, 755, 803, 741, 750, 775, 768, 712, 748, 707, 772, 772, 783, 740, 793, 718, 774, 749, 838, 775, 781, 817, 745] payload_rows_per_dst=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] dup_rows_per_dst=[3072, 3240, 3026, 2978, 2874, 2990, 2940, 3058] combine_rows_per_src=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74584 payload_rows_remote=65306 dup_rows=23720 dup_rows_remote=20804 dedup_ratio=0.241292 router_tokens_per_expert=[786, 757, 731, 790, 757, 780, 788, 736, 768, 785, 754, 758, 803, 736, 759, 734, 749, 758, 810, 755, 708, 755, 753, 729, 738, 781, 806, 785, 789, 804, 769, 763, 799, 747, 760, 778, 786, 805, 754, 789, 753, 778, 740, 780, 800, 759, 771, 768, 801, 748, 770, 759, 749, 806, 773, 721, 787, 778, 831, 754, 780, 781, 734, 739] payload_rows_per_dst=[9266, 9278, 9102, 9482, 9398, 9378, 9328, 9352] dup_rows_per_dst=[2984, 2916, 2932, 2988, 3038, 2920, 2926, 3016] combine_rows_per_src=[9266, 9278, 9102, 9482, 9398, 9378, 9328, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20878 dedup_ratio=0.244059 router_tokens_per_expert=[754, 779, 782, 779, 741, 774, 763, 824, 716, 720, 821, 764, 711, 734, 793, 758, 748, 747, 803, 759, 791, 755, 731, 805, 712, 825, 723, 772, 777, 800, 770, 787, 765, 744, 790, 814, 783, 740, 786, 750, 798, 799, 783, 780, 800, 740, 757, 809, 778, 791, 771, 764, 778, 727, 770, 736, 764, 763, 773, 761, 745, 748, 826, 701] payload_rows_per_dst=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] dup_rows_per_dst=[2968, 2818, 3068, 3004, 3056, 3114, 3016, 2948] combine_rows_per_src=[9424, 9216, 9210, 9328, 9288, 9418, 9214, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65204 dup_rows=23924 dup_rows_remote=21012 dedup_ratio=0.243368 router_tokens_per_expert=[829, 763, 729, 754, 771, 801, 689, 781, 770, 761, 777, 781, 748, 804, 820, 738, 761, 788, 771, 757, 750, 801, 778, 780, 736, 755, 777, 756, 727, 764, 782, 747, 720, 789, 816, 786, 758, 798, 758, 793, 780, 812, 803, 761, 746, 808, 740, 764, 744, 743, 778, 748, 748, 777, 750, 721, 784, 786, 760, 758, 747, 797, 732, 801] payload_rows_per_dst=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] dup_rows_per_dst=[2994, 2994, 3100, 2912, 2946, 3166, 2818, 2994] combine_rows_per_src=[9240, 9404, 9272, 9176, 9490, 9262, 9200, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343888 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267050832 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533757776 weight_movement_bytes=34603008 total_nvlink_bytes=568360784 weight_to_token_ratio=0.064829 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65068 dup_rows=23986 dup_rows_remote=21060 dedup_ratio=0.243998 router_tokens_per_expert=[780, 772, 750, 765, 767, 748, 779, 792, 770, 770, 769, 796, 723, 816, 780, 722, 764, 741, 733, 803, 739, 729, 787, 797, 784, 773, 775, 813, 772, 801, 745, 784, 774, 760, 746, 756, 716, 749, 787, 722, 747, 766, 792, 760, 727, 774, 836, 782, 789, 762, 747, 762, 795, 802, 778, 796, 769, 760, 801, 774, 750, 733, 760, 741] payload_rows_per_dst=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] dup_rows_per_dst=[2992, 3008, 2968, 3078, 2912, 3050, 3052, 2926] combine_rows_per_src=[9314, 9284, 9218, 9416, 9108, 9318, 9410, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=342704 combine_payload_bytes=264847360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265190064 ingress_bytes=334053376 link_bytes=334053376 token_bytes=530037424 weight_movement_bytes=69206016 total_nvlink_bytes=599243440 weight_to_token_ratio=0.130568 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257015960e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267493376 dispatch_meta_bytes=344440 combine_payload_bytes=267493376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267837816 ingress_bytes=302096384 link_bytes=302096384 token_bytes=535331192 weight_movement_bytes=34603008 total_nvlink_bytes=569934200 weight_to_token_ratio=0.064639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136764372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343088 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148912 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954736 weight_movement_bytes=34603008 total_nvlink_bytes=566557744 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64848 dup_rows=24158 dup_rows_remote=21114 dedup_ratio=0.245748 router_tokens_per_expert=[755, 775, 771, 792, 707, 749, 793, 758, 781, 784, 757, 780, 778, 744, 746, 786, 727, 771, 838, 757, 754, 777, 720, 770, 738, 796, 775, 761, 743, 798, 785, 750, 770, 788, 801, 766, 750, 747, 768, 781, 762, 749, 770, 754, 785, 777, 722, 748, 795, 810, 774, 749, 755, 803, 738, 781, 784, 780, 787, 744, 807, 785, 753, 753] payload_rows_per_dst=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] dup_rows_per_dst=[2982, 3154, 2944, 3038, 3044, 2876, 3030, 3090] combine_rows_per_src=[9218, 9158, 9284, 9254, 9298, 9258, 9380, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344864 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267420448 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534496032 weight_movement_bytes=69206016 total_nvlink_bytes=603702048 weight_to_token_ratio=0.129479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64644 dup_rows=24060 dup_rows_remote=20728 dedup_ratio=0.244751 router_tokens_per_expert=[767, 801, 792, 788, 770, 762, 780, 797, 757, 747, 845, 762, 765, 777, 718, 775, 782, 775, 737, 792, 704, 809, 720, 803, 752, 759, 747, 792, 745, 763, 762, 728, 774, 820, 800, 788, 731, 780, 795, 735, 752, 771, 741, 765, 736, 756, 796, 733, 769, 726, 779, 785, 749, 724, 801, 740, 777, 766, 801, 787, 773, 807, 753, 769] payload_rows_per_dst=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] dup_rows_per_dst=[3332, 3028, 2926, 2948, 2890, 2970, 3092, 2874] combine_rows_per_src=[9600, 9366, 9238, 9130, 9134, 9236, 9296, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65192 dup_rows=23798 dup_rows_remote=20790 dedup_ratio=0.242086 router_tokens_per_expert=[807, 751, 795, 806, 708, 756, 740, 779, 751, 783, 783, 754, 692, 757, 719, 782, 771, 749, 763, 742, 800, 764, 769, 775, 792, 806, 753, 761, 757, 764, 772, 764, 753, 741, 792, 754, 801, 767, 752, 750, 732, 756, 765, 800, 755, 802, 807, 759, 784, 764, 749, 755, 767, 780, 780, 782, 749, 785, 778, 763, 856, 750, 787, 772] payload_rows_per_dst=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] dup_rows_per_dst=[2984, 2838, 2988, 2996, 2882, 3014, 3008, 3088] combine_rows_per_src=[9300, 9204, 9278, 9342, 9338, 9338, 9314, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344512 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266863040 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381568 weight_movement_bytes=69206016 total_nvlink_bytes=602587584 weight_to_token_ratio=0.129750 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344480 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265830816 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531317152 weight_movement_bytes=69206016 total_nvlink_bytes=600523168 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344104 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266141736 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939368 weight_movement_bytes=103809024 total_nvlink_bytes=635748392 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343848 combine_payload_bytes=265617408 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265961256 ingress_bytes=334823424 link_bytes=334823424 token_bytes=531578664 weight_movement_bytes=69206016 total_nvlink_bytes=600784680 weight_to_token_ratio=0.130190 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259913589e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341488 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123312 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905136 weight_movement_bytes=34603008 total_nvlink_bytes=564508144 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267493376 dispatch_meta_bytes=344440 combine_payload_bytes=267493376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267837816 ingress_bytes=302096384 link_bytes=302096384 token_bytes=535331192 weight_movement_bytes=34603008 total_nvlink_bytes=569934200 weight_to_token_ratio=0.064639 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136764372e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65176 dup_rows=23820 dup_rows_remote=20810 dedup_ratio=0.242310 router_tokens_per_expert=[752, 788, 806, 788, 800, 803, 766, 784, 749, 739, 760, 745, 792, 773, 778, 736, 760, 764, 793, 783, 731, 775, 744, 784, 740, 736, 747, 773, 774, 802, 769, 785, 775, 706, 808, 736, 765, 741, 746, 770, 786, 785, 797, 785, 778, 732, 759, 750, 748, 760, 752, 782, 767, 776, 784, 790, 817, 785, 761, 779, 720, 784, 740, 769] payload_rows_per_dst=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] dup_rows_per_dst=[3170, 2866, 2990, 2934, 2844, 3006, 3010, 3000] combine_rows_per_src=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343928 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370360 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396792 weight_movement_bytes=69206016 total_nvlink_bytes=603602808 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65030 dup_rows=23970 dup_rows_remote=21016 dedup_ratio=0.243835 router_tokens_per_expert=[762, 729, 829, 774, 783, 713, 729, 757, 786, 780, 722, 786, 758, 750, 735, 761, 790, 730, 758, 765, 776, 757, 798, 755, 745, 823, 786, 796, 735, 788, 818, 762, 793, 757, 767, 752, 746, 832, 752, 762, 770, 797, 731, 784, 715, 757, 774, 766, 777, 803, 776, 764, 741, 814, 754, 784, 784, 755, 782, 751, 796, 748, 769, 763] payload_rows_per_dst=[9212, 9186, 9304, 9428, 9278, 9262, 9332, 9332] dup_rows_per_dst=[2940, 2970, 2954, 3078, 3044, 2926, 3094, 2964] combine_rows_per_src=[9212, 9186, 9304, 9428, 9278, 9262, 9332, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64632 dup_rows=24278 dup_rows_remote=21216 dedup_ratio=0.246969 router_tokens_per_expert=[794, 822, 772, 749, 729, 807, 722, 731, 729, 810, 735, 769, 789, 768, 782, 723, 740, 776, 805, 776, 766, 783, 795, 745, 818, 756, 771, 767, 784, 742, 750, 782, 782, 805, 800, 747, 790, 775, 768, 764, 780, 749, 748, 764, 783, 758, 774, 724, 767, 783, 766, 761, 788, 755, 780, 751, 751, 766, 741, 759, 811, 772, 782, 721] payload_rows_per_dst=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] dup_rows_per_dst=[2730, 3312, 3114, 2686, 3050, 3284, 3040, 3062] combine_rows_per_src=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65200 dup_rows=24050 dup_rows_remote=21124 dedup_ratio=0.244649 router_tokens_per_expert=[777, 774, 823, 767, 828, 761, 760, 767, 756, 811, 722, 764, 757, 742, 731, 707, 799, 791, 732, 762, 785, 738, 772, 723, 811, 795, 755, 779, 783, 776, 755, 800, 795, 816, 715, 790, 757, 797, 772, 798, 731, 761, 769, 773, 730, 739, 755, 740, 721, 793, 753, 806, 778, 771, 799, 759, 751, 741, 791, 790, 754, 778, 769, 757] payload_rows_per_dst=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] dup_rows_per_dst=[3106, 2926, 2908, 3116, 3062, 2912, 3024, 2996] combine_rows_per_src=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64912 dup_rows=24220 dup_rows_remote=21258 dedup_ratio=0.246379 router_tokens_per_expert=[781, 769, 761, 787, 738, 767, 747, 782, 805, 762, 761, 750, 733, 793, 719, 767, 710, 706, 799, 769, 774, 735, 775, 774, 764, 759, 778, 792, 785, 823, 794, 775, 728, 756, 792, 803, 747, 737, 746, 802, 755, 771, 774, 743, 747, 754, 775, 748, 738, 736, 845, 760, 825, 811, 776, 799, 773, 766, 701, 804, 770, 796, 781, 759] payload_rows_per_dst=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] dup_rows_per_dst=[3012, 3010, 2932, 3106, 2984, 2962, 3158, 3056] combine_rows_per_src=[9252, 9170, 9152, 9434, 9238, 9172, 9422, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64726 dup_rows=24266 dup_rows_remote=21280 dedup_ratio=0.246847 router_tokens_per_expert=[762, 776, 750, 760, 811, 778, 761, 760, 759, 768, 768, 789, 740, 760, 752, 789, 773, 733, 743, 800, 784, 773, 769, 780, 780, 791, 757, 792, 746, 819, 774, 758, 754, 759, 729, 730, 837, 760, 768, 812, 787, 770, 762, 759, 770, 753, 714, 778, 782, 783, 772, 763, 748, 766, 820, 704, 748, 745, 759, 773, 767, 760, 773, 792] payload_rows_per_dst=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] dup_rows_per_dst=[3028, 3048, 3152, 3064, 2986, 2886, 3112, 2990] combine_rows_per_src=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65118 dup_rows=23912 dup_rows_remote=20960 dedup_ratio=0.243245 router_tokens_per_expert=[725, 779, 749, 757, 774, 755, 784, 797, 735, 769, 801, 748, 785, 769, 753, 774, 760, 734, 771, 804, 823, 786, 747, 811, 762, 735, 802, 784, 800, 723, 736, 771, 737, 761, 741, 775, 787, 746, 774, 782, 779, 784, 805, 711, 820, 724, 813, 806, 760, 751, 764, 812, 766, 747, 799, 743, 724, 771, 784, 764, 725, 723, 787, 784] payload_rows_per_dst=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] dup_rows_per_dst=[2884, 3002, 3104, 2952, 2944, 3032, 3050, 2944] combine_rows_per_src=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64942 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[812, 817, 791, 798, 763, 745, 758, 786, 743, 767, 745, 787, 776, 757, 721, 788, 765, 790, 734, 757, 749, 753, 763, 795, 747, 765, 789, 734, 833, 806, 773, 765, 728, 808, 760, 784, 790, 718, 792, 762, 776, 743, 757, 752, 760, 764, 764, 763, 820, 742, 768, 751, 744, 798, 754, 734, 758, 763, 772, 774, 758, 798, 724, 801] payload_rows_per_dst=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] dup_rows_per_dst=[3118, 2856, 3174, 3102, 3014, 2978, 2940, 2758] combine_rows_per_src=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64892 dup_rows=24194 dup_rows_remote=21134 dedup_ratio=0.246114 router_tokens_per_expert=[763, 765, 798, 782, 765, 747, 766, 733, 782, 750, 753, 780, 775, 763, 790, 797, 809, 791, 754, 740, 763, 719, 767, 740, 769, 744, 792, 791, 747, 729, 774, 793, 772, 775, 761, 794, 763, 749, 760, 783, 774, 788, 802, 728, 739, 794, 764, 772, 755, 777, 737, 767, 712, 766, 758, 804, 785, 748, 791, 726, 769, 827, 788, 793] payload_rows_per_dst=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] dup_rows_per_dst=[3008, 3040, 2942, 3060, 3050, 3054, 2952, 3088] combine_rows_per_src=[9230, 9340, 9224, 9218, 9264, 9268, 9200, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65200 dup_rows=24050 dup_rows_remote=21124 dedup_ratio=0.244649 router_tokens_per_expert=[777, 774, 823, 767, 828, 761, 760, 767, 756, 811, 722, 764, 757, 742, 731, 707, 799, 791, 732, 762, 785, 738, 772, 723, 811, 795, 755, 779, 783, 776, 755, 800, 795, 816, 715, 790, 757, 797, 772, 798, 731, 761, 769, 773, 730, 739, 755, 740, 721, 793, 753, 806, 778, 771, 799, 759, 751, 741, 791, 790, 754, 778, 769, 757] payload_rows_per_dst=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] dup_rows_per_dst=[3106, 2926, 2908, 3116, 3062, 2912, 3024, 2996] combine_rows_per_src=[9408, 9054, 9296, 9392, 9418, 9084, 9336, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344184 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707064 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069944 weight_movement_bytes=34603008 total_nvlink_bytes=567672952 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=343944 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267304840 ingress_bytes=336166912 link_bytes=336166912 token_bytes=534265736 weight_movement_bytes=69206016 total_nvlink_bytes=603471752 weight_to_token_ratio=0.129535 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264969026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=343392 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265076064 ingress_bytes=333938688 link_bytes=333938688 token_bytes=529808736 weight_movement_bytes=69206016 total_nvlink_bytes=599014752 weight_to_token_ratio=0.130625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64632 dup_rows=24278 dup_rows_remote=21216 dedup_ratio=0.246969 router_tokens_per_expert=[794, 822, 772, 749, 729, 807, 722, 731, 729, 810, 735, 769, 789, 768, 782, 723, 740, 776, 805, 776, 766, 783, 795, 745, 818, 756, 771, 767, 784, 742, 750, 782, 782, 805, 800, 747, 790, 775, 768, 764, 780, 749, 748, 764, 783, 758, 774, 724, 767, 783, 766, 761, 788, 755, 780, 751, 751, 766, 741, 759, 811, 772, 782, 721] payload_rows_per_dst=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] dup_rows_per_dst=[2730, 3312, 3114, 2686, 3050, 3284, 3040, 3062] combine_rows_per_src=[8974, 9416, 9258, 8824, 9354, 9654, 9152, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=345296 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267404496 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463696 weight_movement_bytes=34603008 total_nvlink_bytes=569066704 weight_to_token_ratio=0.064743 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344680 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266224232 ingress_bytes=300482560 link_bytes=300482560 token_bytes=532103784 weight_movement_bytes=34603008 total_nvlink_bytes=566706792 weight_to_token_ratio=0.065031 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130691682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=344024 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265461720 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530579416 weight_movement_bytes=69206016 total_nvlink_bytes=599785432 weight_to_token_ratio=0.130435 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344312 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067640 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790968 weight_movement_bytes=34603008 total_nvlink_bytes=568393976 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64816 dup_rows=24178 dup_rows_remote=21304 dedup_ratio=0.245951 router_tokens_per_expert=[794, 783, 772, 737, 776, 773, 809, 715, 758, 787, 815, 785, 809, 782, 810, 738, 786, 772, 754, 753, 751, 773, 765, 784, 749, 765, 772, 780, 761, 822, 724, 770, 752, 769, 797, 784, 733, 723, 742, 792, 778, 742, 755, 803, 741, 750, 775, 768, 712, 748, 707, 772, 772, 783, 740, 793, 718, 774, 749, 838, 775, 781, 817, 745] payload_rows_per_dst=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] dup_rows_per_dst=[3072, 3240, 3026, 2978, 2874, 2990, 2940, 3058] combine_rows_per_src=[9246, 9328, 9250, 9308, 9310, 9234, 9114, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344104 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266141736 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939368 weight_movement_bytes=103809024 total_nvlink_bytes=635748392 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64942 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[812, 817, 791, 798, 763, 745, 758, 786, 743, 767, 745, 787, 776, 757, 721, 788, 765, 790, 734, 757, 749, 753, 763, 795, 747, 765, 789, 734, 833, 806, 773, 765, 728, 808, 760, 784, 790, 718, 792, 762, 776, 743, 757, 752, 760, 764, 764, 763, 820, 742, 768, 751, 744, 798, 754, 734, 758, 763, 772, 774, 758, 798, 724, 801] payload_rows_per_dst=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] dup_rows_per_dst=[3118, 2856, 3174, 3102, 3014, 2978, 2940, 2758] combine_rows_per_src=[9422, 9312, 9502, 9322, 9270, 9180, 9282, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65114 dup_rows=23920 dup_rows_remote=20858 dedup_ratio=0.243327 router_tokens_per_expert=[770, 735, 772, 759, 755, 752, 753, 743, 754, 808, 750, 769, 793, 756, 757, 722, 746, 729, 814, 797, 801, 768, 741, 811, 779, 758, 741, 818, 759, 759, 772, 818, 804, 714, 752, 773, 753, 810, 778, 740, 776, 800, 763, 790, 768, 718, 775, 763, 790, 783, 785, 775, 741, 760, 789, 743, 749, 800, 759, 827, 718, 812, 759, 726] payload_rows_per_dst=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] dup_rows_per_dst=[2932, 2992, 3084, 3076, 3004, 2906, 3062, 2864] combine_rows_per_src=[9146, 9226, 9330, 9332, 9244, 9400, 9270, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=343392 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265076064 ingress_bytes=333938688 link_bytes=333938688 token_bytes=529808736 weight_movement_bytes=69206016 total_nvlink_bytes=599014752 weight_to_token_ratio=0.130625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256584399e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344480 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265830816 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531317152 weight_movement_bytes=69206016 total_nvlink_bytes=600523168 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343056 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266345488 ingress_bytes=335208448 link_bytes=335208448 token_bytes=532347920 weight_movement_bytes=69206016 total_nvlink_bytes=601553936 weight_to_token_ratio=0.130001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261362403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=345296 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267404496 ingress_bytes=301662208 link_bytes=301662208 token_bytes=534463696 weight_movement_bytes=34603008 total_nvlink_bytes=569066704 weight_to_token_ratio=0.064743 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135130603e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65024 dup_rows=23968 dup_rows_remote=20892 dedup_ratio=0.243815 router_tokens_per_expert=[826, 717, 748, 740, 786, 768, 758, 796, 805, 806, 780, 764, 713, 751, 784, 785, 794, 776, 814, 749, 698, 790, 763, 732, 763, 746, 767, 835, 807, 748, 792, 732, 757, 754, 784, 818, 736, 782, 773, 729, 773, 791, 745, 783, 713, 746, 734, 779, 773, 780, 768, 814, 777, 749, 784, 749, 791, 785, 751, 749, 785, 742, 764, 761] payload_rows_per_dst=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] dup_rows_per_dst=[2984, 3044, 2952, 3076, 2986, 2862, 3076, 2988] combine_rows_per_src=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65054 dup_rows=23930 dup_rows_remote=20840 dedup_ratio=0.243429 router_tokens_per_expert=[780, 779, 784, 809, 762, 762, 749, 812, 762, 739, 738, 755, 793, 713, 741, 756, 764, 820, 788, 775, 746, 778, 754, 780, 749, 799, 753, 709, 780, 710, 756, 755, 789, 797, 768, 725, 784, 798, 752, 759, 773, 739, 760, 742, 809, 800, 798, 767, 772, 767, 784, 757, 824, 758, 789, 780, 755, 739, 775, 746, 746, 793, 778, 779] payload_rows_per_dst=[9382, 9138, 9320, 9138, 9384, 9326, 9448, 9238] dup_rows_per_dst=[3092, 2856, 3090, 2884, 2960, 3050, 3014, 2984] combine_rows_per_src=[9382, 9138, 9320, 9138, 9384, 9326, 9448, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64620 dup_rows=24270 dup_rows_remote=21160 dedup_ratio=0.246887 router_tokens_per_expert=[816, 772, 753, 761, 726, 817, 762, 753, 816, 766, 745, 793, 785, 761, 775, 756, 825, 754, 785, 739, 756, 804, 736, 769, 761, 752, 774, 762, 784, 748, 778, 716, 758, 726, 781, 781, 765, 791, 752, 785, 803, 803, 796, 747, 766, 735, 778, 764, 755, 806, 754, 797, 734, 774, 728, 762, 757, 719, 721, 793, 760, 799, 795, 767] payload_rows_per_dst=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] dup_rows_per_dst=[2856, 3076, 2932, 3210, 2854, 3362, 2870, 3110] combine_rows_per_src=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65044 dup_rows=24096 dup_rows_remote=21038 dedup_ratio=0.245117 router_tokens_per_expert=[809, 784, 769, 753, 781, 757, 740, 803, 760, 788, 773, 789, 759, 777, 815, 758, 778, 760, 763, 771, 773, 791, 810, 765, 768, 785, 792, 735, 740, 716, 781, 771, 785, 771, 761, 777, 736, 761, 734, 786, 759, 756, 743, 768, 750, 775, 742, 762, 837, 749, 783, 733, 783, 763, 744, 733, 798, 797, 771, 754, 755, 758, 779, 735] payload_rows_per_dst=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] dup_rows_per_dst=[3082, 3090, 3060, 2864, 3058, 3036, 3014, 2892] combine_rows_per_src=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65274 dup_rows=23810 dup_rows_remote=20878 dedup_ratio=0.242208 router_tokens_per_expert=[759, 773, 775, 807, 774, 730, 787, 720, 785, 755, 810, 765, 788, 761, 765, 774, 792, 777, 725, 799, 728, 749, 766, 777, 764, 729, 789, 751, 750, 789, 748, 756, 787, 801, 752, 753, 765, 792, 771, 773, 745, 723, 805, 765, 726, 781, 788, 752, 790, 785, 759, 771, 730, 775, 793, 807, 760, 798, 784, 770, 760, 729, 771, 774] payload_rows_per_dst=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] dup_rows_per_dst=[2924, 3042, 2894, 2932, 3012, 2918, 3092, 2996] combine_rows_per_src=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343888 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267050832 ingress_bytes=301309952 link_bytes=301309952 token_bytes=533757776 weight_movement_bytes=34603008 total_nvlink_bytes=568360784 weight_to_token_ratio=0.064829 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133805092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65112 dup_rows=23916 dup_rows_remote=20936 dedup_ratio=0.243286 router_tokens_per_expert=[791, 750, 780, 766, 805, 784, 793, 798, 821, 742, 705, 770, 769, 778, 763, 780, 763, 762, 741, 754, 762, 788, 768, 801, 786, 776, 756, 771, 765, 726, 799, 816, 789, 822, 722, 745, 744, 796, 793, 737, 743, 775, 722, 765, 765, 750, 786, 735, 769, 739, 770, 768, 793, 754, 720, 801, 737, 798, 788, 737, 821, 739, 767, 733] payload_rows_per_dst=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] dup_rows_per_dst=[3122, 2980, 2978, 3050, 2990, 2850, 3000, 2946] combine_rows_per_src=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65260 dup_rows=23932 dup_rows_remote=21038 dedup_ratio=0.243449 router_tokens_per_expert=[787, 786, 768, 797, 746, 772, 742, 770, 705, 729, 805, 773, 752, 751, 784, 780, 770, 765, 809, 755, 783, 709, 766, 819, 762, 757, 753, 756, 770, 788, 750, 764, 759, 800, 798, 809, 751, 784, 760, 804, 719, 748, 737, 754, 777, 748, 774, 746, 748, 787, 752, 768, 753, 781, 791, 775, 780, 757, 790, 725, 771, 799, 800, 784] payload_rows_per_dst=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] dup_rows_per_dst=[2956, 2876, 3040, 2950, 3102, 2894, 3056, 3058] combine_rows_per_src=[9380, 9282, 9312, 9250, 9428, 9112, 9254, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65118 dup_rows=23912 dup_rows_remote=20960 dedup_ratio=0.243245 router_tokens_per_expert=[725, 779, 749, 757, 774, 755, 784, 797, 735, 769, 801, 748, 785, 769, 753, 774, 760, 734, 771, 804, 823, 786, 747, 811, 762, 735, 802, 784, 800, 723, 736, 771, 737, 761, 741, 775, 787, 746, 774, 782, 779, 784, 805, 711, 820, 724, 813, 806, 760, 751, 764, 812, 766, 747, 799, 743, 724, 771, 784, 764, 725, 723, 787, 784] payload_rows_per_dst=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] dup_rows_per_dst=[2884, 3002, 3104, 2952, 2944, 3032, 3050, 2944] combine_rows_per_src=[9356, 9266, 9368, 9274, 9262, 9452, 9234, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343576 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266804760 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265944 weight_movement_bytes=69206016 total_nvlink_bytes=602471960 weight_to_token_ratio=0.129778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64620 dup_rows=24270 dup_rows_remote=21160 dedup_ratio=0.246887 router_tokens_per_expert=[816, 772, 753, 761, 726, 817, 762, 753, 816, 766, 745, 793, 785, 761, 775, 756, 825, 754, 785, 739, 756, 804, 736, 769, 761, 752, 774, 762, 784, 748, 778, 716, 758, 726, 781, 781, 765, 791, 752, 785, 803, 803, 796, 747, 766, 735, 778, 764, 755, 806, 754, 797, 734, 774, 728, 762, 757, 719, 721, 793, 760, 799, 795, 767] payload_rows_per_dst=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] dup_rows_per_dst=[2856, 3076, 2932, 3210, 2854, 3362, 2870, 3110] combine_rows_per_src=[9176, 9384, 9104, 9366, 8920, 9614, 9056, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344192 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042944 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741696 weight_movement_bytes=34603008 total_nvlink_bytes=568344704 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345192 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650152 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955112 weight_movement_bytes=34603008 total_nvlink_bytes=569558120 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74038 payload_rows_remote=64726 dup_rows=24266 dup_rows_remote=21280 dedup_ratio=0.246847 router_tokens_per_expert=[762, 776, 750, 760, 811, 778, 761, 760, 759, 768, 768, 789, 740, 760, 752, 789, 773, 733, 743, 800, 784, 773, 769, 780, 780, 791, 757, 792, 746, 819, 774, 758, 754, 759, 729, 730, 837, 760, 768, 812, 787, 770, 762, 759, 770, 753, 714, 778, 782, 783, 772, 763, 748, 766, 820, 704, 748, 745, 759, 773, 767, 760, 773, 792] payload_rows_per_dst=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] dup_rows_per_dst=[3028, 3048, 3152, 3064, 2986, 2886, 3112, 2990] combine_rows_per_src=[9288, 9202, 9158, 9370, 9312, 9300, 9164, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344312 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067640 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790968 weight_movement_bytes=34603008 total_nvlink_bytes=568393976 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65056 dup_rows=24026 dup_rows_remote=21154 dedup_ratio=0.244405 router_tokens_per_expert=[742, 740, 778, 757, 742, 745, 770, 773, 806, 791, 829, 767, 783, 760, 779, 756, 753, 782, 797, 745, 737, 759, 770, 764, 787, 754, 746, 773, 761, 724, 747, 746, 804, 820, 776, 813, 772, 760, 771, 779, 751, 767, 768, 720, 782, 764, 713, 768, 743, 776, 798, 804, 795, 774, 780, 745, 775, 743, 781, 768, 811, 767, 772, 729] payload_rows_per_dst=[9222, 9416, 9174, 9192, 9382, 9204, 9430, 9258] dup_rows_per_dst=[2872, 3126, 3040, 2884, 3208, 2862, 3000, 3034] combine_rows_per_src=[9222, 9416, 9174, 9192, 9382, 9204, 9430, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65112 dup_rows=23916 dup_rows_remote=20936 dedup_ratio=0.243286 router_tokens_per_expert=[791, 750, 780, 766, 805, 784, 793, 798, 821, 742, 705, 770, 769, 778, 763, 780, 763, 762, 741, 754, 762, 788, 768, 801, 786, 776, 756, 771, 765, 726, 799, 816, 789, 822, 722, 745, 744, 796, 793, 737, 743, 775, 722, 765, 765, 750, 786, 735, 769, 739, 770, 768, 793, 754, 720, 801, 737, 798, 788, 737, 821, 739, 767, 733] payload_rows_per_dst=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] dup_rows_per_dst=[3122, 2980, 2978, 3050, 2990, 2850, 3000, 2946] combine_rows_per_src=[9412, 9276, 9300, 9340, 9306, 9232, 9228, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343664 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266681968 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020272 weight_movement_bytes=69206016 total_nvlink_bytes=602226288 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343576 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266804760 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265944 weight_movement_bytes=69206016 total_nvlink_bytes=602471960 weight_to_token_ratio=0.129778 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65176 dup_rows=23820 dup_rows_remote=20810 dedup_ratio=0.242310 router_tokens_per_expert=[752, 788, 806, 788, 800, 803, 766, 784, 749, 739, 760, 745, 792, 773, 778, 736, 760, 764, 793, 783, 731, 775, 744, 784, 740, 736, 747, 773, 774, 802, 769, 785, 775, 706, 808, 736, 765, 741, 746, 770, 786, 785, 797, 785, 778, 732, 759, 750, 748, 760, 752, 782, 767, 776, 784, 790, 817, 785, 761, 779, 720, 784, 740, 769] payload_rows_per_dst=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] dup_rows_per_dst=[3170, 2866, 2990, 2934, 2844, 3006, 3010, 3000] combine_rows_per_src=[9404, 9278, 9278, 9318, 9250, 9338, 9308, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264683520 dispatch_meta_bytes=343120 combine_payload_bytes=264683520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265026640 ingress_bytes=299286528 link_bytes=299286528 token_bytes=529710160 weight_movement_bytes=34603008 total_nvlink_bytes=564313168 weight_to_token_ratio=0.065324 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126191110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344328 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764552 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184776 weight_movement_bytes=34603008 total_nvlink_bytes=567787784 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344608 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267706912 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069216 weight_movement_bytes=34603008 total_nvlink_bytes=569672224 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345192 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650152 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955112 weight_movement_bytes=34603008 total_nvlink_bytes=569558120 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=344024 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265461720 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530579416 weight_movement_bytes=69206016 total_nvlink_bytes=599785432 weight_to_token_ratio=0.130435 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344840 combine_payload_bytes=266469376 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266814216 ingress_bytes=404881408 link_bytes=404881408 token_bytes=533283592 weight_movement_bytes=138412032 total_nvlink_bytes=671695624 weight_to_token_ratio=0.259547 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523536143e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344192 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267042944 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741696 weight_movement_bytes=34603008 total_nvlink_bytes=568344704 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65086 dup_rows=24032 dup_rows_remote=21112 dedup_ratio=0.244466 router_tokens_per_expert=[753, 761, 756, 816, 754, 759, 779, 745, 747, 771, 785, 781, 750, 739, 788, 796, 751, 778, 784, 750, 753, 774, 848, 748, 775, 843, 767, 711, 781, 748, 759, 767, 792, 742, 822, 750, 758, 786, 781, 760, 753, 751, 744, 741, 808, 764, 712, 760, 755, 772, 777, 749, 749, 708, 770, 773, 781, 780, 808, 752, 835, 757, 760, 785] payload_rows_per_dst=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] dup_rows_per_dst=[3020, 3048, 3096, 3018, 3052, 2904, 2920, 2974] combine_rows_per_src=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=343944 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267304840 ingress_bytes=336166912 link_bytes=336166912 token_bytes=534265736 weight_movement_bytes=69206016 total_nvlink_bytes=603471752 weight_to_token_ratio=0.129535 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264969026e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65120 dup_rows=23884 dup_rows_remote=20980 dedup_ratio=0.242961 router_tokens_per_expert=[715, 792, 744, 782, 790, 809, 751, 755, 731, 782, 755, 778, 741, 750, 771, 803, 778, 764, 781, 790, 783, 739, 731, 736, 765, 781, 783, 782, 769, 792, 742, 750, 756, 758, 752, 763, 783, 766, 765, 765, 747, 825, 754, 741, 751, 746, 788, 777, 758, 760, 791, 807, 700, 841, 754, 757, 761, 811, 785, 803, 789, 747, 769, 767] payload_rows_per_dst=[9170, 9220, 9300, 9318, 9224, 9318, 9398, 9472] dup_rows_per_dst=[3106, 3002, 2904, 3010, 2992, 2940, 2938, 2992] combine_rows_per_src=[9170, 9220, 9300, 9318, 9224, 9318, 9398, 9472] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64816 dup_rows=23988 dup_rows_remote=20922 dedup_ratio=0.244019 router_tokens_per_expert=[753, 777, 776, 811, 778, 786, 753, 749, 777, 709, 774, 804, 799, 747, 762, 754, 773, 731, 822, 719, 770, 773, 737, 772, 747, 759, 747, 764, 775, 785, 773, 824, 765, 753, 772, 785, 753, 745, 767, 788, 786, 812, 778, 775, 723, 755, 755, 765, 726, 770, 770, 778, 762, 696, 788, 770, 833, 734, 772, 773, 790, 807, 775, 751] payload_rows_per_dst=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] dup_rows_per_dst=[2878, 2846, 2958, 3180, 3068, 3038, 2954, 3066] combine_rows_per_src=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65030 dup_rows=24012 dup_rows_remote=21014 dedup_ratio=0.244263 router_tokens_per_expert=[771, 765, 758, 705, 817, 775, 741, 781, 760, 799, 756, 765, 725, 744, 733, 829, 772, 743, 744, 798, 805, 780, 736, 723, 770, 750, 757, 814, 764, 742, 781, 745, 752, 785, 790, 765, 719, 796, 744, 779, 779, 810, 745, 733, 741, 812, 723, 744, 832, 766, 789, 808, 793, 714, 815, 773, 784, 783, 775, 809, 750, 776, 732, 788] payload_rows_per_dst=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] dup_rows_per_dst=[2976, 2954, 2950, 2930, 2998, 2990, 3144, 3070] combine_rows_per_src=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65244 dup_rows=23864 dup_rows_remote=20986 dedup_ratio=0.242757 router_tokens_per_expert=[728, 733, 784, 728, 791, 779, 806, 771, 787, 745, 705, 790, 777, 744, 701, 779, 745, 793, 830, 747, 778, 778, 734, 787, 717, 750, 761, 715, 744, 787, 802, 761, 786, 777, 803, 742, 768, 768, 759, 794, 779, 782, 745, 776, 752, 796, 786, 788, 713, 748, 781, 759, 755, 777, 805, 792, 783, 783, 771, 756, 759, 828, 771, 793] payload_rows_per_dst=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] dup_rows_per_dst=[2974, 2848, 2980, 2878, 3092, 3054, 2938, 3100] combine_rows_per_src=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65178 dup_rows=23828 dup_rows_remote=20872 dedup_ratio=0.242391 router_tokens_per_expert=[709, 733, 790, 792, 754, 748, 788, 805, 795, 759, 779, 756, 774, 792, 763, 772, 780, 782, 790, 783, 776, 778, 754, 765, 743, 767, 791, 762, 745, 775, 769, 787, 780, 736, 798, 732, 788, 809, 768, 791, 771, 776, 777, 696, 794, 770, 786, 757, 764, 749, 765, 735, 738, 756, 747, 796, 776, 768, 776, 723, 780, 771, 764, 759] payload_rows_per_dst=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] dup_rows_per_dst=[2976, 3008, 3020, 2958, 2996, 2956, 2932, 2982] combine_rows_per_src=[9262, 9372, 9396, 9320, 9408, 9298, 9168, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65034 dup_rows=23874 dup_rows_remote=20874 dedup_ratio=0.242859 router_tokens_per_expert=[756, 799, 837, 771, 792, 717, 746, 780, 764, 753, 766, 811, 821, 785, 755, 812, 768, 799, 749, 760, 740, 728, 747, 818, 779, 796, 768, 762, 701, 769, 783, 746, 752, 748, 720, 730, 759, 730, 782, 749, 754, 758, 781, 734, 792, 752, 817, 772, 776, 757, 791, 766, 730, 827, 802, 757, 771, 777, 761, 760, 747, 772, 764, 786] payload_rows_per_dst=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] dup_rows_per_dst=[3000, 3156, 2798, 3000, 2860, 3294, 3018, 2748] combine_rows_per_src=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344792 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266937048 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529304 weight_movement_bytes=34603008 total_nvlink_bytes=568132312 weight_to_token_ratio=0.064857 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65114 dup_rows=23990 dup_rows_remote=21106 dedup_ratio=0.244039 router_tokens_per_expert=[727, 759, 774, 780, 803, 786, 771, 828, 736, 752, 811, 769, 734, 717, 797, 726, 781, 814, 693, 757, 783, 755, 800, 765, 712, 773, 757, 774, 730, 777, 768, 769, 795, 764, 782, 761, 799, 730, 787, 784, 814, 772, 762, 786, 814, 736, 755, 678, 757, 742, 820, 754, 812, 756, 768, 818, 784, 789, 738, 729, 787, 758, 768, 775] payload_rows_per_dst=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] dup_rows_per_dst=[3020, 2884, 3024, 2972, 3022, 2966, 3102, 3000] combine_rows_per_src=[9436, 9200, 9272, 9148, 9382, 9268, 9352, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344400 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075920 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807440 weight_movement_bytes=69206016 total_nvlink_bytes=603013456 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65274 dup_rows=23810 dup_rows_remote=20878 dedup_ratio=0.242208 router_tokens_per_expert=[759, 773, 775, 807, 774, 730, 787, 720, 785, 755, 810, 765, 788, 761, 765, 774, 792, 777, 725, 799, 728, 749, 766, 777, 764, 729, 789, 751, 750, 789, 748, 756, 787, 801, 752, 753, 765, 792, 771, 773, 745, 723, 805, 765, 726, 781, 788, 752, 790, 785, 759, 771, 730, 775, 793, 807, 760, 798, 784, 770, 760, 729, 771, 774] payload_rows_per_dst=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] dup_rows_per_dst=[2924, 3042, 2894, 2932, 3012, 2918, 3092, 2996] combine_rows_per_src=[9326, 9364, 9332, 9220, 9376, 9252, 9328, 9296] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342952 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829288 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531315624 weight_movement_bytes=69206016 total_nvlink_bytes=600521640 weight_to_token_ratio=0.130254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344176 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266707056 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069936 weight_movement_bytes=69206016 total_nvlink_bytes=602275952 weight_to_token_ratio=0.129825 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344920 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584344 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823768 weight_movement_bytes=34603008 total_nvlink_bytes=569426776 weight_to_token_ratio=0.064700 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344400 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075920 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807440 weight_movement_bytes=69206016 total_nvlink_bytes=603013456 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344200 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313288 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282376 weight_movement_bytes=34603008 total_nvlink_bytes=568885384 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343632 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266722896 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102160 weight_movement_bytes=138412032 total_nvlink_bytes=671514192 weight_to_token_ratio=0.259635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=65044 dup_rows=24096 dup_rows_remote=21038 dedup_ratio=0.245117 router_tokens_per_expert=[809, 784, 769, 753, 781, 757, 740, 803, 760, 788, 773, 789, 759, 777, 815, 758, 778, 760, 763, 771, 773, 791, 810, 765, 768, 785, 792, 735, 740, 716, 781, 771, 785, 771, 761, 777, 736, 761, 734, 786, 759, 756, 743, 768, 750, 775, 742, 762, 837, 749, 783, 733, 783, 763, 744, 733, 798, 797, 771, 754, 755, 758, 779, 735] payload_rows_per_dst=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] dup_rows_per_dst=[3082, 3090, 3060, 2864, 3058, 3036, 3014, 2892] combine_rows_per_src=[9310, 9348, 9362, 9312, 9164, 9074, 9236, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64816 dup_rows=23988 dup_rows_remote=20922 dedup_ratio=0.244019 router_tokens_per_expert=[753, 777, 776, 811, 778, 786, 753, 749, 777, 709, 774, 804, 799, 747, 762, 754, 773, 731, 822, 719, 770, 773, 737, 772, 747, 759, 747, 764, 775, 785, 773, 824, 765, 753, 772, 785, 753, 745, 767, 788, 786, 812, 778, 775, 723, 755, 755, 765, 726, 770, 770, 778, 762, 696, 788, 770, 833, 734, 772, 773, 790, 807, 775, 751] payload_rows_per_dst=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] dup_rows_per_dst=[2878, 2846, 2958, 3180, 3068, 3038, 2954, 3066] combine_rows_per_src=[9172, 9014, 9258, 9560, 9280, 9260, 9272, 9500] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=344880 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267051824 ingress_bytes=335912960 link_bytes=335912960 token_bytes=533758768 weight_movement_bytes=69206016 total_nvlink_bytes=602964784 weight_to_token_ratio=0.129658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264013425e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12251498072 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6129706584 ingress_bytes=7436705792 link_bytes=7436705792 total_nvlink_bytes=13566412376 weight_to_token_ratio=0.107327 prefetch_share_of_ingress=0.093060 dedup_ratio=0.244757 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216683704e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267554288 prefetch_bytes=795869184 reduce_bytes=622854144 weight_movement_bytes=1418723328 egress_bytes=6137734640 ingress_bytes=7548542976 link_bytes=7548542976 total_nvlink_bytes=13686277616 weight_to_token_ratio=0.115648 prefetch_share_of_ingress=0.105433 dedup_ratio=0.243694 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=18 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027717 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234980848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74614 payload_rows_remote=65202 dup_rows=23690 dup_rows_remote=20652 dedup_ratio=0.240987 router_tokens_per_expert=[800, 721, 742, 794, 740, 784, 733, 775, 734, 780, 742, 788, 764, 805, 726, 754, 770, 808, 734, 789, 819, 719, 784, 815, 768, 769, 788, 771, 726, 788, 769, 761, 782, 795, 769, 752, 739, 731, 771, 775, 744, 765, 779, 776, 778, 768, 805, 775, 764, 776, 797, 742, 781, 797, 784, 784, 749, 768, 776, 760, 717, 738, 754, 801] payload_rows_per_dst=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] dup_rows_per_dst=[2918, 2938, 3008, 2936, 2930, 3020, 3038, 2902] combine_rows_per_src=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65034 dup_rows=23874 dup_rows_remote=20874 dedup_ratio=0.242859 router_tokens_per_expert=[756, 799, 837, 771, 792, 717, 746, 780, 764, 753, 766, 811, 821, 785, 755, 812, 768, 799, 749, 760, 740, 728, 747, 818, 779, 796, 768, 762, 701, 769, 783, 746, 752, 748, 720, 730, 759, 730, 782, 749, 754, 758, 781, 734, 792, 752, 817, 772, 776, 757, 791, 766, 730, 827, 802, 757, 771, 777, 761, 760, 747, 772, 764, 786] payload_rows_per_dst=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] dup_rows_per_dst=[3000, 3156, 2798, 3000, 2860, 3294, 3018, 2748] combine_rows_per_src=[9396, 9444, 9354, 9208, 9080, 9620, 9394, 8934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344608 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267706912 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069216 weight_movement_bytes=34603008 total_nvlink_bytes=569672224 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65024 dup_rows=23968 dup_rows_remote=20892 dedup_ratio=0.243815 router_tokens_per_expert=[826, 717, 748, 740, 786, 768, 758, 796, 805, 806, 780, 764, 713, 751, 784, 785, 794, 776, 814, 749, 698, 790, 763, 732, 763, 746, 767, 835, 807, 748, 792, 732, 757, 754, 784, 818, 736, 782, 773, 729, 773, 791, 745, 783, 713, 746, 734, 779, 773, 780, 768, 814, 777, 749, 784, 749, 791, 785, 751, 749, 785, 742, 764, 761] payload_rows_per_dst=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] dup_rows_per_dst=[2984, 3044, 2952, 3076, 2986, 2862, 3076, 2988] combine_rows_per_src=[9294, 9332, 9280, 9304, 9280, 9266, 9312, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342952 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829288 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531315624 weight_movement_bytes=69206016 total_nvlink_bytes=600521640 weight_to_token_ratio=0.130254 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344328 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764552 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184776 weight_movement_bytes=34603008 total_nvlink_bytes=567787784 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343632 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266722896 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102160 weight_movement_bytes=138412032 total_nvlink_bytes=671514192 weight_to_token_ratio=0.259635 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=64928 dup_rows=23832 dup_rows_remote=20834 dedup_ratio=0.242432 router_tokens_per_expert=[747, 751, 770, 771, 800, 759, 766, 807, 757, 761, 730, 776, 790, 765, 797, 762, 799, 827, 739, 795, 763, 787, 776, 785, 712, 749, 771, 762, 729, 726, 822, 773, 765, 744, 774, 789, 760, 755, 767, 784, 768, 791, 798, 755, 770, 773, 796, 765, 796, 742, 766, 742, 707, 759, 774, 788, 789, 779, 737, 755, 732, 758, 760, 790] payload_rows_per_dst=[9276, 9328, 9544, 9216, 9284, 9486, 9170, 9168] dup_rows_per_dst=[3066, 2948, 2998, 2872, 2992, 2946, 2978, 3032] combine_rows_per_src=[9276, 9328, 9544, 9216, 9284, 9486, 9170, 9168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21146 dedup_ratio=0.244893 router_tokens_per_expert=[781, 749, 795, 770, 742, 751, 782, 755, 795, 797, 797, 785, 812, 776, 786, 772, 811, 740, 791, 747, 757, 793, 755, 765, 783, 800, 757, 800, 764, 795, 767, 779, 761, 742, 775, 759, 786, 765, 742, 797, 778, 725, 782, 789, 749, 740, 772, 765, 745, 760, 817, 749, 717, 780, 737, 718, 762, 743, 756, 753, 799, 740, 726, 774] payload_rows_per_dst=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] dup_rows_per_dst=[2976, 2942, 3130, 3130, 2910, 2914, 3144, 2928] combine_rows_per_src=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=64832 dup_rows=23934 dup_rows_remote=20842 dedup_ratio=0.243469 router_tokens_per_expert=[753, 801, 768, 740, 741, 834, 766, 724, 803, 829, 817, 758, 783, 770, 775, 758, 799, 786, 704, 802, 783, 737, 747, 751, 727, 774, 748, 787, 732, 776, 770, 805, 782, 801, 789, 778, 734, 741, 748, 752, 785, 770, 793, 741, 785, 750, 740, 757, 759, 718, 772, 746, 754, 769, 768, 763, 796, 787, 781, 768, 757, 750, 807, 763] payload_rows_per_dst=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] dup_rows_per_dst=[3092, 3098, 3034, 3054, 2948, 2814, 2924, 2970] combine_rows_per_src=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343664 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266681968 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020272 weight_movement_bytes=69206016 total_nvlink_bytes=602226288 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65134 dup_rows=23836 dup_rows_remote=20792 dedup_ratio=0.242472 router_tokens_per_expert=[740, 793, 750, 789, 763, 756, 753, 740, 729, 728, 737, 788, 745, 751, 815, 805, 743, 760, 768, 756, 760, 782, 750, 814, 824, 765, 759, 772, 754, 762, 766, 776, 763, 795, 792, 759, 812, 767, 743, 737, 774, 807, 748, 777, 742, 767, 797, 777, 747, 766, 763, 765, 786, 737, 710, 802, 796, 750, 784, 754, 830, 762, 782, 768] payload_rows_per_dst=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] dup_rows_per_dst=[2910, 2944, 3016, 2996, 2992, 3044, 2882, 3052] combine_rows_per_src=[9258, 9252, 9250, 9360, 9344, 9334, 9270, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343416 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267410808 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478200 weight_movement_bytes=34603008 total_nvlink_bytes=569081208 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64706 dup_rows=24130 dup_rows_remote=20972 dedup_ratio=0.245463 router_tokens_per_expert=[758, 773, 764, 792, 751, 754, 788, 755, 807, 711, 790, 835, 771, 846, 747, 806, 718, 759, 713, 736, 745, 729, 744, 769, 769, 793, 769, 829, 755, 721, 775, 741, 723, 785, 738, 789, 818, 743, 799, 774, 769, 761, 817, 807, 754, 746, 781, 730, 752, 758, 719, 755, 787, 764, 771, 765, 788, 762, 800, 762, 760, 814, 771, 777] payload_rows_per_dst=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] dup_rows_per_dst=[3056, 3158, 2880, 3038, 2990, 2976, 2892, 3140] combine_rows_per_src=[9214, 9468, 8946, 9266, 9348, 9354, 9250, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65244 dup_rows=23864 dup_rows_remote=20986 dedup_ratio=0.242757 router_tokens_per_expert=[728, 733, 784, 728, 791, 779, 806, 771, 787, 745, 705, 790, 777, 744, 701, 779, 745, 793, 830, 747, 778, 778, 734, 787, 717, 750, 761, 715, 744, 787, 802, 761, 786, 777, 803, 742, 768, 768, 759, 794, 779, 782, 745, 776, 752, 796, 786, 788, 713, 748, 781, 759, 755, 777, 805, 792, 783, 783, 771, 756, 759, 828, 771, 793] payload_rows_per_dst=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] dup_rows_per_dst=[2974, 2848, 2980, 2878, 3092, 3054, 2938, 3100] combine_rows_per_src=[9266, 9208, 9404, 9196, 9302, 9354, 9322, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233224 weight_movement_bytes=34603008 total_nvlink_bytes=566836232 weight_to_token_ratio=0.065015 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344680 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682984 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021288 weight_movement_bytes=69206016 total_nvlink_bytes=602227304 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=342696 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265894568 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531446440 weight_movement_bytes=34603008 total_nvlink_bytes=566049448 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343704 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267132568 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921432 weight_movement_bytes=103809024 total_nvlink_bytes=637730456 weight_to_token_ratio=0.194428 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21146 dedup_ratio=0.244893 router_tokens_per_expert=[781, 749, 795, 770, 742, 751, 782, 755, 795, 797, 797, 785, 812, 776, 786, 772, 811, 740, 791, 747, 757, 793, 755, 765, 783, 800, 757, 800, 764, 795, 767, 779, 761, 742, 775, 759, 786, 765, 742, 797, 778, 725, 782, 789, 749, 740, 772, 765, 745, 760, 817, 749, 717, 780, 737, 718, 762, 743, 756, 753, 799, 740, 726, 774] payload_rows_per_dst=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] dup_rows_per_dst=[2976, 2942, 3130, 3130, 2910, 2914, 3144, 2928] combine_rows_per_src=[9274, 9198, 9260, 9360, 9312, 9290, 9330, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65030 dup_rows=24012 dup_rows_remote=21014 dedup_ratio=0.244263 router_tokens_per_expert=[771, 765, 758, 705, 817, 775, 741, 781, 760, 799, 756, 765, 725, 744, 733, 829, 772, 743, 744, 798, 805, 780, 736, 723, 770, 750, 757, 814, 764, 742, 781, 745, 752, 785, 790, 765, 719, 796, 744, 779, 779, 810, 745, 733, 741, 812, 723, 744, 832, 766, 789, 808, 793, 714, 815, 773, 784, 783, 775, 809, 750, 776, 732, 788] payload_rows_per_dst=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] dup_rows_per_dst=[2976, 2954, 2950, 2930, 2998, 2990, 3144, 3070] combine_rows_per_src=[9250, 9268, 9252, 9316, 9262, 9184, 9436, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265035776 dispatch_meta_bytes=342712 combine_payload_bytes=265035776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265378488 ingress_bytes=368844800 link_bytes=368844800 token_bytes=530414264 weight_movement_bytes=103809024 total_nvlink_bytes=634223288 weight_to_token_ratio=0.195713 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387933288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262685504 prefetch_bytes=622854144 reduce_bytes=865075200 weight_movement_bytes=1487929344 egress_bytes=6135298880 ingress_bytes=7615315968 link_bytes=7615315968 total_nvlink_bytes=13750614848 weight_to_token_ratio=0.121338 prefetch_share_of_ingress=0.081790 dedup_ratio=0.243745 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=25 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245905257e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344920 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584344 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823768 weight_movement_bytes=34603008 total_nvlink_bytes=569426776 weight_to_token_ratio=0.064700 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254394568 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6131153096 ingress_bytes=7472758784 link_bytes=7472758784 total_nvlink_bytes=13603911880 weight_to_token_ratio=0.110125 prefetch_share_of_ingress=0.092611 dedup_ratio=0.244128 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.222582161e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12235805256 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6121853512 ingress_bytes=7532675072 link_bytes=7532675072 total_nvlink_bytes=13654528584 weight_to_token_ratio=0.115949 prefetch_share_of_ingress=0.096468 dedup_ratio=0.244628 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232384777e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12241040960 prefetch_bytes=553648128 reduce_bytes=1003487232 weight_movement_bytes=1557135360 egress_bytes=6124467776 ingress_bytes=7673708544 link_bytes=7673708544 total_nvlink_bytes=13798176320 weight_to_token_ratio=0.127206 prefetch_share_of_ingress=0.072149 dedup_ratio=0.243948 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=29 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.255458586e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=64832 dup_rows=23934 dup_rows_remote=20842 dedup_ratio=0.243469 router_tokens_per_expert=[753, 801, 768, 740, 741, 834, 766, 724, 803, 829, 817, 758, 783, 770, 775, 758, 799, 786, 704, 802, 783, 737, 747, 751, 727, 774, 748, 787, 732, 776, 770, 805, 782, 801, 789, 778, 734, 741, 748, 752, 785, 770, 793, 741, 785, 750, 740, 757, 759, 718, 772, 746, 754, 769, 768, 763, 796, 787, 781, 768, 757, 750, 807, 763] payload_rows_per_dst=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] dup_rows_per_dst=[3092, 3098, 3034, 3054, 2948, 2814, 2924, 2970] combine_rows_per_src=[9538, 9504, 9184, 9412, 9302, 9184, 9174, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264870264 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6136394104 ingress_bytes=7270375424 link_bytes=7270375424 total_nvlink_bytes=13406769528 weight_to_token_ratio=0.093103 prefetch_share_of_ingress=0.090430 dedup_ratio=0.244031 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189471192e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65086 dup_rows=24032 dup_rows_remote=21112 dedup_ratio=0.244466 router_tokens_per_expert=[753, 761, 756, 816, 754, 759, 779, 745, 747, 771, 785, 781, 750, 739, 788, 796, 751, 778, 784, 750, 753, 774, 848, 748, 775, 843, 767, 711, 781, 748, 759, 767, 792, 742, 822, 750, 758, 786, 781, 760, 753, 751, 744, 741, 808, 764, 712, 760, 755, 772, 777, 749, 749, 708, 770, 773, 781, 780, 808, 752, 835, 757, 760, 785] payload_rows_per_dst=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] dup_rows_per_dst=[3020, 3048, 3096, 3018, 3052, 2904, 2920, 2974] combine_rows_per_src=[9226, 9266, 9276, 9284, 9330, 9162, 9186, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344680 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682984 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021288 weight_movement_bytes=69206016 total_nvlink_bytes=602227304 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259773720 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133845272 ingress_bytes=7371636736 link_bytes=7371636736 total_nvlink_bytes=13505482008 weight_to_token_ratio=0.101609 prefetch_share_of_ingress=0.098576 dedup_ratio=0.244296 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206038068e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none - [2026-08-26 16:56:36] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 792.9 | throughput per GPU (TFLOP/s/GPU): 160.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476632E+00 | loss scale: 1.0 | grad norm: 7.104 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267554288 prefetch_bytes=795869184 reduce_bytes=622854144 weight_movement_bytes=1418723328 egress_bytes=6137734640 ingress_bytes=7548542976 link_bytes=7548542976 total_nvlink_bytes=13686277616 weight_to_token_ratio=0.115648 prefetch_share_of_ingress=0.105433 dedup_ratio=0.243694 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=18 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027717 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234980848e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344176 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266707056 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069936 weight_movement_bytes=69206016 total_nvlink_bytes=602275952 weight_to_token_ratio=0.129825 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=342696 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265894568 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531446440 weight_movement_bytes=34603008 total_nvlink_bytes=566049448 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344792 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266937048 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529304 weight_movement_bytes=34603008 total_nvlink_bytes=568132312 weight_to_token_ratio=0.064857 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12235805256 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6121853512 ingress_bytes=7532675072 link_bytes=7532675072 total_nvlink_bytes=13654528584 weight_to_token_ratio=0.115949 prefetch_share_of_ingress=0.096468 dedup_ratio=0.244628 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232384777e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12251498072 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6129706584 ingress_bytes=7436705792 link_bytes=7436705792 total_nvlink_bytes=13566412376 weight_to_token_ratio=0.107327 prefetch_share_of_ingress=0.093060 dedup_ratio=0.244757 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216683704e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12241040960 prefetch_bytes=553648128 reduce_bytes=1003487232 weight_movement_bytes=1557135360 egress_bytes=6124467776 ingress_bytes=7673708544 link_bytes=7673708544 total_nvlink_bytes=13798176320 weight_to_token_ratio=0.127206 prefetch_share_of_ingress=0.072149 dedup_ratio=0.243948 prefetch_slots=32 unused_prefetch_slots=336 slot_utilization=0.086957 reduce_slots=29 remote_demand=16 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28321 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.255458586e-04 prefetch_bytes_per_gemm_flop=9.057971014e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74614 payload_rows_remote=65202 dup_rows=23690 dup_rows_remote=20652 dedup_ratio=0.240987 router_tokens_per_expert=[800, 721, 742, 794, 740, 784, 733, 775, 734, 780, 742, 788, 764, 805, 726, 754, 770, 808, 734, 789, 819, 719, 784, 815, 768, 769, 788, 771, 726, 788, 769, 761, 782, 795, 769, 752, 739, 731, 771, 775, 744, 765, 779, 776, 778, 768, 805, 775, 764, 776, 797, 742, 781, 797, 784, 784, 749, 768, 776, 760, 717, 738, 754, 801] payload_rows_per_dst=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] dup_rows_per_dst=[2918, 2938, 3008, 2936, 2930, 3020, 3038, 2902] combine_rows_per_src=[9260, 9248, 9468, 9344, 9298, 9360, 9412, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343416 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267410808 ingress_bytes=301670400 link_bytes=301670400 token_bytes=534478200 weight_movement_bytes=34603008 total_nvlink_bytes=569081208 weight_to_token_ratio=0.064742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135161429e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262685504 prefetch_bytes=622854144 reduce_bytes=865075200 weight_movement_bytes=1487929344 egress_bytes=6135298880 ingress_bytes=7615315968 link_bytes=7615315968 total_nvlink_bytes=13750614848 weight_to_token_ratio=0.121338 prefetch_share_of_ingress=0.081790 dedup_ratio=0.243745 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=25 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027500 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245905257e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none + [2026-08-26 18:08:42] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 793.1 | throughput per GPU (TFLOP/s/GPU): 160.9 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 9.476632E+00 | loss scale: 1.0 | grad norm: 7.104 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65032 dup_rows=24054 dup_rows_remote=21120 dedup_ratio=0.244690 router_tokens_per_expert=[756, 733, 753, 797, 773, 751, 730, 746, 768, 788, 770, 763, 736, 755, 829, 825, 771, 775, 776, 767, 807, 772, 822, 749, 781, 730, 732, 790, 800, 754, 713, 776, 788, 769, 764, 761, 784, 768, 763, 739, 743, 776, 792, 777, 786, 773, 766, 734, 762, 769, 739, 788, 730, 765, 728, 799, 760, 772, 761, 760, 790, 780, 780, 798] payload_rows_per_dst=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] dup_rows_per_dst=[2848, 3082, 3106, 2934, 3044, 2968, 3014, 3058] combine_rows_per_src=[9230, 9386, 9372, 9218, 9228, 9326, 9146, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64958 dup_rows=24112 dup_rows_remote=21086 dedup_ratio=0.245280 router_tokens_per_expert=[777, 780, 759, 786, 787, 761, 732, 793, 784, 748, 763, 767, 772, 760, 756, 733, 777, 757, 807, 754, 760, 727, 796, 763, 714, 795, 766, 736, 768, 737, 761, 738, 724, 735, 802, 732, 815, 752, 793, 777, 759, 776, 801, 792, 778, 716, 807, 769, 821, 800, 762, 782, 809, 782, 768, 720, 837, 707, 765, 789, 783, 780, 737, 768] payload_rows_per_dst=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] dup_rows_per_dst=[2988, 2946, 3068, 2842, 3026, 3066, 3070, 3106] combine_rows_per_src=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65208 dup_rows=23840 dup_rows_remote=20938 dedup_ratio=0.242513 router_tokens_per_expert=[749, 771, 742, 765, 830, 777, 776, 764, 789, 759, 811, 777, 749, 745, 757, 805, 743, 773, 745, 757, 763, 770, 781, 790, 766, 726, 734, 738, 790, 769, 794, 800, 793, 736, 768, 817, 772, 728, 762, 763, 793, 764, 761, 770, 733, 770, 738, 750, 758, 763, 788, 773, 767, 776, 778, 744, 805, 789, 779, 752, 779, 780, 776, 722] payload_rows_per_dst=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] dup_rows_per_dst=[3012, 3052, 2932, 2894, 3004, 2902, 3024, 3020] combine_rows_per_src=[9336, 9332, 9312, 9340, 9274, 9256, 9270, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65080 dup_rows=23890 dup_rows_remote=20998 dedup_ratio=0.243022 router_tokens_per_expert=[770, 754, 792, 776, 725, 732, 732, 777, 834, 829, 772, 744, 764, 800, 810, 813, 781, 746, 774, 726, 772, 769, 781, 766, 748, 710, 766, 807, 801, 751, 771, 793, 772, 779, 806, 747, 768, 807, 797, 783, 738, 794, 785, 760, 788, 737, 797, 736, 708, 764, 748, 758, 743, 759, 735, 788, 750, 734, 791, 778, 729, 779, 690, 818] payload_rows_per_dst=[9214, 9202, 9162, 9456, 9100, 9554, 9392, 9334] dup_rows_per_dst=[2902, 3048, 2904, 3146, 2866, 3180, 2952, 2892] combine_rows_per_src=[9214, 9202, 9162, 9456, 9100, 9554, 9392, 9334] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65004 dup_rows=23936 dup_rows_remote=20840 dedup_ratio=0.243490 router_tokens_per_expert=[789, 759, 764, 763, 726, 717, 772, 820, 754, 774, 804, 780, 804, 780, 757, 777, 758, 762, 748, 732, 810, 783, 811, 769, 779, 781, 817, 773, 685, 843, 788, 729, 817, 770, 806, 752, 763, 770, 744, 784, 737, 790, 755, 753, 740, 734, 778, 774, 750, 753, 744, 772, 766, 735, 758, 764, 783, 791, 786, 734, 762, 722, 772, 785] payload_rows_per_dst=[9236, 9364, 9402, 9414, 9346, 9110, 9190, 9306] dup_rows_per_dst=[2984, 3096, 2944, 2976, 3066, 3012, 2894, 2964] combine_rows_per_src=[9236, 9364, 9402, 9414, 9346, 9110, 9190, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64958 dup_rows=24112 dup_rows_remote=21086 dedup_ratio=0.245280 router_tokens_per_expert=[777, 780, 759, 786, 787, 761, 732, 793, 784, 748, 763, 767, 772, 760, 756, 733, 777, 757, 807, 754, 760, 727, 796, 763, 714, 795, 766, 736, 768, 737, 761, 738, 724, 735, 802, 732, 815, 752, 793, 777, 759, 776, 801, 792, 778, 716, 807, 769, 821, 800, 762, 782, 809, 782, 768, 720, 837, 707, 765, 789, 783, 780, 737, 768] payload_rows_per_dst=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] dup_rows_per_dst=[2988, 2946, 3068, 2842, 3026, 3066, 3070, 3106] combine_rows_per_src=[9362, 9220, 9214, 9188, 9234, 9330, 9418, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64972 dup_rows=24112 dup_rows_remote=21216 dedup_ratio=0.245280 router_tokens_per_expert=[755, 777, 795, 757, 769, 777, 754, 737, 787, 795, 813, 728, 791, 759, 735, 776, 745, 775, 725, 754, 728, 791, 770, 770, 746, 777, 728, 769, 774, 748, 789, 725, 756, 771, 788, 774, 823, 782, 761, 778, 798, 771, 757, 721, 806, 779, 750, 799, 788, 783, 719, 764, 729, 783, 813, 753, 761, 780, 797, 774, 784, 775, 736, 780] payload_rows_per_dst=[9290, 9332, 9220, 9176, 9326, 9302, 9218, 9328] dup_rows_per_dst=[2952, 3036, 2896, 2936, 3140, 3060, 3046, 3046] combine_rows_per_src=[9290, 9332, 9220, 9176, 9326, 9302, 9218, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64976 dup_rows=24056 dup_rows_remote=21042 dedup_ratio=0.244710 router_tokens_per_expert=[774, 764, 747, 793, 718, 757, 771, 744, 742, 798, 773, 735, 778, 756, 750, 786, 729, 732, 748, 771, 795, 793, 795, 753, 764, 780, 783, 806, 758, 790, 771, 785, 756, 775, 764, 723, 768, 764, 761, 802, 752, 782, 807, 754, 759, 771, 798, 778, 788, 753, 771, 754, 779, 741, 789, 768, 741, 727, 779, 820, 769, 787, 802, 731] payload_rows_per_dst=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] dup_rows_per_dst=[2930, 2940, 2998, 3100, 3020, 3050, 3014, 3004] combine_rows_per_src=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344608 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715680 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086752 weight_movement_bytes=34603008 total_nvlink_bytes=567689760 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344176 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266412144 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532480112 weight_movement_bytes=103809024 total_nvlink_bytes=636289136 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=344584 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267436552 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534528520 weight_movement_bytes=34603008 total_nvlink_bytes=569131528 weight_to_token_ratio=0.064736 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344312 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266911992 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479672 weight_movement_bytes=34603008 total_nvlink_bytes=568082680 weight_to_token_ratio=0.064863 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64976 dup_rows=24056 dup_rows_remote=21042 dedup_ratio=0.244710 router_tokens_per_expert=[774, 764, 747, 793, 718, 757, 771, 744, 742, 798, 773, 735, 778, 756, 750, 786, 729, 732, 748, 771, 795, 793, 795, 753, 764, 780, 783, 806, 758, 790, 771, 785, 756, 775, 764, 723, 768, 764, 761, 802, 752, 782, 807, 754, 759, 771, 798, 778, 788, 753, 771, 754, 779, 741, 789, 768, 741, 727, 779, 820, 769, 787, 802, 731] payload_rows_per_dst=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] dup_rows_per_dst=[2930, 2940, 2998, 3100, 3020, 3050, 3014, 3004] combine_rows_per_src=[9206, 9296, 9234, 9374, 9206, 9352, 9272, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343376 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266599760 ingress_bytes=404668416 link_bytes=404668416 token_bytes=532856144 weight_movement_bytes=138412032 total_nvlink_bytes=671268176 weight_to_token_ratio=0.259755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522734671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344176 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266412144 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532480112 weight_movement_bytes=103809024 total_nvlink_bytes=636289136 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266470064 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532595376 weight_movement_bytes=69206016 total_nvlink_bytes=601801392 weight_to_token_ratio=0.129941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64848 dup_rows=23888 dup_rows_remote=20642 dedup_ratio=0.243001 router_tokens_per_expert=[757, 792, 725, 770, 748, 770, 811, 767, 804, 798, 744, 789, 782, 794, 739, 789, 756, 752, 786, 779, 816, 736, 816, 829, 798, 704, 760, 780, 755, 784, 785, 789, 787, 782, 780, 769, 784, 801, 766, 757, 778, 711, 699, 763, 735, 788, 714, 730, 732, 767, 792, 780, 790, 830, 765, 736, 740, 752, 740, 783, 686, 761, 760, 790] payload_rows_per_dst=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] dup_rows_per_dst=[3246, 2752, 3206, 3020, 3026, 2744, 3032, 2862] combine_rows_per_src=[9568, 9220, 9334, 9290, 9398, 9092, 9352, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344072 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485768 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627464 weight_movement_bytes=34603008 total_nvlink_bytes=567230472 weight_to_token_ratio=0.064967 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65300 dup_rows=23770 dup_rows_remote=20852 dedup_ratio=0.241801 router_tokens_per_expert=[753, 754, 756, 810, 798, 737, 807, 763, 800, 783, 792, 805, 779, 724, 756, 732, 790, 740, 808, 753, 756, 778, 732, 813, 742, 775, 724, 766, 783, 791, 760, 735, 725, 763, 779, 746, 791, 757, 764, 760, 754, 786, 742, 785, 752, 787, 774, 744, 784, 787, 728, 793, 768, 779, 720, 771, 750, 819, 770, 807, 776, 792, 746, 758] payload_rows_per_dst=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] dup_rows_per_dst=[2988, 3078, 2956, 2918, 2932, 2984, 2912, 3002] combine_rows_per_src=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=341960 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265959368 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531576776 weight_movement_bytes=34603008 total_nvlink_bytes=566179784 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64882 dup_rows=24038 dup_rows_remote=20970 dedup_ratio=0.244527 router_tokens_per_expert=[787, 728, 734, 783, 792, 762, 790, 773, 786, 712, 820, 751, 762, 774, 774, 727, 731, 771, 722, 749, 822, 727, 763, 755, 798, 765, 800, 750, 793, 780, 780, 797, 781, 775, 768, 775, 807, 795, 788, 737, 774, 772, 762, 771, 833, 782, 731, 764, 778, 726, 723, 774, 744, 772, 796, 750, 795, 747, 762, 747, 764, 799, 767, 735] payload_rows_per_dst=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] dup_rows_per_dst=[3094, 2860, 2882, 3086, 3068, 3038, 3038, 2972] combine_rows_per_src=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344072 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485768 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627464 weight_movement_bytes=34603008 total_nvlink_bytes=567230472 weight_to_token_ratio=0.064967 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20954 dedup_ratio=0.243490 router_tokens_per_expert=[769, 773, 757, 736, 755, 730, 740, 768, 806, 791, 762, 773, 754, 793, 813, 784, 790, 778, 790, 708, 761, 812, 782, 754, 757, 776, 761, 795, 786, 816, 732, 794, 731, 810, 777, 761, 794, 767, 743, 804, 776, 803, 751, 779, 745, 770, 810, 763, 790, 747, 797, 752, 768, 725, 704, 770, 731, 732, 767, 787, 729, 764, 751, 758] payload_rows_per_dst=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] dup_rows_per_dst=[2908, 3152, 3020, 3022, 3052, 2982, 2932, 2868] combine_rows_per_src=[9148, 9400, 9330, 9412, 9322, 9412, 9174, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65300 dup_rows=23770 dup_rows_remote=20852 dedup_ratio=0.241801 router_tokens_per_expert=[753, 754, 756, 810, 798, 737, 807, 763, 800, 783, 792, 805, 779, 724, 756, 732, 790, 740, 808, 753, 756, 778, 732, 813, 742, 775, 724, 766, 783, 791, 760, 735, 725, 763, 779, 746, 791, 757, 764, 760, 754, 786, 742, 785, 752, 787, 774, 744, 784, 787, 728, 793, 768, 779, 720, 771, 750, 819, 770, 807, 776, 792, 746, 758] payload_rows_per_dst=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] dup_rows_per_dst=[2988, 3078, 2956, 2918, 2932, 2984, 2912, 3002] combine_rows_per_src=[9368, 9264, 9384, 9234, 9238, 9264, 9348, 9434] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64872 dup_rows=24062 dup_rows_remote=21022 dedup_ratio=0.244771 router_tokens_per_expert=[791, 770, 751, 796, 760, 748, 767, 778, 792, 811, 787, 796, 766, 779, 779, 821, 721, 759, 740, 759, 775, 737, 796, 723, 754, 800, 768, 739, 747, 752, 771, 765, 776, 752, 789, 738, 763, 778, 797, 732, 745, 768, 715, 738, 788, 741, 797, 809, 780, 822, 741, 755, 723, 731, 785, 789, 762, 770, 763, 818, 768, 791, 767, 763] payload_rows_per_dst=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] dup_rows_per_dst=[3004, 2946, 3156, 2924, 2734, 2944, 3314, 3040] combine_rows_per_src=[9318, 9146, 9410, 9286, 8982, 9258, 9472, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65122 dup_rows=23962 dup_rows_remote=21060 dedup_ratio=0.243754 router_tokens_per_expert=[809, 790, 759, 772, 785, 796, 778, 765, 738, 731, 748, 785, 754, 735, 796, 774, 787, 785, 777, 792, 767, 764, 791, 764, 817, 781, 764, 785, 772, 769, 741, 749, 730, 749, 786, 827, 774, 762, 723, 823, 767, 742, 767, 735, 780, 740, 733, 740, 738, 758, 776, 769, 766, 779, 749, 743, 756, 800, 770, 755, 817, 739, 777, 762] payload_rows_per_dst=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] dup_rows_per_dst=[2994, 2902, 3186, 3100, 3016, 2812, 2906, 3046] combine_rows_per_src=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65018 dup_rows=23976 dup_rows_remote=21064 dedup_ratio=0.243896 router_tokens_per_expert=[750, 785, 788, 787, 727, 766, 804, 793, 742, 771, 788, 778, 766, 744, 788, 762, 744, 775, 763, 813, 775, 780, 683, 778, 765, 732, 759, 737, 785, 790, 789, 809, 755, 803, 743, 801, 753, 793, 760, 786, 772, 753, 728, 755, 830, 783, 770, 766, 760, 721, 778, 767, 741, 774, 777, 766, 797, 712, 747, 747, 774, 771, 756, 797] payload_rows_per_dst=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] dup_rows_per_dst=[3028, 2912, 2912, 3110, 3098, 3004, 2920, 2992] combine_rows_per_src=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64980 dup_rows=23898 dup_rows_remote=20848 dedup_ratio=0.243103 router_tokens_per_expert=[765, 738, 755, 813, 736, 782, 786, 771, 798, 810, 751, 762, 785, 819, 726, 753, 736, 707, 775, 729, 721, 746, 824, 791, 737, 768, 787, 776, 737, 775, 798, 728, 836, 812, 754, 763, 753, 730, 755, 758, 770, 790, 758, 803, 767, 748, 799, 728, 813, 800, 758, 777, 776, 811, 743, 760, 768, 777, 760, 697, 801, 749, 786, 767] payload_rows_per_dst=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] dup_rows_per_dst=[3004, 3092, 2880, 2962, 3050, 2978, 3050, 2882] combine_rows_per_src=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=341960 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265959368 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531576776 weight_movement_bytes=34603008 total_nvlink_bytes=566179784 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65018 dup_rows=23976 dup_rows_remote=21064 dedup_ratio=0.243896 router_tokens_per_expert=[750, 785, 788, 787, 727, 766, 804, 793, 742, 771, 788, 778, 766, 744, 788, 762, 744, 775, 763, 813, 775, 780, 683, 778, 765, 732, 759, 737, 785, 790, 789, 809, 755, 803, 743, 801, 753, 793, 760, 786, 772, 753, 728, 755, 830, 783, 770, 766, 760, 721, 778, 767, 741, 774, 777, 766, 797, 712, 747, 747, 774, 771, 756, 797] payload_rows_per_dst=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] dup_rows_per_dst=[3028, 2912, 2912, 3110, 3098, 3004, 2920, 2992] combine_rows_per_src=[9372, 9366, 9310, 9222, 9290, 9310, 9248, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64882 dup_rows=24038 dup_rows_remote=20970 dedup_ratio=0.244527 router_tokens_per_expert=[787, 728, 734, 783, 792, 762, 790, 773, 786, 712, 820, 751, 762, 774, 774, 727, 731, 771, 722, 749, 822, 727, 763, 755, 798, 765, 800, 750, 793, 780, 780, 797, 781, 775, 768, 775, 807, 795, 788, 737, 774, 772, 762, 771, 833, 782, 731, 764, 778, 726, 723, 774, 744, 772, 796, 750, 795, 747, 762, 747, 764, 799, 767, 735] payload_rows_per_dst=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] dup_rows_per_dst=[3094, 2860, 2882, 3086, 3068, 3038, 3038, 2972] combine_rows_per_src=[9204, 9352, 9198, 9440, 9384, 9340, 9088, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65122 dup_rows=23962 dup_rows_remote=21060 dedup_ratio=0.243754 router_tokens_per_expert=[809, 790, 759, 772, 785, 796, 778, 765, 738, 731, 748, 785, 754, 735, 796, 774, 787, 785, 777, 792, 767, 764, 791, 764, 817, 781, 764, 785, 772, 769, 741, 749, 730, 749, 786, 827, 774, 762, 723, 823, 767, 742, 767, 735, 780, 740, 733, 740, 738, 758, 776, 769, 766, 779, 749, 743, 756, 800, 770, 755, 817, 739, 777, 762] payload_rows_per_dst=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] dup_rows_per_dst=[2994, 2902, 3186, 3100, 3016, 2812, 2906, 3046] combine_rows_per_src=[9514, 9220, 9268, 9256, 9332, 9196, 9250, 9306] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267468800 dispatch_meta_bytes=344608 combine_payload_bytes=267468800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267813408 ingress_bytes=302071808 link_bytes=302071808 token_bytes=535282208 weight_movement_bytes=34603008 total_nvlink_bytes=569885216 weight_to_token_ratio=0.064644 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136671895e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343408 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100080 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856752 weight_movement_bytes=69206016 total_nvlink_bytes=601062768 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343640 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266403416 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532463192 weight_movement_bytes=69206016 total_nvlink_bytes=601669208 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343576 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059288 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775000 weight_movement_bytes=69206016 total_nvlink_bytes=600981016 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344728 combine_payload_bytes=266739712 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267084440 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533824152 weight_movement_bytes=138412032 total_nvlink_bytes=672236184 weight_to_token_ratio=0.259284 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344328 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658056 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971784 weight_movement_bytes=34603008 total_nvlink_bytes=567574792 weight_to_token_ratio=0.064925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343408 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100080 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856752 weight_movement_bytes=69206016 total_nvlink_bytes=601062768 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344728 combine_payload_bytes=266739712 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267084440 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533824152 weight_movement_bytes=138412032 total_nvlink_bytes=672236184 weight_to_token_ratio=0.259284 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64980 dup_rows=23898 dup_rows_remote=20848 dedup_ratio=0.243103 router_tokens_per_expert=[765, 738, 755, 813, 736, 782, 786, 771, 798, 810, 751, 762, 785, 819, 726, 753, 736, 707, 775, 729, 721, 746, 824, 791, 737, 768, 787, 776, 737, 775, 798, 728, 836, 812, 754, 763, 753, 730, 755, 758, 770, 790, 758, 803, 767, 748, 799, 728, 813, 800, 758, 777, 776, 811, 743, 760, 768, 777, 760, 697, 801, 749, 786, 767] payload_rows_per_dst=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] dup_rows_per_dst=[3004, 3092, 2880, 2962, 3050, 2978, 3050, 2882] combine_rows_per_src=[9288, 9316, 9178, 9250, 9272, 9348, 9426, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343312 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266501392 ingress_bytes=300761088 link_bytes=300761088 token_bytes=532659472 weight_movement_bytes=34603008 total_nvlink_bytes=567262480 weight_to_token_ratio=0.064963 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131739761e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64994 dup_rows=24144 dup_rows_remote=21238 dedup_ratio=0.245605 router_tokens_per_expert=[733, 786, 757, 698, 719, 761, 808, 745, 738, 772, 773, 776, 772, 802, 769, 776, 780, 749, 741, 772, 768, 779, 784, 744, 791, 723, 742, 734, 781, 760, 774, 736, 759, 743, 761, 751, 782, 748, 784, 784, 743, 745, 837, 795, 816, 780, 808, 788, 771, 800, 778, 718, 742, 739, 797, 768, 818, 749, 757, 792, 765, 826, 762, 803] payload_rows_per_dst=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] dup_rows_per_dst=[2906, 3070, 2938, 2888, 3184, 3104, 2990, 3064] combine_rows_per_src=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64884 dup_rows=24028 dup_rows_remote=20862 dedup_ratio=0.244425 router_tokens_per_expert=[748, 803, 787, 780, 811, 775, 799, 762, 791, 760, 754, 771, 780, 755, 779, 788, 733, 755, 762, 737, 797, 738, 743, 722, 745, 788, 807, 833, 759, 754, 774, 819, 749, 764, 766, 740, 802, 773, 721, 745, 761, 767, 771, 757, 767, 779, 780, 750, 782, 737, 765, 753, 768, 794, 776, 763, 747, 722, 835, 804, 708, 782, 769, 746] payload_rows_per_dst=[9404, 9362, 9120, 9392, 9178, 9330, 9330, 9160] dup_rows_per_dst=[3126, 2994, 2854, 3166, 2942, 2934, 2946, 3066] combine_rows_per_src=[9404, 9362, 9120, 9392, 9178, 9330, 9330, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344928 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560352 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775776 weight_movement_bytes=34603008 total_nvlink_bytes=567378784 weight_to_token_ratio=0.064949 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64848 dup_rows=24170 dup_rows_remote=21080 dedup_ratio=0.245870 router_tokens_per_expert=[784, 787, 721, 762, 825, 759, 783, 779, 761, 766, 753, 753, 738, 741, 733, 795, 735, 729, 773, 785, 766, 815, 799, 776, 781, 795, 716, 754, 729, 776, 743, 752, 744, 804, 724, 827, 769, 767, 793, 760, 773, 772, 766, 833, 729, 760, 796, 805, 748, 694, 800, 778, 713, 788, 742, 815, 775, 749, 756, 761, 748, 781, 793, 825] payload_rows_per_dst=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] dup_rows_per_dst=[3080, 2914, 3072, 2886, 3090, 3080, 2924, 3124] combine_rows_per_src=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65158 dup_rows=23756 dup_rows_remote=20754 dedup_ratio=0.241659 router_tokens_per_expert=[789, 778, 769, 762, 796, 793, 755, 765, 774, 793, 795, 752, 743, 765, 791, 735, 767, 777, 756, 809, 783, 770, 772, 773, 781, 776, 727, 764, 749, 728, 773, 792, 779, 769, 770, 740, 725, 810, 774, 724, 768, 737, 785, 796, 747, 770, 766, 827, 759, 732, 802, 797, 788, 754, 788, 744, 761, 712, 795, 759, 701, 815, 748, 758] payload_rows_per_dst=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] dup_rows_per_dst=[3004, 2858, 3074, 2930, 2944, 3002, 3030, 2914] combine_rows_per_src=[9410, 9438, 9340, 9250, 9238, 9390, 9298, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64994 dup_rows=24144 dup_rows_remote=21238 dedup_ratio=0.245605 router_tokens_per_expert=[733, 786, 757, 698, 719, 761, 808, 745, 738, 772, 773, 776, 772, 802, 769, 776, 780, 749, 741, 772, 768, 779, 784, 744, 791, 723, 742, 734, 781, 760, 774, 736, 759, 743, 761, 751, 782, 748, 784, 784, 743, 745, 837, 795, 816, 780, 808, 788, 771, 800, 778, 718, 742, 739, 797, 768, 818, 749, 757, 792, 765, 826, 762, 803] payload_rows_per_dst=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] dup_rows_per_dst=[2906, 3070, 2938, 2888, 3184, 3104, 2990, 3064] combine_rows_per_src=[9166, 9228, 9296, 9194, 9350, 9330, 9236, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64734 dup_rows=24198 dup_rows_remote=20958 dedup_ratio=0.246155 router_tokens_per_expert=[769, 765, 750, 784, 792, 771, 791, 729, 743, 745, 744, 756, 729, 760, 787, 775, 788, 721, 752, 769, 748, 747, 790, 742, 745, 729, 773, 738, 763, 744, 760, 733, 800, 815, 820, 830, 768, 712, 761, 805, 759, 806, 778, 748, 796, 786, 771, 792, 776, 765, 786, 782, 762, 720, 760, 814, 782, 801, 787, 798, 786, 742, 784, 728] payload_rows_per_dst=[8924, 9186, 9390, 9054, 9128, 9470, 9582, 9372] dup_rows_per_dst=[2802, 2892, 3142, 2898, 2784, 3160, 3280, 3240] combine_rows_per_src=[8924, 9186, 9390, 9054, 9128, 9470, 9582, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74586 payload_rows_remote=65302 dup_rows=23718 dup_rows_remote=20728 dedup_ratio=0.241272 router_tokens_per_expert=[801, 762, 786, 764, 811, 728, 747, 768, 758, 781, 778, 810, 752, 797, 792, 746, 740, 798, 743, 757, 788, 728, 776, 807, 758, 799, 746, 758, 754, 826, 764, 760, 754, 744, 784, 796, 759, 780, 746, 768, 782, 747, 761, 744, 752, 737, 760, 811, 741, 780, 810, 777, 771, 748, 783, 747, 742, 808, 774, 750, 751, 767, 725, 770] payload_rows_per_dst=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] dup_rows_per_dst=[3002, 3062, 2990, 2918, 3006, 2950, 2946, 2844] combine_rows_per_src=[9332, 9366, 9284, 9412, 9256, 9238, 9368, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65018 dup_rows=23908 dup_rows_remote=20986 dedup_ratio=0.243205 router_tokens_per_expert=[774, 772, 788, 770, 742, 780, 781, 764, 829, 722, 771, 813, 744, 757, 748, 766, 786, 782, 788, 760, 725, 753, 724, 751, 748, 791, 775, 810, 828, 788, 762, 821, 750, 731, 850, 753, 767, 724, 770, 818, 738, 815, 743, 786, 799, 718, 739, 739, 726, 720, 766, 764, 784, 736, 770, 790, 776, 727, 737, 816, 778, 769, 768, 772] payload_rows_per_dst=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] dup_rows_per_dst=[2968, 2922, 2894, 3118, 3040, 2926, 2998, 3042] combine_rows_per_src=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64848 dup_rows=24170 dup_rows_remote=21080 dedup_ratio=0.245870 router_tokens_per_expert=[784, 787, 721, 762, 825, 759, 783, 779, 761, 766, 753, 753, 738, 741, 733, 795, 735, 729, 773, 785, 766, 815, 799, 776, 781, 795, 716, 754, 729, 776, 743, 752, 744, 804, 724, 827, 769, 767, 793, 760, 773, 772, 766, 833, 729, 760, 796, 805, 748, 694, 800, 778, 713, 788, 742, 815, 775, 749, 756, 761, 748, 781, 793, 825] payload_rows_per_dst=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] dup_rows_per_dst=[3080, 2914, 3072, 2886, 3090, 3080, 2924, 3124] combine_rows_per_src=[9320, 9166, 9284, 9206, 9286, 9388, 9232, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65018 dup_rows=23908 dup_rows_remote=20986 dedup_ratio=0.243205 router_tokens_per_expert=[774, 772, 788, 770, 742, 780, 781, 764, 829, 722, 771, 813, 744, 757, 748, 766, 786, 782, 788, 760, 725, 753, 724, 751, 748, 791, 775, 810, 828, 788, 762, 821, 750, 731, 850, 753, 767, 724, 770, 818, 738, 815, 743, 786, 799, 718, 739, 739, 726, 720, 766, 764, 784, 736, 770, 790, 776, 727, 737, 816, 778, 769, 768, 772] payload_rows_per_dst=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] dup_rows_per_dst=[2968, 2922, 2894, 3118, 3040, 2926, 2998, 3042] combine_rows_per_src=[9374, 9378, 9244, 9528, 9286, 9228, 9114, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65282 dup_rows=23846 dup_rows_remote=20862 dedup_ratio=0.242574 router_tokens_per_expert=[718, 737, 718, 744, 792, 802, 821, 783, 760, 748, 795, 782, 751, 783, 811, 753, 784, 749, 744, 798, 778, 777, 746, 781, 750, 752, 779, 843, 769, 768, 793, 793, 749, 775, 755, 780, 801, 796, 773, 805, 745, 734, 743, 732, 744, 786, 738, 730, 734, 758, 744, 727, 760, 770, 824, 763, 741, 786, 783, 764, 769, 783, 749, 809] payload_rows_per_dst=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] dup_rows_per_dst=[2916, 2976, 3060, 3154, 3086, 2756, 2984, 2914] combine_rows_per_src=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=342984 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266107848 ingress_bytes=369573888 link_bytes=369573888 token_bytes=531872712 weight_movement_bytes=103809024 total_nvlink_bytes=635681736 weight_to_token_ratio=0.195176 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390676787e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343712 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265961120 ingress_bytes=369426432 link_bytes=369426432 token_bytes=531578528 weight_movement_bytes=103809024 total_nvlink_bytes=635387552 weight_to_token_ratio=0.195284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390121922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343648 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267230816 ingress_bytes=301490176 link_bytes=301490176 token_bytes=534117984 weight_movement_bytes=34603008 total_nvlink_bytes=568720992 weight_to_token_ratio=0.064785 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134483260e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344928 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560352 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775776 weight_movement_bytes=34603008 total_nvlink_bytes=567378784 weight_to_token_ratio=0.064949 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342768 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493232 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530643696 weight_movement_bytes=34603008 total_nvlink_bytes=565246704 weight_to_token_ratio=0.065209 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=344120 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267821112 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535298104 weight_movement_bytes=34603008 total_nvlink_bytes=569901112 weight_to_token_ratio=0.064643 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343712 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265961120 ingress_bytes=369426432 link_bytes=369426432 token_bytes=531578528 weight_movement_bytes=103809024 total_nvlink_bytes=635387552 weight_to_token_ratio=0.195284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390121922e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65282 dup_rows=23846 dup_rows_remote=20862 dedup_ratio=0.242574 router_tokens_per_expert=[718, 737, 718, 744, 792, 802, 821, 783, 760, 748, 795, 782, 751, 783, 811, 753, 784, 749, 744, 798, 778, 777, 746, 781, 750, 752, 779, 843, 769, 768, 793, 793, 749, 775, 755, 780, 801, 796, 773, 805, 745, 734, 743, 732, 744, 786, 738, 730, 734, 758, 744, 727, 760, 770, 824, 763, 741, 786, 783, 764, 769, 783, 749, 809] payload_rows_per_dst=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] dup_rows_per_dst=[2916, 2976, 3060, 3154, 3086, 2756, 2984, 2914] combine_rows_per_src=[9314, 9390, 9254, 9340, 9382, 9148, 9176, 9454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344016 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266657744 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971472 weight_movement_bytes=34603008 total_nvlink_bytes=567574480 weight_to_token_ratio=0.064925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344576 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739648 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535134720 weight_movement_bytes=34603008 total_nvlink_bytes=569737728 weight_to_token_ratio=0.064662 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65202 dup_rows=23764 dup_rows_remote=20774 dedup_ratio=0.241740 router_tokens_per_expert=[803, 738, 755, 756, 795, 796, 768, 732, 774, 749, 705, 782, 798, 799, 753, 817, 762, 764, 787, 818, 799, 772, 726, 723, 782, 752, 783, 719, 810, 743, 755, 726, 786, 804, 820, 756, 793, 842, 774, 754, 744, 805, 786, 756, 734, 749, 783, 795, 728, 774, 790, 764, 733, 759, 759, 741, 794, 767, 689, 760, 720, 747, 811, 794] payload_rows_per_dst=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] dup_rows_per_dst=[2990, 3032, 2944, 2772, 3208, 2920, 2956, 2942] combine_rows_per_src=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343904 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411296 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478688 weight_movement_bytes=103809024 total_nvlink_bytes=638287712 weight_to_token_ratio=0.194225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65144 dup_rows=23920 dup_rows_remote=21084 dedup_ratio=0.243327 router_tokens_per_expert=[786, 749, 778, 809, 744, 774, 801, 753, 785, 774, 826, 795, 855, 730, 778, 705, 737, 760, 764, 760, 795, 754, 813, 744, 755, 723, 784, 771, 746, 747, 765, 747, 750, 760, 769, 742, 752, 781, 771, 770, 752, 789, 796, 825, 764, 736, 793, 745, 773, 781, 802, 724, 765, 754, 746, 753, 788, 749, 737, 742, 754, 860, 758, 764] payload_rows_per_dst=[9382, 9434, 9272, 9240, 9152, 9358, 9208, 9338] dup_rows_per_dst=[3006, 3062, 2982, 2836, 3038, 3042, 2988, 2966] combine_rows_per_src=[9382, 9434, 9272, 9240, 9152, 9358, 9208, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65090 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[741, 769, 783, 760, 809, 766, 795, 783, 747, 771, 767, 732, 756, 777, 788, 750, 758, 777, 778, 782, 761, 731, 742, 753, 765, 757, 752, 781, 770, 758, 734, 739, 786, 733, 706, 817, 768, 765, 775, 777, 770, 742, 772, 793, 714, 797, 759, 754, 791, 774, 782, 729, 804, 796, 795, 763, 799, 786, 773, 812, 790, 746, 777, 775] payload_rows_per_dst=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] dup_rows_per_dst=[3088, 2900, 2852, 2946, 3074, 2926, 3118, 3130] combine_rows_per_src=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65202 dup_rows=23764 dup_rows_remote=20774 dedup_ratio=0.241740 router_tokens_per_expert=[803, 738, 755, 756, 795, 796, 768, 732, 774, 749, 705, 782, 798, 799, 753, 817, 762, 764, 787, 818, 799, 772, 726, 723, 782, 752, 783, 719, 810, 743, 755, 726, 786, 804, 820, 756, 793, 842, 774, 754, 744, 805, 786, 756, 734, 749, 783, 795, 728, 774, 790, 764, 733, 759, 759, 741, 794, 767, 689, 760, 720, 747, 811, 794] payload_rows_per_dst=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] dup_rows_per_dst=[2990, 3032, 2944, 2772, 3208, 2920, 2956, 2942] combine_rows_per_src=[9338, 9368, 9358, 9280, 9450, 9384, 9140, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64816 dup_rows=24166 dup_rows_remote=21092 dedup_ratio=0.245829 router_tokens_per_expert=[797, 829, 776, 742, 739, 739, 731, 767, 793, 790, 763, 784, 776, 781, 815, 778, 781, 753, 733, 761, 752, 794, 759, 795, 729, 770, 763, 755, 780, 761, 779, 759, 810, 766, 792, 705, 733, 723, 782, 792, 772, 749, 801, 764, 764, 770, 759, 819, 829, 718, 707, 761, 805, 755, 732, 737, 762, 795, 804, 766, 750, 795, 728, 783] payload_rows_per_dst=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] dup_rows_per_dst=[3016, 3092, 3042, 2966, 3004, 3074, 2958, 3014] combine_rows_per_src=[9224, 9468, 9214, 9226, 9202, 9322, 9130, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64980 dup_rows=24062 dup_rows_remote=21220 dedup_ratio=0.244771 router_tokens_per_expert=[782, 771, 777, 718, 746, 770, 775, 755, 796, 718, 731, 788, 759, 751, 804, 750, 749, 829, 747, 762, 755, 794, 756, 759, 762, 825, 750, 780, 735, 807, 814, 803, 803, 761, 762, 755, 827, 726, 766, 744, 781, 801, 733, 762, 784, 717, 774, 819, 764, 767, 809, 759, 762, 759, 781, 770, 757, 691, 784, 768, 778, 747, 771, 752] payload_rows_per_dst=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] dup_rows_per_dst=[2844, 2818, 3018, 3136, 3086, 3028, 3290, 2842] combine_rows_per_src=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65232 dup_rows=23688 dup_rows_remote=20810 dedup_ratio=0.240967 router_tokens_per_expert=[791, 759, 773, 751, 800, 780, 746, 792, 746, 769, 782, 750, 830, 751, 787, 797, 797, 745, 780, 757, 735, 795, 741, 781, 732, 762, 774, 738, 740, 800, 780, 813, 753, 766, 776, 810, 802, 748, 782, 748, 792, 747, 779, 821, 741, 746, 752, 775, 732, 743, 725, 771, 778, 800, 733, 752, 752, 767, 779, 735, 742, 775, 786, 770] payload_rows_per_dst=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] dup_rows_per_dst=[2980, 3048, 2878, 2936, 3044, 2946, 2994, 2862] combine_rows_per_src=[9404, 9376, 9384, 9342, 9326, 9360, 9074, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64980 dup_rows=24062 dup_rows_remote=21220 dedup_ratio=0.244771 router_tokens_per_expert=[782, 771, 777, 718, 746, 770, 775, 755, 796, 718, 731, 788, 759, 751, 804, 750, 749, 829, 747, 762, 755, 794, 756, 759, 762, 825, 750, 780, 735, 807, 814, 803, 803, 761, 762, 755, 827, 726, 766, 744, 781, 801, 733, 762, 784, 717, 774, 819, 764, 767, 809, 759, 762, 759, 781, 770, 757, 691, 784, 768, 778, 747, 771, 752] payload_rows_per_dst=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] dup_rows_per_dst=[2844, 2818, 3018, 3136, 3086, 3028, 3290, 2842] combine_rows_per_src=[9142, 9034, 9284, 9416, 9284, 9330, 9490, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65090 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[741, 769, 783, 760, 809, 766, 795, 783, 747, 771, 767, 732, 756, 777, 788, 750, 758, 777, 778, 782, 761, 731, 742, 753, 765, 757, 752, 781, 770, 758, 734, 739, 786, 733, 706, 817, 768, 765, 775, 777, 770, 742, 772, 793, 714, 797, 759, 754, 791, 774, 782, 729, 804, 796, 795, 763, 799, 786, 773, 812, 790, 746, 777, 775] payload_rows_per_dst=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] dup_rows_per_dst=[3088, 2900, 2852, 2946, 3074, 2926, 3118, 3130] combine_rows_per_src=[9324, 9276, 9312, 9166, 9180, 9276, 9350, 9386] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65276 dup_rows=23892 dup_rows_remote=21038 dedup_ratio=0.243042 router_tokens_per_expert=[763, 811, 748, 778, 746, 790, 773, 776, 761, 774, 706, 749, 759, 701, 754, 791, 744, 777, 789, 762, 746, 809, 779, 782, 775, 743, 766, 792, 779, 758, 777, 841, 788, 760, 766, 788, 716, 777, 787, 726, 780, 763, 794, 766, 760, 781, 775, 757, 757, 747, 774, 810, 761, 711, 723, 766, 746, 773, 807, 772, 794, 785, 721, 822] payload_rows_per_dst=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] dup_rows_per_dst=[3034, 2854, 2968, 3064, 2960, 2996, 2920, 3096] combine_rows_per_src=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65098 dup_rows=23982 dup_rows_remote=20894 dedup_ratio=0.243958 router_tokens_per_expert=[741, 782, 783, 775, 780, 751, 767, 763, 760, 860, 738, 697, 794, 822, 757, 754, 771, 801, 772, 714, 790, 756, 767, 738, 765, 787, 797, 744, 751, 724, 757, 756, 739, 738, 784, 770, 762, 779, 773, 772, 785, 796, 740, 737, 792, 764, 778, 774, 742, 837, 789, 707, 775, 800, 756, 750, 799, 747, 769, 790, 762, 774, 774, 784] payload_rows_per_dst=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] dup_rows_per_dst=[3012, 3028, 2932, 2888, 2962, 3012, 3088, 3060] combine_rows_per_src=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344912 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174736 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534004560 weight_movement_bytes=34603008 total_nvlink_bytes=568607568 weight_to_token_ratio=0.064799 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344200 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952840 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561480 weight_movement_bytes=34603008 total_nvlink_bytes=568164488 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65276 dup_rows=23892 dup_rows_remote=21038 dedup_ratio=0.243042 router_tokens_per_expert=[763, 811, 748, 778, 746, 790, 773, 776, 761, 774, 706, 749, 759, 701, 754, 791, 744, 777, 789, 762, 746, 809, 779, 782, 775, 743, 766, 792, 779, 758, 777, 841, 788, 760, 766, 788, 716, 777, 787, 726, 780, 763, 794, 766, 760, 781, 775, 757, 757, 747, 774, 810, 761, 711, 723, 766, 746, 773, 807, 772, 794, 785, 721, 822] payload_rows_per_dst=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] dup_rows_per_dst=[3034, 2854, 2968, 3064, 2960, 2996, 2920, 3096] combine_rows_per_src=[9336, 9136, 9408, 9398, 9256, 9356, 9178, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343904 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411296 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478688 weight_movement_bytes=103809024 total_nvlink_bytes=638287712 weight_to_token_ratio=0.194225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344168 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534440 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724712 weight_movement_bytes=34603008 total_nvlink_bytes=569327720 weight_to_token_ratio=0.064712 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=343632 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265829968 ingress_bytes=334692352 link_bytes=334692352 token_bytes=531316304 weight_movement_bytes=69206016 total_nvlink_bytes=600522320 weight_to_token_ratio=0.130254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259420376e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344800 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502880 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660960 weight_movement_bytes=69206016 total_nvlink_bytes=601866976 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344168 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267534440 ingress_bytes=301793280 link_bytes=301793280 token_bytes=534724712 weight_movement_bytes=34603008 total_nvlink_bytes=569327720 weight_to_token_ratio=0.064712 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135623816e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344200 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952840 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561480 weight_movement_bytes=34603008 total_nvlink_bytes=568164488 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65098 dup_rows=23982 dup_rows_remote=20894 dedup_ratio=0.243958 router_tokens_per_expert=[741, 782, 783, 775, 780, 751, 767, 763, 760, 860, 738, 697, 794, 822, 757, 754, 771, 801, 772, 714, 790, 756, 767, 738, 765, 787, 797, 744, 751, 724, 757, 756, 739, 738, 784, 770, 762, 779, 773, 772, 785, 796, 740, 737, 792, 764, 778, 774, 742, 837, 789, 707, 775, 800, 756, 750, 799, 747, 769, 790, 762, 774, 774, 784] payload_rows_per_dst=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] dup_rows_per_dst=[3012, 3028, 2932, 2888, 2962, 3012, 3088, 3060] combine_rows_per_src=[9272, 9336, 9286, 9274, 9272, 9320, 9224, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=345256 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267715752 ingress_bytes=336576512 link_bytes=336576512 token_bytes=535086248 weight_movement_bytes=69206016 total_nvlink_bytes=604292264 weight_to_token_ratio=0.129336 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266510318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64886 dup_rows=24176 dup_rows_remote=21178 dedup_ratio=0.245931 router_tokens_per_expert=[761, 745, 760, 757, 757, 791, 759, 790, 772, 793, 762, 806, 773, 762, 778, 775, 748, 734, 821, 726, 746, 765, 777, 771, 757, 776, 729, 763, 832, 728, 736, 747, 780, 747, 761, 788, 799, 758, 794, 805, 736, 746, 789, 772, 760, 770, 798, 775, 799, 724, 745, 773, 813, 788, 784, 721, 763, 753, 768, 779, 759, 743, 779, 786] payload_rows_per_dst=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] dup_rows_per_dst=[2998, 3134, 3094, 3138, 3116, 2816, 3040, 2840] combine_rows_per_src=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985376 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626784 weight_movement_bytes=34603008 total_nvlink_bytes=568229792 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344256 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266117312 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531890368 weight_movement_bytes=138412032 total_nvlink_bytes=670302400 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65298 dup_rows=23852 dup_rows_remote=21036 dedup_ratio=0.242635 router_tokens_per_expert=[758, 694, 816, 764, 779, 805, 787, 769, 792, 751, 758, 750, 759, 737, 775, 768, 785, 802, 765, 741, 783, 794, 809, 779, 766, 784, 740, 720, 757, 754, 713, 751, 738, 775, 739, 774, 745, 753, 738, 795, 771, 784, 762, 768, 756, 790, 860, 792, 783, 779, 792, 732, 802, 745, 747, 786, 781, 740, 738, 795, 858, 720, 769, 740] payload_rows_per_dst=[9282, 9240, 9426, 9154, 9216, 9604, 9314, 9216] dup_rows_per_dst=[3062, 2940, 3090, 2816, 2898, 2962, 3018, 3066] combine_rows_per_src=[9282, 9240, 9426, 9154, 9216, 9604, 9314, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65076 dup_rows=24044 dup_rows_remote=21070 dedup_ratio=0.244588 router_tokens_per_expert=[743, 783, 783, 762, 767, 764, 753, 751, 774, 761, 818, 744, 733, 790, 738, 743, 743, 777, 774, 726, 732, 782, 726, 837, 735, 744, 767, 750, 809, 728, 782, 761, 788, 749, 797, 783, 725, 734, 737, 766, 780, 769, 789, 753, 762, 824, 775, 788, 770, 752, 787, 757, 777, 795, 759, 758, 788, 765, 768, 813, 778, 799, 777, 810] payload_rows_per_dst=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] dup_rows_per_dst=[2960, 2884, 2958, 2988, 2974, 3120, 3046, 3114] combine_rows_per_src=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65102 dup_rows=23968 dup_rows_remote=21042 dedup_ratio=0.243815 router_tokens_per_expert=[764, 821, 777, 768, 807, 701, 779, 760, 764, 764, 765, 739, 756, 746, 756, 780, 787, 777, 792, 786, 824, 780, 752, 781, 789, 794, 743, 771, 743, 789, 809, 785, 734, 755, 739, 795, 753, 774, 785, 769, 815, 766, 747, 720, 730, 752, 789, 761, 767, 754, 802, 736, 812, 794, 745, 753, 787, 746, 728, 741, 765, 730, 795, 764] payload_rows_per_dst=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] dup_rows_per_dst=[3054, 2940, 3170, 3070, 2910, 2926, 2968, 2930] combine_rows_per_src=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64884 dup_rows=23948 dup_rows_remote=20804 dedup_ratio=0.243612 router_tokens_per_expert=[800, 750, 732, 775, 765, 774, 742, 815, 780, 796, 741, 765, 687, 782, 767, 782, 822, 806, 756, 729, 762, 741, 737, 782, 749, 771, 768, 783, 701, 775, 814, 776, 775, 756, 765, 757, 797, 737, 786, 796, 755, 748, 782, 756, 800, 714, 801, 751, 779, 782, 748, 758, 750, 736, 742, 748, 790, 766, 808, 775, 814, 782, 777, 796] payload_rows_per_dst=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] dup_rows_per_dst=[2952, 2904, 2994, 3002, 3154, 2962, 2836, 3144] combine_rows_per_src=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65056 dup_rows=23962 dup_rows_remote=21020 dedup_ratio=0.243754 router_tokens_per_expert=[782, 790, 748, 767, 783, 741, 718, 801, 744, 751, 765, 785, 771, 746, 757, 769, 759, 726, 771, 776, 798, 761, 808, 715, 771, 769, 768, 750, 798, 808, 741, 757, 775, 799, 803, 770, 736, 780, 716, 827, 749, 786, 758, 719, 759, 728, 768, 792, 781, 767, 765, 770, 786, 775, 829, 774, 717, 750, 718, 787, 789, 796, 751, 838] payload_rows_per_dst=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] dup_rows_per_dst=[2966, 2964, 2942, 3118, 3016, 2938, 3014, 3004] combine_rows_per_src=[9294, 9212, 9286, 9206, 9396, 9180, 9480, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64886 dup_rows=24176 dup_rows_remote=21178 dedup_ratio=0.245931 router_tokens_per_expert=[761, 745, 760, 757, 757, 791, 759, 790, 772, 793, 762, 806, 773, 762, 778, 775, 748, 734, 821, 726, 746, 765, 777, 771, 757, 776, 729, 763, 832, 728, 736, 747, 780, 747, 761, 788, 799, 758, 794, 805, 736, 746, 789, 772, 760, 770, 798, 775, 799, 724, 745, 773, 813, 788, 784, 721, 763, 753, 768, 779, 759, 743, 779, 786] payload_rows_per_dst=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] dup_rows_per_dst=[2998, 3134, 3094, 3138, 3116, 2816, 3040, 2840] combine_rows_per_src=[9242, 9308, 9378, 9332, 9348, 9142, 9254, 9124] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65076 dup_rows=24044 dup_rows_remote=21070 dedup_ratio=0.244588 router_tokens_per_expert=[743, 783, 783, 762, 767, 764, 753, 751, 774, 761, 818, 744, 733, 790, 738, 743, 743, 777, 774, 726, 732, 782, 726, 837, 735, 744, 767, 750, 809, 728, 782, 761, 788, 749, 797, 783, 725, 734, 737, 766, 780, 769, 789, 753, 762, 824, 775, 788, 770, 752, 787, 757, 777, 795, 759, 758, 788, 765, 768, 813, 778, 799, 777, 810] payload_rows_per_dst=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] dup_rows_per_dst=[2960, 2884, 2958, 2988, 2974, 3120, 3046, 3114] combine_rows_per_src=[9252, 9318, 9236, 9164, 9184, 9360, 9264, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64884 dup_rows=23948 dup_rows_remote=20804 dedup_ratio=0.243612 router_tokens_per_expert=[800, 750, 732, 775, 765, 774, 742, 815, 780, 796, 741, 765, 687, 782, 767, 782, 822, 806, 756, 729, 762, 741, 737, 782, 749, 771, 768, 783, 701, 775, 814, 776, 775, 756, 765, 757, 797, 737, 786, 796, 755, 748, 782, 756, 800, 714, 801, 751, 779, 782, 748, 758, 750, 736, 742, 748, 790, 766, 808, 775, 814, 782, 777, 796] payload_rows_per_dst=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] dup_rows_per_dst=[2952, 2904, 2994, 3002, 3154, 2962, 2836, 3144] combine_rows_per_src=[9064, 9296, 9276, 9272, 9474, 9252, 9250, 9472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65102 dup_rows=23968 dup_rows_remote=21042 dedup_ratio=0.243815 router_tokens_per_expert=[764, 821, 777, 768, 807, 701, 779, 760, 764, 764, 765, 739, 756, 746, 756, 780, 787, 777, 792, 786, 824, 780, 752, 781, 789, 794, 743, 771, 743, 789, 809, 785, 734, 755, 739, 795, 753, 774, 785, 769, 815, 766, 747, 720, 730, 752, 789, 761, 767, 754, 802, 736, 812, 794, 745, 753, 787, 746, 728, 741, 765, 730, 795, 764] payload_rows_per_dst=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] dup_rows_per_dst=[3054, 2940, 3170, 3070, 2910, 2926, 2968, 2930] combine_rows_per_src=[9300, 9200, 9388, 9376, 9298, 9234, 9358, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=345336 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267805944 ingress_bytes=302063616 link_bytes=302063616 token_bytes=535266552 weight_movement_bytes=34603008 total_nvlink_bytes=569869560 weight_to_token_ratio=0.064646 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136641069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344304 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266813680 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533283056 weight_movement_bytes=69206016 total_nvlink_bytes=602489072 weight_to_token_ratio=0.129774 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65146 dup_rows=23778 dup_rows_remote=20810 dedup_ratio=0.241882 router_tokens_per_expert=[777, 799, 723, 800, 791, 754, 779, 734, 743, 750, 742, 792, 761, 796, 813, 777, 755, 759, 716, 747, 794, 712, 764, 807, 787, 744, 806, 751, 788, 835, 757, 771, 747, 759, 756, 773, 761, 786, 754, 779, 790, 735, 774, 751, 764, 786, 756, 745, 755, 772, 809, 750, 744, 724, 781, 793, 767, 807, 785, 797, 736, 794, 743, 755] payload_rows_per_dst=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] dup_rows_per_dst=[2956, 2968, 2906, 3082, 2972, 2928, 2948, 3018] combine_rows_per_src=[9358, 9380, 9202, 9396, 9258, 9274, 9308, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344256 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266117312 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531890368 weight_movement_bytes=138412032 total_nvlink_bytes=670302400 weight_to_token_ratio=0.260227 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65052 dup_rows=23952 dup_rows_remote=20968 dedup_ratio=0.243652 router_tokens_per_expert=[751, 810, 808, 737, 831, 781, 749, 788, 782, 732, 756, 768, 780, 714, 773, 752, 744, 740, 791, 782, 740, 779, 788, 744, 751, 747, 794, 720, 788, 785, 750, 769, 772, 738, 760, 745, 777, 788, 758, 789, 744, 772, 730, 780, 774, 759, 783, 772, 803, 772, 744, 734, 772, 802, 773, 742, 835, 791, 783, 786, 735, 757, 783, 775] payload_rows_per_dst=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] dup_rows_per_dst=[3088, 2896, 3026, 3004, 3016, 2882, 2984, 3056] combine_rows_per_src=[9422, 9218, 9190, 9204, 9238, 9346, 9300, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=345336 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267805944 ingress_bytes=302063616 link_bytes=302063616 token_bytes=535266552 weight_movement_bytes=34603008 total_nvlink_bytes=569869560 weight_to_token_ratio=0.064646 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136641069e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344584 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266895880 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447176 weight_movement_bytes=34603008 total_nvlink_bytes=568050184 weight_to_token_ratio=0.064867 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344576 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002368 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533660160 weight_movement_bytes=69206016 total_nvlink_bytes=602866176 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344304 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266813680 ingress_bytes=335675392 link_bytes=335675392 token_bytes=533283056 weight_movement_bytes=69206016 total_nvlink_bytes=602489072 weight_to_token_ratio=0.129774 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263119476e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=342752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266107616 ingress_bytes=404176896 link_bytes=404176896 token_bytes=531872480 weight_movement_bytes=138412032 total_nvlink_bytes=670284512 weight_to_token_ratio=0.260235 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520885121e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65098 dup_rows=23920 dup_rows_remote=20912 dedup_ratio=0.243327 router_tokens_per_expert=[773, 712, 755, 781, 759, 763, 799, 752, 772, 735, 798, 750, 800, 780, 758, 762, 778, 789, 752, 790, 796, 748, 767, 726, 725, 751, 765, 727, 777, 816, 804, 782, 809, 792, 755, 708, 792, 770, 767, 757, 767, 750, 795, 804, 792, 801, 769, 756, 752, 799, 730, 789, 756, 754, 741, 782, 789, 778, 746, 769, 738, 743, 781, 779] payload_rows_per_dst=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] dup_rows_per_dst=[3008, 3284, 2992, 3140, 2940, 3034, 2926, 2596] combine_rows_per_src=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343824 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181840 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019856 weight_movement_bytes=34603008 total_nvlink_bytes=568622864 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344080 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266797072 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533250064 weight_movement_bytes=34603008 total_nvlink_bytes=567853072 weight_to_token_ratio=0.064891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344040 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266985448 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626856 weight_movement_bytes=69206016 total_nvlink_bytes=602832872 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64900 dup_rows=24186 dup_rows_remote=21154 dedup_ratio=0.246033 router_tokens_per_expert=[812, 748, 772, 754, 776, 774, 776, 760, 769, 793, 794, 731, 755, 776, 765, 776, 756, 759, 779, 737, 773, 744, 763, 775, 775, 796, 782, 787, 763, 751, 722, 759, 793, 775, 808, 744, 750, 768, 737, 750, 758, 769, 773, 757, 781, 742, 768, 750, 805, 754, 767, 792, 787, 765, 794, 765, 759, 769, 772, 776, 745, 763, 777, 787] payload_rows_per_dst=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] dup_rows_per_dst=[3104, 3004, 2920, 3044, 3032, 2974, 3138, 2970] combine_rows_per_src=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74738 payload_rows_remote=65288 dup_rows=23566 dup_rows_remote=20586 dedup_ratio=0.239726 router_tokens_per_expert=[759, 745, 740, 771, 743, 731, 809, 767, 818, 766, 757, 780, 763, 745, 794, 798, 796, 761, 728, 766, 767, 794, 778, 785, 766, 756, 800, 791, 792, 785, 724, 801, 750, 793, 729, 753, 820, 784, 802, 743, 745, 794, 738, 750, 799, 775, 784, 750, 739, 740, 816, 813, 714, 781, 780, 736, 741, 822, 740, 764, 728, 718, 765, 770] payload_rows_per_dst=[9352, 9456, 9326, 9450, 9324, 9272, 9284, 9274] dup_rows_per_dst=[2778, 2986, 3024, 2980, 3024, 2998, 2954, 2822] combine_rows_per_src=[9352, 9456, 9326, 9450, 9324, 9272, 9284, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64850 dup_rows=24060 dup_rows_remote=21062 dedup_ratio=0.244751 router_tokens_per_expert=[751, 756, 753, 772, 771, 781, 815, 784, 764, 777, 773, 778, 787, 772, 789, 683, 744, 734, 794, 769, 780, 835, 762, 778, 762, 803, 740, 789, 788, 754, 789, 759, 784, 730, 731, 731, 792, 753, 766, 753, 774, 770, 752, 772, 753, 755, 748, 759, 758, 762, 730, 754, 790, 758, 791, 787, 777, 754, 770, 807, 775, 808, 748, 774] payload_rows_per_dst=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] dup_rows_per_dst=[3050, 3028, 2998, 3010, 2980, 2986, 2958, 3050] combine_rows_per_src=[9316, 9218, 9394, 9358, 9100, 9180, 9302, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64900 dup_rows=24186 dup_rows_remote=21154 dedup_ratio=0.246033 router_tokens_per_expert=[812, 748, 772, 754, 776, 774, 776, 760, 769, 793, 794, 731, 755, 776, 765, 776, 756, 759, 779, 737, 773, 744, 763, 775, 775, 796, 782, 787, 763, 751, 722, 759, 793, 775, 808, 744, 750, 768, 737, 750, 758, 769, 773, 757, 781, 742, 768, 750, 805, 754, 767, 792, 787, 765, 794, 765, 759, 769, 772, 776, 745, 763, 777, 787] payload_rows_per_dst=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] dup_rows_per_dst=[3104, 3004, 2920, 3044, 3032, 2974, 3138, 2970] combine_rows_per_src=[9240, 9314, 9252, 9226, 9218, 9222, 9320, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65098 dup_rows=23920 dup_rows_remote=20912 dedup_ratio=0.243327 router_tokens_per_expert=[773, 712, 755, 781, 759, 763, 799, 752, 772, 735, 798, 750, 800, 780, 758, 762, 778, 789, 752, 790, 796, 748, 767, 726, 725, 751, 765, 727, 777, 816, 804, 782, 809, 792, 755, 708, 792, 770, 767, 757, 767, 750, 795, 804, 792, 801, 769, 756, 752, 799, 730, 789, 756, 754, 741, 782, 789, 778, 746, 769, 738, 743, 781, 779] payload_rows_per_dst=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] dup_rows_per_dst=[3008, 3284, 2992, 3140, 2940, 3034, 2926, 2596] combine_rows_per_src=[9286, 9660, 9222, 9312, 9360, 9302, 9290, 8952] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64658 dup_rows=24112 dup_rows_remote=20844 dedup_ratio=0.245280 router_tokens_per_expert=[738, 781, 735, 811, 775, 732, 750, 813, 790, 720, 779, 744, 792, 782, 811, 754, 753, 708, 785, 736, 724, 792, 777, 774, 819, 774, 764, 783, 751, 764, 797, 775, 743, 728, 782, 746, 795, 757, 778, 758, 800, 758, 815, 694, 742, 784, 796, 735, 780, 758, 783, 760, 710, 809, 787, 746, 748, 724, 792, 768, 793, 804, 802, 794] payload_rows_per_dst=[9282, 9166, 9270, 9062, 9276, 9326, 9276, 9534] dup_rows_per_dst=[2882, 3072, 2934, 2702, 3046, 3112, 3096, 3268] combine_rows_per_src=[9282, 9166, 9270, 9062, 9276, 9326, 9276, 9534] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64732 dup_rows=24174 dup_rows_remote=20994 dedup_ratio=0.245911 router_tokens_per_expert=[765, 752, 763, 746, 758, 748, 746, 762, 796, 751, 744, 755, 785, 818, 775, 781, 725, 768, 772, 757, 791, 738, 767, 794, 755, 787, 776, 735, 764, 770, 793, 757, 774, 756, 763, 765, 830, 768, 764, 759, 822, 809, 765, 819, 798, 773, 761, 742, 744, 733, 764, 749, 758, 717, 802, 791, 764, 727, 779, 741, 796, 730, 769, 826] payload_rows_per_dst=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] dup_rows_per_dst=[2880, 3116, 2918, 3008, 3016, 3180, 3056, 3000] combine_rows_per_src=[9200, 9294, 9306, 9266, 9342, 9398, 9060, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343648 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969248 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594848 weight_movement_bytes=34603008 total_nvlink_bytes=566197856 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64918 dup_rows=24096 dup_rows_remote=21088 dedup_ratio=0.245117 router_tokens_per_expert=[735, 772, 787, 816, 741, 763, 782, 796, 796, 798, 767, 770, 818, 703, 737, 760, 816, 760, 776, 758, 845, 786, 777, 797, 812, 798, 772, 754, 755, 739, 787, 799, 746, 761, 801, 799, 773, 715, 759, 760, 724, 771, 731, 773, 780, 720, 810, 729, 766, 718, 753, 776, 765, 786, 785, 766, 751, 750, 752, 762, 721, 758, 764, 755] payload_rows_per_dst=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] dup_rows_per_dst=[2988, 3008, 3232, 3104, 3034, 2782, 2988, 2960] combine_rows_per_src=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64820 dup_rows=24132 dup_rows_remote=21020 dedup_ratio=0.245483 router_tokens_per_expert=[747, 755, 776, 783, 757, 756, 822, 750, 759, 774, 727, 750, 766, 769, 752, 788, 751, 740, 772, 773, 746, 762, 733, 823, 797, 735, 759, 828, 764, 733, 770, 770, 735, 775, 766, 794, 747, 752, 768, 787, 804, 745, 761, 794, 826, 726, 765, 754, 748, 750, 753, 762, 780, 837, 801, 801, 789, 770, 760, 740, 771, 734, 768, 802] payload_rows_per_dst=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] dup_rows_per_dst=[2954, 2912, 2964, 3048, 3012, 3020, 3112, 3110] combine_rows_per_src=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=343496 combine_payload_bytes=267419648 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267763144 ingress_bytes=440434688 link_bytes=440434688 token_bytes=535182792 weight_movement_bytes=173015040 total_nvlink_bytes=708197832 weight_to_token_ratio=0.323282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657320273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64820 dup_rows=24132 dup_rows_remote=21020 dedup_ratio=0.245483 router_tokens_per_expert=[747, 755, 776, 783, 757, 756, 822, 750, 759, 774, 727, 750, 766, 769, 752, 788, 751, 740, 772, 773, 746, 762, 733, 823, 797, 735, 759, 828, 764, 733, 770, 770, 735, 775, 766, 794, 747, 752, 768, 787, 804, 745, 761, 794, 826, 726, 765, 754, 748, 750, 753, 762, 780, 837, 801, 801, 789, 770, 760, 740, 771, 734, 768, 802] payload_rows_per_dst=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] dup_rows_per_dst=[2954, 2912, 2964, 3048, 3012, 3020, 3112, 3110] combine_rows_per_src=[9338, 9258, 9236, 9264, 9236, 9330, 9352, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64918 dup_rows=24096 dup_rows_remote=21088 dedup_ratio=0.245117 router_tokens_per_expert=[735, 772, 787, 816, 741, 763, 782, 796, 796, 798, 767, 770, 818, 703, 737, 760, 816, 760, 776, 758, 845, 786, 777, 797, 812, 798, 772, 754, 755, 739, 787, 799, 746, 761, 801, 799, 773, 715, 759, 760, 724, 771, 731, 773, 780, 720, 810, 729, 766, 718, 753, 776, 765, 786, 785, 766, 751, 750, 752, 762, 721, 758, 764, 755] payload_rows_per_dst=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] dup_rows_per_dst=[2988, 3008, 3232, 3104, 3034, 2782, 2988, 2960] combine_rows_per_src=[9396, 9290, 9398, 9328, 9194, 9294, 9242, 9066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344216 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174616 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005016 weight_movement_bytes=34603008 total_nvlink_bytes=566608024 weight_to_token_ratio=0.065043 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343648 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969248 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594848 weight_movement_bytes=34603008 total_nvlink_bytes=566197856 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=343496 combine_payload_bytes=267419648 prefetch_bytes=0 reduce_bytes=173015040 egress_bytes=267763144 ingress_bytes=440434688 link_bytes=440434688 token_bytes=535182792 weight_movement_bytes=173015040 total_nvlink_bytes=708197832 weight_to_token_ratio=0.323282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.657320273e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342008 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181176 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530020344 weight_movement_bytes=34603008 total_nvlink_bytes=564623352 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344040 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266985448 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626856 weight_movement_bytes=69206016 total_nvlink_bytes=602832872 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342904 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485176 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627448 weight_movement_bytes=34603008 total_nvlink_bytes=565230456 weight_to_token_ratio=0.065211 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64966 dup_rows=23956 dup_rows_remote=20794 dedup_ratio=0.243693 router_tokens_per_expert=[730, 813, 765, 827, 808, 771, 775, 775, 764, 729, 790, 761, 740, 795, 733, 768, 738, 778, 768, 788, 731, 723, 809, 731, 780, 773, 776, 808, 750, 786, 777, 796, 786, 784, 771, 783, 760, 778, 791, 782, 746, 725, 802, 745, 757, 804, 792, 744, 750, 752, 774, 754, 783, 782, 761, 727, 773, 776, 761, 743, 746, 743, 767, 754] payload_rows_per_dst=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] dup_rows_per_dst=[3162, 2904, 2884, 3092, 3112, 3014, 2872, 2916] combine_rows_per_src=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344024 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248152 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152280 weight_movement_bytes=69206016 total_nvlink_bytes=601358296 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342008 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181176 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530020344 weight_movement_bytes=34603008 total_nvlink_bytes=564623352 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343360 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846080 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348800 weight_movement_bytes=34603008 total_nvlink_bytes=565951808 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343040 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266443776 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532544512 weight_movement_bytes=69206016 total_nvlink_bytes=601750528 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65234 dup_rows=23844 dup_rows_remote=20936 dedup_ratio=0.242554 router_tokens_per_expert=[778, 784, 744, 761, 783, 788, 770, 757, 785, 808, 758, 816, 785, 747, 791, 749, 746, 772, 766, 753, 748, 772, 694, 803, 790, 780, 760, 752, 747, 736, 761, 769, 724, 771, 730, 787, 749, 742, 776, 788, 798, 788, 767, 761, 718, 725, 776, 832, 764, 734, 811, 783, 806, 815, 779, 762, 715, 777, 813, 789, 743, 730, 772, 774] payload_rows_per_dst=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] dup_rows_per_dst=[3046, 3152, 2850, 2934, 2908, 3008, 3032, 2914] combine_rows_per_src=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344024 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248152 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152280 weight_movement_bytes=69206016 total_nvlink_bytes=601358296 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20982 dedup_ratio=0.243876 router_tokens_per_expert=[781, 780, 758, 744, 799, 818, 762, 782, 735, 737, 759, 753, 793, 783, 791, 742, 765, 714, 798, 769, 777, 772, 778, 781, 759, 760, 762, 805, 817, 769, 771, 798, 783, 791, 764, 813, 771, 723, 782, 783, 741, 789, 751, 782, 778, 714, 747, 742, 752, 789, 786, 738, 804, 794, 731, 764, 743, 740, 735, 768, 787, 785, 750, 720] payload_rows_per_dst=[9370, 9164, 9316, 9432, 9314, 9162, 9388, 9184] dup_rows_per_dst=[3078, 3022, 2992, 3050, 3106, 2926, 2928, 2872] combine_rows_per_src=[9370, 9164, 9316, 9432, 9314, 9162, 9388, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64956 dup_rows=24086 dup_rows_remote=21020 dedup_ratio=0.245015 router_tokens_per_expert=[764, 723, 782, 766, 802, 769, 794, 755, 753, 770, 792, 709, 759, 787, 806, 734, 781, 746, 765, 728, 785, 743, 792, 773, 734, 739, 770, 805, 763, 779, 794, 780, 787, 792, 789, 754, 746, 739, 757, 762, 792, 771, 777, 800, 810, 777, 747, 754, 794, 770, 709, 746, 732, 743, 809, 740, 751, 743, 779, 775, 805, 779, 810, 771] payload_rows_per_dst=[9314, 9278, 9270, 9262, 9284, 9330, 9094, 9386] dup_rows_per_dst=[2996, 2942, 2956, 3066, 2968, 3126, 2992, 3040] combine_rows_per_src=[9314, 9278, 9270, 9262, 9284, 9330, 9094, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64932 dup_rows=23962 dup_rows_remote=20846 dedup_ratio=0.243754 router_tokens_per_expert=[778, 741, 816, 736, 763, 806, 777, 748, 778, 799, 797, 754, 794, 786, 731, 791, 762, 811, 805, 747, 765, 754, 784, 794, 731, 787, 765, 721, 756, 730, 752, 785, 768, 739, 778, 800, 778, 806, 799, 763, 761, 757, 752, 819, 742, 794, 777, 805, 761, 772, 722, 749, 713, 760, 742, 754, 804, 773, 734, 777, 705, 770, 773, 761] payload_rows_per_dst=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] dup_rows_per_dst=[2946, 3122, 3028, 2802, 2916, 3054, 2978, 3116] combine_rows_per_src=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64990 dup_rows=23988 dup_rows_remote=20872 dedup_ratio=0.244019 router_tokens_per_expert=[746, 770, 745, 757, 762, 739, 783, 750, 770, 774, 755, 759, 749, 801, 733, 722, 800, 841, 754, 786, 748, 787, 774, 781, 805, 763, 818, 785, 762, 771, 740, 755, 802, 775, 728, 765, 769, 758, 777, 757, 795, 805, 806, 790, 788, 729, 771, 737, 759, 770, 735, 778, 768, 785, 700, 735, 745, 786, 775, 755, 770, 782, 775, 797] payload_rows_per_dst=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] dup_rows_per_dst=[2938, 2958, 3096, 2990, 3008, 3116, 2850, 3032] combine_rows_per_src=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65234 dup_rows=23844 dup_rows_remote=20936 dedup_ratio=0.242554 router_tokens_per_expert=[778, 784, 744, 761, 783, 788, 770, 757, 785, 808, 758, 816, 785, 747, 791, 749, 746, 772, 766, 753, 748, 772, 694, 803, 790, 780, 760, 752, 747, 736, 761, 769, 724, 771, 730, 787, 749, 742, 776, 788, 798, 788, 767, 761, 718, 725, 776, 832, 764, 734, 811, 783, 806, 815, 779, 762, 715, 777, 813, 789, 743, 730, 772, 774] payload_rows_per_dst=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] dup_rows_per_dst=[3046, 3152, 2850, 2934, 2908, 3008, 3032, 2914] combine_rows_per_src=[9284, 9326, 9258, 9256, 9226, 9322, 9476, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65240 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[754, 725, 771, 794, 787, 764, 784, 775, 789, 694, 788, 706, 763, 775, 768, 778, 710, 727, 774, 734, 765, 792, 735, 791, 773, 799, 759, 803, 776, 730, 775, 779, 807, 812, 801, 784, 744, 785, 767, 806, 744, 718, 745, 724, 780, 744, 794, 773, 763, 758, 789, 753, 769, 760, 778, 795, 761, 787, 745, 803, 788, 788, 801, 749] payload_rows_per_dst=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] dup_rows_per_dst=[3022, 2884, 2794, 3058, 3088, 2868, 3012, 3100] combine_rows_per_src=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65062 dup_rows=23858 dup_rows_remote=20780 dedup_ratio=0.242696 router_tokens_per_expert=[782, 769, 761, 749, 732, 796, 758, 761, 761, 722, 763, 718, 738, 813, 806, 755, 711, 737, 750, 781, 793, 791, 788, 742, 818, 764, 805, 788, 771, 837, 765, 752, 797, 723, 777, 762, 759, 805, 787, 776, 760, 774, 764, 779, 801, 701, 730, 755, 828, 781, 749, 799, 776, 806, 794, 698, 749, 789, 772, 765, 796, 696, 753, 774] payload_rows_per_dst=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] dup_rows_per_dst=[2938, 2884, 2946, 3148, 3094, 2854, 3078, 2916] combine_rows_per_src=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64966 dup_rows=23956 dup_rows_remote=20794 dedup_ratio=0.243693 router_tokens_per_expert=[730, 813, 765, 827, 808, 771, 775, 775, 764, 729, 790, 761, 740, 795, 733, 768, 738, 778, 768, 788, 731, 723, 809, 731, 780, 773, 776, 808, 750, 786, 777, 796, 786, 784, 771, 783, 760, 778, 791, 782, 746, 725, 802, 745, 757, 804, 792, 744, 750, 752, 774, 754, 783, 782, 761, 727, 773, 776, 761, 743, 746, 743, 767, 754] payload_rows_per_dst=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] dup_rows_per_dst=[3162, 2904, 2884, 3092, 3112, 3014, 2872, 2916] combine_rows_per_src=[9382, 9278, 9248, 9362, 9358, 9216, 9294, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343984 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641328 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938672 weight_movement_bytes=34603008 total_nvlink_bytes=567541680 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344680 combine_payload_bytes=267198464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267543144 ingress_bytes=405610496 link_bytes=405610496 token_bytes=534741608 weight_movement_bytes=138412032 total_nvlink_bytes=673153640 weight_to_token_ratio=0.258839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526279642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64906 dup_rows=24016 dup_rows_remote=21058 dedup_ratio=0.244303 router_tokens_per_expert=[741, 830, 766, 766, 765, 776, 758, 756, 832, 719, 745, 771, 776, 748, 773, 749, 746, 823, 728, 779, 768, 726, 821, 762, 739, 767, 767, 846, 787, 791, 714, 746, 778, 826, 755, 811, 736, 765, 715, 794, 773, 775, 733, 756, 798, 786, 798, 741, 805, 775, 753, 766, 769, 765, 784, 780, 753, 749, 742, 748, 734, 805, 733, 770] payload_rows_per_dst=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] dup_rows_per_dst=[2958, 3160, 3144, 2982, 2882, 2988, 3020, 2882] combine_rows_per_src=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64990 dup_rows=23988 dup_rows_remote=20872 dedup_ratio=0.244019 router_tokens_per_expert=[746, 770, 745, 757, 762, 739, 783, 750, 770, 774, 755, 759, 749, 801, 733, 722, 800, 841, 754, 786, 748, 787, 774, 781, 805, 763, 818, 785, 762, 771, 740, 755, 802, 775, 728, 765, 769, 758, 777, 757, 795, 805, 806, 790, 788, 729, 771, 737, 759, 770, 735, 778, 768, 785, 700, 735, 745, 786, 775, 755, 770, 782, 775, 797] payload_rows_per_dst=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] dup_rows_per_dst=[2938, 2958, 3096, 2990, 3008, 3116, 2850, 3032] combine_rows_per_src=[9166, 9168, 9446, 9408, 9254, 9326, 9210, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64932 dup_rows=23962 dup_rows_remote=20846 dedup_ratio=0.243754 router_tokens_per_expert=[778, 741, 816, 736, 763, 806, 777, 748, 778, 799, 797, 754, 794, 786, 731, 791, 762, 811, 805, 747, 765, 754, 784, 794, 731, 787, 765, 721, 756, 730, 752, 785, 768, 739, 778, 800, 778, 806, 799, 763, 761, 757, 752, 819, 742, 794, 777, 805, 761, 772, 722, 749, 713, 760, 742, 754, 804, 773, 734, 777, 705, 770, 773, 761] payload_rows_per_dst=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] dup_rows_per_dst=[2946, 3122, 3028, 2802, 2916, 3054, 2978, 3116] combine_rows_per_src=[9302, 9338, 9416, 8920, 9344, 9442, 9170, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343904 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403680 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463456 weight_movement_bytes=103809024 total_nvlink_bytes=636272480 weight_to_token_ratio=0.194960 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343112 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304584 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266056 weight_movement_bytes=34603008 total_nvlink_bytes=566869064 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65062 dup_rows=23858 dup_rows_remote=20780 dedup_ratio=0.242696 router_tokens_per_expert=[782, 769, 761, 749, 732, 796, 758, 761, 761, 722, 763, 718, 738, 813, 806, 755, 711, 737, 750, 781, 793, 791, 788, 742, 818, 764, 805, 788, 771, 837, 765, 752, 797, 723, 777, 762, 759, 805, 787, 776, 760, 774, 764, 779, 801, 701, 730, 755, 828, 781, 749, 799, 776, 806, 794, 698, 749, 789, 772, 765, 796, 696, 753, 774] payload_rows_per_dst=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] dup_rows_per_dst=[2938, 2884, 2946, 3148, 3094, 2854, 3078, 2916] combine_rows_per_src=[9278, 9268, 9240, 9452, 9278, 9274, 9384, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344680 combine_payload_bytes=267198464 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267543144 ingress_bytes=405610496 link_bytes=405610496 token_bytes=534741608 weight_movement_bytes=138412032 total_nvlink_bytes=673153640 weight_to_token_ratio=0.258839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526279642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65240 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[754, 725, 771, 794, 787, 764, 784, 775, 789, 694, 788, 706, 763, 775, 768, 778, 710, 727, 774, 734, 765, 792, 735, 791, 773, 799, 759, 803, 776, 730, 775, 779, 807, 812, 801, 784, 744, 785, 767, 806, 744, 718, 745, 724, 780, 744, 794, 773, 763, 758, 789, 753, 769, 760, 778, 795, 761, 787, 745, 803, 788, 788, 801, 749] payload_rows_per_dst=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] dup_rows_per_dst=[3022, 2884, 2794, 3058, 3088, 2868, 3012, 3100] combine_rows_per_src=[9286, 9238, 9262, 9330, 9524, 9176, 9318, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343040 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266443776 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532544512 weight_movement_bytes=69206016 total_nvlink_bytes=601750528 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343448 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542488 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741528 weight_movement_bytes=34603008 total_nvlink_bytes=567344536 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343112 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304584 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266056 weight_movement_bytes=34603008 total_nvlink_bytes=566869064 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343368 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266837320 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533331272 weight_movement_bytes=34603008 total_nvlink_bytes=567934280 weight_to_token_ratio=0.064881 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343856 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266198832 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532053808 weight_movement_bytes=69206016 total_nvlink_bytes=601259824 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65000 dup_rows=24036 dup_rows_remote=21062 dedup_ratio=0.244507 router_tokens_per_expert=[769, 791, 776, 795, 782, 766, 771, 767, 791, 734, 792, 799, 744, 753, 776, 756, 763, 735, 796, 779, 766, 753, 743, 786, 798, 794, 698, 800, 811, 751, 769, 759, 789, 736, 744, 778, 761, 716, 766, 750, 783, 794, 787, 753, 763, 787, 706, 780, 807, 773, 749, 766, 773, 789, 756, 791, 744, 787, 752, 780, 741, 771, 756, 761] payload_rows_per_dst=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] dup_rows_per_dst=[3126, 3030, 2974, 2966, 2894, 2924, 3094, 3028] combine_rows_per_src=[9308, 9260, 9268, 9394, 9186, 9382, 9314, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65146 dup_rows=23902 dup_rows_remote=20848 dedup_ratio=0.243144 router_tokens_per_expert=[735, 789, 775, 742, 779, 757, 804, 763, 769, 755, 813, 801, 768, 773, 780, 740, 745, 759, 742, 739, 819, 748, 786, 761, 734, 761, 752, 785, 780, 777, 754, 771, 742, 771, 761, 797, 805, 759, 761, 759, 748, 777, 713, 815, 739, 759, 781, 740, 769, 815, 804, 822, 772, 755, 785, 774, 831, 805, 726, 769, 742, 723, 769, 708] payload_rows_per_dst=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] dup_rows_per_dst=[2928, 3034, 2972, 2986, 3054, 2950, 3092, 2886] combine_rows_per_src=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21068 dedup_ratio=0.244161 router_tokens_per_expert=[822, 760, 774, 783, 739, 770, 740, 758, 777, 759, 761, 751, 802, 756, 771, 760, 744, 761, 778, 751, 715, 776, 749, 801, 780, 773, 720, 758, 745, 811, 764, 769, 754, 846, 809, 780, 810, 749, 760, 788, 722, 748, 800, 775, 765, 753, 762, 771, 693, 793, 775, 792, 782, 732, 761, 767, 781, 732, 733, 761, 794, 798, 794, 794] payload_rows_per_dst=[9274, 9278, 9236, 9306, 9448, 9230, 9180, 9350] dup_rows_per_dst=[3018, 2996, 2914, 2934, 3144, 2962, 3010, 3024] combine_rows_per_src=[9274, 9278, 9236, 9306, 9448, 9230, 9180, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64726 dup_rows=24112 dup_rows_remote=20872 dedup_ratio=0.245280 router_tokens_per_expert=[753, 745, 724, 778, 769, 733, 766, 808, 746, 776, 776, 758, 747, 770, 771, 790, 778, 774, 764, 751, 717, 779, 749, 776, 798, 784, 755, 736, 799, 763, 810, 757, 732, 747, 817, 749, 760, 792, 825, 802, 746, 740, 742, 804, 835, 789, 804, 750, 741, 725, 787, 762, 779, 756, 774, 781, 724, 780, 784, 760, 806, 709, 767, 783] payload_rows_per_dst=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] dup_rows_per_dst=[2968, 3006, 3030, 2770, 2850, 3350, 2898, 3240] combine_rows_per_src=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65146 dup_rows=23902 dup_rows_remote=20848 dedup_ratio=0.243144 router_tokens_per_expert=[735, 789, 775, 742, 779, 757, 804, 763, 769, 755, 813, 801, 768, 773, 780, 740, 745, 759, 742, 739, 819, 748, 786, 761, 734, 761, 752, 785, 780, 777, 754, 771, 742, 771, 761, 797, 805, 759, 761, 759, 748, 777, 713, 815, 739, 759, 781, 740, 769, 815, 804, 822, 772, 755, 785, 774, 831, 805, 726, 769, 742, 723, 769, 708] payload_rows_per_dst=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] dup_rows_per_dst=[2928, 3034, 2972, 2986, 3054, 2950, 3092, 2886] combine_rows_per_src=[9360, 9364, 9226, 9242, 9256, 9194, 9500, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344248 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584248 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824248 weight_movement_bytes=34603008 total_nvlink_bytes=567427256 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64906 dup_rows=24016 dup_rows_remote=21058 dedup_ratio=0.244303 router_tokens_per_expert=[741, 830, 766, 766, 765, 776, 758, 756, 832, 719, 745, 771, 776, 748, 773, 749, 746, 823, 728, 779, 768, 726, 821, 762, 739, 767, 767, 846, 787, 791, 714, 746, 778, 826, 755, 811, 736, 765, 715, 794, 773, 775, 733, 756, 798, 786, 798, 741, 805, 775, 753, 766, 769, 765, 784, 780, 753, 749, 742, 748, 734, 805, 733, 770] payload_rows_per_dst=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] dup_rows_per_dst=[2958, 3160, 3144, 2982, 2882, 2988, 3020, 2882] combine_rows_per_src=[9382, 9400, 9290, 9214, 9110, 9332, 9374, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65264 dup_rows=23898 dup_rows_remote=20916 dedup_ratio=0.243103 router_tokens_per_expert=[782, 734, 804, 738, 778, 729, 778, 782, 780, 799, 759, 785, 789, 784, 764, 738, 755, 795, 712, 786, 816, 768, 797, 744, 783, 751, 766, 756, 801, 742, 802, 817, 809, 767, 776, 808, 748, 778, 767, 757, 755, 744, 746, 754, 757, 741, 821, 744, 754, 772, 766, 776, 746, 795, 750, 772, 807, 729, 725, 760, 746, 734, 778, 756] payload_rows_per_dst=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] dup_rows_per_dst=[2996, 2976, 3004, 3114, 2896, 2982, 3002, 2928] combine_rows_per_src=[9254, 9420, 9342, 9322, 9524, 9142, 9260, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64988 dup_rows=24002 dup_rows_remote=20834 dedup_ratio=0.244161 router_tokens_per_expert=[722, 731, 758, 803, 753, 744, 760, 819, 745, 786, 746, 751, 740, 749, 759, 780, 763, 772, 754, 776, 771, 763, 735, 784, 758, 818, 767, 766, 779, 819, 759, 797, 747, 790, 797, 787, 758, 718, 771, 786, 749, 771, 691, 734, 767, 760, 788, 813, 777, 777, 805, 772, 760, 792, 743, 815, 786, 801, 785, 751, 714, 772, 801, 747] payload_rows_per_dst=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] dup_rows_per_dst=[2976, 2818, 2966, 3128, 3012, 2948, 3168, 2986] combine_rows_per_src=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344256 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567872 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791488 weight_movement_bytes=103809024 total_nvlink_bytes=636600512 weight_to_token_ratio=0.194840 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344248 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584248 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824248 weight_movement_bytes=34603008 total_nvlink_bytes=567427256 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64914 dup_rows=24024 dup_rows_remote=21000 dedup_ratio=0.244385 router_tokens_per_expert=[727, 786, 774, 781, 726, 798, 786, 750, 737, 808, 737, 775, 742, 758, 771, 746, 752, 796, 783, 746, 774, 747, 797, 782, 791, 736, 786, 777, 766, 795, 803, 757, 775, 732, 744, 751, 778, 802, 782, 784, 752, 761, 760, 762, 774, 773, 761, 805, 747, 805, 755, 764, 800, 814, 763, 731, 812, 777, 786, 745, 731, 759, 742, 735] payload_rows_per_dst=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] dup_rows_per_dst=[3024, 2964, 3172, 2986, 3090, 2968, 2876, 2944] combine_rows_per_src=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65030 dup_rows=23934 dup_rows_remote=20838 dedup_ratio=0.243469 router_tokens_per_expert=[768, 723, 775, 760, 792, 765, 791, 785, 774, 817, 799, 786, 708, 776, 774, 784, 780, 754, 796, 759, 758, 725, 792, 800, 738, 838, 778, 799, 708, 761, 764, 777, 783, 750, 788, 792, 681, 787, 781, 764, 780, 730, 731, 737, 768, 755, 813, 768, 743, 764, 755, 771, 758, 728, 796, 757, 765, 764, 768, 784, 801, 783, 776, 727] payload_rows_per_dst=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] dup_rows_per_dst=[3048, 3096, 3040, 2996, 2954, 2948, 2832, 3020] combine_rows_per_src=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343976 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181992 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534020008 weight_movement_bytes=69206016 total_nvlink_bytes=603226024 weight_to_token_ratio=0.129594 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64726 dup_rows=24112 dup_rows_remote=20872 dedup_ratio=0.245280 router_tokens_per_expert=[753, 745, 724, 778, 769, 733, 766, 808, 746, 776, 776, 758, 747, 770, 771, 790, 778, 774, 764, 751, 717, 779, 749, 776, 798, 784, 755, 736, 799, 763, 810, 757, 732, 747, 817, 749, 760, 792, 825, 802, 746, 740, 742, 804, 835, 789, 804, 750, 741, 725, 787, 762, 779, 756, 774, 781, 724, 780, 784, 760, 806, 709, 767, 783] payload_rows_per_dst=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] dup_rows_per_dst=[2968, 3006, 3030, 2770, 2850, 3350, 2898, 3240] combine_rows_per_src=[9184, 9244, 9146, 9130, 9118, 9582, 9322, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64988 dup_rows=24002 dup_rows_remote=20834 dedup_ratio=0.244161 router_tokens_per_expert=[722, 731, 758, 803, 753, 744, 760, 819, 745, 786, 746, 751, 740, 749, 759, 780, 763, 772, 754, 776, 771, 763, 735, 784, 758, 818, 767, 766, 779, 819, 759, 797, 747, 790, 797, 787, 758, 718, 771, 786, 749, 771, 691, 734, 767, 760, 788, 813, 777, 777, 805, 772, 760, 792, 743, 815, 786, 801, 785, 751, 714, 772, 801, 747] payload_rows_per_dst=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] dup_rows_per_dst=[2976, 2818, 2966, 3128, 3012, 2948, 3168, 2986] combine_rows_per_src=[9204, 9294, 9270, 9398, 9296, 9198, 9314, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343976 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181992 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534020008 weight_movement_bytes=69206016 total_nvlink_bytes=603226024 weight_to_token_ratio=0.129594 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65030 dup_rows=23934 dup_rows_remote=20838 dedup_ratio=0.243469 router_tokens_per_expert=[768, 723, 775, 760, 792, 765, 791, 785, 774, 817, 799, 786, 708, 776, 774, 784, 780, 754, 796, 759, 758, 725, 792, 800, 738, 838, 778, 799, 708, 761, 764, 777, 783, 750, 788, 792, 681, 787, 781, 764, 780, 730, 731, 737, 768, 755, 813, 768, 743, 764, 755, 771, 758, 728, 796, 757, 765, 764, 768, 784, 801, 783, 776, 727] payload_rows_per_dst=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] dup_rows_per_dst=[3048, 3096, 3040, 2996, 2954, 2948, 2832, 3020] combine_rows_per_src=[9270, 9340, 9288, 9330, 9298, 9216, 9312, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342392 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265460088 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577784 weight_movement_bytes=34603008 total_nvlink_bytes=565180792 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343856 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266198832 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532053808 weight_movement_bytes=69206016 total_nvlink_bytes=601259824 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344720 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267666064 ingress_bytes=301924352 link_bytes=301924352 token_bytes=534987408 weight_movement_bytes=34603008 total_nvlink_bytes=569590416 weight_to_token_ratio=0.064680 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136117030e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342392 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265460088 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577784 weight_movement_bytes=34603008 total_nvlink_bytes=565180792 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343288 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266534136 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532724984 weight_movement_bytes=103809024 total_nvlink_bytes=636534008 weight_to_token_ratio=0.194864 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343656 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231400 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119144 weight_movement_bytes=34603008 total_nvlink_bytes=566722152 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65240 dup_rows=23834 dup_rows_remote=20942 dedup_ratio=0.242452 router_tokens_per_expert=[700, 761, 826, 763, 772, 787, 784, 757, 741, 795, 775, 718, 791, 706, 786, 754, 776, 759, 745, 743, 766, 772, 726, 774, 772, 784, 754, 796, 788, 730, 747, 765, 773, 762, 785, 729, 743, 819, 787, 816, 757, 757, 765, 807, 848, 755, 714, 768, 791, 738, 767, 755, 769, 788, 761, 773, 768, 744, 748, 802, 769, 774, 771, 836] payload_rows_per_dst=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] dup_rows_per_dst=[3020, 2900, 2892, 2980, 3004, 2942, 2988, 3108] combine_rows_per_src=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343472 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266706352 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069232 weight_movement_bytes=69206016 total_nvlink_bytes=602275248 weight_to_token_ratio=0.129826 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64968 dup_rows=24020 dup_rows_remote=20906 dedup_ratio=0.244344 router_tokens_per_expert=[785, 807, 795, 749, 780, 803, 749, 762, 793, 727, 801, 817, 763, 766, 750, 785, 763, 746, 782, 743, 776, 792, 762, 797, 752, 828, 806, 757, 737, 752, 796, 787, 758, 749, 752, 745, 733, 766, 767, 837, 745, 859, 773, 719, 777, 746, 807, 768, 750, 755, 795, 731, 743, 745, 795, 757, 685, 775, 738, 766, 734, 791, 750, 733] payload_rows_per_dst=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] dup_rows_per_dst=[3138, 3142, 3000, 3114, 2884, 3064, 2866, 2812] combine_rows_per_src=[9322, 9262, 9322, 9316, 9330, 9324, 9276, 9132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65240 dup_rows=23834 dup_rows_remote=20942 dedup_ratio=0.242452 router_tokens_per_expert=[700, 761, 826, 763, 772, 787, 784, 757, 741, 795, 775, 718, 791, 706, 786, 754, 776, 759, 745, 743, 766, 772, 726, 774, 772, 784, 754, 796, 788, 730, 747, 765, 773, 762, 785, 729, 743, 819, 787, 816, 757, 757, 765, 807, 848, 755, 714, 768, 791, 738, 767, 755, 769, 788, 761, 773, 768, 744, 748, 802, 769, 774, 771, 836] payload_rows_per_dst=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] dup_rows_per_dst=[3020, 2900, 2892, 2980, 3004, 2942, 2988, 3108] combine_rows_per_src=[9280, 9232, 9230, 9292, 9424, 9400, 9296, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65292 dup_rows=23780 dup_rows_remote=20894 dedup_ratio=0.241903 router_tokens_per_expert=[779, 812, 771, 733, 794, 788, 795, 757, 781, 748, 753, 758, 725, 742, 765, 757, 761, 741, 788, 813, 765, 744, 798, 759, 745, 806, 753, 818, 768, 735, 772, 743, 771, 761, 741, 801, 739, 749, 766, 731, 816, 791, 718, 781, 746, 755, 810, 763, 735, 751, 833, 788, 787, 784, 759, 778, 740, 758, 699, 785, 782, 779, 808, 780] payload_rows_per_dst=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] dup_rows_per_dst=[3088, 2858, 2996, 3008, 2886, 2920, 3030, 2994] combine_rows_per_src=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20790 dedup_ratio=0.243083 router_tokens_per_expert=[797, 765, 744, 764, 744, 757, 718, 789, 781, 735, 769, 793, 730, 778, 774, 826, 781, 759, 796, 792, 771, 797, 747, 735, 808, 761, 774, 765, 757, 735, 828, 805, 733, 767, 746, 807, 739, 779, 730, 777, 757, 806, 786, 740, 805, 769, 750, 721, 748, 759, 812, 751, 777, 745, 763, 758, 751, 825, 744, 804, 771, 740, 755, 762] payload_rows_per_dst=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] dup_rows_per_dst=[2846, 3100, 3034, 3024, 2904, 2854, 3028, 3106] combine_rows_per_src=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65292 dup_rows=23780 dup_rows_remote=20894 dedup_ratio=0.241903 router_tokens_per_expert=[779, 812, 771, 733, 794, 788, 795, 757, 781, 748, 753, 758, 725, 742, 765, 757, 761, 741, 788, 813, 765, 744, 798, 759, 745, 806, 753, 818, 768, 735, 772, 743, 771, 761, 741, 801, 739, 749, 766, 731, 816, 791, 718, 781, 746, 755, 810, 763, 735, 751, 833, 788, 787, 784, 759, 778, 740, 758, 699, 785, 782, 779, 808, 780] payload_rows_per_dst=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] dup_rows_per_dst=[3088, 2858, 2996, 3008, 2886, 2920, 3030, 2994] combine_rows_per_src=[9370, 9200, 9342, 9272, 9232, 9440, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343496 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452424 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561352 weight_movement_bytes=34603008 total_nvlink_bytes=567164360 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64914 dup_rows=24024 dup_rows_remote=21000 dedup_ratio=0.244385 router_tokens_per_expert=[727, 786, 774, 781, 726, 798, 786, 750, 737, 808, 737, 775, 742, 758, 771, 746, 752, 796, 783, 746, 774, 747, 797, 782, 791, 736, 786, 777, 766, 795, 803, 757, 775, 732, 744, 751, 778, 802, 782, 784, 752, 761, 760, 762, 774, 773, 761, 805, 747, 805, 755, 764, 800, 814, 763, 731, 812, 777, 786, 745, 731, 759, 742, 735] payload_rows_per_dst=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] dup_rows_per_dst=[3024, 2964, 3172, 2986, 3090, 2968, 2876, 2944] combine_rows_per_src=[9366, 9184, 9456, 9302, 9284, 9328, 9130, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65054 dup_rows=23940 dup_rows_remote=20800 dedup_ratio=0.243530 router_tokens_per_expert=[736, 779, 727, 785, 737, 722, 746, 805, 772, 771, 778, 800, 768, 742, 816, 770, 777, 735, 743, 755, 777, 719, 745, 741, 763, 713, 775, 748, 777, 743, 778, 758, 765, 740, 804, 786, 763, 772, 789, 757, 761, 783, 790, 750, 764, 811, 755, 811, 784, 785, 824, 765, 783, 767, 765, 757, 775, 793, 767, 794, 772, 779, 781, 759] payload_rows_per_dst=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] dup_rows_per_dst=[2866, 2916, 2874, 2878, 3070, 3140, 3100, 3096] combine_rows_per_src=[9208, 9518, 9110, 9232, 9282, 9310, 9360, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20790 dedup_ratio=0.243083 router_tokens_per_expert=[797, 765, 744, 764, 744, 757, 718, 789, 781, 735, 769, 793, 730, 778, 774, 826, 781, 759, 796, 792, 771, 797, 747, 735, 808, 761, 774, 765, 757, 735, 828, 805, 733, 767, 746, 807, 739, 779, 730, 777, 757, 806, 786, 740, 805, 769, 750, 721, 748, 759, 812, 751, 777, 745, 763, 758, 751, 825, 744, 804, 771, 740, 755, 762] payload_rows_per_dst=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] dup_rows_per_dst=[2846, 3100, 3034, 3024, 2904, 2854, 3028, 3106] combine_rows_per_src=[9310, 9180, 9322, 9442, 9252, 9222, 9290, 9390] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=64980 dup_rows=23860 dup_rows_remote=20862 dedup_ratio=0.242716 router_tokens_per_expert=[769, 746, 731, 774, 822, 790, 751, 766, 750, 733, 772, 750, 749, 789, 736, 817, 823, 739, 754, 787, 830, 762, 792, 768, 755, 765, 760, 738, 790, 747, 783, 754, 761, 767, 752, 735, 768, 734, 755, 775, 748, 804, 794, 748, 820, 744, 758, 752, 788, 791, 791, 784, 804, 802, 744, 727, 759, 768, 744, 732, 795, 797, 750, 769] payload_rows_per_dst=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] dup_rows_per_dst=[2984, 2980, 3064, 2914, 2862, 3058, 2998, 3000] combine_rows_per_src=[9314, 9212, 9446, 9270, 9232, 9278, 9464, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343496 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452424 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561352 weight_movement_bytes=34603008 total_nvlink_bytes=567164360 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64846 dup_rows=24034 dup_rows_remote=20820 dedup_ratio=0.244486 router_tokens_per_expert=[775, 750, 749, 741, 761, 779, 783, 806, 801, 775, 749, 775, 828, 790, 738, 768, 767, 745, 780, 708, 676, 791, 727, 759, 774, 729, 764, 784, 760, 790, 766, 795, 794, 778, 750, 791, 810, 781, 782, 832, 772, 801, 791, 713, 755, 733, 782, 781, 812, 804, 713, 736, 807, 800, 740, 735, 724, 760, 767, 799, 781, 735, 752, 758] payload_rows_per_dst=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] dup_rows_per_dst=[3214, 3002, 2950, 3044, 3168, 2884, 2814, 2958] combine_rows_per_src=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344728 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267567768 ingress_bytes=301826048 link_bytes=301826048 token_bytes=534790808 weight_movement_bytes=34603008 total_nvlink_bytes=569393816 weight_to_token_ratio=0.064704 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135747120e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344744 combine_payload_bytes=267436032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267780776 ingress_bytes=371245056 link_bytes=371245056 token_bytes=535216808 weight_movement_bytes=103809024 total_nvlink_bytes=639025832 weight_to_token_ratio=0.193957 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396965258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64862 dup_rows=24138 dup_rows_remote=21092 dedup_ratio=0.245544 router_tokens_per_expert=[727, 744, 736, 795, 746, 741, 725, 746, 766, 742, 789, 774, 757, 797, 755, 795, 850, 759, 777, 775, 775, 765, 812, 784, 745, 760, 780, 750, 805, 747, 788, 728, 793, 773, 763, 784, 826, 810, 786, 718, 767, 762, 723, 746, 770, 759, 741, 785, 797, 748, 753, 777, 748, 794, 785, 730, 758, 765, 778, 784, 803, 736, 806, 749] payload_rows_per_dst=[9088, 9304, 9434, 9258, 9298, 9220, 9236, 9328] dup_rows_per_dst=[2832, 3046, 3160, 2948, 3208, 2886, 3028, 3030] combine_rows_per_src=[9088, 9304, 9434, 9258, 9298, 9220, 9236, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344744 combine_payload_bytes=267436032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267780776 ingress_bytes=371245056 link_bytes=371245056 token_bytes=535216808 weight_movement_bytes=103809024 total_nvlink_bytes=639025832 weight_to_token_ratio=0.193957 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396965258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343656 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231400 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119144 weight_movement_bytes=34603008 total_nvlink_bytes=566722152 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65016 dup_rows=24002 dup_rows_remote=20954 dedup_ratio=0.244161 router_tokens_per_expert=[731, 780, 773, 841, 784, 725, 750, 732, 780, 761, 759, 787, 783, 722, 758, 775, 801, 817, 748, 749, 759, 766, 743, 784, 795, 779, 742, 831, 782, 743, 730, 764, 761, 761, 744, 737, 741, 775, 770, 735, 792, 797, 735, 815, 763, 797, 755, 826, 764, 814, 743, 751, 752, 780, 759, 799, 772, 743, 766, 757, 769, 766, 773, 766] payload_rows_per_dst=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] dup_rows_per_dst=[2960, 3054, 3048, 3022, 2844, 3114, 3032, 2928] combine_rows_per_src=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343416 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266804600 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533265784 weight_movement_bytes=103809024 total_nvlink_bytes=637074808 weight_to_token_ratio=0.194667 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343232 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266656960 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532970688 weight_movement_bytes=34603008 total_nvlink_bytes=567573696 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343416 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266804600 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533265784 weight_movement_bytes=103809024 total_nvlink_bytes=637074808 weight_to_token_ratio=0.194667 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343368 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266501448 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659528 weight_movement_bytes=69206016 total_nvlink_bytes=601865544 weight_to_token_ratio=0.129925 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=342664 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265951880 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561096 weight_movement_bytes=34603008 total_nvlink_bytes=566164104 weight_to_token_ratio=0.065097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343816 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266018568 ingress_bytes=334880768 link_bytes=334880768 token_bytes=531693320 weight_movement_bytes=69206016 total_nvlink_bytes=600899336 weight_to_token_ratio=0.130162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260129370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65052 dup_rows=23780 dup_rows_remote=20682 dedup_ratio=0.241903 router_tokens_per_expert=[762, 776, 751, 809, 769, 773, 805, 792, 765, 776, 727, 769, 752, 708, 737, 757, 754, 822, 762, 768, 789, 815, 778, 739, 801, 748, 789, 796, 815, 796, 774, 766, 785, 785, 778, 724, 808, 781, 760, 771, 729, 739, 724, 795, 720, 757, 759, 780, 748, 753, 756, 784, 804, 732, 796, 778, 723, 712, 796, 755, 775, 772, 775, 758] payload_rows_per_dst=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] dup_rows_per_dst=[3106, 2816, 3020, 3098, 3006, 2824, 2978, 2932] combine_rows_per_src=[9368, 9166, 9434, 9472, 9378, 9182, 9324, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65016 dup_rows=24002 dup_rows_remote=20954 dedup_ratio=0.244161 router_tokens_per_expert=[731, 780, 773, 841, 784, 725, 750, 732, 780, 761, 759, 787, 783, 722, 758, 775, 801, 817, 748, 749, 759, 766, 743, 784, 795, 779, 742, 831, 782, 743, 730, 764, 761, 761, 744, 737, 741, 775, 770, 735, 792, 797, 735, 815, 763, 797, 755, 826, 764, 814, 743, 751, 752, 780, 759, 799, 772, 743, 766, 757, 769, 766, 773, 766] payload_rows_per_dst=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] dup_rows_per_dst=[2960, 3054, 3048, 3022, 2844, 3114, 3032, 2928] combine_rows_per_src=[9272, 9196, 9286, 9310, 9204, 9446, 9292, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343816 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266018568 ingress_bytes=334880768 link_bytes=334880768 token_bytes=531693320 weight_movement_bytes=69206016 total_nvlink_bytes=600899336 weight_to_token_ratio=0.130162 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260129370e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74060 payload_rows_remote=64714 dup_rows=24244 dup_rows_remote=21200 dedup_ratio=0.246623 router_tokens_per_expert=[766, 744, 699, 757, 785, 781, 755, 780, 811, 749, 763, 789, 768, 721, 768, 710, 793, 764, 769, 784, 780, 792, 699, 772, 803, 789, 751, 784, 786, 837, 744, 765, 799, 757, 772, 802, 772, 755, 778, 760, 745, 765, 737, 806, 772, 737, 781, 734, 786, 774, 779, 785, 787, 741, 737, 783, 794, 747, 769, 784, 779, 781, 765, 731] payload_rows_per_dst=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] dup_rows_per_dst=[2966, 2888, 3098, 3166, 3044, 2994, 3060, 3028] combine_rows_per_src=[9168, 9270, 9208, 9352, 9346, 9160, 9284, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64608 dup_rows=24018 dup_rows_remote=20878 dedup_ratio=0.244324 router_tokens_per_expert=[785, 780, 745, 749, 812, 789, 797, 768, 768, 752, 723, 777, 783, 746, 773, 738, 735, 716, 760, 765, 780, 765, 758, 725, 780, 806, 756, 766, 779, 784, 770, 786, 753, 793, 758, 780, 735, 772, 757, 757, 769, 774, 749, 763, 742, 771, 784, 760, 793, 760, 735, 761, 735, 767, 735, 800, 740, 833, 785, 812, 835, 780, 756, 792] payload_rows_per_dst=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] dup_rows_per_dst=[2968, 3042, 2874, 3006, 2784, 3298, 2906, 3140] combine_rows_per_src=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=342936 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266795928 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533248920 weight_movement_bytes=69206016 total_nvlink_bytes=602454936 weight_to_token_ratio=0.129782 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64846 dup_rows=24034 dup_rows_remote=20820 dedup_ratio=0.244486 router_tokens_per_expert=[775, 750, 749, 741, 761, 779, 783, 806, 801, 775, 749, 775, 828, 790, 738, 768, 767, 745, 780, 708, 676, 791, 727, 759, 774, 729, 764, 784, 760, 790, 766, 795, 794, 778, 750, 791, 810, 781, 782, 832, 772, 801, 791, 713, 755, 733, 782, 781, 812, 804, 713, 736, 807, 800, 740, 735, 724, 760, 767, 799, 781, 735, 752, 758] payload_rows_per_dst=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] dup_rows_per_dst=[3214, 3002, 2950, 3044, 3168, 2884, 2814, 2958] combine_rows_per_src=[9424, 9452, 9128, 9312, 9268, 9362, 9130, 9194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64912 dup_rows=24064 dup_rows_remote=21156 dedup_ratio=0.244792 router_tokens_per_expert=[771, 732, 730, 806, 783, 776, 773, 788, 761, 739, 811, 748, 775, 789, 774, 796, 743, 752, 766, 762, 751, 763, 780, 791, 709, 749, 766, 811, 766, 779, 806, 749, 761, 773, 807, 771, 755, 789, 803, 758, 753, 786, 734, 786, 780, 729, 721, 812, 748, 777, 796, 775, 770, 720, 771, 761, 760, 770, 737, 783, 775, 740, 788, 768] payload_rows_per_dst=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] dup_rows_per_dst=[3096, 3096, 2994, 3056, 3030, 2868, 2908, 3016] combine_rows_per_src=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343880 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649416 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954952 weight_movement_bytes=34603008 total_nvlink_bytes=567557960 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64608 dup_rows=24018 dup_rows_remote=20878 dedup_ratio=0.244324 router_tokens_per_expert=[785, 780, 745, 749, 812, 789, 797, 768, 768, 752, 723, 777, 783, 746, 773, 738, 735, 716, 760, 765, 780, 765, 758, 725, 780, 806, 756, 766, 779, 784, 770, 786, 753, 793, 758, 780, 735, 772, 757, 757, 769, 774, 749, 763, 742, 771, 784, 760, 793, 760, 735, 761, 735, 767, 735, 800, 740, 833, 785, 812, 835, 780, 756, 792] payload_rows_per_dst=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] dup_rows_per_dst=[2968, 3042, 2874, 3006, 2784, 3298, 2906, 3140] combine_rows_per_src=[9362, 9078, 9134, 9314, 8920, 9472, 9328, 9678] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=65018 dup_rows=24102 dup_rows_remote=21188 dedup_ratio=0.245178 router_tokens_per_expert=[798, 739, 790, 761, 749, 809, 801, 761, 752, 757, 760, 761, 773, 747, 797, 738, 760, 783, 737, 800, 763, 731, 800, 772, 768, 785, 767, 757, 725, 730, 732, 750, 775, 787, 802, 787, 795, 759, 786, 769, 736, 771, 735, 762, 783, 783, 731, 748, 748, 774, 799, 751, 759, 772, 759, 795, 762, 710, 791, 797, 819, 775, 787, 792] payload_rows_per_dst=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] dup_rows_per_dst=[3034, 3002, 3040, 2920, 3098, 2914, 3068, 3026] combine_rows_per_src=[9382, 9168, 9252, 9108, 9422, 9184, 9246, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65078 dup_rows=24084 dup_rows_remote=21138 dedup_ratio=0.244995 router_tokens_per_expert=[722, 733, 783, 781, 751, 756, 754, 764, 774, 787, 817, 821, 771, 756, 728, 803, 754, 709, 760, 791, 746, 808, 749, 752, 751, 767, 761, 748, 800, 778, 748, 749, 804, 724, 776, 806, 802, 770, 759, 796, 758, 764, 765, 761, 752, 774, 730, 735, 768, 774, 750, 735, 836, 775, 807, 805, 772, 746, 782, 753, 782, 770, 776, 773] payload_rows_per_dst=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] dup_rows_per_dst=[2946, 3396, 3076, 2940, 3088, 2906, 2996, 2736] combine_rows_per_src=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343656 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265412200 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530480744 weight_movement_bytes=69206016 total_nvlink_bytes=599686760 weight_to_token_ratio=0.130459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64912 dup_rows=24064 dup_rows_remote=21156 dedup_ratio=0.244792 router_tokens_per_expert=[771, 732, 730, 806, 783, 776, 773, 788, 761, 739, 811, 748, 775, 789, 774, 796, 743, 752, 766, 762, 751, 763, 780, 791, 709, 749, 766, 811, 766, 779, 806, 749, 761, 773, 807, 771, 755, 789, 803, 758, 753, 786, 734, 786, 780, 729, 721, 812, 748, 777, 796, 775, 770, 720, 771, 761, 760, 770, 737, 783, 775, 740, 788, 768] payload_rows_per_dst=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] dup_rows_per_dst=[3096, 3096, 2994, 3056, 3030, 2868, 2908, 3016] combine_rows_per_src=[9222, 9290, 9222, 9214, 9404, 9334, 9328, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65154 dup_rows=23944 dup_rows_remote=20938 dedup_ratio=0.243571 router_tokens_per_expert=[765, 794, 761, 769, 722, 750, 768, 827, 752, 798, 752, 804, 774, 724, 752, 750, 780, 755, 768, 743, 741, 773, 788, 822, 778, 726, 772, 785, 756, 729, 732, 753, 759, 774, 797, 733, 795, 759, 789, 769, 836, 737, 793, 750, 773, 773, 773, 783, 778, 781, 750, 765, 789, 770, 732, 768, 747, 782, 802, 772, 795, 769, 776, 720] payload_rows_per_dst=[9322, 9206, 9396, 9146, 9346, 9374, 9174, 9396] dup_rows_per_dst=[2990, 3006, 2944, 2916, 3004, 3062, 3092, 2930] combine_rows_per_src=[9322, 9206, 9396, 9146, 9346, 9374, 9174, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=343656 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265412200 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530480744 weight_movement_bytes=69206016 total_nvlink_bytes=599686760 weight_to_token_ratio=0.130459 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65132 dup_rows=23950 dup_rows_remote=20968 dedup_ratio=0.243632 router_tokens_per_expert=[784, 781, 755, 763, 749, 723, 775, 793, 791, 749, 763, 749, 731, 766, 767, 756, 762, 793, 730, 767, 758, 736, 752, 804, 814, 783, 772, 730, 769, 762, 749, 748, 772, 778, 744, 781, 749, 718, 746, 800, 750, 755, 766, 801, 766, 783, 804, 754, 785, 759, 778, 768, 732, 751, 824, 772, 798, 773, 813, 769, 779, 780, 774, 806] payload_rows_per_dst=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] dup_rows_per_dst=[2958, 2890, 2982, 2954, 2986, 3020, 3036, 3124] combine_rows_per_src=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=342664 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265951880 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561096 weight_movement_bytes=34603008 total_nvlink_bytes=566164104 weight_to_token_ratio=0.065097 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=341944 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264976312 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529610680 weight_movement_bytes=69206016 total_nvlink_bytes=598816696 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344824 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658552 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532972280 weight_movement_bytes=34603008 total_nvlink_bytes=567575288 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=64960 dup_rows=23842 dup_rows_remote=20804 dedup_ratio=0.242533 router_tokens_per_expert=[776, 783, 772, 733, 757, 776, 777, 761, 758, 822, 778, 718, 772, 767, 770, 736, 723, 774, 741, 744, 765, 756, 754, 785, 792, 807, 713, 780, 828, 784, 800, 766, 775, 745, 780, 761, 770, 803, 791, 756, 750, 741, 723, 772, 796, 745, 747, 747, 783, 760, 800, 796, 741, 747, 762, 757, 753, 765, 773, 787, 806, 749, 808, 795] payload_rows_per_dst=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] dup_rows_per_dst=[2904, 2992, 2834, 3038, 3098, 2872, 3028, 3076] combine_rows_per_src=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344272 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223824 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103376 weight_movement_bytes=69206016 total_nvlink_bytes=601309392 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344824 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658552 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532972280 weight_movement_bytes=34603008 total_nvlink_bytes=567575288 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344864 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266904352 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533463840 weight_movement_bytes=103809024 total_nvlink_bytes=637272864 weight_to_token_ratio=0.194594 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344400 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267125072 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905744 weight_movement_bytes=34603008 total_nvlink_bytes=568508752 weight_to_token_ratio=0.064811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344368 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215152 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085936 weight_movement_bytes=34603008 total_nvlink_bytes=568688944 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=64960 dup_rows=23842 dup_rows_remote=20804 dedup_ratio=0.242533 router_tokens_per_expert=[776, 783, 772, 733, 757, 776, 777, 761, 758, 822, 778, 718, 772, 767, 770, 736, 723, 774, 741, 744, 765, 756, 754, 785, 792, 807, 713, 780, 828, 784, 800, 766, 775, 745, 780, 761, 770, 803, 791, 756, 750, 741, 723, 772, 796, 745, 747, 747, 783, 760, 800, 796, 741, 747, 762, 757, 753, 765, 773, 787, 806, 749, 808, 795] payload_rows_per_dst=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] dup_rows_per_dst=[2904, 2992, 2834, 3038, 3098, 2872, 3028, 3076] combine_rows_per_src=[9366, 9250, 9250, 9502, 9264, 9170, 9264, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65132 dup_rows=23950 dup_rows_remote=20968 dedup_ratio=0.243632 router_tokens_per_expert=[784, 781, 755, 763, 749, 723, 775, 793, 791, 749, 763, 749, 731, 766, 767, 756, 762, 793, 730, 767, 758, 736, 752, 804, 814, 783, 772, 730, 769, 762, 749, 748, 772, 778, 744, 781, 749, 718, 746, 800, 750, 755, 766, 801, 766, 783, 804, 754, 785, 759, 778, 768, 732, 751, 824, 772, 798, 773, 813, 769, 779, 780, 774, 806] payload_rows_per_dst=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] dup_rows_per_dst=[2958, 2890, 2982, 2954, 2986, 3020, 3036, 3124] combine_rows_per_src=[9288, 9254, 9222, 9300, 9190, 9338, 9302, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64786 dup_rows=24218 dup_rows_remote=21396 dedup_ratio=0.246358 router_tokens_per_expert=[814, 755, 752, 671, 755, 783, 760, 791, 834, 773, 807, 754, 791, 813, 791, 752, 739, 742, 739, 719, 781, 805, 789, 794, 723, 781, 763, 757, 776, 773, 753, 768, 751, 745, 774, 767, 732, 801, 745, 746, 765, 746, 783, 779, 784, 748, 739, 793, 801, 766, 776, 762, 762, 755, 751, 784, 767, 783, 774, 791, 759, 767, 770, 788] payload_rows_per_dst=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] dup_rows_per_dst=[3036, 3154, 2950, 3026, 2822, 3082, 3050, 3098] combine_rows_per_src=[9126, 9476, 9266, 9162, 9300, 9192, 9264, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64880 dup_rows=24060 dup_rows_remote=20926 dedup_ratio=0.244751 router_tokens_per_expert=[740, 777, 735, 707, 736, 797, 774, 774, 720, 769, 744, 761, 748, 756, 748, 754, 770, 794, 769, 758, 812, 809, 804, 758, 744, 748, 775, 809, 763, 755, 755, 797, 751, 746, 778, 815, 760, 754, 741, 730, 740, 748, 691, 780, 791, 790, 777, 816, 819, 772, 744, 792, 810, 763, 784, 751, 765, 780, 751, 826, 760, 755, 848, 764] payload_rows_per_dst=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] dup_rows_per_dst=[2976, 2600, 3442, 2940, 2934, 2948, 3086, 3134] combine_rows_per_src=[9104, 8736, 9770, 9352, 9216, 9318, 9384, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343056 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419216 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495376 weight_movement_bytes=34603008 total_nvlink_bytes=567098384 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344400 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267125072 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905744 weight_movement_bytes=34603008 total_nvlink_bytes=568508752 weight_to_token_ratio=0.064811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65078 dup_rows=24084 dup_rows_remote=21138 dedup_ratio=0.244995 router_tokens_per_expert=[722, 733, 783, 781, 751, 756, 754, 764, 774, 787, 817, 821, 771, 756, 728, 803, 754, 709, 760, 791, 746, 808, 749, 752, 751, 767, 761, 748, 800, 778, 748, 749, 804, 724, 776, 806, 802, 770, 759, 796, 758, 764, 765, 761, 752, 774, 730, 735, 768, 774, 750, 735, 836, 775, 807, 805, 772, 746, 782, 753, 782, 770, 776, 773] payload_rows_per_dst=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] dup_rows_per_dst=[2946, 3396, 3076, 2940, 3088, 2906, 2996, 2736] combine_rows_per_src=[9142, 9630, 9278, 9264, 9386, 9172, 9288, 9060] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65126 dup_rows=23938 dup_rows_remote=20942 dedup_ratio=0.243510 router_tokens_per_expert=[744, 822, 799, 751, 767, 707, 749, 743, 758, 764, 751, 776, 777, 777, 770, 758, 764, 726, 784, 794, 816, 751, 764, 743, 720, 761, 753, 799, 761, 726, 773, 733, 729, 790, 799, 784, 755, 728, 807, 765, 778, 786, 763, 722, 765, 756, 771, 777, 791, 817, 807, 802, 770, 795, 796, 738, 758, 770, 782, 791, 783, 756, 801, 739] payload_rows_per_dst=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] dup_rows_per_dst=[2888, 2920, 3028, 2906, 3006, 2996, 3196, 2998] combine_rows_per_src=[9276, 9342, 9256, 9146, 9308, 9240, 9436, 9362] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65040 dup_rows=23960 dup_rows_remote=20920 dedup_ratio=0.243734 router_tokens_per_expert=[744, 798, 778, 743, 781, 761, 700, 766, 812, 745, 803, 780, 783, 770, 809, 779, 755, 759, 744, 781, 749, 746, 798, 729, 695, 741, 755, 743, 780, 793, 755, 795, 777, 800, 775, 761, 798, 802, 815, 722, 748, 755, 771, 753, 767, 819, 749, 796, 771, 739, 773, 777, 738, 784, 790, 800, 791, 732, 743, 739, 789, 822, 745, 741] payload_rows_per_dst=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] dup_rows_per_dst=[3032, 3064, 2918, 2796, 3112, 3036, 3040, 2962] combine_rows_per_src=[9110, 9498, 9204, 9318, 9388, 9280, 9304, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64600 dup_rows=24154 dup_rows_remote=20828 dedup_ratio=0.245707 router_tokens_per_expert=[736, 760, 801, 783, 755, 734, 759, 766, 768, 776, 789, 783, 752, 774, 821, 809, 746, 757, 757, 706, 783, 784, 726, 769, 763, 766, 678, 809, 767, 729, 768, 795, 795, 722, 760, 791, 750, 793, 772, 791, 756, 766, 815, 755, 789, 789, 778, 752, 748, 752, 752, 760, 760, 769, 835, 771, 794, 788, 789, 732, 771, 794, 767, 727] payload_rows_per_dst=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] dup_rows_per_dst=[3326, 3310, 2942, 3126, 3010, 3104, 2856, 2480] combine_rows_per_src=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64902 dup_rows=24092 dup_rows_remote=20958 dedup_ratio=0.245076 router_tokens_per_expert=[769, 754, 789, 770, 758, 785, 780, 736, 783, 770, 737, 759, 748, 808, 798, 788, 762, 786, 790, 786, 786, 794, 779, 739, 729, 792, 762, 760, 818, 770, 770, 757, 766, 760, 761, 691, 793, 800, 754, 814, 777, 723, 732, 763, 753, 737, 802, 744, 764, 762, 791, 756, 787, 770, 758, 775, 715, 800, 770, 743, 772, 727, 770, 810] payload_rows_per_dst=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] dup_rows_per_dst=[3038, 3042, 3134, 3064, 3034, 2856, 3004, 2920] combine_rows_per_src=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65146 dup_rows=23872 dup_rows_remote=20784 dedup_ratio=0.242839 router_tokens_per_expert=[782, 800, 756, 784, 781, 777, 758, 744, 760, 786, 804, 774, 743, 735, 790, 795, 714, 712, 761, 726, 757, 809, 810, 774, 761, 774, 751, 802, 761, 798, 803, 761, 817, 762, 753, 726, 760, 764, 775, 793, 781, 754, 764, 808, 765, 767, 767, 795, 752, 726, 720, 724, 759, 735, 761, 800, 783, 767, 764, 749, 732, 822, 737, 827] payload_rows_per_dst=[9344, 9286, 9202, 9350, 9294, 9448, 9170, 9338] dup_rows_per_dst=[3020, 3088, 2924, 3072, 3006, 2954, 2784, 3024] combine_rows_per_src=[9344, 9286, 9202, 9350, 9294, 9448, 9170, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344728 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708184 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071640 weight_movement_bytes=34603008 total_nvlink_bytes=565674648 weight_to_token_ratio=0.065157 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343224 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266091704 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840184 weight_movement_bytes=69206016 total_nvlink_bytes=601046200 weight_to_token_ratio=0.130126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343056 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419216 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495376 weight_movement_bytes=34603008 total_nvlink_bytes=567098384 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344728 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708184 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071640 weight_movement_bytes=34603008 total_nvlink_bytes=565674648 weight_to_token_ratio=0.065157 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343224 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266091704 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840184 weight_movement_bytes=69206016 total_nvlink_bytes=601046200 weight_to_token_ratio=0.130126 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344272 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100368 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856464 weight_movement_bytes=34603008 total_nvlink_bytes=568459472 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344864 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266904352 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533463840 weight_movement_bytes=103809024 total_nvlink_bytes=637272864 weight_to_token_ratio=0.194594 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343840 combine_payload_bytes=266403840 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266747680 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151520 weight_movement_bytes=103809024 total_nvlink_bytes=636960544 weight_to_token_ratio=0.194708 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264601600 dispatch_meta_bytes=341712 combine_payload_bytes=264601600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264943312 ingress_bytes=299204608 link_bytes=299204608 token_bytes=529544912 weight_movement_bytes=34603008 total_nvlink_bytes=564147920 weight_to_token_ratio=0.065345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125882852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343440 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182032 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532020624 weight_movement_bytes=69206016 total_nvlink_bytes=601226640 weight_to_token_ratio=0.130081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343720 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267181736 ingress_bytes=336044032 link_bytes=336044032 token_bytes=534019752 weight_movement_bytes=69206016 total_nvlink_bytes=603225768 weight_to_token_ratio=0.129594 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264506639e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64950 dup_rows=24036 dup_rows_remote=21046 dedup_ratio=0.244507 router_tokens_per_expert=[815, 813, 746, 762, 765, 794, 831, 807, 740, 795, 783, 771, 746, 750, 749, 772, 734, 801, 809, 740, 776, 767, 802, 769, 790, 810, 784, 746, 771, 770, 711, 772, 716, 735, 742, 763, 798, 752, 790, 778, 794, 808, 758, 702, 792, 755, 759, 757, 785, 723, 765, 755, 737, 795, 804, 756, 720, 746, 832, 728, 752, 775, 757, 732] payload_rows_per_dst=[9490, 9228, 9328, 9318, 9210, 9262, 9252, 9180] dup_rows_per_dst=[3176, 2984, 3068, 2990, 2938, 2988, 2988, 2904] combine_rows_per_src=[9490, 9228, 9328, 9318, 9210, 9262, 9252, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64902 dup_rows=24092 dup_rows_remote=20958 dedup_ratio=0.245076 router_tokens_per_expert=[769, 754, 789, 770, 758, 785, 780, 736, 783, 770, 737, 759, 748, 808, 798, 788, 762, 786, 790, 786, 786, 794, 779, 739, 729, 792, 762, 760, 818, 770, 770, 757, 766, 760, 761, 691, 793, 800, 754, 814, 777, 723, 732, 763, 753, 737, 802, 744, 764, 762, 791, 756, 787, 770, 758, 775, 715, 800, 770, 743, 772, 727, 770, 810] payload_rows_per_dst=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] dup_rows_per_dst=[3038, 3042, 3134, 3064, 3034, 2856, 3004, 2920] combine_rows_per_src=[9244, 9340, 9310, 9252, 9244, 9206, 9322, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64964 dup_rows=23920 dup_rows_remote=20972 dedup_ratio=0.243327 router_tokens_per_expert=[775, 780, 739, 762, 753, 781, 816, 738, 769, 792, 788, 744, 735, 797, 811, 756, 793, 793, 838, 744, 726, 722, 826, 788, 711, 760, 840, 724, 762, 800, 792, 717, 754, 761, 772, 760, 788, 742, 783, 824, 770, 792, 776, 758, 735, 795, 712, 767, 794, 704, 818, 785, 798, 733, 706, 767, 752, 778, 782, 786, 747, 721, 768, 752] payload_rows_per_dst=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] dup_rows_per_dst=[2904, 3056, 3054, 2994, 2948, 2966, 2972, 3026] combine_rows_per_src=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64676 dup_rows=24020 dup_rows_remote=20548 dedup_ratio=0.244344 router_tokens_per_expert=[769, 834, 777, 758, 778, 703, 753, 759, 758, 789, 770, 781, 758, 768, 748, 745, 746, 757, 792, 707, 795, 775, 772, 774, 744, 743, 823, 740, 784, 730, 748, 788, 799, 744, 745, 774, 751, 760, 800, 747, 784, 787, 792, 789, 802, 774, 771, 777, 736, 747, 771, 782, 730, 742, 791, 772, 780, 802, 798, 790, 759, 765, 748, 777] payload_rows_per_dst=[8906, 9200, 9082, 9508, 9300, 9264, 9416, 9608] dup_rows_per_dst=[2656, 2942, 2866, 3072, 2940, 2948, 3124, 3472] combine_rows_per_src=[8906, 9200, 9082, 9508, 9300, 9264, 9416, 9608] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343984 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266379184 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414384 weight_movement_bytes=34603008 total_nvlink_bytes=567017392 weight_to_token_ratio=0.064993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343440 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182032 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532020624 weight_movement_bytes=69206016 total_nvlink_bytes=601226640 weight_to_token_ratio=0.130081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64964 dup_rows=23920 dup_rows_remote=20972 dedup_ratio=0.243327 router_tokens_per_expert=[775, 780, 739, 762, 753, 781, 816, 738, 769, 792, 788, 744, 735, 797, 811, 756, 793, 793, 838, 744, 726, 722, 826, 788, 711, 760, 840, 724, 762, 800, 792, 717, 754, 761, 772, 760, 788, 742, 783, 824, 770, 792, 776, 758, 735, 795, 712, 767, 794, 704, 818, 785, 798, 733, 706, 767, 752, 778, 782, 786, 747, 721, 768, 752] payload_rows_per_dst=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] dup_rows_per_dst=[2904, 3056, 3054, 2994, 2948, 2966, 2972, 3026] combine_rows_per_src=[9384, 9328, 9406, 9218, 9420, 9244, 9238, 9146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64924 dup_rows=23958 dup_rows_remote=20788 dedup_ratio=0.243713 router_tokens_per_expert=[733, 801, 763, 768, 802, 753, 760, 812, 762, 788, 782, 770, 775, 813, 777, 774, 733, 773, 741, 796, 722, 775, 761, 751, 784, 774, 716, 713, 775, 784, 786, 732, 770, 763, 754, 771, 786, 777, 788, 735, 740, 789, 795, 767, 775, 839, 771, 820, 754, 750, 778, 747, 771, 774, 685, 809, 778, 755, 756, 775, 735, 758, 794, 744] payload_rows_per_dst=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] dup_rows_per_dst=[3112, 2976, 2942, 2848, 3024, 3170, 2910, 2976] combine_rows_per_src=[9272, 9506, 9162, 9280, 9264, 9422, 9226, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65198 dup_rows=23842 dup_rows_remote=20906 dedup_ratio=0.242533 router_tokens_per_expert=[756, 753, 770, 740, 829, 787, 745, 787, 770, 772, 759, 773, 788, 757, 793, 737, 782, 782, 777, 771, 778, 753, 761, 744, 770, 728, 736, 818, 774, 760, 810, 755, 800, 751, 756, 727, 794, 761, 748, 727, 751, 806, 782, 776, 805, 761, 780, 782, 790, 772, 780, 715, 780, 752, 764, 747, 798, 757, 789, 785, 745, 755, 757, 744] payload_rows_per_dst=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] dup_rows_per_dst=[2940, 3018, 3066, 2986, 2954, 3022, 2936, 2920] combine_rows_per_src=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64816 dup_rows=24236 dup_rows_remote=21212 dedup_ratio=0.246541 router_tokens_per_expert=[786, 780, 739, 718, 788, 771, 749, 802, 765, 808, 762, 725, 762, 773, 761, 752, 771, 773, 754, 763, 746, 711, 756, 777, 767, 779, 783, 745, 775, 785, 721, 772, 769, 774, 796, 770, 813, 746, 778, 778, 772, 801, 769, 811, 737, 786, 778, 817, 786, 770, 757, 784, 791, 756, 831, 771, 720, 736, 764, 744, 795, 706, 770, 757] payload_rows_per_dst=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] dup_rows_per_dst=[3024, 2916, 2956, 3104, 3188, 3002, 3208, 2838] combine_rows_per_src=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343744 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436288 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528832 weight_movement_bytes=34603008 total_nvlink_bytes=567131840 weight_to_token_ratio=0.064979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64600 dup_rows=24154 dup_rows_remote=20828 dedup_ratio=0.245707 router_tokens_per_expert=[736, 760, 801, 783, 755, 734, 759, 766, 768, 776, 789, 783, 752, 774, 821, 809, 746, 757, 757, 706, 783, 784, 726, 769, 763, 766, 678, 809, 767, 729, 768, 795, 795, 722, 760, 791, 750, 793, 772, 791, 756, 766, 815, 755, 789, 789, 778, 752, 748, 752, 752, 760, 760, 769, 835, 771, 794, 788, 789, 732, 771, 794, 767, 727] payload_rows_per_dst=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] dup_rows_per_dst=[3326, 3310, 2942, 3126, 3010, 3104, 2856, 2480] combine_rows_per_src=[9550, 9444, 9114, 9316, 9370, 9306, 9104, 8946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=21006 dedup_ratio=0.242676 router_tokens_per_expert=[734, 751, 796, 757, 777, 736, 737, 774, 759, 781, 763, 775, 751, 781, 781, 748, 733, 742, 775, 767, 720, 780, 757, 780, 759, 765, 814, 758, 764, 740, 773, 798, 800, 765, 767, 825, 763, 766, 786, 814, 778, 744, 727, 731, 719, 783, 785, 779, 790, 803, 774, 768, 799, 764, 751, 742, 778, 765, 768, 742, 747, 803, 813, 787] payload_rows_per_dst=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] dup_rows_per_dst=[2918, 3016, 2850, 3026, 3112, 2936, 2980, 3018] combine_rows_per_src=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65196 dup_rows=23892 dup_rows_remote=20910 dedup_ratio=0.243042 router_tokens_per_expert=[761, 812, 790, 791, 798, 818, 741, 745, 714, 812, 742, 769, 760, 813, 761, 728, 801, 780, 787, 788, 773, 809, 753, 786, 731, 772, 773, 739, 738, 781, 803, 765, 831, 786, 749, 760, 786, 740, 734, 716, 754, 721, 767, 755, 779, 802, 752, 746, 733, 760, 756, 770, 759, 790, 781, 731, 759, 783, 776, 731, 816, 791, 767, 737] payload_rows_per_dst=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] dup_rows_per_dst=[3080, 2982, 3088, 3000, 2964, 2880, 2920, 2978] combine_rows_per_src=[9432, 9216, 9466, 9204, 9240, 9272, 9240, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=340896 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265253792 ingress_bytes=299515904 link_bytes=299515904 token_bytes=530166688 weight_movement_bytes=34603008 total_nvlink_bytes=564769696 weight_to_token_ratio=0.065268 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127054234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65198 dup_rows=23842 dup_rows_remote=20906 dedup_ratio=0.242533 router_tokens_per_expert=[756, 753, 770, 740, 829, 787, 745, 787, 770, 772, 759, 773, 788, 757, 793, 737, 782, 782, 777, 771, 778, 753, 761, 744, 770, 728, 736, 818, 774, 760, 810, 755, 800, 751, 756, 727, 794, 761, 748, 727, 751, 806, 782, 776, 805, 761, 780, 782, 790, 772, 780, 715, 780, 752, 764, 747, 798, 757, 789, 785, 745, 755, 757, 744] payload_rows_per_dst=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] dup_rows_per_dst=[2940, 3018, 3066, 2986, 2954, 3022, 2936, 2920] combine_rows_per_src=[9394, 9280, 9230, 9316, 9174, 9464, 9264, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343984 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266379184 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414384 weight_movement_bytes=34603008 total_nvlink_bytes=567017392 weight_to_token_ratio=0.064993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=340896 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265253792 ingress_bytes=299515904 link_bytes=299515904 token_bytes=530166688 weight_movement_bytes=34603008 total_nvlink_bytes=564769696 weight_to_token_ratio=0.065268 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127054234e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343744 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436288 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528832 weight_movement_bytes=34603008 total_nvlink_bytes=567131840 weight_to_token_ratio=0.064979 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=342848 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266271552 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532200256 weight_movement_bytes=103809024 total_nvlink_bytes=636009280 weight_to_token_ratio=0.195056 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264601600 dispatch_meta_bytes=341712 combine_payload_bytes=264601600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264943312 ingress_bytes=299204608 link_bytes=299204608 token_bytes=529544912 weight_movement_bytes=34603008 total_nvlink_bytes=564147920 weight_to_token_ratio=0.065345 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125882852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344784 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267363024 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534381264 weight_movement_bytes=34603008 total_nvlink_bytes=568984272 weight_to_token_ratio=0.064753 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344416 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267395424 ingress_bytes=301654016 link_bytes=301654016 token_bytes=534446432 weight_movement_bytes=34603008 total_nvlink_bytes=569049440 weight_to_token_ratio=0.064746 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135099777e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344112 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265830448 ingress_bytes=369295360 link_bytes=369295360 token_bytes=531316784 weight_movement_bytes=103809024 total_nvlink_bytes=635125808 weight_to_token_ratio=0.195381 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389628709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344424 combine_payload_bytes=267042816 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267387240 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534430056 weight_movement_bytes=69206016 total_nvlink_bytes=603636072 weight_to_token_ratio=0.129495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65004 dup_rows=23844 dup_rows_remote=20794 dedup_ratio=0.242554 router_tokens_per_expert=[803, 750, 730, 811, 770, 741, 767, 741, 756, 752, 772, 771, 761, 773, 784, 771, 748, 741, 726, 749, 758, 776, 791, 780, 764, 742, 729, 782, 794, 797, 825, 820, 772, 736, 757, 806, 738, 769, 752, 757, 750, 822, 748, 762, 797, 765, 741, 758, 759, 757, 730, 705, 812, 753, 783, 756, 767, 829, 769, 815, 754, 783, 775, 800] payload_rows_per_dst=[9302, 9356, 9194, 9456, 9306, 9268, 9162, 9416] dup_rows_per_dst=[2924, 2924, 2944, 3050, 2868, 3018, 2948, 3168] combine_rows_per_src=[9302, 9356, 9194, 9456, 9306, 9268, 9162, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74448 payload_rows_remote=65190 dup_rows=23856 dup_rows_remote=21006 dedup_ratio=0.242676 router_tokens_per_expert=[734, 751, 796, 757, 777, 736, 737, 774, 759, 781, 763, 775, 751, 781, 781, 748, 733, 742, 775, 767, 720, 780, 757, 780, 759, 765, 814, 758, 764, 740, 773, 798, 800, 765, 767, 825, 763, 766, 786, 814, 778, 744, 727, 731, 719, 783, 785, 779, 790, 803, 774, 768, 799, 764, 751, 742, 778, 765, 768, 742, 747, 803, 813, 787] payload_rows_per_dst=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] dup_rows_per_dst=[2918, 3016, 2850, 3026, 3112, 2936, 2980, 3018] combine_rows_per_src=[9206, 9262, 9258, 9316, 9460, 9156, 9402, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64814 dup_rows=24180 dup_rows_remote=21250 dedup_ratio=0.245972 router_tokens_per_expert=[763, 805, 748, 772, 717, 781, 769, 725, 749, 764, 744, 819, 804, 789, 832, 749, 762, 806, 762, 710, 758, 790, 731, 779, 803, 777, 774, 729, 790, 736, 776, 758, 745, 750, 760, 760, 815, 768, 772, 750, 774, 745, 764, 799, 765, 765, 811, 760, 765, 799, 724, 765, 823, 737, 769, 757, 770, 773, 789, 775, 749, 738, 783, 762] payload_rows_per_dst=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] dup_rows_per_dst=[2920, 3110, 3032, 3128, 2930, 3042, 3002, 3016] combine_rows_per_src=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73972 payload_rows_remote=64714 dup_rows=24332 dup_rows_remote=21360 dedup_ratio=0.247518 router_tokens_per_expert=[762, 734, 791, 739, 701, 809, 754, 784, 790, 755, 802, 766, 792, 759, 773, 826, 751, 785, 780, 773, 775, 789, 764, 785, 759, 754, 758, 804, 774, 750, 799, 787, 755, 770, 769, 785, 734, 760, 746, 790, 767, 754, 786, 769, 764, 761, 766, 741, 766, 769, 769, 736, 767, 785, 722, 782, 751, 758, 752, 758, 748, 808, 780, 760] payload_rows_per_dst=[9194, 9130, 9398, 9254, 9262, 9194, 9282, 9258] dup_rows_per_dst=[2954, 3030, 3200, 3116, 2956, 3022, 3082, 2972] combine_rows_per_src=[9194, 9130, 9398, 9254, 9262, 9194, 9282, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343192 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266599576 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855960 weight_movement_bytes=69206016 total_nvlink_bytes=602061976 weight_to_token_ratio=0.129878 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344784 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267363024 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534381264 weight_movement_bytes=34603008 total_nvlink_bytes=568984272 weight_to_token_ratio=0.064753 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64884 dup_rows=24220 dup_rows_remote=21224 dedup_ratio=0.246379 router_tokens_per_expert=[776, 766, 747, 810, 723, 775, 801, 739, 770, 758, 748, 714, 802, 783, 771, 785, 774, 791, 769, 746, 779, 766, 803, 770, 848, 756, 756, 794, 759, 759, 793, 777, 784, 764, 743, 761, 730, 772, 835, 741, 757, 730, 720, 782, 779, 760, 789, 781, 809, 757, 760, 765, 763, 775, 754, 745, 736, 774, 734, 762, 826, 783, 748, 725] payload_rows_per_dst=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] dup_rows_per_dst=[3050, 3024, 3056, 3164, 2984, 2996, 2940, 3006] combine_rows_per_src=[9224, 9238, 9340, 9320, 9276, 9200, 9316, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64814 dup_rows=24180 dup_rows_remote=21250 dedup_ratio=0.245972 router_tokens_per_expert=[763, 805, 748, 772, 717, 781, 769, 725, 749, 764, 744, 819, 804, 789, 832, 749, 762, 806, 762, 710, 758, 790, 731, 779, 803, 777, 774, 729, 790, 736, 776, 758, 745, 750, 760, 760, 815, 768, 772, 750, 774, 745, 764, 799, 765, 765, 811, 760, 765, 799, 724, 765, 823, 737, 769, 757, 770, 773, 789, 775, 749, 738, 783, 762] payload_rows_per_dst=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] dup_rows_per_dst=[2920, 3110, 3032, 3128, 2930, 3042, 3002, 3016] combine_rows_per_src=[9240, 9390, 9164, 9158, 9310, 9324, 9276, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65010 dup_rows=23874 dup_rows_remote=20918 dedup_ratio=0.242859 router_tokens_per_expert=[770, 766, 735, 760, 777, 746, 812, 733, 761, 752, 806, 738, 770, 773, 772, 774, 740, 712, 781, 802, 806, 762, 793, 792, 785, 753, 795, 774, 781, 744, 795, 781, 755, 789, 768, 752, 771, 769, 767, 770, 748, 772, 743, 773, 711, 792, 742, 760, 760, 792, 779, 791, 796, 766, 775, 761, 759, 773, 762, 784, 761, 757, 734, 779] payload_rows_per_dst=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] dup_rows_per_dst=[2954, 2876, 2956, 3130, 2998, 2948, 3030, 2982] combine_rows_per_src=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64816 dup_rows=24236 dup_rows_remote=21212 dedup_ratio=0.246541 router_tokens_per_expert=[786, 780, 739, 718, 788, 771, 749, 802, 765, 808, 762, 725, 762, 773, 761, 752, 771, 773, 754, 763, 746, 711, 756, 777, 767, 779, 783, 745, 775, 785, 721, 772, 769, 774, 796, 770, 813, 746, 778, 778, 772, 801, 769, 811, 737, 786, 778, 817, 786, 770, 757, 784, 791, 756, 831, 771, 720, 736, 764, 744, 795, 706, 770, 757] payload_rows_per_dst=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] dup_rows_per_dst=[3024, 2916, 2956, 3104, 3188, 3002, 3208, 2838] combine_rows_per_src=[9252, 9290, 9146, 9150, 9462, 9338, 9284, 9146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20810 dedup_ratio=0.243408 router_tokens_per_expert=[764, 768, 771, 757, 736, 774, 800, 818, 773, 754, 788, 736, 788, 785, 734, 785, 739, 818, 802, 813, 771, 744, 771, 754, 749, 751, 726, 735, 772, 801, 766, 736, 756, 781, 765, 739, 733, 746, 777, 767, 760, 793, 764, 741, 761, 768, 741, 771, 731, 753, 772, 769, 845, 796, 759, 778, 762, 783, 791, 760, 772, 754, 814, 771] payload_rows_per_dst=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] dup_rows_per_dst=[3030, 2930, 3024, 2922, 2944, 2950, 3118, 3010] combine_rows_per_src=[9346, 9356, 9400, 9150, 9184, 9248, 9288, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64732 dup_rows=24108 dup_rows_remote=20918 dedup_ratio=0.245239 router_tokens_per_expert=[759, 773, 775, 731, 742, 755, 782, 813, 707, 809, 824, 726, 766, 761, 767, 769, 754, 739, 772, 762, 769, 798, 747, 800, 766, 761, 781, 742, 777, 784, 824, 753, 760, 716, 788, 779, 763, 707, 734, 747, 737, 791, 771, 786, 747, 769, 775, 810, 776, 741, 741, 766, 770, 788, 759, 775, 789, 792, 774, 779, 769, 780, 817, 768] payload_rows_per_dst=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] dup_rows_per_dst=[3190, 2992, 3006, 2900, 3200, 3158, 2932, 2730] combine_rows_per_src=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344256 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265822400 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531300544 weight_movement_bytes=34603008 total_nvlink_bytes=565903552 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343192 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266599576 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855960 weight_movement_bytes=69206016 total_nvlink_bytes=602061976 weight_to_token_ratio=0.129878 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65080 dup_rows=23984 dup_rows_remote=21014 dedup_ratio=0.243978 router_tokens_per_expert=[736, 767, 786, 772, 811, 758, 801, 740, 762, 767, 781, 756, 806, 771, 760, 702, 805, 723, 808, 770, 736, 731, 769, 739, 775, 782, 775, 758, 756, 754, 813, 753, 755, 781, 797, 789, 745, 740, 796, 728, 757, 732, 744, 765, 771, 798, 792, 781, 754, 805, 789, 722, 767, 802, 795, 704, 816, 735, 763, 826, 761, 791, 784, 744] payload_rows_per_dst=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] dup_rows_per_dst=[3014, 2970, 2904, 3052, 3012, 3006, 2988, 3038] combine_rows_per_src=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344296 combine_payload_bytes=265068544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265412840 ingress_bytes=334274560 link_bytes=334274560 token_bytes=530481384 weight_movement_bytes=69206016 total_nvlink_bytes=599687400 weight_to_token_ratio=0.130459 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257848258e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65080 dup_rows=23984 dup_rows_remote=21014 dedup_ratio=0.243978 router_tokens_per_expert=[736, 767, 786, 772, 811, 758, 801, 740, 762, 767, 781, 756, 806, 771, 760, 702, 805, 723, 808, 770, 736, 731, 769, 739, 775, 782, 775, 758, 756, 754, 813, 753, 755, 781, 797, 789, 745, 740, 796, 728, 757, 732, 744, 765, 771, 798, 792, 781, 754, 805, 789, 722, 767, 802, 795, 704, 816, 735, 763, 826, 761, 791, 784, 744] payload_rows_per_dst=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] dup_rows_per_dst=[3014, 2970, 2904, 3052, 3012, 3006, 2988, 3038] combine_rows_per_src=[9328, 9240, 9258, 9280, 9250, 9274, 9288, 9402] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344432 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266109296 ingress_bytes=300367872 link_bytes=300367872 token_bytes=531874160 weight_movement_bytes=34603008 total_nvlink_bytes=566477168 weight_to_token_ratio=0.065059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130260121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344432 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266109296 ingress_bytes=300367872 link_bytes=300367872 token_bytes=531874160 weight_movement_bytes=34603008 total_nvlink_bytes=566477168 weight_to_token_ratio=0.065059 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130260121e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344256 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265822400 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531300544 weight_movement_bytes=34603008 total_nvlink_bytes=565903552 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343712 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266624672 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532905632 weight_movement_bytes=69206016 total_nvlink_bytes=602111648 weight_to_token_ratio=0.129865 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344112 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265830448 ingress_bytes=369295360 link_bytes=369295360 token_bytes=531316784 weight_movement_bytes=103809024 total_nvlink_bytes=635125808 weight_to_token_ratio=0.195381 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389628709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343592 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944040 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533544488 weight_movement_bytes=34603008 total_nvlink_bytes=568147496 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342600 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265484872 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627144 weight_movement_bytes=34603008 total_nvlink_bytes=565230152 weight_to_token_ratio=0.065212 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344376 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266912056 ingress_bytes=370376704 link_bytes=370376704 token_bytes=533479736 weight_movement_bytes=103809024 total_nvlink_bytes=637288760 weight_to_token_ratio=0.194589 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393697719e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65124 dup_rows=24072 dup_rows_remote=21120 dedup_ratio=0.244873 router_tokens_per_expert=[763, 792, 758, 730, 791, 762, 754, 780, 783, 734, 766, 750, 767, 824, 765, 780, 753, 800, 767, 779, 756, 786, 772, 795, 748, 734, 762, 745, 787, 743, 775, 736, 779, 753, 771, 746, 732, 820, 739, 732, 801, 746, 764, 786, 819, 793, 795, 776, 809, 720, 803, 747, 736, 724, 765, 723, 797, 830, 769, 743, 789, 744, 743, 821] payload_rows_per_dst=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] dup_rows_per_dst=[2978, 3070, 3008, 2952, 2982, 3128, 2920, 3034] combine_rows_per_src=[9282, 9268, 9408, 9108, 9162, 9432, 9134, 9438] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65010 dup_rows=23874 dup_rows_remote=20918 dedup_ratio=0.242859 router_tokens_per_expert=[770, 766, 735, 760, 777, 746, 812, 733, 761, 752, 806, 738, 770, 773, 772, 774, 740, 712, 781, 802, 806, 762, 793, 792, 785, 753, 795, 774, 781, 744, 795, 781, 755, 789, 768, 752, 771, 769, 767, 770, 748, 772, 743, 773, 711, 792, 742, 760, 760, 792, 779, 791, 796, 766, 775, 761, 759, 773, 762, 784, 761, 757, 734, 779] payload_rows_per_dst=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] dup_rows_per_dst=[2954, 2876, 2956, 3130, 2998, 2948, 3030, 2982] combine_rows_per_src=[9244, 9416, 9420, 9286, 9284, 9134, 9410, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64972 dup_rows=24044 dup_rows_remote=21032 dedup_ratio=0.244588 router_tokens_per_expert=[717, 752, 736, 768, 769, 755, 723, 747, 761, 740, 752, 794, 768, 739, 770, 782, 784, 737, 758, 807, 802, 777, 789, 762, 751, 764, 738, 753, 765, 726, 802, 765, 745, 807, 749, 738, 796, 787, 768, 760, 750, 797, 782, 788, 754, 746, 807, 780, 726, 789, 750, 746, 778, 764, 778, 797, 822, 782, 789, 785, 756, 770, 795, 818] payload_rows_per_dst=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] dup_rows_per_dst=[2864, 3018, 3088, 2938, 3012, 3006, 2908, 3210] combine_rows_per_src=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65226 dup_rows=23944 dup_rows_remote=21088 dedup_ratio=0.243571 router_tokens_per_expert=[802, 775, 758, 789, 804, 816, 754, 770, 733, 771, 771, 738, 780, 809, 758, 771, 753, 763, 758, 794, 769, 749, 795, 759, 794, 791, 737, 740, 764, 792, 771, 763, 708, 769, 794, 842, 767, 757, 753, 732, 762, 800, 770, 754, 789, 782, 778, 770, 727, 758, 746, 788, 799, 763, 802, 758, 781, 742, 777, 740, 716, 743, 754, 740] payload_rows_per_dst=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] dup_rows_per_dst=[3078, 3008, 3018, 2994, 2970, 2830, 3190, 2856] combine_rows_per_src=[9458, 9254, 9262, 9310, 9274, 9202, 9466, 9134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=344976 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267092880 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533840784 weight_movement_bytes=34603008 total_nvlink_bytes=568443792 weight_to_token_ratio=0.064819 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64964 dup_rows=23996 dup_rows_remote=20960 dedup_ratio=0.244100 router_tokens_per_expert=[790, 727, 743, 794, 775, 757, 803, 750, 783, 737, 772, 778, 777, 774, 774, 769, 802, 760, 746, 804, 734, 800, 764, 769, 779, 794, 757, 753, 771, 756, 756, 788, 795, 754, 753, 789, 748, 829, 721, 699, 762, 823, 714, 770, 787, 765, 806, 763, 766, 710, 782, 729, 778, 776, 767, 787, 752, 770, 788, 740, 781, 788, 763, 761] payload_rows_per_dst=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] dup_rows_per_dst=[3018, 2984, 3054, 3064, 2944, 3036, 2958, 2938] combine_rows_per_src=[9260, 9344, 9304, 9244, 9232, 9344, 9232, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343712 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266624672 ingress_bytes=335486976 link_bytes=335486976 token_bytes=532905632 weight_movement_bytes=69206016 total_nvlink_bytes=602111648 weight_to_token_ratio=0.129865 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262410482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64972 dup_rows=24044 dup_rows_remote=21032 dedup_ratio=0.244588 router_tokens_per_expert=[717, 752, 736, 768, 769, 755, 723, 747, 761, 740, 752, 794, 768, 739, 770, 782, 784, 737, 758, 807, 802, 777, 789, 762, 751, 764, 738, 753, 765, 726, 802, 765, 745, 807, 749, 738, 796, 787, 768, 760, 750, 797, 782, 788, 754, 746, 807, 780, 726, 789, 750, 746, 778, 764, 778, 797, 822, 782, 789, 785, 756, 770, 795, 818] payload_rows_per_dst=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] dup_rows_per_dst=[2864, 3018, 3088, 2938, 3012, 3006, 2908, 3210] combine_rows_per_src=[9070, 9194, 9344, 9190, 9288, 9402, 9348, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64920 dup_rows=24158 dup_rows_remote=21182 dedup_ratio=0.245748 router_tokens_per_expert=[760, 745, 777, 776, 780, 783, 739, 758, 742, 751, 769, 790, 773, 806, 740, 793, 745, 766, 787, 733, 813, 789, 713, 755, 774, 759, 801, 741, 769, 765, 727, 747, 822, 798, 763, 783, 812, 768, 772, 741, 733, 761, 785, 759, 768, 738, 776, 729, 764, 731, 803, 770, 804, 767, 770, 771, 799, 771, 795, 773, 740, 788, 760, 772] payload_rows_per_dst=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] dup_rows_per_dst=[3062, 2984, 2976, 2962, 3116, 2972, 3062, 3024] combine_rows_per_src=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64732 dup_rows=24108 dup_rows_remote=20918 dedup_ratio=0.245239 router_tokens_per_expert=[759, 773, 775, 731, 742, 755, 782, 813, 707, 809, 824, 726, 766, 761, 767, 769, 754, 739, 772, 762, 769, 798, 747, 800, 766, 761, 781, 742, 777, 784, 824, 753, 760, 716, 788, 779, 763, 707, 734, 747, 737, 791, 771, 786, 747, 769, 775, 810, 776, 741, 741, 766, 770, 788, 759, 775, 789, 792, 774, 779, 769, 780, 817, 768] payload_rows_per_dst=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] dup_rows_per_dst=[3190, 2992, 3006, 2900, 3200, 3158, 2932, 2730] combine_rows_per_src=[9464, 9336, 9276, 9012, 9250, 9512, 9300, 9046] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65252 dup_rows=23758 dup_rows_remote=20776 dedup_ratio=0.241679 router_tokens_per_expert=[737, 732, 772, 743, 766, 771, 788, 739, 804, 859, 729, 763, 730, 780, 751, 749, 774, 772, 717, 818, 761, 765, 807, 834, 762, 753, 777, 796, 789, 755, 771, 780, 824, 703, 753, 731, 771, 751, 743, 813, 789, 763, 739, 755, 722, 769, 771, 759, 814, 793, 771, 725, 759, 737, 752, 787, 771, 793, 773, 755, 763, 839, 761, 759] payload_rows_per_dst=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] dup_rows_per_dst=[2902, 3012, 3032, 3026, 2948, 2888, 2982, 2968] combine_rows_per_src=[9194, 9318, 9464, 9340, 9230, 9246, 9294, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344016 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469328 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594640 weight_movement_bytes=34603008 total_nvlink_bytes=567197648 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64636 dup_rows=24110 dup_rows_remote=20830 dedup_ratio=0.245260 router_tokens_per_expert=[759, 758, 784, 738, 756, 772, 758, 813, 757, 743, 797, 740, 809, 767, 731, 794, 774, 749, 756, 789, 780, 763, 774, 747, 799, 723, 782, 794, 791, 782, 766, 769, 776, 777, 784, 773, 736, 800, 777, 777, 790, 811, 809, 768, 709, 788, 784, 746, 717, 761, 790, 733, 734, 812, 795, 712, 772, 735, 789, 694, 765, 769, 804, 751] payload_rows_per_dst=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] dup_rows_per_dst=[3280, 3054, 2810, 3114, 3146, 3012, 2904, 2790] combine_rows_per_src=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65304 dup_rows=23760 dup_rows_remote=20782 dedup_ratio=0.241699 router_tokens_per_expert=[799, 759, 738, 774, 761, 743, 782, 726, 728, 717, 825, 750, 752, 793, 743, 801, 790, 740, 786, 779, 778, 744, 728, 773, 790, 736, 755, 731, 791, 773, 738, 824, 785, 762, 771, 775, 768, 776, 779, 774, 761, 752, 794, 821, 747, 756, 771, 751, 795, 769, 801, 814, 739, 786, 733, 725, 734, 795, 817, 729, 789, 788, 756, 792] payload_rows_per_dst=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] dup_rows_per_dst=[2902, 2978, 2840, 3036, 3108, 3016, 2972, 2908] combine_rows_per_src=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=345256 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267510952 ingress_bytes=405577728 link_bytes=405577728 token_bytes=534676648 weight_movement_bytes=138412032 total_nvlink_bytes=673088680 weight_to_token_ratio=0.258871 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.526156339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266436240 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528784 weight_movement_bytes=103809024 total_nvlink_bytes=636337808 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74544 payload_rows_remote=65304 dup_rows=23760 dup_rows_remote=20782 dedup_ratio=0.241699 router_tokens_per_expert=[799, 759, 738, 774, 761, 743, 782, 726, 728, 717, 825, 750, 752, 793, 743, 801, 790, 740, 786, 779, 778, 744, 728, 773, 790, 736, 755, 731, 791, 773, 738, 824, 785, 762, 771, 775, 768, 776, 779, 774, 761, 752, 794, 821, 747, 756, 771, 751, 795, 769, 801, 814, 739, 786, 733, 725, 734, 795, 817, 729, 789, 788, 756, 792] payload_rows_per_dst=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] dup_rows_per_dst=[2902, 2978, 2840, 3036, 3108, 3016, 2972, 2908] combine_rows_per_src=[9262, 9240, 9396, 9240, 9272, 9290, 9352, 9492] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344016 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469328 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594640 weight_movement_bytes=34603008 total_nvlink_bytes=567197648 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344408 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256728 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532169048 weight_movement_bytes=34603008 total_nvlink_bytes=566772056 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342600 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265484872 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627144 weight_movement_bytes=34603008 total_nvlink_bytes=565230152 weight_to_token_ratio=0.065212 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=344112 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267616304 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888496 weight_movement_bytes=34603008 total_nvlink_bytes=569491504 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=341864 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265090920 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529839976 weight_movement_bytes=34603008 total_nvlink_bytes=564442984 weight_to_token_ratio=0.065308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64978 dup_rows=23958 dup_rows_remote=21066 dedup_ratio=0.243713 router_tokens_per_expert=[786, 737, 776, 778, 793, 771, 793, 791, 816, 774, 763, 735, 802, 768, 741, 778, 746, 748, 746, 764, 788, 775, 767, 731, 746, 709, 773, 786, 787, 790, 756, 783, 720, 782, 780, 787, 756, 775, 773, 768, 783, 754, 797, 749, 669, 774, 724, 780, 759, 759, 750, 811, 780, 780, 776, 781, 783, 800, 760, 731, 777, 764, 774, 799] payload_rows_per_dst=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] dup_rows_per_dst=[2994, 3114, 2910, 2892, 3032, 2910, 3084, 3022] combine_rows_per_src=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344344 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267829528 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314712 weight_movement_bytes=34603008 total_nvlink_bytes=569917720 weight_to_token_ratio=0.064640 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64920 dup_rows=24158 dup_rows_remote=21182 dedup_ratio=0.245748 router_tokens_per_expert=[760, 745, 777, 776, 780, 783, 739, 758, 742, 751, 769, 790, 773, 806, 740, 793, 745, 766, 787, 733, 813, 789, 713, 755, 774, 759, 801, 741, 769, 765, 727, 747, 822, 798, 763, 783, 812, 768, 772, 741, 733, 761, 785, 759, 768, 738, 776, 729, 764, 731, 803, 770, 804, 767, 770, 771, 799, 771, 795, 773, 740, 788, 760, 772] payload_rows_per_dst=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] dup_rows_per_dst=[3062, 2984, 2976, 2962, 3116, 2972, 3062, 3024] combine_rows_per_src=[9174, 9344, 9226, 9204, 9402, 9126, 9298, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65036 dup_rows=23994 dup_rows_remote=21100 dedup_ratio=0.244080 router_tokens_per_expert=[793, 813, 788, 777, 756, 768, 789, 705, 757, 751, 746, 755, 724, 756, 782, 743, 780, 768, 802, 782, 830, 723, 738, 790, 780, 735, 748, 812, 776, 806, 759, 829, 755, 751, 756, 755, 793, 758, 776, 740, 749, 740, 802, 780, 776, 776, 787, 788, 790, 758, 754, 767, 772, 807, 753, 740, 761, 783, 709, 732, 736, 820, 767, 760] payload_rows_per_dst=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] dup_rows_per_dst=[3042, 2906, 3048, 3006, 2894, 3164, 2984, 2950] combine_rows_per_src=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64978 dup_rows=23958 dup_rows_remote=21066 dedup_ratio=0.243713 router_tokens_per_expert=[786, 737, 776, 778, 793, 771, 793, 791, 816, 774, 763, 735, 802, 768, 741, 778, 746, 748, 746, 764, 788, 775, 767, 731, 746, 709, 773, 786, 787, 790, 756, 783, 720, 782, 780, 787, 756, 775, 773, 768, 783, 754, 797, 749, 669, 774, 724, 780, 759, 759, 750, 811, 780, 780, 776, 781, 783, 800, 760, 731, 777, 764, 774, 799] payload_rows_per_dst=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] dup_rows_per_dst=[2994, 3114, 2910, 2892, 3032, 2910, 3084, 3022] combine_rows_per_src=[9456, 9240, 9220, 9368, 9250, 9150, 9308, 9354] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64990 dup_rows=24134 dup_rows_remote=21234 dedup_ratio=0.245504 router_tokens_per_expert=[771, 785, 793, 746, 741, 744, 767, 799, 763, 803, 788, 754, 737, 737, 756, 716, 782, 755, 709, 799, 789, 742, 773, 798, 736, 777, 760, 772, 769, 755, 781, 729, 805, 807, 731, 776, 753, 806, 746, 763, 754, 794, 768, 785, 767, 749, 812, 833, 784, 773, 797, 788, 771, 743, 823, 758, 741, 712, 769, 778, 802, 728, 769, 741] payload_rows_per_dst=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] dup_rows_per_dst=[2770, 3000, 3014, 3062, 3074, 3178, 3136, 2900] combine_rows_per_src=[9266, 9108, 9280, 9352, 9300, 9346, 9338, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65082 dup_rows=23964 dup_rows_remote=21172 dedup_ratio=0.243774 router_tokens_per_expert=[766, 772, 739, 795, 801, 792, 732, 787, 777, 733, 766, 769, 778, 780, 770, 781, 799, 775, 775, 739, 724, 774, 754, 760, 751, 710, 778, 799, 753, 752, 782, 775, 758, 776, 769, 758, 794, 781, 792, 782, 753, 786, 783, 771, 729, 696, 768, 739, 788, 784, 792, 757, 798, 778, 758, 749, 782, 766, 716, 783, 770, 789, 789, 780] payload_rows_per_dst=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] dup_rows_per_dst=[3022, 3016, 3016, 3004, 3016, 2792, 3032, 3066] combine_rows_per_src=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344176 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494064 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643952 weight_movement_bytes=34603008 total_nvlink_bytes=567246960 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=344408 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266256728 ingress_bytes=300515328 link_bytes=300515328 token_bytes=532169048 weight_movement_bytes=34603008 total_nvlink_bytes=566772056 weight_to_token_ratio=0.065023 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130814986e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65082 dup_rows=23964 dup_rows_remote=21172 dedup_ratio=0.243774 router_tokens_per_expert=[766, 772, 739, 795, 801, 792, 732, 787, 777, 733, 766, 769, 778, 780, 770, 781, 799, 775, 775, 739, 724, 774, 754, 760, 751, 710, 778, 799, 753, 752, 782, 775, 758, 776, 769, 758, 794, 781, 792, 782, 753, 786, 783, 771, 729, 696, 768, 739, 788, 784, 792, 757, 798, 778, 758, 749, 782, 766, 716, 783, 770, 789, 789, 780] payload_rows_per_dst=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] dup_rows_per_dst=[3022, 3016, 3016, 3004, 3016, 2792, 3032, 3066] combine_rows_per_src=[9346, 9292, 9184, 9196, 9404, 9258, 9376, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65036 dup_rows=23994 dup_rows_remote=21100 dedup_ratio=0.244080 router_tokens_per_expert=[793, 813, 788, 777, 756, 768, 789, 705, 757, 751, 746, 755, 724, 756, 782, 743, 780, 768, 802, 782, 830, 723, 738, 790, 780, 735, 748, 812, 776, 806, 759, 829, 755, 751, 756, 755, 793, 758, 776, 740, 749, 740, 802, 780, 776, 776, 787, 788, 790, 758, 754, 767, 772, 807, 753, 740, 761, 783, 709, 732, 736, 820, 767, 760] payload_rows_per_dst=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] dup_rows_per_dst=[3042, 2906, 3048, 3006, 2894, 3164, 2984, 2950] combine_rows_per_src=[9336, 9122, 9378, 9484, 9274, 9232, 9298, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65110 dup_rows=23974 dup_rows_remote=20996 dedup_ratio=0.243876 router_tokens_per_expert=[772, 769, 814, 745, 741, 743, 791, 740, 685, 777, 793, 800, 747, 807, 779, 820, 808, 745, 719, 767, 770, 757, 733, 800, 782, 741, 753, 800, 799, 778, 764, 805, 774, 783, 751, 783, 698, 727, 796, 770, 753, 765, 789, 750, 740, 759, 778, 753, 763, 745, 769, 778, 828, 799, 743, 776, 780, 770, 724, 839, 741, 740, 755, 789] payload_rows_per_dst=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] dup_rows_per_dst=[2954, 3076, 2978, 3102, 2990, 2980, 2936, 2958] combine_rows_per_src=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64636 dup_rows=24110 dup_rows_remote=20830 dedup_ratio=0.245260 router_tokens_per_expert=[759, 758, 784, 738, 756, 772, 758, 813, 757, 743, 797, 740, 809, 767, 731, 794, 774, 749, 756, 789, 780, 763, 774, 747, 799, 723, 782, 794, 791, 782, 766, 769, 776, 777, 784, 773, 736, 800, 777, 777, 790, 811, 809, 768, 709, 788, 784, 746, 717, 761, 790, 733, 734, 812, 795, 712, 772, 735, 789, 694, 765, 769, 804, 751] payload_rows_per_dst=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] dup_rows_per_dst=[3280, 3054, 2810, 3114, 3146, 3012, 2904, 2790] combine_rows_per_src=[9558, 9270, 8900, 9332, 9454, 9390, 9204, 9086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344896 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266543936 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742976 weight_movement_bytes=69206016 total_nvlink_bytes=601948992 weight_to_token_ratio=0.129905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64902 dup_rows=24086 dup_rows_remote=21054 dedup_ratio=0.245015 router_tokens_per_expert=[797, 761, 766, 765, 742, 734, 762, 801, 759, 752, 745, 768, 758, 804, 767, 785, 762, 721, 733, 717, 791, 759, 808, 727, 763, 754, 751, 820, 827, 774, 745, 761, 771, 797, 795, 731, 717, 772, 771, 747, 816, 747, 820, 782, 751, 742, 826, 729, 786, 764, 791, 771, 783, 755, 786, 738, 740, 742, 814, 749, 838, 779, 760, 763] payload_rows_per_dst=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] dup_rows_per_dst=[3006, 2982, 2878, 3048, 2966, 3068, 3032, 3106] combine_rows_per_src=[9250, 9294, 9158, 9342, 9236, 9358, 9316, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732000 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119456 weight_movement_bytes=34603008 total_nvlink_bytes=567722464 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64978 dup_rows=24056 dup_rows_remote=21006 dedup_ratio=0.244710 router_tokens_per_expert=[782, 761, 753, 754, 819, 761, 781, 744, 777, 753, 760, 750, 744, 765, 794, 792, 789, 792, 766, 747, 781, 696, 747, 785, 756, 770, 759, 745, 754, 808, 770, 746, 813, 771, 701, 712, 766, 767, 740, 761, 767, 781, 759, 795, 722, 758, 750, 812, 738, 838, 801, 794, 771, 795, 733, 769, 783, 778, 808, 769, 774, 790, 758, 777] payload_rows_per_dst=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] dup_rows_per_dst=[3050, 3074, 3236, 2894, 2938, 3158, 2746, 2960] combine_rows_per_src=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344896 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266543936 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742976 weight_movement_bytes=69206016 total_nvlink_bytes=601948992 weight_to_token_ratio=0.129905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65008 dup_rows=24042 dup_rows_remote=20996 dedup_ratio=0.244568 router_tokens_per_expert=[763, 816, 783, 762, 732, 741, 737, 796, 742, 795, 785, 754, 729, 808, 754, 783, 770, 737, 795, 783, 749, 759, 797, 714, 751, 778, 745, 802, 763, 730, 787, 728, 747, 760, 743, 782, 809, 764, 726, 772, 825, 786, 749, 792, 722, 740, 782, 761, 750, 791, 760, 793, 749, 775, 780, 797, 782, 762, 815, 763, 813, 759, 748, 787] payload_rows_per_dst=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] dup_rows_per_dst=[2950, 3046, 2958, 3006, 2910, 3012, 3080, 3080] combine_rows_per_src=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732000 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119456 weight_movement_bytes=34603008 total_nvlink_bytes=567722464 weight_to_token_ratio=0.064907 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=345016 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920888 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496760 weight_movement_bytes=34603008 total_nvlink_bytes=568099768 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65008 dup_rows=24042 dup_rows_remote=20996 dedup_ratio=0.244568 router_tokens_per_expert=[763, 816, 783, 762, 732, 741, 737, 796, 742, 795, 785, 754, 729, 808, 754, 783, 770, 737, 795, 783, 749, 759, 797, 714, 751, 778, 745, 802, 763, 730, 787, 728, 747, 760, 743, 782, 809, 764, 726, 772, 825, 786, 749, 792, 722, 740, 782, 761, 750, 791, 760, 793, 749, 775, 780, 797, 782, 762, 815, 763, 813, 759, 748, 787] payload_rows_per_dst=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] dup_rows_per_dst=[2950, 3046, 2958, 3006, 2910, 3012, 3080, 3080] combine_rows_per_src=[9310, 9254, 9250, 9162, 9296, 9302, 9310, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343824 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266182416 ingress_bytes=438853632 link_bytes=438853632 token_bytes=532021008 weight_movement_bytes=173015040 total_nvlink_bytes=705036048 weight_to_token_ratio=0.325203 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651370887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343936 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493824 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643712 weight_movement_bytes=69206016 total_nvlink_bytes=601849728 weight_to_token_ratio=0.129929 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344424 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267034984 ingress_bytes=335896576 link_bytes=335896576 token_bytes=533725544 weight_movement_bytes=69206016 total_nvlink_bytes=602931560 weight_to_token_ratio=0.129666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343824 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266182416 ingress_bytes=438853632 link_bytes=438853632 token_bytes=532021008 weight_movement_bytes=173015040 total_nvlink_bytes=705036048 weight_to_token_ratio=0.325203 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.651370887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65214 dup_rows=23830 dup_rows_remote=20824 dedup_ratio=0.242411 router_tokens_per_expert=[802, 764, 805, 765, 788, 757, 734, 743, 811, 775, 788, 765, 782, 758, 781, 747, 716, 757, 838, 783, 773, 744, 780, 772, 769, 793, 749, 749, 772, 768, 799, 734, 786, 813, 796, 799, 714, 758, 787, 793, 767, 709, 755, 797, 731, 752, 773, 700, 773, 780, 776, 759, 712, 756, 801, 781, 779, 776, 756, 800, 753, 751, 764, 744] payload_rows_per_dst=[9380, 9330, 9330, 9260, 9402, 9162, 9292, 9318] dup_rows_per_dst=[2936, 3084, 2996, 3006, 3090, 2806, 2984, 2928] combine_rows_per_src=[9380, 9330, 9330, 9260, 9402, 9162, 9292, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=341864 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265090920 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529839976 weight_movement_bytes=34603008 total_nvlink_bytes=564442984 weight_to_token_ratio=0.065308 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344016 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616784 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889552 weight_movement_bytes=34603008 total_nvlink_bytes=567492560 weight_to_token_ratio=0.064935 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65110 dup_rows=23974 dup_rows_remote=20996 dedup_ratio=0.243876 router_tokens_per_expert=[772, 769, 814, 745, 741, 743, 791, 740, 685, 777, 793, 800, 747, 807, 779, 820, 808, 745, 719, 767, 770, 757, 733, 800, 782, 741, 753, 800, 799, 778, 764, 805, 774, 783, 751, 783, 698, 727, 796, 770, 753, 765, 789, 750, 740, 759, 778, 753, 763, 745, 769, 778, 828, 799, 743, 776, 780, 770, 724, 839, 741, 740, 755, 789] payload_rows_per_dst=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] dup_rows_per_dst=[2954, 3076, 2978, 3102, 2990, 2980, 2936, 2958] combine_rows_per_src=[9276, 9340, 9220, 9342, 9174, 9194, 9466, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64964 dup_rows=24152 dup_rows_remote=21220 dedup_ratio=0.245687 router_tokens_per_expert=[747, 732, 765, 783, 808, 724, 775, 768, 774, 733, 800, 789, 794, 770, 782, 775, 746, 744, 798, 817, 803, 771, 795, 752, 799, 772, 764, 719, 798, 759, 747, 821, 732, 805, 739, 770, 782, 753, 748, 731, 779, 750, 743, 726, 751, 793, 734, 768, 734, 776, 742, 746, 778, 780, 790, 747, 703, 763, 806, 796, 806, 793, 793, 771] payload_rows_per_dst=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] dup_rows_per_dst=[3022, 3072, 3114, 2978, 2932, 2880, 2972, 3182] combine_rows_per_src=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64644 dup_rows=24098 dup_rows_remote=20916 dedup_ratio=0.245138 router_tokens_per_expert=[774, 779, 772, 812, 768, 803, 796, 786, 769, 744, 813, 776, 784, 746, 742, 773, 725, 781, 774, 797, 746, 786, 765, 790, 818, 767, 739, 718, 798, 765, 744, 774, 779, 746, 804, 801, 781, 758, 756, 742, 720, 788, 731, 773, 763, 782, 787, 776, 747, 740, 725, 760, 747, 737, 741, 708, 766, 744, 776, 778, 800, 794, 816, 762] payload_rows_per_dst=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] dup_rows_per_dst=[3090, 2882, 2536, 2852, 3090, 3580, 2886, 3182] combine_rows_per_src=[9306, 9238, 8672, 9092, 9330, 9780, 9226, 9562] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65156 dup_rows=23822 dup_rows_remote=20860 dedup_ratio=0.242330 router_tokens_per_expert=[799, 770, 738, 778, 788, 727, 771, 789, 749, 752, 761, 760, 783, 807, 763, 760, 774, 806, 766, 770, 743, 740, 772, 835, 782, 737, 763, 772, 750, 778, 785, 711, 731, 789, 783, 755, 739, 752, 785, 766, 736, 756, 789, 705, 774, 753, 840, 791, 784, 762, 786, 766, 776, 783, 762, 770, 752, 817, 758, 778, 748, 779, 752, 756] payload_rows_per_dst=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] dup_rows_per_dst=[2946, 2976, 2984, 2972, 2972, 2962, 3030, 2980] combine_rows_per_src=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=344152 combine_payload_bytes=267116544 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267460696 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534577240 weight_movement_bytes=69206016 total_nvlink_bytes=603783256 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344424 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267034984 ingress_bytes=335896576 link_bytes=335896576 token_bytes=533725544 weight_movement_bytes=69206016 total_nvlink_bytes=602931560 weight_to_token_ratio=0.129666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263951774e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64744 dup_rows=24272 dup_rows_remote=21280 dedup_ratio=0.246908 router_tokens_per_expert=[729, 783, 803, 750, 746, 766, 750, 778, 710, 782, 780, 771, 818, 768, 785, 756, 788, 819, 773, 798, 789, 790, 738, 755, 753, 793, 777, 786, 743, 757, 806, 777, 750, 761, 759, 742, 833, 767, 794, 742, 741, 760, 752, 746, 753, 764, 742, 754, 776, 803, 740, 720, 720, 758, 843, 780, 753, 800, 783, 732, 750, 757, 768, 792] payload_rows_per_dst=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] dup_rows_per_dst=[3002, 3126, 3158, 3128, 2966, 2860, 2992, 3040] combine_rows_per_src=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344736 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437280 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532529824 weight_movement_bytes=34603008 total_nvlink_bytes=567132832 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65156 dup_rows=23822 dup_rows_remote=20860 dedup_ratio=0.242330 router_tokens_per_expert=[799, 770, 738, 778, 788, 727, 771, 789, 749, 752, 761, 760, 783, 807, 763, 760, 774, 806, 766, 770, 743, 740, 772, 835, 782, 737, 763, 772, 750, 778, 785, 711, 731, 789, 783, 755, 739, 752, 785, 766, 736, 756, 789, 705, 774, 753, 840, 791, 784, 762, 786, 766, 776, 783, 762, 770, 752, 817, 758, 778, 748, 779, 752, 756] payload_rows_per_dst=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] dup_rows_per_dst=[2946, 2976, 2984, 2972, 2972, 2962, 3030, 2980] combine_rows_per_src=[9374, 9294, 9428, 9184, 9228, 9326, 9348, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64964 dup_rows=24152 dup_rows_remote=21220 dedup_ratio=0.245687 router_tokens_per_expert=[747, 732, 765, 783, 808, 724, 775, 768, 774, 733, 800, 789, 794, 770, 782, 775, 746, 744, 798, 817, 803, 771, 795, 752, 799, 772, 764, 719, 798, 759, 747, 821, 732, 805, 739, 770, 782, 753, 748, 731, 779, 750, 743, 726, 751, 793, 734, 768, 734, 776, 742, 746, 778, 780, 790, 747, 703, 763, 806, 796, 806, 793, 793, 771] payload_rows_per_dst=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] dup_rows_per_dst=[3022, 3072, 3114, 2978, 2932, 2880, 2972, 3182] combine_rows_per_src=[9182, 9362, 9338, 9380, 9188, 9208, 9214, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=342240 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265124064 ingress_bytes=333987840 link_bytes=333987840 token_bytes=529905888 weight_movement_bytes=69206016 total_nvlink_bytes=599111904 weight_to_token_ratio=0.130601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256769354e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65134 dup_rows=23858 dup_rows_remote=21036 dedup_ratio=0.242696 router_tokens_per_expert=[762, 765, 755, 778, 816, 768, 777, 799, 791, 771, 771, 774, 726, 756, 793, 789, 784, 768, 746, 713, 733, 792, 747, 784, 774, 799, 758, 777, 774, 764, 782, 752, 747, 724, 783, 743, 784, 784, 808, 838, 753, 757, 758, 762, 769, 774, 763, 744, 772, 716, 763, 740, 754, 728, 761, 742, 774, 798, 815, 778, 814, 781, 730, 757] payload_rows_per_dst=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] dup_rows_per_dst=[3118, 2972, 2822, 3056, 3088, 2880, 2810, 3112] combine_rows_per_src=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74032 payload_rows_remote=64744 dup_rows=24272 dup_rows_remote=21280 dedup_ratio=0.246908 router_tokens_per_expert=[729, 783, 803, 750, 746, 766, 750, 778, 710, 782, 780, 771, 818, 768, 785, 756, 788, 819, 773, 798, 789, 790, 738, 755, 753, 793, 777, 786, 743, 757, 806, 777, 750, 761, 759, 742, 833, 767, 794, 742, 741, 760, 752, 746, 753, 764, 742, 754, 776, 803, 740, 720, 720, 758, 843, 780, 753, 800, 783, 732, 750, 757, 768, 792] payload_rows_per_dst=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] dup_rows_per_dst=[3002, 3126, 3158, 3128, 2966, 2860, 2992, 3040] combine_rows_per_src=[9208, 9214, 9342, 9256, 9330, 9164, 9288, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64922 dup_rows=24100 dup_rows_remote=21012 dedup_ratio=0.245158 router_tokens_per_expert=[766, 792, 762, 772, 744, 783, 761, 787, 803, 772, 740, 742, 748, 787, 714, 741, 806, 771, 801, 777, 769, 758, 744, 743, 761, 842, 817, 795, 775, 760, 773, 762, 770, 764, 751, 768, 763, 777, 726, 732, 735, 774, 736, 844, 783, 788, 724, 746, 765, 811, 775, 788, 734, 765, 753, 732, 786, 797, 818, 739, 771, 750, 758, 761] payload_rows_per_dst=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] dup_rows_per_dst=[3088, 2922, 3382, 3136, 2906, 2976, 2980, 2710] combine_rows_per_src=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64978 dup_rows=24056 dup_rows_remote=21006 dedup_ratio=0.244710 router_tokens_per_expert=[782, 761, 753, 754, 819, 761, 781, 744, 777, 753, 760, 750, 744, 765, 794, 792, 789, 792, 766, 747, 781, 696, 747, 785, 756, 770, 759, 745, 754, 808, 770, 746, 813, 771, 701, 712, 766, 767, 740, 761, 767, 781, 759, 795, 722, 758, 750, 812, 738, 838, 801, 794, 771, 795, 733, 769, 783, 778, 808, 769, 774, 790, 758, 777] payload_rows_per_dst=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] dup_rows_per_dst=[3050, 3074, 3236, 2894, 2938, 3158, 2746, 2960] combine_rows_per_src=[9270, 9528, 9572, 9322, 9196, 9278, 8904, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65088 dup_rows=23936 dup_rows_remote=21000 dedup_ratio=0.243490 router_tokens_per_expert=[724, 780, 761, 783, 797, 746, 784, 754, 756, 798, 788, 724, 805, 775, 753, 709, 768, 758, 755, 724, 786, 738, 788, 774, 789, 754, 737, 804, 767, 742, 755, 761, 797, 743, 765, 803, 816, 766, 809, 756, 793, 741, 788, 690, 785, 773, 749, 752, 798, 759, 753, 741, 744, 815, 829, 773, 788, 803, 794, 786, 762, 765, 736, 743] payload_rows_per_dst=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] dup_rows_per_dst=[2966, 2936, 2922, 2960, 3082, 2946, 3080, 3044] combine_rows_per_src=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=342240 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265124064 ingress_bytes=333987840 link_bytes=333987840 token_bytes=529905888 weight_movement_bytes=69206016 total_nvlink_bytes=599111904 weight_to_token_ratio=0.130601 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256769354e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=344064 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267223040 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534102016 weight_movement_bytes=34603008 total_nvlink_bytes=568705024 weight_to_token_ratio=0.064787 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65088 dup_rows=23936 dup_rows_remote=21000 dedup_ratio=0.243490 router_tokens_per_expert=[724, 780, 761, 783, 797, 746, 784, 754, 756, 798, 788, 724, 805, 775, 753, 709, 768, 758, 755, 724, 786, 738, 788, 774, 789, 754, 737, 804, 767, 742, 755, 761, 797, 743, 765, 803, 816, 766, 809, 756, 793, 741, 788, 690, 785, 773, 749, 752, 798, 759, 753, 741, 744, 815, 829, 773, 788, 803, 794, 786, 762, 765, 736, 743] payload_rows_per_dst=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] dup_rows_per_dst=[2966, 2936, 2922, 2960, 3082, 2946, 3080, 3044] combine_rows_per_src=[9292, 9280, 9260, 9258, 9428, 9196, 9344, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=344096 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535520 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726944 weight_movement_bytes=34603008 total_nvlink_bytes=565329952 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344736 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437280 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532529824 weight_movement_bytes=34603008 total_nvlink_bytes=567132832 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343736 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266264248 ingress_bytes=369729536 link_bytes=369729536 token_bytes=532184760 weight_movement_bytes=103809024 total_nvlink_bytes=635993784 weight_to_token_ratio=0.195062 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391262478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344352 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944800 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545248 weight_movement_bytes=69206016 total_nvlink_bytes=602751264 weight_to_token_ratio=0.129710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344680 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133544 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922408 weight_movement_bytes=34603008 total_nvlink_bytes=568525416 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=344096 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535520 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726944 weight_movement_bytes=34603008 total_nvlink_bytes=565329952 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65048 dup_rows=23872 dup_rows_remote=20868 dedup_ratio=0.242839 router_tokens_per_expert=[815, 774, 789, 725, 747, 748, 755, 747, 725, 763, 777, 746, 780, 786, 736, 740, 781, 756, 717, 833, 797, 728, 765, 800, 770, 782, 799, 770, 789, 773, 749, 762, 769, 743, 770, 793, 734, 782, 764, 762, 821, 764, 783, 779, 759, 751, 719, 735, 778, 790, 806, 740, 790, 764, 758, 758, 767, 735, 770, 794, 780, 780, 775, 815] payload_rows_per_dst=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] dup_rows_per_dst=[2978, 2974, 2952, 3004, 3000, 2996, 3004, 2964] combine_rows_per_src=[9222, 9132, 9402, 9384, 9234, 9226, 9364, 9468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65134 dup_rows=23858 dup_rows_remote=21036 dedup_ratio=0.242696 router_tokens_per_expert=[762, 765, 755, 778, 816, 768, 777, 799, 791, 771, 771, 774, 726, 756, 793, 789, 784, 768, 746, 713, 733, 792, 747, 784, 774, 799, 758, 777, 774, 764, 782, 752, 747, 724, 783, 743, 784, 784, 808, 838, 753, 757, 758, 762, 769, 774, 763, 744, 772, 716, 763, 740, 754, 728, 761, 742, 774, 798, 815, 778, 814, 781, 730, 757] payload_rows_per_dst=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] dup_rows_per_dst=[3118, 2972, 2822, 3056, 3088, 2880, 2810, 3112] combine_rows_per_src=[9322, 9370, 9312, 9304, 9334, 9280, 9142, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65282 dup_rows=23776 dup_rows_remote=20934 dedup_ratio=0.241862 router_tokens_per_expert=[759, 764, 766, 773, 795, 728, 753, 815, 763, 760, 794, 740, 757, 777, 788, 755, 805, 799, 730, 771, 814, 756, 723, 792, 713, 775, 698, 805, 783, 783, 748, 736, 716, 780, 754, 709, 740, 764, 823, 758, 807, 819, 753, 798, 748, 773, 764, 798, 710, 799, 786, 707, 779, 807, 731, 779, 767, 788, 825, 744, 795, 739, 782, 792] payload_rows_per_dst=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] dup_rows_per_dst=[2924, 3004, 3060, 2878, 2842, 3044, 2970, 3054] combine_rows_per_src=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343936 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493824 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643712 weight_movement_bytes=69206016 total_nvlink_bytes=601849728 weight_to_token_ratio=0.129929 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344352 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944800 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545248 weight_movement_bytes=69206016 total_nvlink_bytes=602751264 weight_to_token_ratio=0.129710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64964 dup_rows=24012 dup_rows_remote=20950 dedup_ratio=0.244263 router_tokens_per_expert=[745, 773, 794, 765, 701, 765, 784, 741, 769, 737, 781, 773, 785, 754, 779, 786, 773, 746, 783, 729, 764, 768, 752, 750, 762, 781, 730, 797, 781, 752, 790, 758, 794, 745, 788, 797, 724, 763, 743, 798, 771, 811, 757, 792, 757, 784, 781, 727, 762, 806, 755, 764, 808, 763, 739, 788, 769, 801, 740, 789, 807, 801, 739, 741] payload_rows_per_dst=[9194, 8932, 9158, 9278, 9282, 9344, 9776, 9328] dup_rows_per_dst=[2942, 2648, 2972, 3024, 3022, 3000, 3342, 3062] combine_rows_per_src=[9194, 8932, 9158, 9278, 9282, 9344, 9776, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64890 dup_rows=24206 dup_rows_remote=21172 dedup_ratio=0.246236 router_tokens_per_expert=[763, 769, 814, 770, 774, 753, 759, 755, 790, 766, 744, 746, 750, 781, 782, 776, 798, 769, 731, 754, 758, 747, 797, 757, 754, 779, 758, 762, 808, 771, 755, 811, 781, 771, 776, 761, 781, 781, 758, 770, 764, 806, 777, 774, 717, 782, 767, 734, 757, 773, 796, 780, 716, 773, 769, 780, 792, 760, 758, 753, 751, 739, 763, 791] payload_rows_per_dst=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] dup_rows_per_dst=[2986, 2986, 3010, 3098, 3058, 3034, 2932, 3102] combine_rows_per_src=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344680 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133544 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922408 weight_movement_bytes=34603008 total_nvlink_bytes=568525416 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343664 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780272 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533216880 weight_movement_bytes=69206016 total_nvlink_bytes=602422896 weight_to_token_ratio=0.129790 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65178 dup_rows=23954 dup_rows_remote=20972 dedup_ratio=0.243673 router_tokens_per_expert=[743, 800, 751, 760, 775, 734, 723, 755, 753, 803, 782, 818, 779, 786, 762, 791, 811, 763, 779, 809, 769, 790, 785, 745, 777, 752, 754, 783, 778, 801, 774, 735, 790, 817, 694, 780, 805, 760, 742, 771, 776, 772, 738, 748, 802, 764, 752, 747, 791, 727, 746, 807, 703, 754, 756, 793, 762, 763, 739, 772, 796, 769, 766, 730] payload_rows_per_dst=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] dup_rows_per_dst=[2856, 3172, 3092, 2952, 2980, 3004, 2982, 2916] combine_rows_per_src=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64890 dup_rows=24206 dup_rows_remote=21172 dedup_ratio=0.246236 router_tokens_per_expert=[763, 769, 814, 770, 774, 753, 759, 755, 790, 766, 744, 746, 750, 781, 782, 776, 798, 769, 731, 754, 758, 747, 797, 757, 754, 779, 758, 762, 808, 771, 755, 811, 781, 771, 776, 761, 781, 781, 758, 770, 764, 806, 777, 774, 717, 782, 767, 734, 757, 773, 796, 780, 716, 773, 769, 780, 792, 760, 758, 753, 751, 739, 763, 791] payload_rows_per_dst=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] dup_rows_per_dst=[2986, 2986, 3010, 3098, 3058, 3034, 2932, 3102] combine_rows_per_src=[9328, 9284, 9212, 9298, 9300, 9208, 9356, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65282 dup_rows=23776 dup_rows_remote=20934 dedup_ratio=0.241862 router_tokens_per_expert=[759, 764, 766, 773, 795, 728, 753, 815, 763, 760, 794, 740, 757, 777, 788, 755, 805, 799, 730, 771, 814, 756, 723, 792, 713, 775, 698, 805, 783, 783, 748, 736, 716, 780, 754, 709, 740, 764, 823, 758, 807, 819, 753, 798, 748, 773, 764, 798, 710, 799, 786, 707, 779, 807, 731, 779, 767, 788, 825, 744, 795, 739, 782, 792] payload_rows_per_dst=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] dup_rows_per_dst=[2924, 3004, 3060, 2878, 2842, 3044, 2970, 3054] combine_rows_per_src=[9382, 9264, 9320, 9204, 9246, 9476, 9226, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343656 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436200 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528744 weight_movement_bytes=103809024 total_nvlink_bytes=636337768 weight_to_token_ratio=0.194936 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65178 dup_rows=23954 dup_rows_remote=20972 dedup_ratio=0.243673 router_tokens_per_expert=[743, 800, 751, 760, 775, 734, 723, 755, 753, 803, 782, 818, 779, 786, 762, 791, 811, 763, 779, 809, 769, 790, 785, 745, 777, 752, 754, 783, 778, 801, 774, 735, 790, 817, 694, 780, 805, 760, 742, 771, 776, 772, 738, 748, 802, 764, 752, 747, 791, 727, 746, 807, 703, 754, 756, 793, 762, 763, 739, 772, 796, 769, 766, 730] payload_rows_per_dst=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] dup_rows_per_dst=[2856, 3172, 3092, 2952, 2980, 3004, 2982, 2916] combine_rows_per_src=[9226, 9376, 9410, 9356, 9338, 9194, 9172, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344864 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739936 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535135008 weight_movement_bytes=69206016 total_nvlink_bytes=604341024 weight_to_token_ratio=0.129324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65048 dup_rows=23922 dup_rows_remote=21008 dedup_ratio=0.243347 router_tokens_per_expert=[795, 734, 755, 782, 725, 776, 750, 790, 772, 777, 773, 790, 742, 727, 765, 756, 819, 733, 775, 780, 752, 739, 745, 781, 772, 791, 745, 803, 727, 810, 770, 815, 713, 739, 727, 772, 756, 805, 736, 734, 776, 788, 770, 762, 778, 739, 805, 802, 728, 798, 772, 770, 801, 759, 826, 775, 724, 796, 807, 785, 730, 751, 788, 774] payload_rows_per_dst=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] dup_rows_per_dst=[2982, 2878, 2914, 3110, 2866, 3120, 3076, 2976] combine_rows_per_src=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64922 dup_rows=24100 dup_rows_remote=21012 dedup_ratio=0.245158 router_tokens_per_expert=[766, 792, 762, 772, 744, 783, 761, 787, 803, 772, 740, 742, 748, 787, 714, 741, 806, 771, 801, 777, 769, 758, 744, 743, 761, 842, 817, 795, 775, 760, 773, 762, 770, 764, 751, 768, 763, 777, 726, 732, 735, 774, 736, 844, 783, 788, 724, 746, 765, 811, 775, 788, 734, 765, 753, 732, 786, 797, 818, 739, 771, 750, 758, 761] payload_rows_per_dst=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] dup_rows_per_dst=[3088, 2922, 3382, 3136, 2906, 2976, 2980, 2710] combine_rows_per_src=[9282, 9172, 9644, 9398, 9196, 9284, 9266, 8962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64928 dup_rows=24110 dup_rows_remote=21108 dedup_ratio=0.245260 router_tokens_per_expert=[781, 788, 771, 768, 777, 756, 794, 792, 764, 741, 754, 741, 801, 812, 752, 769, 770, 776, 789, 753, 777, 761, 750, 761, 782, 752, 748, 711, 773, 744, 803, 723, 770, 726, 754, 787, 787, 800, 762, 783, 787, 757, 766, 823, 818, 791, 769, 750, 717, 749, 799, 758, 763, 740, 764, 761, 740, 775, 804, 766, 739, 774, 800, 739] payload_rows_per_dst=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] dup_rows_per_dst=[3056, 3002, 3086, 2978, 3054, 3080, 2912, 2942] combine_rows_per_src=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343656 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436200 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528744 weight_movement_bytes=103809024 total_nvlink_bytes=636337768 weight_to_token_ratio=0.194936 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64986 dup_rows=24062 dup_rows_remote=20946 dedup_ratio=0.244771 router_tokens_per_expert=[761, 779, 727, 778, 748, 769, 745, 806, 758, 772, 824, 792, 753, 750, 839, 806, 770, 745, 745, 726, 734, 744, 804, 774, 774, 774, 819, 776, 748, 804, 772, 770, 769, 798, 751, 765, 772, 787, 801, 779, 757, 750, 794, 750, 799, 741, 762, 706, 770, 800, 734, 756, 748, 754, 751, 750, 777, 776, 771, 781, 729, 791, 746, 751] payload_rows_per_dst=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] dup_rows_per_dst=[3116, 3366, 3002, 3096, 2940, 2868, 2830, 2844] combine_rows_per_src=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344248 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266133688 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923128 weight_movement_bytes=69206016 total_nvlink_bytes=601129144 weight_to_token_ratio=0.130105 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64928 dup_rows=24110 dup_rows_remote=21108 dedup_ratio=0.245260 router_tokens_per_expert=[781, 788, 771, 768, 777, 756, 794, 792, 764, 741, 754, 741, 801, 812, 752, 769, 770, 776, 789, 753, 777, 761, 750, 761, 782, 752, 748, 711, 773, 744, 803, 723, 770, 726, 754, 787, 787, 800, 762, 783, 787, 757, 766, 823, 818, 791, 769, 750, 717, 749, 799, 758, 763, 740, 764, 761, 740, 775, 804, 766, 739, 774, 800, 739] payload_rows_per_dst=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] dup_rows_per_dst=[3056, 3002, 3086, 2978, 3054, 3080, 2912, 2942] combine_rows_per_src=[9398, 9266, 9188, 9094, 9284, 9442, 9190, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344600 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313688 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282776 weight_movement_bytes=34603008 total_nvlink_bytes=568885784 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344864 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739936 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535135008 weight_movement_bytes=69206016 total_nvlink_bytes=604341024 weight_to_token_ratio=0.129324 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343728 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526384 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709040 weight_movement_bytes=34603008 total_nvlink_bytes=567312048 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344144 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289232 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234320 weight_movement_bytes=69206016 total_nvlink_bytes=601440336 weight_to_token_ratio=0.130029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65048 dup_rows=23922 dup_rows_remote=21008 dedup_ratio=0.243347 router_tokens_per_expert=[795, 734, 755, 782, 725, 776, 750, 790, 772, 777, 773, 790, 742, 727, 765, 756, 819, 733, 775, 780, 752, 739, 745, 781, 772, 791, 745, 803, 727, 810, 770, 815, 713, 739, 727, 772, 756, 805, 736, 734, 776, 788, 770, 762, 778, 739, 805, 802, 728, 798, 772, 770, 801, 759, 826, 775, 724, 796, 807, 785, 730, 751, 788, 774] payload_rows_per_dst=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] dup_rows_per_dst=[2982, 2878, 2914, 3110, 2866, 3120, 3076, 2976] combine_rows_per_src=[9232, 9326, 9334, 9356, 9098, 9320, 9382, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344600 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313688 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282776 weight_movement_bytes=34603008 total_nvlink_bytes=568885784 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344224 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780832 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533217440 weight_movement_bytes=69206016 total_nvlink_bytes=602423456 weight_to_token_ratio=0.129789 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64842 dup_rows=24192 dup_rows_remote=21222 dedup_ratio=0.246094 router_tokens_per_expert=[798, 739, 760, 745, 740, 777, 794, 748, 741, 792, 774, 792, 793, 798, 780, 798, 781, 791, 810, 782, 800, 719, 750, 806, 783, 739, 737, 796, 770, 747, 794, 754, 771, 786, 754, 751, 772, 735, 755, 758, 722, 727, 758, 742, 739, 767, 781, 759, 797, 786, 789, 782, 777, 810, 753, 777, 793, 754, 769, 774, 781, 737, 729, 739] payload_rows_per_dst=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] dup_rows_per_dst=[2892, 3128, 3214, 2970, 3008, 2808, 3120, 3052] combine_rows_per_src=[9310, 9408, 9264, 9270, 9156, 9182, 9422, 9100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21056 dedup_ratio=0.244893 router_tokens_per_expert=[774, 767, 789, 755, 779, 765, 710, 764, 724, 824, 777, 716, 753, 760, 784, 754, 796, 786, 765, 800, 776, 846, 786, 821, 755, 737, 731, 728, 795, 775, 736, 773, 744, 755, 803, 776, 752, 757, 767, 758, 741, 772, 751, 752, 797, 780, 797, 725, 753, 767, 784, 753, 747, 831, 775, 770, 788, 795, 772, 784, 740, 750, 769, 746] payload_rows_per_dst=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] dup_rows_per_dst=[2944, 2920, 3206, 2968, 3018, 2924, 3060, 3034] combine_rows_per_src=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343736 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266264248 ingress_bytes=369729536 link_bytes=369729536 token_bytes=532184760 weight_movement_bytes=103809024 total_nvlink_bytes=635993784 weight_to_token_ratio=0.195062 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391262478e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344144 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289232 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234320 weight_movement_bytes=69206016 total_nvlink_bytes=601440336 weight_to_token_ratio=0.130029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64926 dup_rows=24004 dup_rows_remote=20770 dedup_ratio=0.244181 router_tokens_per_expert=[764, 799, 791, 759, 800, 776, 735, 756, 783, 758, 784, 805, 830, 735, 771, 757, 741, 773, 763, 741, 711, 764, 782, 774, 808, 774, 770, 791, 730, 767, 756, 782, 761, 789, 739, 745, 763, 729, 784, 730, 767, 778, 752, 784, 738, 776, 765, 801, 779, 727, 805, 820, 757, 771, 758, 763, 793, 719, 793, 711, 822, 763, 750, 790] payload_rows_per_dst=[9378, 9186, 9070, 9146, 9412, 9438, 9296, 9374] dup_rows_per_dst=[2982, 2980, 2772, 2820, 3058, 3094, 3064, 3234] combine_rows_per_src=[9378, 9186, 9070, 9146, 9412, 9438, 9296, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344256 combine_payload_bytes=265592832 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265937088 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529920 weight_movement_bytes=34603008 total_nvlink_bytes=566132928 weight_to_token_ratio=0.065101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=65024 dup_rows=24074 dup_rows_remote=21056 dedup_ratio=0.244893 router_tokens_per_expert=[774, 767, 789, 755, 779, 765, 710, 764, 724, 824, 777, 716, 753, 760, 784, 754, 796, 786, 765, 800, 776, 846, 786, 821, 755, 737, 731, 728, 795, 775, 736, 773, 744, 755, 803, 776, 752, 757, 767, 758, 741, 772, 751, 752, 797, 780, 797, 725, 753, 767, 784, 753, 747, 831, 775, 770, 788, 795, 772, 784, 740, 750, 769, 746] payload_rows_per_dst=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] dup_rows_per_dst=[2944, 2920, 3206, 2968, 3018, 2924, 3060, 3034] combine_rows_per_src=[9262, 9264, 9546, 9092, 9206, 9306, 9300, 9254] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65134 dup_rows=23864 dup_rows_remote=20856 dedup_ratio=0.242757 router_tokens_per_expert=[789, 722, 780, 773, 743, 767, 750, 754, 801, 736, 750, 747, 766, 726, 768, 793, 807, 769, 744, 749, 717, 804, 761, 771, 786, 724, 759, 772, 742, 793, 739, 781, 792, 750, 779, 793, 778, 773, 787, 800, 816, 752, 779, 783, 747, 782, 759, 739, 792, 746, 767, 711, 800, 783, 781, 787, 785, 779, 784, 756, 793, 782, 761, 753] payload_rows_per_dst=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] dup_rows_per_dst=[2910, 2980, 3054, 2860, 3082, 3008, 2968, 3002] combine_rows_per_src=[9246, 9194, 9190, 9332, 9422, 9306, 9366, 9384] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=342784 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266279680 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532216576 weight_movement_bytes=34603008 total_nvlink_bytes=566819584 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344224 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266780832 ingress_bytes=335642624 link_bytes=335642624 token_bytes=533217440 weight_movement_bytes=69206016 total_nvlink_bytes=602423456 weight_to_token_ratio=0.129789 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262996173e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65046 dup_rows=24086 dup_rows_remote=21272 dedup_ratio=0.245015 router_tokens_per_expert=[786, 804, 754, 756, 742, 775, 786, 786, 754, 740, 818, 782, 745, 791, 773, 748, 752, 713, 799, 696, 769, 795, 699, 770, 812, 796, 754, 752, 820, 779, 805, 799, 740, 739, 755, 749, 773, 720, 828, 734, 788, 784, 726, 730, 786, 769, 781, 776, 770, 815, 712, 818, 771, 801, 789, 763, 740, 818, 762, 756, 754, 774, 727, 754] payload_rows_per_dst=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] dup_rows_per_dst=[3094, 2938, 2814, 3178, 2966, 3004, 3202, 2890] combine_rows_per_src=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65006 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[744, 801, 749, 778, 790, 755, 734, 741, 795, 757, 803, 806, 762, 760, 775, 741, 788, 782, 761, 772, 799, 731, 778, 715, 768, 768, 803, 792, 804, 786, 767, 714, 757, 781, 747, 745, 749, 767, 810, 775, 802, 756, 782, 763, 758, 823, 772, 787, 757, 776, 743, 784, 767, 807, 748, 772, 805, 729, 738, 778, 714, 744, 760, 737] payload_rows_per_dst=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] dup_rows_per_dst=[2936, 3058, 2950, 3066, 2946, 3154, 2974, 2880] combine_rows_per_src=[9248, 9340, 9302, 9338, 9316, 9332, 9334, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64986 dup_rows=24062 dup_rows_remote=20946 dedup_ratio=0.244771 router_tokens_per_expert=[761, 779, 727, 778, 748, 769, 745, 806, 758, 772, 824, 792, 753, 750, 839, 806, 770, 745, 745, 726, 734, 744, 804, 774, 774, 774, 819, 776, 748, 804, 772, 770, 769, 798, 751, 765, 772, 787, 801, 779, 757, 750, 794, 750, 799, 741, 762, 706, 770, 800, 734, 756, 748, 754, 751, 750, 777, 776, 771, 781, 729, 791, 746, 751] payload_rows_per_dst=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] dup_rows_per_dst=[3116, 3366, 3002, 3096, 2940, 2868, 2830, 2844] combine_rows_per_src=[9256, 9556, 9306, 9300, 9302, 9228, 9296, 8998] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682624 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020928 weight_movement_bytes=34603008 total_nvlink_bytes=567623936 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344256 combine_payload_bytes=265592832 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265937088 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529920 weight_movement_bytes=34603008 total_nvlink_bytes=566132928 weight_to_token_ratio=0.065101 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64866 dup_rows=24174 dup_rows_remote=21144 dedup_ratio=0.245911 router_tokens_per_expert=[796, 763, 753, 729, 766, 733, 776, 746, 740, 798, 805, 764, 772, 782, 754, 732, 774, 775, 759, 719, 762, 741, 811, 748, 833, 759, 747, 752, 782, 767, 773, 777, 808, 753, 790, 768, 736, 794, 792, 752, 738, 763, 800, 729, 764, 788, 763, 749, 842, 733, 773, 777, 801, 792, 771, 757, 787, 761, 792, 729, 764, 756, 784, 758] payload_rows_per_dst=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] dup_rows_per_dst=[2912, 3030, 2950, 3152, 3038, 3002, 3086, 3004] combine_rows_per_src=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64642 dup_rows=24114 dup_rows_remote=20804 dedup_ratio=0.245300 router_tokens_per_expert=[722, 790, 819, 752, 774, 791, 766, 712, 735, 744, 834, 773, 774, 767, 759, 773, 767, 746, 772, 755, 782, 806, 755, 772, 833, 750, 806, 771, 755, 790, 818, 775, 776, 750, 785, 770, 785, 760, 772, 767, 777, 730, 754, 785, 737, 789, 739, 751, 715, 744, 759, 762, 762, 770, 739, 792, 772, 766, 764, 756, 813, 768, 775, 730] payload_rows_per_dst=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] dup_rows_per_dst=[3310, 3042, 3012, 3070, 3108, 3014, 2572, 2986] combine_rows_per_src=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=342784 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266279680 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532216576 weight_movement_bytes=34603008 total_nvlink_bytes=566819584 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343960 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132824 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921688 weight_movement_bytes=34603008 total_nvlink_bytes=568524696 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64866 dup_rows=24174 dup_rows_remote=21144 dedup_ratio=0.245911 router_tokens_per_expert=[796, 763, 753, 729, 766, 733, 776, 746, 740, 798, 805, 764, 772, 782, 754, 732, 774, 775, 759, 719, 762, 741, 811, 748, 833, 759, 747, 752, 782, 767, 773, 777, 808, 753, 790, 768, 736, 794, 792, 752, 738, 763, 800, 729, 764, 788, 763, 749, 842, 733, 773, 777, 801, 792, 771, 757, 787, 761, 792, 729, 764, 756, 784, 758] payload_rows_per_dst=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] dup_rows_per_dst=[2912, 3030, 2950, 3152, 3038, 3002, 3086, 3004] combine_rows_per_src=[9212, 9264, 9228, 9228, 9348, 9186, 9406, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343984 combine_payload_bytes=266264576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266608560 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532873136 weight_movement_bytes=69206016 total_nvlink_bytes=602079152 weight_to_token_ratio=0.129873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341784 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265115416 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529889048 weight_movement_bytes=34603008 total_nvlink_bytes=564492056 weight_to_token_ratio=0.065302 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65046 dup_rows=24086 dup_rows_remote=21272 dedup_ratio=0.245015 router_tokens_per_expert=[786, 804, 754, 756, 742, 775, 786, 786, 754, 740, 818, 782, 745, 791, 773, 748, 752, 713, 799, 696, 769, 795, 699, 770, 812, 796, 754, 752, 820, 779, 805, 799, 740, 739, 755, 749, 773, 720, 828, 734, 788, 784, 726, 730, 786, 769, 781, 776, 770, 815, 712, 818, 771, 801, 789, 763, 740, 818, 762, 756, 754, 774, 727, 754] payload_rows_per_dst=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] dup_rows_per_dst=[3094, 2938, 2814, 3178, 2966, 3004, 3202, 2890] combine_rows_per_src=[9284, 9364, 9172, 9456, 9110, 9276, 9276, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=344040 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266035176 ingress_bytes=369500160 link_bytes=369500160 token_bytes=531726312 weight_movement_bytes=103809024 total_nvlink_bytes=635535336 weight_to_token_ratio=0.195230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390399355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64894 dup_rows=24080 dup_rows_remote=21012 dedup_ratio=0.244954 router_tokens_per_expert=[803, 740, 779, 761, 790, 719, 783, 806, 730, 764, 761, 761, 784, 763, 797, 806, 766, 781, 803, 777, 740, 776, 758, 756, 771, 719, 758, 782, 783, 750, 779, 693, 705, 785, 797, 761, 796, 784, 797, 774, 782, 750, 759, 749, 853, 797, 783, 749, 761, 797, 777, 729, 748, 745, 751, 782, 761, 794, 777, 750, 751, 761, 761, 747] payload_rows_per_dst=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] dup_rows_per_dst=[3026, 3108, 3066, 2912, 3068, 3076, 2944, 2880] combine_rows_per_src=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=345272 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773688 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533202104 weight_movement_bytes=34603008 total_nvlink_bytes=567805112 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=21068 dedup_ratio=0.243530 router_tokens_per_expert=[801, 728, 802, 829, 820, 801, 766, 775, 795, 771, 721, 709, 753, 774, 751, 764, 792, 766, 822, 745, 774, 765, 733, 755, 788, 749, 722, 785, 756, 772, 746, 744, 772, 753, 735, 758, 755, 783, 765, 799, 792, 727, 785, 714, 804, 786, 775, 764, 778, 756, 716, 797, 751, 762, 755, 781, 776, 732, 787, 769, 757, 796, 807, 791] payload_rows_per_dst=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] dup_rows_per_dst=[3102, 2798, 3084, 2872, 2970, 3008, 3010, 3096] combine_rows_per_src=[9542, 9278, 9220, 9252, 9270, 9286, 9182, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343984 combine_payload_bytes=266264576 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266608560 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532873136 weight_movement_bytes=69206016 total_nvlink_bytes=602079152 weight_to_token_ratio=0.129873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343728 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526384 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709040 weight_movement_bytes=34603008 total_nvlink_bytes=567312048 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=344040 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266035176 ingress_bytes=369500160 link_bytes=369500160 token_bytes=531726312 weight_movement_bytes=103809024 total_nvlink_bytes=635535336 weight_to_token_ratio=0.195230 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390399355e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64782 dup_rows=24132 dup_rows_remote=20882 dedup_ratio=0.245483 router_tokens_per_expert=[747, 772, 749, 806, 800, 756, 787, 804, 813, 744, 768, 768, 714, 749, 790, 730, 720, 761, 756, 743, 791, 783, 788, 795, 767, 796, 749, 798, 774, 770, 790, 762, 748, 759, 793, 754, 764, 770, 791, 800, 767, 780, 795, 779, 753, 751, 727, 799, 793, 812, 771, 748, 726, 764, 726, 758, 747, 761, 767, 767, 780, 747, 768, 747] payload_rows_per_dst=[9204, 9214, 9226, 9328, 9294, 9040, 9476, 9390] dup_rows_per_dst=[2792, 2938, 3048, 3084, 3064, 2790, 3166, 3250] combine_rows_per_src=[9204, 9214, 9226, 9328, 9294, 9040, 9476, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65180 dup_rows=23878 dup_rows_remote=20890 dedup_ratio=0.242900 router_tokens_per_expert=[771, 743, 775, 772, 779, 782, 727, 790, 797, 750, 768, 731, 812, 802, 741, 743, 779, 735, 818, 736, 735, 758, 783, 779, 783, 791, 774, 787, 780, 798, 737, 808, 774, 753, 742, 765, 712, 755, 792, 795, 738, 774, 750, 817, 746, 816, 719, 757, 757, 777, 757, 749, 762, 761, 791, 777, 809, 805, 786, 784, 728, 743, 712, 785] payload_rows_per_dst=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] dup_rows_per_dst=[3002, 3006, 2966, 3056, 2882, 2988, 2892, 3086] combine_rows_per_src=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=345272 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773688 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533202104 weight_movement_bytes=34603008 total_nvlink_bytes=567805112 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73978 payload_rows_remote=64976 dup_rows=24326 dup_rows_remote=21368 dedup_ratio=0.247457 router_tokens_per_expert=[755, 758, 752, 758, 735, 756, 783, 801, 780, 757, 762, 786, 748, 743, 792, 795, 778, 816, 774, 805, 727, 776, 774, 783, 741, 741, 751, 829, 745, 749, 793, 763, 761, 777, 801, 759, 795, 774, 824, 765, 797, 763, 758, 743, 817, 769, 804, 767, 720, 755, 754, 774, 730, 761, 764, 722, 751, 754, 806, 772, 727, 761, 761, 760] payload_rows_per_dst=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] dup_rows_per_dst=[2970, 3076, 3142, 3004, 3082, 3092, 2958, 3002] combine_rows_per_src=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64906 dup_rows=24090 dup_rows_remote=21120 dedup_ratio=0.245056 router_tokens_per_expert=[764, 790, 776, 741, 795, 728, 740, 755, 776, 803, 730, 795, 791, 747, 811, 760, 772, 759, 732, 755, 793, 764, 774, 787, 760, 776, 732, 766, 742, 807, 788, 787, 786, 777, 738, 761, 759, 784, 751, 742, 782, 837, 803, 761, 789, 758, 753, 790, 792, 721, 756, 744, 768, 757, 777, 764, 768, 786, 788, 697, 794, 759, 749, 765] payload_rows_per_dst=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] dup_rows_per_dst=[2970, 3054, 3190, 2860, 3034, 3012, 2982, 2988] combine_rows_per_src=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343624 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149448 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955272 weight_movement_bytes=34603008 total_nvlink_bytes=566558280 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344720 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267043472 ingress_bytes=370507776 link_bytes=370507776 token_bytes=533742224 weight_movement_bytes=103809024 total_nvlink_bytes=637551248 weight_to_token_ratio=0.194493 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394190933e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64894 dup_rows=24080 dup_rows_remote=21012 dedup_ratio=0.244954 router_tokens_per_expert=[803, 740, 779, 761, 790, 719, 783, 806, 730, 764, 761, 761, 784, 763, 797, 806, 766, 781, 803, 777, 740, 776, 758, 756, 771, 719, 758, 782, 783, 750, 779, 693, 705, 785, 797, 761, 796, 784, 797, 774, 782, 750, 759, 749, 853, 797, 783, 749, 761, 797, 777, 729, 748, 745, 751, 782, 761, 794, 777, 750, 751, 761, 761, 747] payload_rows_per_dst=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] dup_rows_per_dst=[3026, 3108, 3066, 2912, 3068, 3076, 2944, 2880] combine_rows_per_src=[9336, 9224, 9248, 9158, 9330, 9368, 9236, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65180 dup_rows=23878 dup_rows_remote=20890 dedup_ratio=0.242900 router_tokens_per_expert=[771, 743, 775, 772, 779, 782, 727, 790, 797, 750, 768, 731, 812, 802, 741, 743, 779, 735, 818, 736, 735, 758, 783, 779, 783, 791, 774, 787, 780, 798, 737, 808, 774, 753, 742, 765, 712, 755, 792, 795, 738, 774, 750, 817, 746, 816, 719, 757, 757, 777, 757, 749, 762, 761, 791, 777, 809, 805, 786, 784, 728, 743, 712, 785] payload_rows_per_dst=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] dup_rows_per_dst=[3002, 3006, 2966, 3056, 2882, 2988, 2892, 3086] combine_rows_per_src=[9276, 9282, 9280, 9460, 9294, 9246, 9370, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=342656 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265689728 ingress_bytes=299950080 link_bytes=299950080 token_bytes=531036800 weight_movement_bytes=34603008 total_nvlink_bytes=565639808 weight_to_token_ratio=0.065161 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128688003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344280 combine_payload_bytes=266977280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267321560 ingress_bytes=405389312 link_bytes=405389312 token_bytes=534298840 weight_movement_bytes=138412032 total_nvlink_bytes=672710872 weight_to_token_ratio=0.259054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525447345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65124 dup_rows=23828 dup_rows_remote=20824 dedup_ratio=0.242391 router_tokens_per_expert=[726, 768, 741, 795, 750, 794, 765, 747, 786, 721, 795, 777, 797, 756, 769, 777, 734, 738, 793, 759, 784, 772, 738, 763, 809, 787, 725, 748, 789, 785, 750, 765, 806, 760, 721, 765, 742, 791, 782, 774, 813, 816, 747, 770, 794, 770, 745, 757, 829, 760, 816, 782, 731, 752, 794, 752, 783, 780, 732, 739, 764, 760, 771, 751] payload_rows_per_dst=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] dup_rows_per_dst=[2930, 3004, 2824, 2936, 3092, 3106, 2974, 2962] combine_rows_per_src=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=345376 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266487072 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532628768 weight_movement_bytes=34603008 total_nvlink_bytes=567231776 weight_to_token_ratio=0.064966 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344104 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266199080 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532054056 weight_movement_bytes=69206016 total_nvlink_bytes=601260072 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64958 dup_rows=24062 dup_rows_remote=21010 dedup_ratio=0.244771 router_tokens_per_expert=[758, 783, 798, 751, 810, 787, 778, 752, 749, 734, 797, 797, 744, 740, 728, 791, 785, 803, 772, 753, 789, 766, 753, 747, 765, 765, 753, 758, 749, 791, 788, 754, 767, 809, 755, 751, 789, 749, 789, 790, 780, 783, 817, 793, 705, 766, 741, 731, 799, 751, 736, 772, 739, 779, 764, 755, 745, 800, 783, 796, 759, 752, 753, 766] payload_rows_per_dst=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] dup_rows_per_dst=[3074, 2920, 3052, 2992, 3084, 2962, 2974, 3004] combine_rows_per_src=[9360, 9240, 9284, 9254, 9314, 9270, 9216, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343792 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267091696 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839600 weight_movement_bytes=34603008 total_nvlink_bytes=568442608 weight_to_token_ratio=0.064819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73978 payload_rows_remote=64976 dup_rows=24326 dup_rows_remote=21368 dedup_ratio=0.247457 router_tokens_per_expert=[755, 758, 752, 758, 735, 756, 783, 801, 780, 757, 762, 786, 748, 743, 792, 795, 778, 816, 774, 805, 727, 776, 774, 783, 741, 741, 751, 829, 745, 749, 793, 763, 761, 777, 801, 759, 795, 774, 824, 765, 797, 763, 758, 743, 817, 769, 804, 767, 720, 755, 754, 774, 730, 761, 764, 722, 751, 754, 806, 772, 727, 761, 761, 760] payload_rows_per_dst=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] dup_rows_per_dst=[2970, 3076, 3142, 3004, 3082, 3092, 2958, 3002] combine_rows_per_src=[9226, 9250, 9324, 9220, 9430, 9344, 9002, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343624 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149448 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955272 weight_movement_bytes=34603008 total_nvlink_bytes=566558280 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64642 dup_rows=24114 dup_rows_remote=20804 dedup_ratio=0.245300 router_tokens_per_expert=[722, 790, 819, 752, 774, 791, 766, 712, 735, 744, 834, 773, 774, 767, 759, 773, 767, 746, 772, 755, 782, 806, 755, 772, 833, 750, 806, 771, 755, 790, 818, 775, 776, 750, 785, 770, 785, 760, 772, 767, 777, 730, 754, 785, 737, 789, 739, 751, 715, 744, 759, 762, 762, 770, 739, 792, 772, 766, 764, 756, 813, 768, 775, 730] payload_rows_per_dst=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] dup_rows_per_dst=[3310, 3042, 3012, 3070, 3108, 3014, 2572, 2986] combine_rows_per_src=[9548, 9306, 9298, 9526, 9222, 9110, 8878, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64932 dup_rows=24104 dup_rows_remote=21048 dedup_ratio=0.245199 router_tokens_per_expert=[726, 778, 764, 774, 799, 797, 778, 741, 790, 729, 777, 748, 819, 773, 762, 686, 753, 755, 733, 778, 800, 820, 740, 754, 786, 776, 766, 775, 756, 749, 763, 773, 754, 759, 777, 775, 788, 794, 768, 747, 698, 739, 737, 800, 767, 765, 794, 771, 753, 770, 795, 786, 795, 795, 803, 777, 766, 761, 761, 723, 789, 770, 802, 755] payload_rows_per_dst=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] dup_rows_per_dst=[2936, 3020, 2940, 3076, 3056, 2946, 3102, 3028] combine_rows_per_src=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344280 combine_payload_bytes=266977280 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267321560 ingress_bytes=405389312 link_bytes=405389312 token_bytes=534298840 weight_movement_bytes=138412032 total_nvlink_bytes=672710872 weight_to_token_ratio=0.259054 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525447345e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65124 dup_rows=23828 dup_rows_remote=20824 dedup_ratio=0.242391 router_tokens_per_expert=[726, 768, 741, 795, 750, 794, 765, 747, 786, 721, 795, 777, 797, 756, 769, 777, 734, 738, 793, 759, 784, 772, 738, 763, 809, 787, 725, 748, 789, 785, 750, 765, 806, 760, 721, 765, 742, 791, 782, 774, 813, 816, 747, 770, 794, 770, 745, 757, 829, 760, 816, 782, 731, 752, 794, 752, 783, 780, 732, 739, 764, 760, 771, 751] payload_rows_per_dst=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] dup_rows_per_dst=[2930, 3004, 2824, 2936, 3092, 3106, 2974, 2962] combine_rows_per_src=[9242, 9352, 9338, 9380, 9190, 9318, 9458, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343872 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411840 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479808 weight_movement_bytes=69206016 total_nvlink_bytes=601685824 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=345376 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266487072 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532628768 weight_movement_bytes=34603008 total_nvlink_bytes=567231776 weight_to_token_ratio=0.064966 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64994 dup_rows=23966 dup_rows_remote=20860 dedup_ratio=0.243795 router_tokens_per_expert=[736, 770, 803, 779, 797, 751, 735, 775, 776, 797, 763, 746, 764, 752, 822, 769, 754, 763, 748, 800, 774, 756, 757, 787, 790, 800, 738, 804, 844, 763, 720, 766, 755, 772, 763, 725, 759, 754, 762, 761, 747, 771, 770, 775, 764, 818, 777, 766, 785, 764, 727, 732, 790, 797, 768, 739, 717, 741, 726, 770, 807, 805, 758, 788] payload_rows_per_dst=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] dup_rows_per_dst=[2996, 3084, 2992, 3106, 2852, 3024, 2972, 2940] combine_rows_per_src=[9296, 9294, 9286, 9344, 9250, 9352, 9232, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341784 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265115416 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529889048 weight_movement_bytes=34603008 total_nvlink_bytes=564492056 weight_to_token_ratio=0.065302 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65256 dup_rows=23948 dup_rows_remote=21010 dedup_ratio=0.243612 router_tokens_per_expert=[784, 733, 748, 733, 750, 798, 733, 785, 752, 757, 755, 756, 754, 767, 759, 785, 791, 757, 794, 769, 807, 760, 769, 747, 805, 762, 766, 776, 789, 753, 753, 789, 765, 813, 767, 793, 808, 771, 745, 775, 722, 768, 779, 751, 821, 789, 782, 773, 754, 733, 788, 787, 780, 759, 831, 745, 763, 732, 715, 728, 800, 712, 776, 791] payload_rows_per_dst=[9244, 9164, 9256, 9422, 9456, 9366, 9348, 9100] dup_rows_per_dst=[2884, 3006, 3012, 3084, 3018, 2996, 3010, 2938] combine_rows_per_src=[9244, 9164, 9256, 9422, 9456, 9366, 9348, 9100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343792 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267091696 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533839600 weight_movement_bytes=34603008 total_nvlink_bytes=568442608 weight_to_token_ratio=0.064819 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343416 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266558840 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774264 weight_movement_bytes=69206016 total_nvlink_bytes=601980280 weight_to_token_ratio=0.129897 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64932 dup_rows=24104 dup_rows_remote=21048 dedup_ratio=0.245199 router_tokens_per_expert=[726, 778, 764, 774, 799, 797, 778, 741, 790, 729, 777, 748, 819, 773, 762, 686, 753, 755, 733, 778, 800, 820, 740, 754, 786, 776, 766, 775, 756, 749, 763, 773, 754, 759, 777, 775, 788, 794, 768, 747, 698, 739, 737, 800, 767, 765, 794, 771, 753, 770, 795, 786, 795, 795, 803, 777, 766, 761, 761, 723, 789, 770, 802, 755] payload_rows_per_dst=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] dup_rows_per_dst=[2936, 3020, 2940, 3076, 3056, 2946, 3102, 3028] combine_rows_per_src=[9378, 9148, 9326, 9212, 9268, 9196, 9446, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20966 dedup_ratio=0.243917 router_tokens_per_expert=[726, 771, 749, 780, 749, 755, 825, 745, 759, 775, 784, 816, 748, 748, 748, 727, 754, 783, 750, 746, 770, 764, 753, 737, 739, 814, 788, 762, 747, 757, 800, 720, 763, 769, 735, 711, 777, 804, 782, 747, 812, 759, 772, 786, 768, 785, 731, 765, 779, 765, 796, 797, 775, 764, 803, 782, 784, 823, 744, 817, 719, 784, 761, 804] payload_rows_per_dst=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] dup_rows_per_dst=[2986, 2970, 2960, 2946, 2930, 3012, 3112, 3062] combine_rows_per_src=[9214, 9240, 9154, 9308, 9246, 9344, 9410, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343872 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411840 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479808 weight_movement_bytes=69206016 total_nvlink_bytes=601685824 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345064 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267633640 ingress_bytes=336494592 link_bytes=336494592 token_bytes=534922216 weight_movement_bytes=69206016 total_nvlink_bytes=604128232 weight_to_token_ratio=0.129376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266202060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64894 dup_rows=24170 dup_rows_remote=21190 dedup_ratio=0.245870 router_tokens_per_expert=[768, 771, 791, 774, 774, 781, 809, 743, 797, 735, 761, 772, 753, 803, 772, 758, 780, 723, 779, 754, 748, 756, 781, 789, 766, 789, 810, 735, 779, 766, 730, 748, 765, 714, 765, 740, 798, 758, 814, 764, 786, 789, 771, 713, 787, 784, 805, 777, 811, 776, 730, 780, 739, 755, 756, 741, 761, 713, 786, 741, 764, 816, 791, 767] payload_rows_per_dst=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] dup_rows_per_dst=[3074, 3066, 2980, 2912, 2980, 3068, 3038, 3052] combine_rows_per_src=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74616 payload_rows_remote=65424 dup_rows=23688 dup_rows_remote=20794 dedup_ratio=0.240967 router_tokens_per_expert=[811, 760, 728, 785, 795, 791, 763, 733, 796, 761, 758, 779, 739, 809, 739, 780, 718, 785, 744, 809, 697, 785, 783, 778, 752, 752, 783, 779, 785, 781, 797, 779, 752, 789, 724, 798, 777, 763, 743, 742, 798, 768, 744, 766, 811, 753, 799, 740, 751, 726, 754, 799, 763, 790, 707, 753, 772, 749, 780, 804, 767, 774, 799, 763] payload_rows_per_dst=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] dup_rows_per_dst=[3010, 2972, 2972, 2940, 2870, 3068, 2894, 2962] combine_rows_per_src=[9322, 9350, 9226, 9476, 9306, 9290, 9192, 9454] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343920 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305392 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266864 weight_movement_bytes=34603008 total_nvlink_bytes=566869872 weight_to_token_ratio=0.065011 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64906 dup_rows=24090 dup_rows_remote=21120 dedup_ratio=0.245056 router_tokens_per_expert=[764, 790, 776, 741, 795, 728, 740, 755, 776, 803, 730, 795, 791, 747, 811, 760, 772, 759, 732, 755, 793, 764, 774, 787, 760, 776, 732, 766, 742, 807, 788, 787, 786, 777, 738, 761, 759, 784, 751, 742, 782, 837, 803, 761, 789, 758, 753, 790, 792, 721, 756, 744, 768, 757, 777, 764, 768, 786, 788, 697, 794, 759, 749, 765] payload_rows_per_dst=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] dup_rows_per_dst=[2970, 3054, 3190, 2860, 3034, 3012, 2982, 2988] combine_rows_per_src=[9308, 9418, 9362, 9148, 9228, 9350, 9176, 9224] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=20958 dedup_ratio=0.244975 router_tokens_per_expert=[728, 750, 775, 719, 734, 779, 747, 776, 794, 751, 779, 815, 778, 783, 766, 759, 802, 739, 754, 793, 758, 816, 785, 772, 746, 776, 736, 772, 806, 776, 740, 772, 750, 770, 777, 784, 784, 762, 753, 718, 772, 737, 754, 775, 759, 767, 780, 794, 786, 742, 792, 700, 799, 812, 773, 776, 791, 779, 800, 779, 770, 768, 725, 748] payload_rows_per_dst=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] dup_rows_per_dst=[3124, 3138, 3082, 2710, 3004, 3022, 3020, 2982] combine_rows_per_src=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343416 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266558840 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774264 weight_movement_bytes=69206016 total_nvlink_bytes=601980280 weight_to_token_ratio=0.129897 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345064 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267633640 ingress_bytes=336494592 link_bytes=336494592 token_bytes=534922216 weight_movement_bytes=69206016 total_nvlink_bytes=604128232 weight_to_token_ratio=0.129376 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266202060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343792 combine_payload_bytes=266166272 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266510064 ingress_bytes=369975296 link_bytes=369975296 token_bytes=532676336 weight_movement_bytes=103809024 total_nvlink_bytes=636485360 weight_to_token_ratio=0.194882 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392187253e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344872 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321576 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298280 weight_movement_bytes=69206016 total_nvlink_bytes=605504296 weight_to_token_ratio=0.129044 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65184 dup_rows=23854 dup_rows_remote=20852 dedup_ratio=0.242655 router_tokens_per_expert=[759, 767, 793, 760, 795, 774, 709, 729, 711, 792, 786, 784, 777, 760, 758, 766, 784, 811, 752, 751, 735, 767, 792, 810, 726, 798, 798, 787, 800, 734, 725, 788, 693, 787, 799, 787, 731, 768, 782, 785, 760, 727, 785, 738, 774, 746, 828, 744, 773, 779, 772, 736, 725, 779, 781, 770, 798, 758, 787, 784, 742, 799, 743, 814] payload_rows_per_dst=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] dup_rows_per_dst=[2988, 3002, 2938, 2988, 3010, 2918, 3002, 3008] combine_rows_per_src=[9184, 9266, 9466, 9324, 9254, 9286, 9228, 9442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343488 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231232 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118976 weight_movement_bytes=34603008 total_nvlink_bytes=566721984 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64894 dup_rows=24170 dup_rows_remote=21190 dedup_ratio=0.245870 router_tokens_per_expert=[768, 771, 791, 774, 774, 781, 809, 743, 797, 735, 761, 772, 753, 803, 772, 758, 780, 723, 779, 754, 748, 756, 781, 789, 766, 789, 810, 735, 779, 766, 730, 748, 765, 714, 765, 740, 798, 758, 814, 764, 786, 789, 771, 713, 787, 784, 805, 777, 811, 776, 730, 780, 739, 755, 756, 741, 761, 713, 786, 741, 764, 816, 791, 767] payload_rows_per_dst=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] dup_rows_per_dst=[3074, 3066, 2980, 2912, 2980, 3068, 3038, 3052] combine_rows_per_src=[9348, 9236, 9240, 9334, 9256, 9356, 9138, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344144 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267337808 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331472 weight_movement_bytes=34603008 total_nvlink_bytes=568934480 weight_to_token_ratio=0.064759 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65142 dup_rows=23952 dup_rows_remote=20978 dedup_ratio=0.243652 router_tokens_per_expert=[748, 754, 799, 750, 783, 806, 774, 722, 752, 731, 737, 792, 729, 766, 774, 756, 773, 740, 732, 815, 804, 713, 777, 698, 761, 778, 779, 834, 778, 770, 776, 809, 740, 735, 820, 759, 750, 748, 758, 782, 769, 755, 799, 794, 763, 772, 760, 754, 761, 806, 791, 750, 756, 774, 761, 798, 757, 762, 769, 791, 786, 825, 724, 773] payload_rows_per_dst=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] dup_rows_per_dst=[2922, 2862, 2872, 3108, 2974, 3068, 3034, 3112] combine_rows_per_src=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344336 combine_payload_bytes=265805824 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266150160 ingress_bytes=404217856 link_bytes=404217856 token_bytes=531955984 weight_movement_bytes=138412032 total_nvlink_bytes=670368016 weight_to_token_ratio=0.260195 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521039250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65202 dup_rows=23826 dup_rows_remote=20818 dedup_ratio=0.242371 router_tokens_per_expert=[773, 782, 749, 796, 724, 748, 798, 734, 768, 735, 758, 768, 807, 762, 755, 739, 799, 751, 759, 790, 780, 746, 691, 776, 790, 785, 767, 746, 753, 751, 785, 765, 817, 750, 777, 764, 794, 805, 772, 798, 797, 785, 753, 737, 771, 784, 771, 759, 759, 781, 775, 768, 775, 736, 741, 748, 775, 763, 822, 770, 781, 798, 709, 787] payload_rows_per_dst=[9288, 9292, 9170, 9276, 9464, 9334, 9286, 9368] dup_rows_per_dst=[2920, 2892, 3014, 3008, 3090, 2980, 2880, 3042] combine_rows_per_src=[9288, 9292, 9170, 9276, 9464, 9334, 9286, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344872 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321576 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298280 weight_movement_bytes=69206016 total_nvlink_bytes=605504296 weight_to_token_ratio=0.129044 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65032 dup_rows=24008 dup_rows_remote=21072 dedup_ratio=0.244222 router_tokens_per_expert=[818, 717, 752, 773, 710, 760, 747, 743, 753, 834, 724, 766, 742, 762, 739, 744, 788, 786, 815, 799, 779, 749, 726, 761, 824, 782, 775, 774, 823, 763, 752, 758, 762, 847, 804, 807, 734, 753, 756, 793, 810, 798, 792, 764, 768, 808, 776, 789, 740, 747, 762, 740, 749, 708, 731, 779, 720, 797, 796, 713, 750, 771, 760, 790] payload_rows_per_dst=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] dup_rows_per_dst=[2908, 2878, 3038, 2912, 3170, 3108, 3058, 2936] combine_rows_per_src=[9132, 9250, 9250, 9156, 9398, 9502, 9344, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344336 combine_payload_bytes=265805824 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266150160 ingress_bytes=404217856 link_bytes=404217856 token_bytes=531955984 weight_movement_bytes=138412032 total_nvlink_bytes=670368016 weight_to_token_ratio=0.260195 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521039250e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=65006 dup_rows=24172 dup_rows_remote=21210 dedup_ratio=0.245890 router_tokens_per_expert=[836, 761, 744, 766, 816, 729, 766, 801, 813, 738, 822, 784, 747, 757, 778, 785, 787, 746, 825, 754, 799, 796, 778, 732, 783, 731, 776, 731, 794, 747, 756, 776, 814, 792, 772, 760, 744, 743, 770, 750, 737, 809, 724, 796, 751, 723, 750, 746, 785, 772, 751, 758, 752, 753, 708, 765, 688, 771, 822, 750, 766, 748, 796, 832] payload_rows_per_dst=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] dup_rows_per_dst=[3090, 3088, 3102, 2974, 2994, 2888, 2962, 3074] combine_rows_per_src=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65230 dup_rows=23860 dup_rows_remote=20804 dedup_ratio=0.242716 router_tokens_per_expert=[794, 752, 744, 796, 775, 766, 817, 768, 752, 731, 741, 766, 813, 708, 792, 745, 800, 772, 768, 753, 781, 790, 829, 780, 797, 767, 830, 758, 766, 767, 833, 737, 714, 774, 739, 740, 737, 763, 767, 750, 754, 728, 798, 754, 785, 797, 754, 765, 748, 723, 803, 781, 757, 749, 763, 757, 773, 784, 795, 786, 739, 721, 819, 747] payload_rows_per_dst=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] dup_rows_per_dst=[2996, 2902, 3092, 3098, 2838, 3056, 2880, 2998] combine_rows_per_src=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344104 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266199080 ingress_bytes=335060992 link_bytes=335060992 token_bytes=532054056 weight_movement_bytes=69206016 total_nvlink_bytes=601260072 weight_to_token_ratio=0.130073 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260807538e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344144 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267337808 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331472 weight_movement_bytes=34603008 total_nvlink_bytes=568934480 weight_to_token_ratio=0.064759 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344480 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166112 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987744 weight_movement_bytes=34603008 total_nvlink_bytes=568590752 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65142 dup_rows=23952 dup_rows_remote=20978 dedup_ratio=0.243652 router_tokens_per_expert=[748, 754, 799, 750, 783, 806, 774, 722, 752, 731, 737, 792, 729, 766, 774, 756, 773, 740, 732, 815, 804, 713, 777, 698, 761, 778, 779, 834, 778, 770, 776, 809, 740, 735, 820, 759, 750, 748, 758, 782, 769, 755, 799, 794, 763, 772, 760, 754, 761, 806, 791, 750, 756, 774, 761, 798, 757, 762, 769, 791, 786, 825, 724, 773] payload_rows_per_dst=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] dup_rows_per_dst=[2922, 2862, 2872, 3108, 2974, 3068, 3034, 3112] combine_rows_per_src=[9350, 9212, 9232, 9462, 9210, 9264, 9360, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344080 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267411472 ingress_bytes=336273408 link_bytes=336273408 token_bytes=534478864 weight_movement_bytes=69206016 total_nvlink_bytes=603684880 weight_to_token_ratio=0.129483 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265369762e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64678 dup_rows=24234 dup_rows_remote=21260 dedup_ratio=0.246521 router_tokens_per_expert=[755, 718, 773, 776, 788, 767, 804, 756, 788, 823, 748, 781, 769, 745, 797, 776, 734, 763, 755, 750, 781, 754, 808, 739, 784, 755, 832, 752, 749, 749, 764, 774, 760, 800, 795, 829, 759, 749, 751, 753, 759, 745, 734, 772, 771, 747, 725, 784, 780, 824, 758, 735, 776, 835, 789, 762, 749, 728, 772, 704, 801, 780, 762, 757] payload_rows_per_dst=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] dup_rows_per_dst=[2974, 3128, 2984, 3048, 3058, 2984, 3124, 2934] combine_rows_per_src=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65230 dup_rows=23860 dup_rows_remote=20804 dedup_ratio=0.242716 router_tokens_per_expert=[794, 752, 744, 796, 775, 766, 817, 768, 752, 731, 741, 766, 813, 708, 792, 745, 800, 772, 768, 753, 781, 790, 829, 780, 797, 767, 830, 758, 766, 767, 833, 737, 714, 774, 739, 740, 737, 763, 767, 750, 754, 728, 798, 754, 785, 797, 754, 765, 748, 723, 803, 781, 757, 749, 763, 757, 773, 784, 795, 786, 739, 721, 819, 747] payload_rows_per_dst=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] dup_rows_per_dst=[2996, 2902, 3092, 3098, 2838, 3056, 2880, 2998] combine_rows_per_src=[9428, 9194, 9454, 9412, 9130, 9214, 9282, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344416 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266715488 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533086560 weight_movement_bytes=103809024 total_nvlink_bytes=636895584 weight_to_token_ratio=0.194732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344864 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266609440 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532874016 weight_movement_bytes=34603008 total_nvlink_bytes=567477024 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344136 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526216 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708296 weight_movement_bytes=34603008 total_nvlink_bytes=569311304 weight_to_token_ratio=0.064714 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65042 dup_rows=23768 dup_rows_remote=20682 dedup_ratio=0.241781 router_tokens_per_expert=[760, 698, 773, 766, 779, 782, 728, 781, 784, 838, 771, 783, 748, 842, 734, 790, 738, 775, 784, 792, 796, 797, 772, 772, 749, 759, 755, 814, 740, 773, 722, 748, 783, 702, 768, 728, 765, 772, 758, 780, 796, 754, 810, 774, 758, 756, 743, 706, 782, 812, 780, 780, 765, 741, 770, 769, 763, 759, 775, 790, 774, 768, 767, 761] payload_rows_per_dst=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] dup_rows_per_dst=[2926, 3086, 3120, 2886, 2886, 2940, 2922, 3002] combine_rows_per_src=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343752 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264840 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185928 weight_movement_bytes=34603008 total_nvlink_bytes=564788936 weight_to_token_ratio=0.065266 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64948 dup_rows=24104 dup_rows_remote=21120 dedup_ratio=0.245199 router_tokens_per_expert=[774, 823, 776, 746, 768, 812, 767, 771, 760, 769, 746, 743, 778, 716, 789, 714, 786, 702, 773, 783, 742, 787, 811, 734, 745, 764, 755, 795, 784, 743, 782, 775, 816, 792, 790, 734, 766, 786, 774, 769, 730, 768, 778, 813, 737, 766, 744, 741, 789, 786, 758, 760, 735, 772, 783, 809, 767, 740, 784, 735, 773, 787, 780, 777] payload_rows_per_dst=[9360, 9130, 9252, 9298, 9358, 9180, 9308, 9314] dup_rows_per_dst=[3114, 2900, 2984, 2988, 3096, 2974, 3076, 2972] combine_rows_per_src=[9360, 9130, 9252, 9298, 9358, 9180, 9308, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342896 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266754928 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533166960 weight_movement_bytes=34603008 total_nvlink_bytes=567769968 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64770 dup_rows=24268 dup_rows_remote=21306 dedup_ratio=0.246867 router_tokens_per_expert=[836, 778, 761, 776, 764, 806, 737, 752, 782, 750, 781, 745, 772, 800, 744, 741, 769, 796, 735, 789, 800, 755, 733, 781, 794, 784, 737, 813, 724, 779, 768, 779, 764, 813, 754, 776, 762, 741, 761, 743, 791, 754, 774, 758, 751, 720, 802, 798, 784, 770, 759, 722, 780, 738, 809, 796, 751, 737, 791, 779, 727, 772, 739, 775] payload_rows_per_dst=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] dup_rows_per_dst=[3080, 3004, 3126, 3090, 2962, 3000, 3048, 2958] combine_rows_per_src=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344480 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267166112 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987744 weight_movement_bytes=34603008 total_nvlink_bytes=568590752 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=65006 dup_rows=24172 dup_rows_remote=21210 dedup_ratio=0.245890 router_tokens_per_expert=[836, 761, 744, 766, 816, 729, 766, 801, 813, 738, 822, 784, 747, 757, 778, 785, 787, 746, 825, 754, 799, 796, 778, 732, 783, 731, 776, 731, 794, 747, 756, 776, 814, 792, 772, 760, 744, 743, 770, 750, 737, 809, 724, 796, 751, 723, 750, 746, 785, 772, 751, 758, 752, 753, 708, 765, 688, 771, 822, 750, 766, 748, 796, 832] payload_rows_per_dst=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] dup_rows_per_dst=[3090, 3088, 3102, 2974, 2994, 2888, 2962, 3074] combine_rows_per_src=[9348, 9360, 9332, 9214, 9296, 9184, 9126, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344136 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526216 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708296 weight_movement_bytes=34603008 total_nvlink_bytes=569311304 weight_to_token_ratio=0.064714 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=20958 dedup_ratio=0.244975 router_tokens_per_expert=[728, 750, 775, 719, 734, 779, 747, 776, 794, 751, 779, 815, 778, 783, 766, 759, 802, 739, 754, 793, 758, 816, 785, 772, 746, 776, 736, 772, 806, 776, 740, 772, 750, 770, 777, 784, 784, 762, 753, 718, 772, 737, 754, 775, 759, 767, 780, 794, 786, 742, 792, 700, 799, 812, 773, 776, 791, 779, 800, 779, 770, 768, 725, 748] payload_rows_per_dst=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] dup_rows_per_dst=[3124, 3138, 3082, 2710, 3004, 3022, 3020, 2982] combine_rows_per_src=[9308, 9490, 9358, 9122, 9386, 9252, 9146, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344272 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371280 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398288 weight_movement_bytes=34603008 total_nvlink_bytes=567001296 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65042 dup_rows=23768 dup_rows_remote=20682 dedup_ratio=0.241781 router_tokens_per_expert=[760, 698, 773, 766, 779, 782, 728, 781, 784, 838, 771, 783, 748, 842, 734, 790, 738, 775, 784, 792, 796, 797, 772, 772, 749, 759, 755, 814, 740, 773, 722, 748, 783, 702, 768, 728, 765, 772, 758, 780, 796, 754, 810, 774, 758, 756, 743, 706, 782, 812, 780, 780, 765, 741, 770, 769, 763, 759, 775, 790, 774, 768, 767, 761] payload_rows_per_dst=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] dup_rows_per_dst=[2926, 3086, 3120, 2886, 2886, 2940, 2922, 3002] combine_rows_per_src=[9208, 9494, 9332, 9234, 9226, 9254, 9476, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344864 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266609440 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532874016 weight_movement_bytes=34603008 total_nvlink_bytes=567477024 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64974 dup_rows=23992 dup_rows_remote=21038 dedup_ratio=0.244059 router_tokens_per_expert=[743, 720, 750, 789, 800, 766, 794, 784, 800, 730, 743, 812, 740, 812, 736, 740, 773, 774, 799, 755, 750, 749, 759, 766, 770, 782, 789, 765, 752, 763, 746, 779, 783, 746, 787, 765, 744, 789, 735, 770, 746, 758, 783, 781, 775, 789, 747, 811, 768, 769, 762, 802, 800, 758, 828, 768, 752, 799, 749, 738, 779, 748, 771, 722] payload_rows_per_dst=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] dup_rows_per_dst=[2936, 2976, 2972, 2954, 2982, 3146, 3140, 2886] combine_rows_per_src=[9356, 9250, 9278, 9338, 9256, 9234, 9370, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=65022 dup_rows=24100 dup_rows_remote=21112 dedup_ratio=0.245158 router_tokens_per_expert=[809, 770, 751, 779, 721, 761, 761, 750, 759, 767, 739, 812, 741, 719, 735, 763, 769, 737, 749, 759, 764, 807, 754, 808, 728, 834, 768, 764, 748, 792, 785, 797, 805, 783, 789, 769, 807, 814, 723, 767, 804, 767, 805, 795, 755, 735, 769, 767, 787, 798, 805, 729, 738, 743, 773, 759, 733, 758, 758, 780, 758, 738, 791, 750] payload_rows_per_dst=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] dup_rows_per_dst=[2894, 2986, 3094, 3086, 2758, 3298, 2996, 2988] combine_rows_per_src=[9062, 9082, 9448, 9346, 9178, 9674, 9232, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65012 dup_rows=24120 dup_rows_remote=21114 dedup_ratio=0.245361 router_tokens_per_expert=[753, 778, 719, 795, 712, 763, 773, 784, 771, 755, 818, 802, 815, 811, 733, 827, 797, 765, 723, 759, 793, 780, 786, 757, 749, 774, 773, 767, 755, 755, 778, 731, 801, 778, 764, 801, 800, 815, 744, 794, 736, 686, 789, 787, 717, 703, 771, 760, 716, 720, 784, 776, 761, 791, 762, 779, 788, 779, 780, 770, 773, 736, 730, 810] payload_rows_per_dst=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] dup_rows_per_dst=[2942, 3098, 2980, 2912, 3220, 2842, 3006, 3120] combine_rows_per_src=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344272 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371280 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398288 weight_movement_bytes=34603008 total_nvlink_bytes=567001296 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65046 dup_rows=24090 dup_rows_remote=21186 dedup_ratio=0.245056 router_tokens_per_expert=[810, 763, 770, 751, 736, 849, 782, 722, 796, 695, 755, 772, 749, 797, 775, 799, 769, 783, 748, 791, 747, 734, 827, 780, 809, 766, 790, 769, 754, 778, 754, 727, 764, 739, 723, 749, 771, 775, 727, 775, 738, 782, 739, 752, 691, 768, 761, 805, 778, 820, 768, 819, 750, 775, 774, 750, 740, 828, 750, 800, 760, 799, 762, 773] payload_rows_per_dst=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] dup_rows_per_dst=[3066, 3030, 3110, 3022, 2818, 2904, 3112, 3028] combine_rows_per_src=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343488 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231232 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118976 weight_movement_bytes=34603008 total_nvlink_bytes=566721984 weight_to_token_ratio=0.065029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344304 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265642224 ingress_bytes=334503936 link_bytes=334503936 token_bytes=530940144 weight_movement_bytes=69206016 total_nvlink_bytes=600146160 weight_to_token_ratio=0.130346 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258711381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65094 dup_rows=23944 dup_rows_remote=20982 dedup_ratio=0.243571 router_tokens_per_expert=[743, 765, 766, 783, 747, 771, 785, 748, 784, 750, 757, 759, 803, 812, 732, 761, 746, 726, 770, 775, 792, 816, 703, 723, 728, 747, 732, 779, 712, 764, 808, 733, 748, 777, 770, 783, 731, 760, 815, 816, 778, 734, 768, 799, 765, 753, 800, 809, 773, 759, 783, 807, 766, 736, 746, 783, 755, 779, 800, 785, 797, 791, 786, 780] payload_rows_per_dst=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] dup_rows_per_dst=[2962, 2938, 2976, 2952, 3040, 3060, 2900, 3116] combine_rows_per_src=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342896 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266754928 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533166960 weight_movement_bytes=34603008 total_nvlink_bytes=567769968 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64770 dup_rows=24268 dup_rows_remote=21306 dedup_ratio=0.246867 router_tokens_per_expert=[836, 778, 761, 776, 764, 806, 737, 752, 782, 750, 781, 745, 772, 800, 744, 741, 769, 796, 735, 789, 800, 755, 733, 781, 794, 784, 737, 813, 724, 779, 768, 779, 764, 813, 754, 776, 762, 741, 761, 743, 791, 754, 774, 758, 751, 720, 802, 798, 784, 770, 759, 722, 780, 738, 809, 796, 751, 737, 791, 779, 727, 772, 739, 775] payload_rows_per_dst=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] dup_rows_per_dst=[3080, 3004, 3126, 3090, 2962, 3000, 3048, 2958] combine_rows_per_src=[9340, 9226, 9190, 9266, 9266, 9296, 9268, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344048 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477552 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611056 weight_movement_bytes=34603008 total_nvlink_bytes=567214064 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65046 dup_rows=24090 dup_rows_remote=21186 dedup_ratio=0.245056 router_tokens_per_expert=[810, 763, 770, 751, 736, 849, 782, 722, 796, 695, 755, 772, 749, 797, 775, 799, 769, 783, 748, 791, 747, 734, 827, 780, 809, 766, 790, 769, 754, 778, 754, 727, 764, 739, 723, 749, 771, 775, 727, 775, 738, 782, 739, 752, 691, 768, 761, 805, 778, 820, 768, 819, 750, 775, 774, 750, 740, 828, 750, 800, 760, 799, 762, 773] payload_rows_per_dst=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] dup_rows_per_dst=[3066, 3030, 3110, 3022, 2818, 2904, 3112, 3028] combine_rows_per_src=[9300, 9246, 9248, 9272, 9228, 9168, 9356, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344536 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674648 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004760 weight_movement_bytes=34603008 total_nvlink_bytes=567607768 weight_to_token_ratio=0.064921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344504 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266633656 ingress_bytes=439304192 link_bytes=439304192 token_bytes=532922808 weight_movement_bytes=173015040 total_nvlink_bytes=705937848 weight_to_token_ratio=0.324653 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653066308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344928 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773344 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201760 weight_movement_bytes=34603008 total_nvlink_bytes=567804768 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73986 payload_rows_remote=64660 dup_rows=24318 dup_rows_remote=21150 dedup_ratio=0.247375 router_tokens_per_expert=[786, 773, 777, 771, 751, 762, 840, 747, 773, 775, 742, 819, 785, 776, 769, 808, 746, 754, 776, 743, 778, 768, 788, 737, 785, 722, 790, 748, 729, 843, 732, 751, 760, 814, 760, 795, 782, 789, 718, 756, 817, 714, 772, 797, 732, 713, 772, 743, 779, 810, 784, 766, 743, 790, 703, 755, 767, 770, 773, 759, 717, 776, 789, 793] payload_rows_per_dst=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] dup_rows_per_dst=[3154, 3168, 3024, 2996, 3084, 2928, 3012, 2952] combine_rows_per_src=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344304 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969328 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594352 weight_movement_bytes=69206016 total_nvlink_bytes=602800368 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64968 dup_rows=24116 dup_rows_remote=21106 dedup_ratio=0.245321 router_tokens_per_expert=[763, 769, 753, 788, 756, 812, 804, 765, 712, 780, 792, 761, 734, 807, 761, 793, 764, 738, 756, 764, 762, 792, 762, 777, 785, 763, 773, 749, 792, 743, 757, 748, 731, 723, 778, 761, 773, 765, 801, 801, 785, 762, 782, 746, 834, 755, 848, 736, 768, 741, 710, 767, 749, 779, 759, 741, 835, 801, 775, 799, 713, 719, 777, 763] payload_rows_per_dst=[9316, 9260, 9220, 9214, 9322, 9384, 9160, 9312] dup_rows_per_dst=[3104, 3020, 3010, 3006, 2944, 3112, 2868, 3052] combine_rows_per_src=[9316, 9260, 9220, 9214, 9322, 9384, 9160, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=343240 combine_payload_bytes=264847360 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265190600 ingress_bytes=299450368 link_bytes=299450368 token_bytes=530037960 weight_movement_bytes=34603008 total_nvlink_bytes=564640968 weight_to_token_ratio=0.065284 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126807627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64980 dup_rows=24016 dup_rows_remote=20994 dedup_ratio=0.244303 router_tokens_per_expert=[769, 758, 813, 791, 718, 754, 747, 772, 743, 744, 727, 801, 732, 794, 764, 766, 830, 777, 761, 780, 752, 752, 757, 790, 784, 796, 806, 737, 741, 763, 785, 761, 789, 788, 750, 768, 765, 827, 724, 754, 795, 751, 732, 797, 785, 773, 801, 757, 789, 745, 782, 735, 725, 766, 789, 777, 773, 769, 774, 763, 686, 800, 791, 767] payload_rows_per_dst=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] dup_rows_per_dst=[2930, 2916, 3064, 2972, 3022, 3070, 3084, 2958] combine_rows_per_src=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65012 dup_rows=24120 dup_rows_remote=21114 dedup_ratio=0.245361 router_tokens_per_expert=[753, 778, 719, 795, 712, 763, 773, 784, 771, 755, 818, 802, 815, 811, 733, 827, 797, 765, 723, 759, 793, 780, 786, 757, 749, 774, 773, 767, 755, 755, 778, 731, 801, 778, 764, 801, 800, 815, 744, 794, 736, 686, 789, 787, 717, 703, 771, 760, 716, 720, 784, 776, 761, 791, 762, 779, 788, 779, 780, 770, 773, 736, 730, 810] payload_rows_per_dst=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] dup_rows_per_dst=[2942, 3098, 2980, 2912, 3220, 2842, 3006, 3120] combine_rows_per_src=[9212, 9566, 9340, 9252, 9374, 9056, 9172, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=344304 combine_payload_bytes=265297920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265642224 ingress_bytes=334503936 link_bytes=334503936 token_bytes=530940144 weight_movement_bytes=69206016 total_nvlink_bytes=600146160 weight_to_token_ratio=0.130346 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258711381e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65204 dup_rows=23762 dup_rows_remote=20894 dedup_ratio=0.241720 router_tokens_per_expert=[810, 784, 705, 780, 799, 803, 778, 754, 785, 747, 750, 734, 759, 758, 734, 784, 731, 784, 796, 745, 754, 795, 784, 757, 825, 769, 764, 743, 752, 738, 766, 746, 785, 818, 775, 781, 784, 760, 764, 783, 812, 797, 806, 775, 789, 724, 777, 741, 783, 772, 721, 721, 829, 701, 732, 824, 758, 781, 729, 771, 748, 774, 764, 760] payload_rows_per_dst=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] dup_rows_per_dst=[3058, 2868, 3026, 2868, 3054, 3084, 2902, 2902] combine_rows_per_src=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344928 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773344 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201760 weight_movement_bytes=34603008 total_nvlink_bytes=567804768 weight_to_token_ratio=0.064897 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344296 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453224 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562152 weight_movement_bytes=69206016 total_nvlink_bytes=601768168 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64678 dup_rows=24234 dup_rows_remote=21260 dedup_ratio=0.246521 router_tokens_per_expert=[755, 718, 773, 776, 788, 767, 804, 756, 788, 823, 748, 781, 769, 745, 797, 776, 734, 763, 755, 750, 781, 754, 808, 739, 784, 755, 832, 752, 749, 749, 764, 774, 760, 800, 795, 829, 759, 749, 751, 753, 759, 745, 734, 772, 771, 747, 725, 784, 780, 824, 758, 735, 776, 835, 789, 762, 749, 728, 772, 704, 801, 780, 762, 757] payload_rows_per_dst=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] dup_rows_per_dst=[2974, 3128, 2984, 3048, 3058, 2984, 3124, 2934] combine_rows_per_src=[9392, 9364, 9092, 9270, 9334, 9104, 9394, 9120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73986 payload_rows_remote=64660 dup_rows=24318 dup_rows_remote=21150 dedup_ratio=0.247375 router_tokens_per_expert=[786, 773, 777, 771, 751, 762, 840, 747, 773, 775, 742, 819, 785, 776, 769, 808, 746, 754, 776, 743, 778, 768, 788, 737, 785, 722, 790, 748, 729, 843, 732, 751, 760, 814, 760, 795, 782, 789, 718, 756, 817, 714, 772, 797, 732, 713, 772, 743, 779, 810, 784, 766, 743, 790, 703, 755, 767, 770, 773, 759, 717, 776, 789, 793] payload_rows_per_dst=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] dup_rows_per_dst=[3154, 3168, 3024, 2996, 3084, 2928, 3012, 2952] combine_rows_per_src=[9260, 9326, 9156, 9204, 9264, 9192, 9248, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344504 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266633656 ingress_bytes=439304192 link_bytes=439304192 token_bytes=532922808 weight_movement_bytes=173015040 total_nvlink_bytes=705937848 weight_to_token_ratio=0.324653 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653066308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65204 dup_rows=23762 dup_rows_remote=20894 dedup_ratio=0.241720 router_tokens_per_expert=[810, 784, 705, 780, 799, 803, 778, 754, 785, 747, 750, 734, 759, 758, 734, 784, 731, 784, 796, 745, 754, 795, 784, 757, 825, 769, 764, 743, 752, 738, 766, 746, 785, 818, 775, 781, 784, 760, 764, 783, 812, 797, 806, 775, 789, 724, 777, 741, 783, 772, 721, 721, 829, 701, 732, 824, 758, 781, 729, 771, 748, 774, 764, 760] payload_rows_per_dst=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] dup_rows_per_dst=[3058, 2868, 3026, 2868, 3054, 3084, 2902, 2902] combine_rows_per_src=[9368, 9234, 9266, 9338, 9446, 9358, 9264, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64734 dup_rows=24156 dup_rows_remote=20992 dedup_ratio=0.245728 router_tokens_per_expert=[715, 724, 782, 773, 772, 787, 777, 726, 779, 730, 784, 815, 761, 769, 760, 762, 782, 783, 790, 806, 738, 765, 829, 784, 779, 716, 713, 819, 773, 758, 733, 725, 808, 768, 778, 760, 794, 740, 785, 774, 743, 748, 716, 795, 774, 767, 786, 736, 771, 791, 784, 770, 797, 777, 765, 706, 776, 753, 837, 745, 779, 771, 785, 764] payload_rows_per_dst=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] dup_rows_per_dst=[2982, 3030, 3096, 2888, 3086, 2826, 3084, 3164] combine_rows_per_src=[9070, 9290, 9416, 9144, 9328, 9098, 9388, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64850 dup_rows=24146 dup_rows_remote=21144 dedup_ratio=0.245626 router_tokens_per_expert=[800, 768, 760, 748, 789, 730, 744, 819, 757, 803, 770, 732, 789, 758, 751, 740, 776, 780, 789, 798, 767, 743, 721, 732, 768, 741, 787, 786, 749, 765, 775, 768, 794, 822, 735, 776, 764, 783, 770, 732, 786, 751, 764, 782, 746, 751, 761, 780, 741, 748, 742, 781, 761, 782, 838, 762, 791, 756, 783, 741, 784, 751, 808, 783] payload_rows_per_dst=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] dup_rows_per_dst=[2984, 2948, 3018, 2996, 3100, 3022, 3002, 3076] combine_rows_per_src=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64988 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[725, 749, 776, 763, 731, 759, 786, 827, 783, 736, 773, 762, 797, 847, 759, 760, 774, 764, 752, 779, 753, 749, 808, 770, 747, 764, 735, 763, 775, 758, 756, 784, 793, 786, 751, 778, 760, 775, 791, 774, 767, 739, 753, 773, 779, 732, 788, 795, 733, 768, 743, 721, 742, 829, 777, 762, 761, 765, 761, 777, 793, 772, 803, 747] payload_rows_per_dst=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] dup_rows_per_dst=[2950, 3052, 3056, 3010, 3050, 3034, 2874, 3072] combine_rows_per_src=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262822600 prefetch_bytes=761266176 reduce_bytes=415236096 weight_movement_bytes=1176502272 egress_bytes=6135370440 ingress_bytes=7303954432 link_bytes=7303954432 total_nvlink_bytes=13439324872 weight_to_token_ratio=0.095941 prefetch_share_of_ingress=0.104227 dedup_ratio=0.243957 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=12 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.194964892e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64720 dup_rows=24246 dup_rows_remote=21156 dedup_ratio=0.246643 router_tokens_per_expert=[756, 816, 727, 788, 770, 761, 802, 781, 734, 767, 756, 756, 747, 781, 753, 723, 794, 769, 733, 807, 748, 728, 787, 733, 786, 743, 786, 682, 754, 744, 787, 766, 772, 750, 789, 704, 765, 767, 800, 809, 752, 762, 796, 756, 767, 796, 769, 799, 823, 762, 776, 808, 757, 768, 781, 786, 785, 699, 804, 774, 771, 808, 752, 780] payload_rows_per_dst=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] dup_rows_per_dst=[3090, 3112, 2910, 3290, 2758, 3144, 3084, 2858] combine_rows_per_src=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343752 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264840 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185928 weight_movement_bytes=34603008 total_nvlink_bytes=564788936 weight_to_token_ratio=0.065266 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343896 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501976 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660056 weight_movement_bytes=69206016 total_nvlink_bytes=601866072 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264847360 dispatch_meta_bytes=343240 combine_payload_bytes=264847360 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265190600 ingress_bytes=299450368 link_bytes=299450368 token_bytes=530037960 weight_movement_bytes=34603008 total_nvlink_bytes=564640968 weight_to_token_ratio=0.065284 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126807627e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64980 dup_rows=24016 dup_rows_remote=20994 dedup_ratio=0.244303 router_tokens_per_expert=[769, 758, 813, 791, 718, 754, 747, 772, 743, 744, 727, 801, 732, 794, 764, 766, 830, 777, 761, 780, 752, 752, 757, 790, 784, 796, 806, 737, 741, 763, 785, 761, 789, 788, 750, 768, 765, 827, 724, 754, 795, 751, 732, 797, 785, 773, 801, 757, 789, 745, 782, 735, 725, 766, 789, 777, 773, 769, 774, 763, 686, 800, 791, 767] payload_rows_per_dst=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] dup_rows_per_dst=[2930, 2916, 3064, 2972, 3022, 3070, 3084, 2958] combine_rows_per_src=[9314, 9226, 9334, 9374, 9308, 9312, 9132, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344392 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267419976 ingress_bytes=301678592 link_bytes=301678592 token_bytes=534495560 weight_movement_bytes=34603008 total_nvlink_bytes=569098568 weight_to_token_ratio=0.064740 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135192255e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262822600 prefetch_bytes=761266176 reduce_bytes=415236096 weight_movement_bytes=1176502272 egress_bytes=6135370440 ingress_bytes=7303954432 link_bytes=7303954432 total_nvlink_bytes=13439324872 weight_to_token_ratio=0.095941 prefetch_share_of_ingress=0.104227 dedup_ratio=0.243957 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=12 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028152 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.194964892e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64988 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[725, 749, 776, 763, 731, 759, 786, 827, 783, 736, 773, 762, 797, 847, 759, 760, 774, 764, 752, 779, 753, 749, 808, 770, 747, 764, 735, 763, 775, 758, 756, 784, 793, 786, 751, 778, 760, 775, 791, 774, 767, 739, 753, 773, 779, 732, 788, 795, 733, 768, 743, 721, 742, 829, 777, 762, 761, 765, 761, 777, 793, 772, 803, 747] payload_rows_per_dst=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] dup_rows_per_dst=[2950, 3052, 3056, 3010, 3050, 3034, 2874, 3072] combine_rows_per_src=[9282, 9382, 9242, 9154, 9366, 9218, 9276, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=342904 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265493368 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530643832 weight_movement_bytes=69206016 total_nvlink_bytes=599849848 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343976 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969576 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595176 weight_movement_bytes=34603008 total_nvlink_bytes=566198184 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64850 dup_rows=24146 dup_rows_remote=21144 dedup_ratio=0.245626 router_tokens_per_expert=[800, 768, 760, 748, 789, 730, 744, 819, 757, 803, 770, 732, 789, 758, 751, 740, 776, 780, 789, 798, 767, 743, 721, 732, 768, 741, 787, 786, 749, 765, 775, 768, 794, 822, 735, 776, 764, 783, 770, 732, 786, 751, 764, 782, 746, 751, 761, 780, 741, 748, 742, 781, 761, 782, 838, 762, 791, 756, 783, 741, 784, 751, 808, 783] payload_rows_per_dst=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] dup_rows_per_dst=[2984, 2948, 3018, 2996, 3100, 3022, 3002, 3076] combine_rows_per_src=[9332, 9252, 9194, 9282, 9252, 9220, 9308, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343896 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501976 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660056 weight_movement_bytes=69206016 total_nvlink_bytes=601866072 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344208 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266535056 ingress_bytes=369999872 link_bytes=369999872 token_bytes=532725904 weight_movement_bytes=103809024 total_nvlink_bytes=636534928 weight_to_token_ratio=0.194864 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392279731e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65068 dup_rows=23966 dup_rows_remote=20890 dedup_ratio=0.243795 router_tokens_per_expert=[752, 745, 768, 736, 780, 777, 766, 769, 750, 767, 781, 774, 762, 767, 757, 815, 757, 747, 797, 743, 738, 787, 746, 767, 779, 831, 734, 786, 784, 762, 779, 725, 745, 789, 803, 754, 765, 798, 765, 752, 731, 782, 779, 739, 747, 779, 753, 776, 797, 800, 769, 764, 774, 749, 775, 792, 774, 778, 750, 786, 760, 791, 738, 770] payload_rows_per_dst=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] dup_rows_per_dst=[2922, 3076, 2962, 3012, 3048, 2966, 3074, 2906] combine_rows_per_src=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12252499184 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6130207984 ingress_bytes=7437205504 link_bytes=7437205504 total_nvlink_bytes=13567413488 weight_to_token_ratio=0.107318 prefetch_share_of_ingress=0.093054 dedup_ratio=0.244641 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216765459e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343504 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265436624 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530529744 weight_movement_bytes=69206016 total_nvlink_bytes=599735760 weight_to_token_ratio=0.130447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12267437256 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6137674952 ingress_bytes=7548485632 link_bytes=7548485632 total_nvlink_bytes=13686160584 weight_to_token_ratio=0.115650 prefetch_share_of_ingress=0.082514 dedup_ratio=0.243349 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234971467e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233130616 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6120514168 ingress_bytes=7565942784 link_bytes=7565942784 total_nvlink_bytes=13686456952 weight_to_token_ratio=0.118802 prefetch_share_of_ingress=0.086897 dedup_ratio=0.244675 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.237827547e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261720312 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6134817016 ingress_bytes=7476420608 link_bytes=7476420608 total_nvlink_bytes=13611237624 weight_to_token_ratio=0.110059 prefetch_share_of_ingress=0.097194 dedup_ratio=0.244081 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223181254e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65094 dup_rows=23944 dup_rows_remote=20982 dedup_ratio=0.243571 router_tokens_per_expert=[743, 765, 766, 783, 747, 771, 785, 748, 784, 750, 757, 759, 803, 812, 732, 761, 746, 726, 770, 775, 792, 816, 703, 723, 728, 747, 732, 779, 712, 764, 808, 733, 748, 777, 770, 783, 731, 760, 815, 816, 778, 734, 768, 799, 765, 753, 800, 809, 773, 759, 783, 807, 766, 736, 746, 783, 755, 779, 800, 785, 797, 791, 786, 780] payload_rows_per_dst=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] dup_rows_per_dst=[2962, 2938, 2976, 2952, 3040, 3060, 2900, 3116] combine_rows_per_src=[9266, 9378, 9126, 9328, 9420, 9352, 9060, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343976 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969576 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595176 weight_movement_bytes=34603008 total_nvlink_bytes=566198184 weight_to_token_ratio=0.065093 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262195000 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135054136 ingress_bytes=7511261184 link_bytes=7511261184 total_nvlink_bytes=13646315320 weight_to_token_ratio=0.112877 prefetch_share_of_ingress=0.082923 dedup_ratio=0.243981 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228881353e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12252499184 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6130207984 ingress_bytes=7437205504 link_bytes=7437205504 total_nvlink_bytes=13567413488 weight_to_token_ratio=0.107318 prefetch_share_of_ingress=0.093054 dedup_ratio=0.244641 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.216765459e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65068 dup_rows=23966 dup_rows_remote=20890 dedup_ratio=0.243795 router_tokens_per_expert=[752, 745, 768, 736, 780, 777, 766, 769, 750, 767, 781, 774, 762, 767, 757, 815, 757, 747, 797, 743, 738, 787, 746, 767, 779, 831, 734, 786, 784, 762, 779, 725, 745, 789, 803, 754, 765, 798, 765, 752, 731, 782, 779, 739, 747, 779, 753, 776, 797, 800, 769, 764, 774, 749, 775, 792, 774, 778, 750, 786, 760, 791, 738, 770] payload_rows_per_dst=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] dup_rows_per_dst=[2922, 3076, 2962, 3012, 3048, 2966, 3074, 2906] combine_rows_per_src=[9264, 9270, 9202, 9348, 9294, 9206, 9366, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344304 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969328 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594352 weight_movement_bytes=69206016 total_nvlink_bytes=602800368 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261720312 prefetch_bytes=726663168 reduce_bytes=622854144 weight_movement_bytes=1349517312 egress_bytes=6134817016 ingress_bytes=7476420608 link_bytes=7476420608 total_nvlink_bytes=13611237624 weight_to_token_ratio=0.110059 prefetch_share_of_ingress=0.097194 dedup_ratio=0.244081 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=18 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223181254e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343832 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266862360 ingress_bytes=301121536 link_bytes=301121536 token_bytes=533380888 weight_movement_bytes=34603008 total_nvlink_bytes=567983896 weight_to_token_ratio=0.064875 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133096098e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233329296 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6120614544 ingress_bytes=7531438080 link_bytes=7531438080 total_nvlink_bytes=13652052624 weight_to_token_ratio=0.115972 prefetch_share_of_ingress=0.096484 dedup_ratio=0.244823 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232182398e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266946016 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6137429472 ingress_bytes=7548239872 link_bytes=7548239872 total_nvlink_bytes=13685669344 weight_to_token_ratio=0.115654 prefetch_share_of_ingress=0.087101 dedup_ratio=0.243662 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234931259e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:56:37] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 792.1 | throughput per GPU (TFLOP/s/GPU): 161.1 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.098793E+01 | loss scale: 1.0 | grad norm: 435.853 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64720 dup_rows=24246 dup_rows_remote=21156 dedup_ratio=0.246643 router_tokens_per_expert=[756, 816, 727, 788, 770, 761, 802, 781, 734, 767, 756, 756, 747, 781, 753, 723, 794, 769, 733, 807, 748, 728, 787, 733, 786, 743, 786, 682, 754, 744, 787, 766, 772, 750, 789, 704, 765, 767, 800, 809, 752, 762, 796, 756, 767, 796, 769, 799, 823, 762, 776, 808, 757, 768, 781, 786, 785, 699, 804, 774, 771, 808, 752, 780] payload_rows_per_dst=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] dup_rows_per_dst=[3090, 3112, 2910, 3290, 2758, 3144, 3084, 2858] combine_rows_per_src=[9338, 9368, 9288, 9408, 8926, 9250, 9438, 9042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343504 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265436624 ingress_bytes=334299136 link_bytes=334299136 token_bytes=530529744 weight_movement_bytes=69206016 total_nvlink_bytes=599735760 weight_to_token_ratio=0.130447 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257940735e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233329296 prefetch_bytes=726663168 reduce_bytes=692060160 weight_movement_bytes=1418723328 egress_bytes=6120614544 ingress_bytes=7531438080 link_bytes=7531438080 total_nvlink_bytes=13652052624 weight_to_token_ratio=0.115972 prefetch_share_of_ingress=0.096484 dedup_ratio=0.244823 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=20 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232182398e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-26 18:08:42] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 797.4 | throughput per GPU (TFLOP/s/GPU): 160.0 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.098793E+01 | loss scale: 1.0 | grad norm: 435.853 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64914 dup_rows=24148 dup_rows_remote=21040 dedup_ratio=0.245646 router_tokens_per_expert=[818, 694, 803, 808, 744, 780, 761, 714, 771, 761, 803, 735, 753, 775, 760, 771, 741, 724, 760, 785, 788, 749, 753, 775, 717, 800, 763, 772, 780, 808, 772, 763, 790, 769, 753, 717, 781, 806, 775, 740, 816, 757, 713, 762, 756, 797, 739, 831, 832, 771, 815, 791, 750, 741, 757, 768, 800, 742, 761, 786, 706, 807, 745, 777] payload_rows_per_dst=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] dup_rows_per_dst=[2998, 2996, 2892, 3108, 3032, 3036, 3108, 2978] combine_rows_per_src=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65184 dup_rows=23834 dup_rows_remote=20870 dedup_ratio=0.242452 router_tokens_per_expert=[799, 805, 774, 730, 767, 836, 752, 791, 770, 755, 816, 735, 758, 777, 746, 768, 796, 732, 748, 766, 769, 785, 770, 784, 729, 778, 726, 786, 760, 736, 760, 805, 765, 773, 752, 738, 753, 773, 759, 796, 750, 816, 772, 742, 802, 767, 714, 743, 724, 736, 763, 775, 793, 848, 784, 763, 789, 755, 774, 769, 724, 771, 767, 793] payload_rows_per_dst=[9378, 9286, 9306, 9250, 9250, 9286, 9312, 9402] dup_rows_per_dst=[3130, 2964, 2994, 2910, 2968, 2926, 3060, 2882] combine_rows_per_src=[9378, 9286, 9306, 9250, 9250, 9286, 9312, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64890 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[772, 747, 781, 773, 733, 763, 744, 735, 744, 742, 786, 805, 806, 768, 844, 805, 757, 754, 751, 791, 778, 725, 759, 757, 697, 730, 785, 796, 803, 723, 783, 787, 784, 806, 793, 735, 792, 767, 740, 801, 770, 791, 795, 761, 770, 741, 760, 748, 779, 812, 753, 762, 813, 781, 779, 771, 718, 748, 768, 746, 763, 771, 734, 776] payload_rows_per_dst=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] dup_rows_per_dst=[2960, 3120, 2964, 3028, 3108, 3008, 3118, 2844] combine_rows_per_src=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65140 dup_rows=23926 dup_rows_remote=21004 dedup_ratio=0.243388 router_tokens_per_expert=[763, 770, 806, 740, 761, 758, 779, 781, 738, 802, 733, 780, 736, 730, 778, 813, 806, 776, 756, 859, 794, 789, 784, 770, 794, 736, 778, 740, 755, 743, 796, 779, 806, 763, 752, 726, 795, 764, 725, 749, 747, 775, 758, 705, 806, 797, 767, 763, 767, 803, 750, 745, 807, 727, 767, 739, 782, 752, 802, 754, 762, 770, 753, 751] payload_rows_per_dst=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] dup_rows_per_dst=[3026, 2948, 3122, 3026, 2922, 3030, 2922, 2930] combine_rows_per_src=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64644 dup_rows=24128 dup_rows_remote=20738 dedup_ratio=0.245443 router_tokens_per_expert=[814, 758, 812, 743, 732, 739, 779, 759, 745, 712, 752, 790, 812, 792, 832, 777, 706, 727, 769, 730, 770, 722, 760, 727, 779, 751, 754, 741, 725, 807, 760, 758, 760, 739, 796, 761, 723, 753, 789, 777, 784, 790, 832, 801, 775, 812, 767, 751, 749, 768, 785, 733, 799, 770, 788, 769, 823, 785, 771, 788, 758, 778, 772, 772] payload_rows_per_dst=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] dup_rows_per_dst=[3018, 2826, 2798, 2984, 2962, 3222, 2928, 3390] combine_rows_per_src=[9254, 9132, 9024, 9166, 9234, 9440, 9394, 9532] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64914 dup_rows=24148 dup_rows_remote=21040 dedup_ratio=0.245646 router_tokens_per_expert=[818, 694, 803, 808, 744, 780, 761, 714, 771, 761, 803, 735, 753, 775, 760, 771, 741, 724, 760, 785, 788, 749, 753, 775, 717, 800, 763, 772, 780, 808, 772, 763, 790, 769, 753, 717, 781, 806, 775, 740, 816, 757, 713, 762, 756, 797, 739, 831, 832, 771, 815, 791, 750, 741, 757, 768, 800, 742, 761, 786, 706, 807, 745, 777] payload_rows_per_dst=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] dup_rows_per_dst=[2998, 2996, 2892, 3108, 3032, 3036, 3108, 2978] combine_rows_per_src=[9246, 9262, 9258, 9242, 9230, 9306, 9342, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64890 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[772, 747, 781, 773, 733, 763, 744, 735, 744, 742, 786, 805, 806, 768, 844, 805, 757, 754, 751, 791, 778, 725, 759, 757, 697, 730, 785, 796, 803, 723, 783, 787, 784, 806, 793, 735, 792, 767, 740, 801, 770, 791, 795, 761, 770, 741, 760, 748, 779, 812, 753, 762, 813, 781, 779, 771, 718, 748, 768, 746, 763, 771, 734, 776] payload_rows_per_dst=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] dup_rows_per_dst=[2960, 3120, 2964, 3028, 3108, 3008, 3118, 2844] combine_rows_per_src=[9136, 9480, 9180, 9180, 9328, 9264, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65140 dup_rows=23926 dup_rows_remote=21004 dedup_ratio=0.243388 router_tokens_per_expert=[763, 770, 806, 740, 761, 758, 779, 781, 738, 802, 733, 780, 736, 730, 778, 813, 806, 776, 756, 859, 794, 789, 784, 770, 794, 736, 778, 740, 755, 743, 796, 779, 806, 763, 752, 726, 795, 764, 725, 749, 747, 775, 758, 705, 806, 797, 767, 763, 767, 803, 750, 745, 807, 727, 767, 739, 782, 752, 802, 754, 762, 770, 753, 751] payload_rows_per_dst=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] dup_rows_per_dst=[3026, 2948, 3122, 3026, 2922, 3030, 2922, 2930] combine_rows_per_src=[9290, 9272, 9546, 9216, 9238, 9206, 9288, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65038 dup_rows=23872 dup_rows_remote=20734 dedup_ratio=0.242839 router_tokens_per_expert=[733, 778, 772, 776, 729, 737, 761, 768, 781, 751, 773, 771, 793, 768, 771, 719, 810, 792, 781, 768, 759, 846, 744, 766, 796, 751, 810, 771, 767, 755, 771, 744, 801, 767, 735, 766, 776, 742, 741, 758, 790, 792, 743, 716, 793, 771, 785, 750, 759, 750, 774, 789, 827, 759, 765, 752, 817, 768, 764, 794, 742, 770, 739, 745] payload_rows_per_dst=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] dup_rows_per_dst=[2914, 3090, 3138, 2946, 2876, 2956, 3022, 2930] combine_rows_per_src=[9194, 9164, 9394, 9384, 9296, 9324, 9328, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344216 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267337880 ingress_bytes=370802688 link_bytes=370802688 token_bytes=534331544 weight_movement_bytes=103809024 total_nvlink_bytes=638140568 weight_to_token_ratio=0.194278 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395300663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64966 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[813, 748, 796, 752, 804, 702, 799, 703, 787, 778, 800, 754, 777, 780, 774, 719, 781, 769, 744, 793, 777, 818, 798, 798, 756, 795, 745, 809, 756, 756, 757, 749, 737, 745, 778, 738, 783, 779, 782, 798, 794, 787, 762, 756, 727, 774, 760, 766, 770, 765, 768, 797, 769, 769, 764, 755, 741, 779, 764, 778, 756, 748, 719, 757] payload_rows_per_dst=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] dup_rows_per_dst=[2864, 3074, 3058, 3076, 2960, 3038, 3022, 2954] combine_rows_per_src=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344128 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133568 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923008 weight_movement_bytes=69206016 total_nvlink_bytes=601129024 weight_to_token_ratio=0.130105 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341528 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123352 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905176 weight_movement_bytes=34603008 total_nvlink_bytes=564508184 weight_to_token_ratio=0.065300 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343816 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231560 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119304 weight_movement_bytes=34603008 total_nvlink_bytes=566722312 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344216 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267337880 ingress_bytes=370802688 link_bytes=370802688 token_bytes=534331544 weight_movement_bytes=103809024 total_nvlink_bytes=638140568 weight_to_token_ratio=0.194278 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395300663e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344576 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158016 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971456 weight_movement_bytes=34603008 total_nvlink_bytes=568574464 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=341528 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265123352 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529905176 weight_movement_bytes=34603008 total_nvlink_bytes=564508184 weight_to_token_ratio=0.065300 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344128 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133568 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923008 weight_movement_bytes=69206016 total_nvlink_bytes=601129024 weight_to_token_ratio=0.130105 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64990 dup_rows=23978 dup_rows_remote=20788 dedup_ratio=0.243917 router_tokens_per_expert=[774, 782, 791, 792, 725, 798, 724, 724, 784, 795, 738, 776, 763, 775, 823, 758, 821, 765, 778, 790, 758, 720, 771, 803, 790, 776, 734, 750, 751, 765, 745, 726, 769, 783, 784, 780, 760, 767, 746, 758, 772, 790, 793, 808, 736, 745, 745, 778, 735, 786, 839, 781, 796, 710, 765, 767, 763, 774, 775, 794, 743, 725, 755, 765] payload_rows_per_dst=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] dup_rows_per_dst=[3190, 3084, 2964, 3110, 3148, 2834, 2708, 2940] combine_rows_per_src=[9336, 9342, 9168, 9508, 9432, 9214, 9078, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343088 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738736 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134384 weight_movement_bytes=69206016 total_nvlink_bytes=602340400 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343960 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266444696 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532545432 weight_movement_bytes=103809024 total_nvlink_bytes=636354456 weight_to_token_ratio=0.194930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64966 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[813, 748, 796, 752, 804, 702, 799, 703, 787, 778, 800, 754, 777, 780, 774, 719, 781, 769, 744, 793, 777, 818, 798, 798, 756, 795, 745, 809, 756, 756, 757, 749, 737, 745, 778, 738, 783, 779, 782, 798, 794, 787, 762, 756, 727, 774, 760, 766, 770, 765, 768, 797, 769, 769, 764, 755, 741, 779, 764, 778, 756, 748, 719, 757] payload_rows_per_dst=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] dup_rows_per_dst=[2864, 3074, 3058, 3076, 2960, 3038, 3022, 2954] combine_rows_per_src=[9370, 9264, 9498, 9170, 9320, 9214, 9292, 9130] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65098 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[770, 774, 740, 753, 740, 787, 742, 778, 746, 756, 774, 767, 755, 810, 743, 770, 780, 811, 748, 776, 757, 800, 800, 783, 717, 724, 817, 756, 780, 761, 792, 759, 822, 767, 804, 752, 776, 802, 748, 756, 747, 752, 756, 768, 802, 752, 744, 762, 746, 750, 800, 791, 795, 756, 778, 746, 722, 822, 759, 787, 779, 735, 766, 744] payload_rows_per_dst=[9276, 9220, 9386, 9200, 9402, 9172, 9386, 9276] dup_rows_per_dst=[2892, 3022, 3124, 3012, 3052, 2994, 2938, 2952] combine_rows_per_src=[9276, 9220, 9386, 9200, 9402, 9172, 9386, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65138 dup_rows=23950 dup_rows_remote=21076 dedup_ratio=0.243632 router_tokens_per_expert=[769, 743, 772, 781, 800, 789, 765, 741, 793, 780, 773, 787, 772, 673, 772, 794, 746, 769, 798, 748, 745, 794, 761, 816, 797, 778, 743, 771, 777, 753, 778, 804, 722, 761, 785, 705, 764, 759, 798, 772, 743, 766, 710, 773, 779, 751, 748, 775, 802, 780, 795, 814, 728, 766, 757, 780, 768, 770, 727, 766, 777, 815, 801, 713] payload_rows_per_dst=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] dup_rows_per_dst=[3000, 3008, 3000, 3012, 2896, 2874, 3116, 3044] combine_rows_per_src=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343112 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542152 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741192 weight_movement_bytes=34603008 total_nvlink_bytes=567344200 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64802 dup_rows=24136 dup_rows_remote=20994 dedup_ratio=0.245524 router_tokens_per_expert=[712, 735, 777, 734, 742, 744, 770, 782, 741, 756, 763, 783, 815, 802, 779, 766, 789, 740, 747, 810, 754, 735, 782, 733, 775, 782, 765, 763, 752, 792, 769, 831, 747, 833, 783, 785, 730, 780, 763, 741, 735, 796, 754, 792, 785, 765, 776, 778, 779, 806, 772, 782, 733, 787, 724, 796, 729, 739, 789, 773, 779, 767, 758, 776] payload_rows_per_dst=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] dup_rows_per_dst=[2786, 2990, 2692, 3108, 3018, 3292, 3108, 3142] combine_rows_per_src=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64994 dup_rows=24006 dup_rows_remote=21062 dedup_ratio=0.244202 router_tokens_per_expert=[767, 733, 753, 771, 764, 779, 762, 780, 808, 780, 792, 776, 744, 812, 787, 828, 737, 743, 796, 768, 768, 790, 754, 804, 802, 773, 737, 734, 793, 780, 751, 754, 784, 764, 754, 794, 718, 797, 746, 779, 772, 769, 778, 747, 760, 740, 797, 779, 786, 776, 757, 745, 733, 780, 786, 812, 731, 736, 740, 796, 697, 791, 771, 717] payload_rows_per_dst=[9310, 9364, 9334, 9304, 9432, 9304, 9230, 9020] dup_rows_per_dst=[2908, 3290, 2986, 2944, 2840, 2980, 3120, 2938] combine_rows_per_src=[9310, 9364, 9334, 9304, 9432, 9304, 9230, 9020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65102 dup_rows=24064 dup_rows_remote=21106 dedup_ratio=0.244792 router_tokens_per_expert=[779, 737, 802, 773, 748, 734, 716, 743, 746, 772, 750, 759, 759, 760, 796, 759, 786, 763, 770, 811, 808, 789, 803, 744, 772, 765, 771, 800, 784, 767, 766, 763, 760, 769, 737, 760, 697, 780, 800, 745, 761, 768, 760, 761, 755, 773, 793, 798, 769, 764, 797, 772, 731, 790, 781, 787, 790, 770, 776, 755, 736, 791, 776, 755] payload_rows_per_dst=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] dup_rows_per_dst=[2842, 3008, 3044, 3044, 2958, 3024, 3094, 3050] combine_rows_per_src=[9222, 9194, 9504, 9332, 9138, 9314, 9288, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344248 combine_payload_bytes=266641408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266985656 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533627064 weight_movement_bytes=103809024 total_nvlink_bytes=637436088 weight_to_token_ratio=0.194535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344856 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267150104 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955352 weight_movement_bytes=34603008 total_nvlink_bytes=568558360 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343112 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266542152 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532741192 weight_movement_bytes=34603008 total_nvlink_bytes=567344200 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65138 dup_rows=23950 dup_rows_remote=21076 dedup_ratio=0.243632 router_tokens_per_expert=[769, 743, 772, 781, 800, 789, 765, 741, 793, 780, 773, 787, 772, 673, 772, 794, 746, 769, 798, 748, 745, 794, 761, 816, 797, 778, 743, 771, 777, 753, 778, 804, 722, 761, 785, 705, 764, 759, 798, 772, 743, 766, 710, 773, 779, 751, 748, 775, 802, 780, 795, 814, 728, 766, 757, 780, 768, 770, 727, 766, 777, 815, 801, 713] payload_rows_per_dst=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] dup_rows_per_dst=[3000, 3008, 3000, 3012, 2896, 2874, 3116, 3044] combine_rows_per_src=[9320, 9280, 9354, 9390, 9236, 9216, 9328, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64802 dup_rows=24136 dup_rows_remote=20994 dedup_ratio=0.245524 router_tokens_per_expert=[712, 735, 777, 734, 742, 744, 770, 782, 741, 756, 763, 783, 815, 802, 779, 766, 789, 740, 747, 810, 754, 735, 782, 733, 775, 782, 765, 763, 752, 792, 769, 831, 747, 833, 783, 785, 730, 780, 763, 741, 735, 796, 754, 792, 785, 765, 776, 778, 779, 806, 772, 782, 733, 787, 724, 796, 729, 739, 789, 773, 779, 767, 758, 776] payload_rows_per_dst=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] dup_rows_per_dst=[2786, 2990, 2692, 3108, 3018, 3292, 3108, 3142] combine_rows_per_src=[9206, 9262, 8782, 9368, 9306, 9556, 9322, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64938 dup_rows=23992 dup_rows_remote=20916 dedup_ratio=0.244059 router_tokens_per_expert=[817, 792, 791, 759, 801, 822, 740, 793, 830, 754, 764, 755, 741, 760, 761, 765, 797, 741, 794, 790, 769, 791, 796, 747, 769, 759, 758, 756, 781, 758, 751, 757, 769, 768, 763, 767, 780, 772, 743, 763, 763, 781, 726, 770, 777, 789, 783, 768, 750, 808, 711, 731, 725, 801, 756, 763, 713, 753, 789, 746, 801, 742, 755, 767] payload_rows_per_dst=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] dup_rows_per_dst=[3186, 3032, 3076, 2984, 2978, 2922, 2880, 2934] combine_rows_per_src=[9444, 9228, 9374, 9194, 9272, 9392, 9210, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343184 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265772176 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531201168 weight_movement_bytes=34603008 total_nvlink_bytes=565804176 weight_to_token_ratio=0.065141 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343960 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266444696 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532545432 weight_movement_bytes=103809024 total_nvlink_bytes=636354456 weight_to_token_ratio=0.194930 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344248 combine_payload_bytes=266641408 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266985656 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533627064 weight_movement_bytes=103809024 total_nvlink_bytes=637436088 weight_to_token_ratio=0.194535 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344224 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559648 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532775072 weight_movement_bytes=69206016 total_nvlink_bytes=601981088 weight_to_token_ratio=0.129897 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344832 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002624 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660416 weight_movement_bytes=34603008 total_nvlink_bytes=568263424 weight_to_token_ratio=0.064841 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65328 dup_rows=23716 dup_rows_remote=20658 dedup_ratio=0.241252 router_tokens_per_expert=[738, 787, 743, 744, 816, 793, 775, 800, 796, 771, 746, 746, 762, 771, 734, 768, 783, 792, 795, 785, 761, 749, 796, 723, 787, 775, 785, 738, 759, 770, 784, 741, 763, 728, 733, 747, 779, 751, 776, 820, 779, 804, 708, 739, 744, 745, 725, 762, 782, 807, 787, 722, 783, 758, 753, 767, 770, 744, 804, 854, 768, 766, 760, 811] payload_rows_per_dst=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] dup_rows_per_dst=[2936, 2904, 3036, 3012, 2916, 2888, 3058, 2966] combine_rows_per_src=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344856 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267150104 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955352 weight_movement_bytes=34603008 total_nvlink_bytes=568558360 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343184 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265772176 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531201168 weight_movement_bytes=34603008 total_nvlink_bytes=565804176 weight_to_token_ratio=0.065141 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343416 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329464 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315512 weight_movement_bytes=103809024 total_nvlink_bytes=636124536 weight_to_token_ratio=0.195014 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64792 dup_rows=24038 dup_rows_remote=20992 dedup_ratio=0.244527 router_tokens_per_expert=[807, 790, 733, 789, 753, 766, 763, 758, 736, 768, 781, 760, 817, 727, 785, 735, 764, 768, 767, 768, 781, 750, 762, 780, 765, 796, 781, 770, 748, 739, 804, 765, 778, 781, 772, 792, 773, 729, 751, 699, 783, 736, 735, 777, 730, 775, 813, 777, 793, 746, 771, 761, 806, 745, 823, 754, 763, 786, 769, 791, 805, 731, 767, 764] payload_rows_per_dst=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] dup_rows_per_dst=[3046, 2982, 2950, 2972, 3154, 3086, 2900, 2948] combine_rows_per_src=[9474, 9236, 9330, 9162, 9428, 9166, 9066, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65136 dup_rows=23906 dup_rows_remote=21076 dedup_ratio=0.243184 router_tokens_per_expert=[769, 806, 792, 816, 772, 769, 795, 714, 832, 754, 742, 764, 744, 733, 754, 723, 761, 748, 814, 772, 791, 786, 806, 738, 762, 769, 807, 754, 795, 790, 790, 796, 719, 797, 754, 763, 776, 737, 693, 778, 791, 777, 761, 749, 779, 734, 755, 758, 767, 787, 729, 815, 777, 779, 767, 733, 773, 775, 750, 727, 772, 790, 750, 782] payload_rows_per_dst=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] dup_rows_per_dst=[3026, 2830, 3008, 3222, 2806, 2962, 3042, 3010] combine_rows_per_src=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=343944 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267927432 ingress_bytes=336789504 link_bytes=336789504 token_bytes=535510920 weight_movement_bytes=69206016 total_nvlink_bytes=604716936 weight_to_token_ratio=0.129234 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267311790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65034 dup_rows=23958 dup_rows_remote=21016 dedup_ratio=0.243713 router_tokens_per_expert=[737, 796, 795, 817, 773, 725, 805, 788, 790, 784, 730, 805, 766, 801, 748, 759, 725, 755, 791, 756, 796, 759, 777, 747, 761, 713, 808, 762, 781, 759, 805, 775, 743, 764, 764, 778, 772, 787, 751, 730, 764, 800, 801, 731, 731, 778, 730, 792, 722, 775, 771, 746, 749, 779, 778, 776, 717, 813, 760, 791, 735, 799, 765, 771] payload_rows_per_dst=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] dup_rows_per_dst=[3068, 3020, 3014, 2984, 2914, 2942, 2964, 3052] combine_rows_per_src=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65136 dup_rows=23906 dup_rows_remote=21076 dedup_ratio=0.243184 router_tokens_per_expert=[769, 806, 792, 816, 772, 769, 795, 714, 832, 754, 742, 764, 744, 733, 754, 723, 761, 748, 814, 772, 791, 786, 806, 738, 762, 769, 807, 754, 795, 790, 790, 796, 719, 797, 754, 763, 776, 737, 693, 778, 791, 777, 761, 749, 779, 734, 755, 758, 767, 787, 729, 815, 777, 779, 767, 733, 773, 775, 750, 727, 772, 790, 750, 782] payload_rows_per_dst=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] dup_rows_per_dst=[3026, 2830, 3008, 3222, 2806, 2962, 3042, 3010] combine_rows_per_src=[9440, 9262, 9424, 9304, 9228, 9246, 9266, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65028 dup_rows=24044 dup_rows_remote=20944 dedup_ratio=0.244588 router_tokens_per_expert=[746, 766, 793, 758, 753, 789, 778, 762, 747, 765, 784, 806, 737, 789, 793, 788, 827, 746, 762, 784, 763, 756, 738, 755, 778, 807, 772, 734, 748, 752, 791, 776, 774, 809, 772, 719, 758, 772, 803, 759, 718, 774, 785, 784, 778, 739, 787, 789, 751, 736, 735, 824, 736, 799, 783, 770, 755, 778, 750, 740, 785, 731, 762, 754] payload_rows_per_dst=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] dup_rows_per_dst=[2934, 3036, 3044, 2968, 3100, 3058, 3014, 2890] combine_rows_per_src=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=65028 dup_rows=24104 dup_rows_remote=21146 dedup_ratio=0.245199 router_tokens_per_expert=[776, 760, 750, 775, 786, 778, 729, 785, 764, 771, 780, 777, 806, 779, 735, 756, 815, 741, 851, 755, 840, 854, 784, 795, 769, 803, 753, 710, 758, 759, 742, 771, 795, 722, 752, 740, 710, 754, 725, 732, 753, 784, 819, 691, 744, 814, 773, 725, 760, 784, 800, 739, 763, 782, 742, 788, 725, 761, 758, 751, 755, 765, 834, 805] payload_rows_per_dst=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] dup_rows_per_dst=[2972, 2946, 3322, 2958, 2860, 2970, 3000, 3076] combine_rows_per_src=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343136 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731168 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119200 weight_movement_bytes=34603008 total_nvlink_bytes=565722208 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65328 dup_rows=23716 dup_rows_remote=20658 dedup_ratio=0.241252 router_tokens_per_expert=[738, 787, 743, 744, 816, 793, 775, 800, 796, 771, 746, 746, 762, 771, 734, 768, 783, 792, 795, 785, 761, 749, 796, 723, 787, 775, 785, 738, 759, 770, 784, 741, 763, 728, 733, 747, 779, 751, 776, 820, 779, 804, 708, 739, 744, 745, 725, 762, 782, 807, 787, 722, 783, 758, 753, 767, 770, 744, 804, 854, 768, 766, 760, 811] payload_rows_per_dst=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] dup_rows_per_dst=[2936, 2904, 3036, 3012, 2916, 2888, 3058, 2966] combine_rows_per_src=[9456, 9284, 9332, 9266, 9278, 9124, 9260, 9588] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64848 dup_rows=24054 dup_rows_remote=20880 dedup_ratio=0.244690 router_tokens_per_expert=[781, 730, 780, 744, 782, 774, 725, 735, 764, 771, 792, 790, 763, 755, 791, 780, 759, 758, 768, 772, 781, 778, 768, 791, 753, 777, 736, 755, 746, 717, 757, 765, 749, 825, 726, 772, 771, 803, 814, 756, 757, 738, 785, 777, 788, 753, 788, 782, 754, 759, 731, 736, 767, 783, 790, 795, 740, 781, 746, 776, 784, 806, 799, 783] payload_rows_per_dst=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] dup_rows_per_dst=[2920, 2968, 2564, 3190, 3394, 2878, 2966, 3174] combine_rows_per_src=[8998, 9200, 8766, 9506, 9802, 9312, 9264, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343136 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731168 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119200 weight_movement_bytes=34603008 total_nvlink_bytes=565722208 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344848 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141904 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938960 weight_movement_bytes=69206016 total_nvlink_bytes=603144976 weight_to_token_ratio=0.129614 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=65028 dup_rows=24104 dup_rows_remote=21146 dedup_ratio=0.245199 router_tokens_per_expert=[776, 760, 750, 775, 786, 778, 729, 785, 764, 771, 780, 777, 806, 779, 735, 756, 815, 741, 851, 755, 840, 854, 784, 795, 769, 803, 753, 710, 758, 759, 742, 771, 795, 722, 752, 740, 710, 754, 725, 732, 753, 784, 819, 691, 744, 814, 773, 725, 760, 784, 800, 739, 763, 782, 742, 788, 725, 761, 758, 751, 755, 765, 834, 805] payload_rows_per_dst=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] dup_rows_per_dst=[2972, 2946, 3322, 2958, 2860, 2970, 3000, 3076] combine_rows_per_src=[9306, 9390, 9548, 9172, 9000, 9236, 9316, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65028 dup_rows=24044 dup_rows_remote=20944 dedup_ratio=0.244588 router_tokens_per_expert=[746, 766, 793, 758, 753, 789, 778, 762, 747, 765, 784, 806, 737, 789, 793, 788, 827, 746, 762, 784, 763, 756, 738, 755, 778, 807, 772, 734, 748, 752, 791, 776, 774, 809, 772, 719, 758, 772, 803, 759, 718, 774, 785, 784, 778, 739, 787, 789, 751, 736, 735, 824, 736, 799, 783, 770, 755, 778, 750, 740, 785, 731, 762, 754] payload_rows_per_dst=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] dup_rows_per_dst=[2934, 3036, 3044, 2968, 3100, 3058, 3014, 2890] combine_rows_per_src=[9356, 9382, 9218, 9348, 9232, 9250, 9254, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344200 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723464 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102728 weight_movement_bytes=69206016 total_nvlink_bytes=602308744 weight_to_token_ratio=0.129817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=64958 dup_rows=23844 dup_rows_remote=20750 dedup_ratio=0.242554 router_tokens_per_expert=[771, 717, 838, 736, 711, 794, 773, 763, 754, 771, 772, 789, 742, 759, 785, 809, 812, 801, 805, 780, 726, 814, 790, 770, 767, 739, 743, 789, 773, 789, 758, 734, 721, 770, 744, 803, 762, 790, 753, 765, 794, 757, 790, 784, 755, 722, 723, 767, 760, 768, 800, 740, 757, 778, 776, 784, 742, 766, 753, 735, 759, 798, 761, 801] payload_rows_per_dst=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] dup_rows_per_dst=[2878, 3008, 3094, 2900, 2978, 3020, 3028, 2938] combine_rows_per_src=[9328, 9354, 9502, 9284, 9238, 9164, 9298, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=342912 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960320 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531577728 weight_movement_bytes=34603008 total_nvlink_bytes=566180736 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344696 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699384 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533054072 weight_movement_bytes=34603008 total_nvlink_bytes=567657080 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65190 dup_rows=23824 dup_rows_remote=20890 dedup_ratio=0.242350 router_tokens_per_expert=[783, 780, 748, 779, 787, 785, 765, 763, 758, 799, 756, 755, 795, 795, 724, 772, 799, 809, 734, 793, 759, 750, 797, 732, 735, 773, 769, 758, 745, 752, 716, 734, 751, 771, 747, 738, 776, 797, 780, 798, 808, 770, 763, 817, 767, 749, 777, 747, 775, 713, 778, 789, 760, 744, 752, 801, 789, 810, 729, 774, 777, 782, 815, 709] payload_rows_per_dst=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] dup_rows_per_dst=[3072, 3020, 3030, 2720, 3044, 2986, 2934, 3018] combine_rows_per_src=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65034 dup_rows=23958 dup_rows_remote=21016 dedup_ratio=0.243713 router_tokens_per_expert=[737, 796, 795, 817, 773, 725, 805, 788, 790, 784, 730, 805, 766, 801, 748, 759, 725, 755, 791, 756, 796, 759, 777, 747, 761, 713, 808, 762, 781, 759, 805, 775, 743, 764, 764, 778, 772, 787, 751, 730, 764, 800, 801, 731, 731, 778, 730, 792, 722, 775, 771, 746, 749, 779, 778, 776, 717, 813, 760, 791, 735, 799, 765, 771] payload_rows_per_dst=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] dup_rows_per_dst=[3068, 3020, 3014, 2984, 2914, 2942, 2964, 3052] combine_rows_per_src=[9404, 9346, 9198, 9344, 9264, 9312, 9228, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344848 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141904 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938960 weight_movement_bytes=69206016 total_nvlink_bytes=603144976 weight_to_token_ratio=0.129614 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=343888 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698576 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053264 weight_movement_bytes=34603008 total_nvlink_bytes=567656272 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344696 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699384 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533054072 weight_movement_bytes=34603008 total_nvlink_bytes=567657080 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267583488 dispatch_meta_bytes=343944 combine_payload_bytes=267583488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267927432 ingress_bytes=336789504 link_bytes=336789504 token_bytes=535510920 weight_movement_bytes=69206016 total_nvlink_bytes=604716936 weight_to_token_ratio=0.129234 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267311790e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342832 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266410800 ingress_bytes=369876992 link_bytes=369876992 token_bytes=532478768 weight_movement_bytes=103809024 total_nvlink_bytes=636287792 weight_to_token_ratio=0.194954 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391817343e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65046 dup_rows=24122 dup_rows_remote=21084 dedup_ratio=0.245382 router_tokens_per_expert=[761, 785, 745, 743, 781, 826, 763, 742, 782, 756, 783, 732, 709, 785, 788, 752, 756, 770, 781, 748, 797, 790, 756, 800, 799, 749, 760, 740, 778, 788, 750, 785, 777, 751, 769, 776, 757, 756, 754, 802, 829, 752, 773, 772, 758, 771, 763, 777, 730, 771, 779, 789, 734, 769, 740, 791, 782, 766, 754, 765, 732, 745, 784, 804] payload_rows_per_dst=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] dup_rows_per_dst=[3074, 3038, 3016, 3074, 3056, 3030, 2886, 2948] combine_rows_per_src=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=342912 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960320 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531577728 weight_movement_bytes=34603008 total_nvlink_bytes=566180736 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344200 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723464 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102728 weight_movement_bytes=69206016 total_nvlink_bytes=602308744 weight_to_token_ratio=0.129817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65134 dup_rows=23866 dup_rows_remote=20876 dedup_ratio=0.242778 router_tokens_per_expert=[785, 733, 749, 772, 770, 776, 806, 756, 764, 749, 804, 762, 756, 749, 773, 770, 741, 799, 775, 769, 774, 785, 713, 772, 742, 767, 745, 803, 765, 770, 738, 750, 758, 748, 775, 797, 779, 749, 709, 771, 753, 760, 780, 828, 773, 835, 814, 734, 807, 758, 795, 750, 781, 774, 789, 768, 792, 790, 798, 767, 707, 731, 765, 735] payload_rows_per_dst=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] dup_rows_per_dst=[2990, 3008, 2998, 2870, 2910, 3034, 3146, 2910] combine_rows_per_src=[9304, 9246, 9258, 9290, 9262, 9524, 9294, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344320 combine_payload_bytes=267018240 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267362560 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380800 weight_movement_bytes=34603008 total_nvlink_bytes=568983808 weight_to_token_ratio=0.064753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65006 dup_rows=23846 dup_rows_remote=20756 dedup_ratio=0.242574 router_tokens_per_expert=[738, 811, 743, 824, 726, 733, 783, 756, 785, 802, 738, 725, 757, 741, 790, 767, 794, 828, 811, 790, 761, 769, 771, 726, 756, 750, 744, 762, 762, 721, 810, 765, 779, 726, 782, 761, 743, 769, 766, 746, 826, 799, 784, 775, 736, 763, 820, 768, 748, 820, 784, 765, 782, 791, 759, 739, 759, 784, 726, 751, 769, 747, 773, 773] payload_rows_per_dst=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] dup_rows_per_dst=[2894, 2954, 3038, 2984, 2908, 3090, 3026, 2952] combine_rows_per_src=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344520 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772936 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201352 weight_movement_bytes=34603008 total_nvlink_bytes=567804360 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64628 dup_rows=24016 dup_rows_remote=20678 dedup_ratio=0.244303 router_tokens_per_expert=[784, 805, 781, 751, 780, 756, 736, 789, 772, 781, 733, 787, 797, 793, 757, 763, 767, 750, 756, 776, 806, 746, 775, 787, 764, 739, 793, 789, 731, 755, 794, 756, 780, 788, 789, 799, 751, 783, 801, 760, 766, 776, 754, 734, 775, 722, 762, 739, 752, 767, 789, 778, 747, 735, 758, 778, 783, 767, 738, 770, 800, 769, 759, 734] payload_rows_per_dst=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] dup_rows_per_dst=[2974, 3060, 2982, 2990, 3068, 2582, 3022, 3338] combine_rows_per_src=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64960 dup_rows=24086 dup_rows_remote=21102 dedup_ratio=0.245015 router_tokens_per_expert=[786, 732, 753, 766, 763, 745, 808, 751, 766, 793, 754, 758, 777, 760, 794, 754, 781, 720, 726, 761, 783, 742, 750, 747, 745, 730, 751, 769, 804, 811, 748, 763, 831, 761, 801, 776, 727, 803, 732, 768, 786, 753, 766, 783, 759, 781, 820, 785, 790, 756, 763, 779, 783, 744, 767, 788, 823, 785, 786, 717, 769, 721, 746, 812] payload_rows_per_dst=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] dup_rows_per_dst=[3042, 3132, 2844, 2984, 3038, 3038, 3056, 2952] combine_rows_per_src=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343048 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266607624 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532872200 weight_movement_bytes=69206016 total_nvlink_bytes=602078216 weight_to_token_ratio=0.129874 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65046 dup_rows=24122 dup_rows_remote=21084 dedup_ratio=0.245382 router_tokens_per_expert=[761, 785, 745, 743, 781, 826, 763, 742, 782, 756, 783, 732, 709, 785, 788, 752, 756, 770, 781, 748, 797, 790, 756, 800, 799, 749, 760, 740, 778, 788, 750, 785, 777, 751, 769, 776, 757, 756, 754, 802, 829, 752, 773, 772, 758, 771, 763, 777, 730, 771, 779, 789, 734, 769, 740, 791, 782, 766, 754, 765, 732, 745, 784, 804] payload_rows_per_dst=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] dup_rows_per_dst=[3074, 3038, 3016, 3074, 3056, 3030, 2886, 2948] combine_rows_per_src=[9218, 9136, 9380, 9224, 9228, 9360, 9320, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65026 dup_rows=23992 dup_rows_remote=21032 dedup_ratio=0.244059 router_tokens_per_expert=[756, 801, 772, 771, 719, 804, 753, 779, 732, 769, 801, 732, 784, 771, 764, 700, 754, 801, 747, 722, 724, 767, 769, 778, 793, 777, 832, 791, 713, 789, 745, 749, 746, 805, 751, 751, 754, 730, 784, 802, 790, 741, 737, 803, 793, 773, 768, 762, 755, 807, 762, 765, 820, 807, 754, 783, 787, 784, 764, 735, 771, 733, 803, 773] payload_rows_per_dst=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] dup_rows_per_dst=[2972, 2944, 2916, 3120, 2960, 2990, 3064, 3026] combine_rows_per_src=[9338, 9162, 9208, 9258, 9286, 9344, 9442, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64960 dup_rows=24086 dup_rows_remote=21102 dedup_ratio=0.245015 router_tokens_per_expert=[786, 732, 753, 766, 763, 745, 808, 751, 766, 793, 754, 758, 777, 760, 794, 754, 781, 720, 726, 761, 783, 742, 750, 747, 745, 730, 751, 769, 804, 811, 748, 763, 831, 761, 801, 776, 727, 803, 732, 768, 786, 753, 766, 783, 759, 781, 820, 785, 790, 756, 763, 779, 783, 744, 767, 788, 823, 785, 786, 717, 769, 721, 746, 812] payload_rows_per_dst=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] dup_rows_per_dst=[3042, 3132, 2844, 2984, 3038, 3038, 3056, 2952] combine_rows_per_src=[9166, 9180, 9176, 9258, 9360, 9428, 9284, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65190 dup_rows=23824 dup_rows_remote=20890 dedup_ratio=0.242350 router_tokens_per_expert=[783, 780, 748, 779, 787, 785, 765, 763, 758, 799, 756, 755, 795, 795, 724, 772, 799, 809, 734, 793, 759, 750, 797, 732, 735, 773, 769, 758, 745, 752, 716, 734, 751, 771, 747, 738, 776, 797, 780, 798, 808, 770, 763, 817, 767, 749, 777, 747, 775, 713, 778, 789, 760, 744, 752, 801, 789, 810, 729, 774, 777, 782, 815, 709] payload_rows_per_dst=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] dup_rows_per_dst=[3072, 3020, 3030, 2720, 3044, 2986, 2934, 3018] combine_rows_per_src=[9308, 9288, 9316, 9244, 9272, 9410, 9290, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65088 dup_rows=24034 dup_rows_remote=21118 dedup_ratio=0.244486 router_tokens_per_expert=[813, 817, 768, 806, 772, 757, 773, 749, 732, 793, 757, 787, 727, 771, 777, 787, 764, 694, 787, 723, 764, 761, 784, 772, 761, 733, 755, 742, 776, 753, 755, 782, 764, 794, 732, 822, 710, 756, 776, 781, 770, 775, 747, 779, 786, 806, 758, 768, 744, 785, 792, 788, 747, 815, 847, 745, 761, 811, 764, 714, 801, 745, 730, 747] payload_rows_per_dst=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] dup_rows_per_dst=[2994, 3036, 2916, 2920, 2990, 3108, 3162, 2908] combine_rows_per_src=[9516, 9226, 9182, 9194, 9280, 9270, 9364, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264716288 dispatch_meta_bytes=341224 combine_payload_bytes=264716288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265057512 ingress_bytes=299319296 link_bytes=299319296 token_bytes=529773800 weight_movement_bytes=34603008 total_nvlink_bytes=564376808 weight_to_token_ratio=0.065317 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126314414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64628 dup_rows=24016 dup_rows_remote=20678 dedup_ratio=0.244303 router_tokens_per_expert=[784, 805, 781, 751, 780, 756, 736, 789, 772, 781, 733, 787, 797, 793, 757, 763, 767, 750, 756, 776, 806, 746, 775, 787, 764, 739, 793, 789, 731, 755, 794, 756, 780, 788, 789, 799, 751, 783, 801, 760, 766, 776, 754, 734, 775, 722, 762, 739, 752, 767, 789, 778, 747, 735, 758, 778, 783, 767, 738, 770, 800, 769, 759, 734] payload_rows_per_dst=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] dup_rows_per_dst=[2974, 3060, 2982, 2990, 3068, 2582, 3022, 3338] combine_rows_per_src=[9340, 9306, 9344, 9252, 9434, 8716, 9236, 9660] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65186 dup_rows=23996 dup_rows_remote=21074 dedup_ratio=0.244100 router_tokens_per_expert=[762, 792, 731, 793, 728, 749, 746, 758, 754, 810, 752, 761, 775, 769, 824, 770, 785, 737, 846, 776, 845, 737, 776, 739, 774, 766, 802, 764, 785, 780, 745, 795, 764, 788, 806, 713, 821, 814, 764, 760, 729, 778, 765, 760, 793, 765, 747, 750, 793, 696, 726, 800, 795, 720, 729, 763, 789, 748, 749, 775, 765, 744, 763, 754] payload_rows_per_dst=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] dup_rows_per_dst=[2900, 3096, 3036, 3040, 3070, 3008, 2922, 2924] combine_rows_per_src=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344248 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420408 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496568 weight_movement_bytes=34603008 total_nvlink_bytes=567099576 weight_to_token_ratio=0.064983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65006 dup_rows=23846 dup_rows_remote=20756 dedup_ratio=0.242574 router_tokens_per_expert=[738, 811, 743, 824, 726, 733, 783, 756, 785, 802, 738, 725, 757, 741, 790, 767, 794, 828, 811, 790, 761, 769, 771, 726, 756, 750, 744, 762, 762, 721, 810, 765, 779, 726, 782, 761, 743, 769, 766, 746, 826, 799, 784, 775, 736, 763, 820, 768, 748, 820, 784, 765, 782, 791, 759, 739, 759, 784, 726, 751, 769, 747, 773, 773] payload_rows_per_dst=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] dup_rows_per_dst=[2894, 2954, 3038, 2984, 2908, 3090, 3026, 2952] combine_rows_per_src=[9334, 9256, 9462, 9156, 9236, 9452, 9350, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344520 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772936 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201352 weight_movement_bytes=34603008 total_nvlink_bytes=567804360 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344248 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420408 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496568 weight_movement_bytes=34603008 total_nvlink_bytes=567099576 weight_to_token_ratio=0.064983 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344232 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266690728 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037224 weight_movement_bytes=103809024 total_nvlink_bytes=636846248 weight_to_token_ratio=0.194750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65192 dup_rows=23844 dup_rows_remote=20752 dedup_ratio=0.242554 router_tokens_per_expert=[817, 790, 732, 792, 781, 794, 761, 812, 784, 775, 781, 777, 748, 766, 746, 803, 776, 726, 783, 772, 766, 751, 775, 747, 800, 778, 713, 721, 769, 782, 817, 776, 764, 790, 752, 773, 751, 803, 813, 742, 724, 821, 754, 774, 765, 722, 725, 710, 774, 768, 772, 786, 783, 781, 775, 763, 690, 756, 812, 782, 760, 766, 768, 722] payload_rows_per_dst=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] dup_rows_per_dst=[3090, 3092, 2908, 2934, 3018, 2868, 3078, 2856] combine_rows_per_src=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344320 combine_payload_bytes=267018240 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267362560 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380800 weight_movement_bytes=34603008 total_nvlink_bytes=568983808 weight_to_token_ratio=0.064753 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264716288 dispatch_meta_bytes=341224 combine_payload_bytes=264716288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265057512 ingress_bytes=299319296 link_bytes=299319296 token_bytes=529773800 weight_movement_bytes=34603008 total_nvlink_bytes=564376808 weight_to_token_ratio=0.065317 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126314414e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343048 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266607624 ingress_bytes=335470592 link_bytes=335470592 token_bytes=532872200 weight_movement_bytes=69206016 total_nvlink_bytes=602078216 weight_to_token_ratio=0.129874 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262348830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65088 dup_rows=23928 dup_rows_remote=20892 dedup_ratio=0.243408 router_tokens_per_expert=[806, 782, 757, 740, 759, 761, 804, 744, 790, 754, 774, 721, 764, 798, 791, 746, 726, 746, 762, 798, 752, 739, 765, 810, 737, 784, 757, 784, 749, 760, 779, 792, 761, 784, 751, 790, 779, 746, 822, 752, 796, 770, 771, 776, 802, 736, 739, 839, 776, 770, 718, 776, 764, 818, 795, 820, 762, 711, 753, 737, 707, 759, 788, 753] payload_rows_per_dst=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] dup_rows_per_dst=[3036, 3118, 3028, 2982, 2900, 3138, 2896, 2830] combine_rows_per_src=[9288, 9326, 9330, 9486, 9100, 9584, 9152, 9110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345040 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267346896 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348752 weight_movement_bytes=103809024 total_nvlink_bytes=638157776 weight_to_token_ratio=0.194272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65316 dup_rows=23892 dup_rows_remote=20986 dedup_ratio=0.243042 router_tokens_per_expert=[761, 718, 745, 744, 746, 802, 801, 779, 816, 764, 793, 765, 767, 755, 779, 714, 769, 764, 756, 752, 751, 787, 800, 812, 813, 736, 782, 742, 771, 760, 795, 752, 775, 759, 788, 758, 812, 765, 775, 765, 773, 707, 758, 767, 743, 764, 743, 746, 782, 785, 755, 797, 797, 761, 776, 771, 767, 737, 780, 795, 790, 725, 767, 778] payload_rows_per_dst=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] dup_rows_per_dst=[2966, 2964, 3064, 2998, 2934, 2906, 3066, 2994] combine_rows_per_src=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343776 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370208 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396640 weight_movement_bytes=69206016 total_nvlink_bytes=603602656 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64812 dup_rows=23970 dup_rows_remote=20748 dedup_ratio=0.243835 router_tokens_per_expert=[775, 759, 741, 749, 726, 801, 789, 766, 779, 781, 744, 737, 771, 801, 737, 776, 807, 742, 789, 735, 766, 822, 735, 762, 804, 758, 722, 772, 747, 799, 759, 784, 761, 783, 769, 752, 722, 725, 756, 768, 768, 777, 791, 772, 772, 772, 765, 755, 797, 769, 806, 779, 753, 794, 753, 811, 767, 769, 769, 721, 817, 761, 744, 799] payload_rows_per_dst=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] dup_rows_per_dst=[2864, 2748, 2986, 2948, 2890, 3086, 3226, 3222] combine_rows_per_src=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65192 dup_rows=23844 dup_rows_remote=20752 dedup_ratio=0.242554 router_tokens_per_expert=[817, 790, 732, 792, 781, 794, 761, 812, 784, 775, 781, 777, 748, 766, 746, 803, 776, 726, 783, 772, 766, 751, 775, 747, 800, 778, 713, 721, 769, 782, 817, 776, 764, 790, 752, 773, 751, 803, 813, 742, 724, 821, 754, 774, 765, 722, 725, 710, 774, 768, 772, 786, 783, 781, 775, 763, 690, 756, 812, 782, 760, 766, 768, 722] payload_rows_per_dst=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] dup_rows_per_dst=[3090, 3092, 2908, 2934, 3018, 2868, 3078, 2856] combine_rows_per_src=[9468, 9268, 9284, 9378, 9358, 9122, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64942 dup_rows=24060 dup_rows_remote=20966 dedup_ratio=0.244751 router_tokens_per_expert=[744, 761, 745, 753, 772, 790, 754, 756, 778, 775, 800, 763, 736, 769, 793, 825, 743, 779, 742, 777, 764, 744, 758, 778, 834, 734, 758, 802, 746, 779, 786, 728, 794, 769, 792, 760, 800, 728, 804, 751, 719, 780, 792, 738, 761, 751, 777, 762, 746, 763, 782, 794, 741, 776, 734, 785, 796, 744, 753, 721, 830, 764, 831, 748] payload_rows_per_dst=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] dup_rows_per_dst=[2914, 3030, 2956, 2938, 3094, 3064, 3032, 3032] combine_rows_per_src=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65140 dup_rows=23844 dup_rows_remote=21018 dedup_ratio=0.242554 router_tokens_per_expert=[752, 733, 785, 808, 780, 784, 752, 773, 772, 788, 792, 763, 778, 735, 786, 778, 731, 778, 753, 751, 772, 760, 780, 769, 730, 737, 768, 798, 759, 737, 747, 797, 809, 752, 790, 780, 789, 808, 732, 767, 805, 762, 737, 813, 736, 726, 774, 754, 796, 734, 731, 767, 830, 731, 754, 791, 759, 773, 809, 745, 720, 825, 734, 793] payload_rows_per_dst=[9234, 9410, 9250, 9320, 9402, 9210, 9320, 9314] dup_rows_per_dst=[3100, 2974, 2938, 2826, 3052, 3004, 2948, 3002] combine_rows_per_src=[9234, 9410, 9250, 9320, 9402, 9210, 9320, 9314] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343920 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944368 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544816 weight_movement_bytes=69206016 total_nvlink_bytes=602750832 weight_to_token_ratio=0.129710 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64942 dup_rows=24060 dup_rows_remote=20966 dedup_ratio=0.244751 router_tokens_per_expert=[744, 761, 745, 753, 772, 790, 754, 756, 778, 775, 800, 763, 736, 769, 793, 825, 743, 779, 742, 777, 764, 744, 758, 778, 834, 734, 758, 802, 746, 779, 786, 728, 794, 769, 792, 760, 800, 728, 804, 751, 719, 780, 792, 738, 761, 751, 777, 762, 746, 763, 782, 794, 741, 776, 734, 785, 796, 744, 753, 721, 830, 764, 831, 748] payload_rows_per_dst=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] dup_rows_per_dst=[2914, 3030, 2956, 2938, 3094, 3064, 3032, 3032] combine_rows_per_src=[9236, 9448, 9214, 9396, 9302, 9096, 9210, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345208 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879544 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535413880 weight_movement_bytes=34603008 total_nvlink_bytes=570016888 weight_to_token_ratio=0.064629 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65082 dup_rows=23900 dup_rows_remote=21024 dedup_ratio=0.243123 router_tokens_per_expert=[788, 778, 739, 777, 747, 807, 739, 750, 788, 778, 733, 780, 782, 765, 828, 833, 729, 774, 754, 731, 781, 744, 756, 830, 771, 812, 764, 726, 794, 811, 722, 802, 765, 728, 760, 802, 746, 736, 803, 751, 749, 807, 753, 729, 753, 801, 802, 803, 786, 781, 775, 753, 743, 736, 762, 765, 755, 759, 804, 757, 783, 726, 739, 727] payload_rows_per_dst=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] dup_rows_per_dst=[3022, 3182, 2876, 2976, 2978, 3076, 2918, 2872] combine_rows_per_src=[9228, 9392, 9322, 9428, 9204, 9318, 9284, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=342240 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812192 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282144 weight_movement_bytes=34603008 total_nvlink_bytes=565885152 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64812 dup_rows=23970 dup_rows_remote=20748 dedup_ratio=0.243835 router_tokens_per_expert=[775, 759, 741, 749, 726, 801, 789, 766, 779, 781, 744, 737, 771, 801, 737, 776, 807, 742, 789, 735, 766, 822, 735, 762, 804, 758, 722, 772, 747, 799, 759, 784, 761, 783, 769, 752, 722, 725, 756, 768, 768, 777, 791, 772, 772, 772, 765, 755, 797, 769, 806, 779, 753, 794, 753, 811, 767, 769, 769, 721, 817, 761, 744, 799] payload_rows_per_dst=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] dup_rows_per_dst=[2864, 2748, 2986, 2948, 2890, 3086, 3226, 3222] combine_rows_per_src=[9114, 9054, 9226, 9446, 9182, 9258, 9532, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65186 dup_rows=23996 dup_rows_remote=21074 dedup_ratio=0.244100 router_tokens_per_expert=[762, 792, 731, 793, 728, 749, 746, 758, 754, 810, 752, 761, 775, 769, 824, 770, 785, 737, 846, 776, 845, 737, 776, 739, 774, 766, 802, 764, 785, 780, 745, 795, 764, 788, 806, 713, 821, 814, 764, 760, 729, 778, 765, 760, 793, 765, 747, 750, 793, 696, 726, 800, 795, 720, 729, 763, 789, 748, 749, 775, 765, 744, 763, 754] payload_rows_per_dst=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] dup_rows_per_dst=[2900, 3096, 3036, 3040, 3070, 3008, 2922, 2924] combine_rows_per_src=[9218, 9334, 9446, 9382, 9390, 9166, 9122, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64958 dup_rows=24104 dup_rows_remote=20998 dedup_ratio=0.245199 router_tokens_per_expert=[789, 784, 785, 799, 777, 724, 792, 744, 764, 751, 723, 741, 780, 819, 732, 781, 726, 762, 808, 802, 812, 752, 775, 808, 790, 742, 840, 724, 754, 750, 771, 736, 754, 772, 827, 760, 799, 739, 720, 748, 742, 778, 790, 768, 760, 777, 815, 757, 771, 738, 785, 751, 782, 810, 778, 759, 717, 773, 753, 754, 782, 739, 767, 750] payload_rows_per_dst=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] dup_rows_per_dst=[2988, 2916, 3186, 2964, 2972, 3120, 3106, 2852] combine_rows_per_src=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344632 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158072 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971512 weight_movement_bytes=34603008 total_nvlink_bytes=568574520 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65316 dup_rows=23892 dup_rows_remote=20986 dedup_ratio=0.243042 router_tokens_per_expert=[761, 718, 745, 744, 746, 802, 801, 779, 816, 764, 793, 765, 767, 755, 779, 714, 769, 764, 756, 752, 751, 787, 800, 812, 813, 736, 782, 742, 771, 760, 795, 752, 775, 759, 788, 758, 812, 765, 775, 765, 773, 707, 758, 767, 743, 764, 743, 746, 782, 785, 755, 797, 797, 761, 776, 771, 767, 737, 780, 795, 790, 725, 767, 778] payload_rows_per_dst=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] dup_rows_per_dst=[2966, 2964, 3064, 2998, 2934, 2906, 3066, 2994] combine_rows_per_src=[9226, 9342, 9318, 9304, 9460, 9096, 9382, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=343776 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267370208 ingress_bytes=336232448 link_bytes=336232448 token_bytes=534396640 weight_movement_bytes=69206016 total_nvlink_bytes=603602656 weight_to_token_ratio=0.129503 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265215633e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344632 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267158072 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971512 weight_movement_bytes=34603008 total_nvlink_bytes=568574520 weight_to_token_ratio=0.064803 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343632 combine_payload_bytes=266002432 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266346064 ingress_bytes=369811456 link_bytes=369811456 token_bytes=532348496 weight_movement_bytes=103809024 total_nvlink_bytes=636157520 weight_to_token_ratio=0.195002 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391570737e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64940 dup_rows=24112 dup_rows_remote=21072 dedup_ratio=0.245280 router_tokens_per_expert=[767, 735, 770, 792, 762, 758, 784, 746, 770, 777, 752, 776, 789, 749, 742, 791, 807, 762, 715, 763, 840, 799, 814, 784, 757, 739, 759, 747, 772, 745, 764, 819, 777, 755, 762, 749, 773, 733, 773, 728, 742, 731, 797, 769, 804, 774, 746, 725, 750, 742, 771, 797, 793, 826, 748, 787, 792, 761, 729, 718, 797, 777, 809, 771] payload_rows_per_dst=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] dup_rows_per_dst=[3014, 3040, 3146, 2980, 2932, 2966, 3026, 3008] combine_rows_per_src=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344424 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266920296 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533496168 weight_movement_bytes=69206016 total_nvlink_bytes=602702184 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=342240 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812192 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282144 weight_movement_bytes=34603008 total_nvlink_bytes=565885152 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345040 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267346896 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348752 weight_movement_bytes=103809024 total_nvlink_bytes=638157776 weight_to_token_ratio=0.194272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=345208 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267879544 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535413880 weight_movement_bytes=34603008 total_nvlink_bytes=570016888 weight_to_token_ratio=0.064629 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65184 dup_rows=23772 dup_rows_remote=20666 dedup_ratio=0.241821 router_tokens_per_expert=[811, 787, 785, 776, 753, 758, 799, 755, 812, 773, 715, 714, 771, 771, 767, 767, 777, 772, 756, 841, 785, 746, 763, 788, 774, 772, 809, 755, 755, 737, 759, 738, 784, 718, 774, 761, 771, 777, 804, 737, 755, 737, 775, 767, 745, 752, 736, 742, 713, 746, 810, 764, 775, 797, 807, 809, 780, 781, 750, 773, 775, 763, 742, 791] payload_rows_per_dst=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] dup_rows_per_dst=[3106, 2992, 3002, 2878, 2908, 3200, 2724, 2962] combine_rows_per_src=[9348, 9226, 9470, 9320, 9284, 9626, 8910, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343824 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411792 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479760 weight_movement_bytes=69206016 total_nvlink_bytes=601685776 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65026 dup_rows=23828 dup_rows_remote=20626 dedup_ratio=0.242391 router_tokens_per_expert=[737, 797, 766, 752, 768, 759, 792, 744, 811, 775, 747, 748, 761, 768, 763, 777, 745, 756, 766, 721, 770, 774, 778, 766, 770, 743, 729, 727, 782, 797, 774, 792, 777, 781, 751, 761, 771, 778, 736, 744, 743, 814, 794, 827, 814, 810, 791, 733, 813, 764, 768, 787, 771, 789, 753, 743, 733, 784, 788, 751, 737, 759, 778, 754] payload_rows_per_dst=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] dup_rows_per_dst=[2978, 3002, 2936, 2910, 2932, 3202, 2982, 2886] combine_rows_per_src=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344048 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338288 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332528 weight_movement_bytes=69206016 total_nvlink_bytes=601538544 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64940 dup_rows=24112 dup_rows_remote=21072 dedup_ratio=0.245280 router_tokens_per_expert=[767, 735, 770, 792, 762, 758, 784, 746, 770, 777, 752, 776, 789, 749, 742, 791, 807, 762, 715, 763, 840, 799, 814, 784, 757, 739, 759, 747, 772, 745, 764, 819, 777, 755, 762, 749, 773, 733, 773, 728, 742, 731, 797, 769, 804, 774, 746, 725, 750, 742, 771, 797, 793, 826, 748, 787, 792, 761, 729, 718, 797, 777, 809, 771] payload_rows_per_dst=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] dup_rows_per_dst=[3014, 3040, 3146, 2980, 2932, 2966, 3026, 3008] combine_rows_per_src=[9214, 9252, 9422, 9224, 9168, 9210, 9402, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=343400 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267337064 ingress_bytes=336199680 link_bytes=336199680 token_bytes=534330728 weight_movement_bytes=69206016 total_nvlink_bytes=603536744 weight_to_token_ratio=0.129519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265092330e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64974 dup_rows=24078 dup_rows_remote=21134 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 775, 728, 762, 785, 794, 782, 779, 763, 753, 745, 752, 779, 729, 758, 803, 755, 792, 721, 760, 782, 794, 762, 722, 765, 794, 773, 760, 755, 783, 746, 781, 787, 719, 779, 796, 795, 758, 774, 736, 750, 766, 753, 823, 761, 772, 756, 744, 796, 736, 810, 749, 786, 789, 750, 769, 777, 812, 769, 779, 743, 776, 748] payload_rows_per_dst=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] dup_rows_per_dst=[3028, 2922, 3110, 2944, 3088, 3044, 2952, 2990] combine_rows_per_src=[9348, 9194, 9228, 9252, 9290, 9190, 9368, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64894 dup_rows=23888 dup_rows_remote=20742 dedup_ratio=0.243001 router_tokens_per_expert=[770, 763, 794, 757, 723, 723, 769, 763, 739, 785, 794, 800, 825, 744, 749, 777, 761, 816, 738, 797, 771, 781, 783, 749, 799, 766, 803, 818, 745, 770, 792, 786, 746, 776, 770, 745, 785, 767, 775, 733, 721, 780, 758, 779, 747, 812, 712, 753, 762, 733, 755, 729, 800, 792, 728, 759, 821, 802, 757, 770, 768, 772, 734, 761] payload_rows_per_dst=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] dup_rows_per_dst=[2894, 3022, 2870, 2986, 2864, 2972, 3134, 3146] combine_rows_per_src=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=342608 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689104 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533035600 weight_movement_bytes=34603008 total_nvlink_bytes=567638608 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64970 dup_rows=23948 dup_rows_remote=20838 dedup_ratio=0.243612 router_tokens_per_expert=[809, 764, 721, 756, 752, 749, 727, 766, 768, 792, 766, 778, 751, 794, 770, 745, 791, 784, 794, 785, 782, 767, 758, 787, 805, 727, 750, 828, 774, 756, 831, 704, 780, 804, 765, 763, 762, 774, 771, 759, 814, 758, 762, 763, 756, 765, 751, 782, 786, 759, 736, 761, 700, 749, 749, 802, 741, 786, 797, 743, 781, 778, 720, 804] payload_rows_per_dst=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] dup_rows_per_dst=[2974, 3054, 3110, 2978, 2920, 2976, 2848, 3088] combine_rows_per_src=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64794 dup_rows=24146 dup_rows_remote=21186 dedup_ratio=0.245626 router_tokens_per_expert=[773, 744, 769, 749, 723, 769, 753, 753, 768, 795, 765, 771, 793, 805, 760, 765, 747, 754, 774, 812, 780, 792, 775, 766, 791, 780, 796, 714, 783, 754, 772, 769, 800, 779, 769, 797, 788, 751, 712, 766, 788, 719, 787, 749, 748, 770, 777, 791, 785, 755, 754, 787, 742, 790, 772, 761, 756, 766, 760, 763, 805, 718, 735, 798] payload_rows_per_dst=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] dup_rows_per_dst=[2952, 3042, 3134, 3118, 2960, 3004, 3030, 2906] combine_rows_per_src=[9114, 9402, 9266, 9200, 9364, 9254, 9262, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64894 dup_rows=23888 dup_rows_remote=20742 dedup_ratio=0.243001 router_tokens_per_expert=[770, 763, 794, 757, 723, 723, 769, 763, 739, 785, 794, 800, 825, 744, 749, 777, 761, 816, 738, 797, 771, 781, 783, 749, 799, 766, 803, 818, 745, 770, 792, 786, 746, 776, 770, 745, 785, 767, 775, 733, 721, 780, 758, 779, 747, 812, 712, 753, 762, 733, 755, 729, 800, 792, 728, 759, 821, 802, 757, 770, 768, 772, 734, 761] payload_rows_per_dst=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] dup_rows_per_dst=[2894, 3022, 2870, 2986, 2864, 2972, 3134, 3146] combine_rows_per_src=[9230, 9356, 9168, 9274, 9330, 9226, 9310, 9522] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64958 dup_rows=24104 dup_rows_remote=20998 dedup_ratio=0.245199 router_tokens_per_expert=[789, 784, 785, 799, 777, 724, 792, 744, 764, 751, 723, 741, 780, 819, 732, 781, 726, 762, 808, 802, 812, 752, 775, 808, 790, 742, 840, 724, 754, 750, 771, 736, 754, 772, 827, 760, 799, 739, 720, 748, 742, 778, 790, 768, 760, 777, 815, 757, 771, 738, 785, 751, 782, 810, 778, 759, 717, 773, 753, 754, 782, 739, 767, 750] payload_rows_per_dst=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] dup_rows_per_dst=[2988, 2916, 3186, 2964, 2972, 3120, 3106, 2852] combine_rows_per_src=[9400, 9266, 9304, 9250, 9266, 9254, 9242, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65026 dup_rows=23828 dup_rows_remote=20626 dedup_ratio=0.242391 router_tokens_per_expert=[737, 797, 766, 752, 768, 759, 792, 744, 811, 775, 747, 748, 761, 768, 763, 777, 745, 756, 766, 721, 770, 774, 778, 766, 770, 743, 729, 727, 782, 797, 774, 792, 777, 781, 751, 761, 771, 778, 736, 744, 743, 814, 794, 827, 814, 810, 791, 733, 813, 764, 768, 787, 771, 789, 753, 743, 733, 784, 788, 751, 737, 759, 778, 754] payload_rows_per_dst=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] dup_rows_per_dst=[2978, 3002, 2936, 2910, 2932, 3202, 2982, 2886] combine_rows_per_src=[9252, 9298, 9216, 9318, 9266, 9450, 9394, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64970 dup_rows=23948 dup_rows_remote=20838 dedup_ratio=0.243612 router_tokens_per_expert=[809, 764, 721, 756, 752, 749, 727, 766, 768, 792, 766, 778, 751, 794, 770, 745, 791, 784, 794, 785, 782, 767, 758, 787, 805, 727, 750, 828, 774, 756, 831, 704, 780, 804, 765, 763, 762, 774, 771, 759, 814, 758, 762, 763, 756, 765, 751, 782, 786, 759, 736, 761, 700, 749, 749, 802, 741, 786, 797, 743, 781, 778, 720, 804] payload_rows_per_dst=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] dup_rows_per_dst=[2974, 3054, 3110, 2978, 2920, 2976, 2848, 3088] combine_rows_per_src=[9114, 9274, 9386, 9372, 9436, 9326, 9236, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344048 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338288 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332528 weight_movement_bytes=69206016 total_nvlink_bytes=601538544 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344432 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266477936 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532611440 weight_movement_bytes=69206016 total_nvlink_bytes=601817456 weight_to_token_ratio=0.129937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65112 dup_rows=23982 dup_rows_remote=21064 dedup_ratio=0.243958 router_tokens_per_expert=[760, 764, 797, 760, 750, 783, 754, 802, 773, 730, 764, 773, 744, 746, 776, 773, 803, 766, 786, 756, 749, 769, 798, 748, 768, 806, 787, 761, 754, 800, 756, 758, 725, 761, 796, 824, 761, 812, 809, 779, 776, 760, 750, 775, 744, 774, 767, 797, 775, 725, 748, 787, 743, 731, 777, 778, 753, 777, 741, 736, 767, 771, 725, 794] payload_rows_per_dst=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] dup_rows_per_dst=[3020, 2910, 3016, 3002, 3118, 3080, 2918, 2918] combine_rows_per_src=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343232 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266460352 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577472 weight_movement_bytes=138412032 total_nvlink_bytes=670989504 weight_to_token_ratio=0.259891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343920 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265740144 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531136368 weight_movement_bytes=69206016 total_nvlink_bytes=600342384 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=342544 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148368 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954192 weight_movement_bytes=34603008 total_nvlink_bytes=566557200 weight_to_token_ratio=0.065049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343232 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266460352 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577472 weight_movement_bytes=138412032 total_nvlink_bytes=670989504 weight_to_token_ratio=0.259891 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64934 dup_rows=24038 dup_rows_remote=20998 dedup_ratio=0.244527 router_tokens_per_expert=[734, 741, 743, 745, 715, 804, 755, 764, 780, 745, 796, 751, 816, 781, 757, 760, 759, 781, 745, 787, 756, 793, 767, 777, 800, 793, 776, 728, 726, 726, 828, 769, 732, 780, 765, 774, 822, 783, 750, 759, 797, 729, 764, 762, 725, 790, 789, 784, 762, 781, 789, 820, 786, 737, 729, 745, 789, 777, 768, 752, 748, 818, 771, 777] payload_rows_per_dst=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] dup_rows_per_dst=[2870, 3040, 3036, 3012, 3004, 2944, 3072, 3060] combine_rows_per_src=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343824 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411792 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479760 weight_movement_bytes=69206016 total_nvlink_bytes=601685776 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=342608 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689104 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533035600 weight_movement_bytes=34603008 total_nvlink_bytes=567638608 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64796 dup_rows=23968 dup_rows_remote=20656 dedup_ratio=0.243815 router_tokens_per_expert=[761, 770, 757, 714, 748, 814, 774, 791, 801, 726, 776, 754, 775, 734, 767, 799, 807, 782, 789, 801, 769, 786, 764, 780, 732, 812, 759, 792, 811, 772, 726, 743, 778, 749, 765, 759, 793, 758, 793, 764, 750, 794, 760, 774, 749, 785, 751, 736, 787, 766, 768, 737, 781, 773, 773, 731, 764, 759, 765, 761, 749, 744, 796, 754] payload_rows_per_dst=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] dup_rows_per_dst=[3312, 2974, 2802, 2958, 2976, 2914, 3082, 2950] combine_rows_per_src=[9540, 9314, 9160, 9410, 9244, 9284, 9150, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344704 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043456 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742208 weight_movement_bytes=34603008 total_nvlink_bytes=568345216 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64874 dup_rows=24134 dup_rows_remote=21188 dedup_ratio=0.245504 router_tokens_per_expert=[772, 796, 785, 775, 764, 777, 728, 761, 792, 729, 760, 749, 795, 785, 790, 749, 758, 806, 766, 772, 820, 777, 760, 836, 703, 766, 766, 738, 809, 736, 708, 802, 773, 776, 781, 786, 781, 769, 798, 770, 778, 801, 707, 785, 797, 767, 749, 737, 751, 773, 728, 731, 734, 772, 725, 761, 797, 767, 814, 747, 763, 808, 749, 747] payload_rows_per_dst=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] dup_rows_per_dst=[3020, 3014, 3126, 2896, 3214, 2946, 2834, 3084] combine_rows_per_src=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343728 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313392 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283056 weight_movement_bytes=69206016 total_nvlink_bytes=601489072 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64934 dup_rows=24038 dup_rows_remote=20998 dedup_ratio=0.244527 router_tokens_per_expert=[734, 741, 743, 745, 715, 804, 755, 764, 780, 745, 796, 751, 816, 781, 757, 760, 759, 781, 745, 787, 756, 793, 767, 777, 800, 793, 776, 728, 726, 726, 828, 769, 732, 780, 765, 774, 822, 783, 750, 759, 797, 729, 764, 762, 725, 790, 789, 784, 762, 781, 789, 820, 786, 737, 729, 745, 789, 777, 768, 752, 748, 818, 771, 777] payload_rows_per_dst=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] dup_rows_per_dst=[2870, 3040, 3036, 3012, 3004, 2944, 3072, 3060] combine_rows_per_src=[9132, 9332, 9294, 9280, 9326, 9336, 9226, 9340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341808 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746224 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150640 weight_movement_bytes=34603008 total_nvlink_bytes=565753648 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65030 dup_rows=23982 dup_rows_remote=21082 dedup_ratio=0.243958 router_tokens_per_expert=[772, 720, 738, 753, 770, 788, 761, 752, 833, 741, 751, 762, 754, 792, 792, 815, 788, 802, 744, 802, 726, 770, 756, 763, 780, 741, 789, 771, 753, 720, 757, 785, 721, 724, 772, 710, 815, 703, 780, 742, 719, 762, 830, 800, 770, 785, 773, 753, 772, 799, 754, 757, 770, 805, 782, 773, 776, 773, 763, 804, 752, 744, 854, 774] payload_rows_per_dst=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] dup_rows_per_dst=[2926, 3042, 2968, 2900, 2892, 3050, 3084, 3120] combine_rows_per_src=[9182, 9438, 9334, 9292, 9042, 9334, 9340, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64842 dup_rows=24054 dup_rows_remote=21008 dedup_ratio=0.244690 router_tokens_per_expert=[713, 743, 774, 783, 785, 764, 729, 757, 715, 748, 761, 794, 788, 776, 782, 757, 774, 734, 762, 763, 794, 809, 768, 823, 787, 761, 778, 782, 786, 795, 754, 830, 812, 757, 765, 761, 803, 782, 787, 725, 756, 732, 815, 789, 777, 757, 750, 767, 758, 768, 787, 770, 742, 793, 762, 716, 768, 764, 770, 753, 754, 720, 772, 751] payload_rows_per_dst=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] dup_rows_per_dst=[2944, 3006, 3046, 3122, 2990, 2998, 3026, 2922] combine_rows_per_src=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64944 dup_rows=23994 dup_rows_remote=21006 dedup_ratio=0.244080 router_tokens_per_expert=[740, 795, 752, 745, 777, 749, 790, 750, 755, 785, 771, 735, 707, 800, 837, 784, 748, 760, 771, 753, 762, 756, 802, 747, 729, 757, 736, 776, 789, 798, 756, 796, 784, 765, 813, 739, 751, 765, 797, 763, 768, 773, 799, 762, 759, 774, 774, 845, 756, 763, 753, 796, 737, 769, 792, 783, 767, 785, 721, 764, 770, 773, 747, 737] payload_rows_per_dst=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] dup_rows_per_dst=[2952, 3026, 2988, 2982, 2988, 3082, 3040, 2936] combine_rows_per_src=[9244, 9322, 9210, 9292, 9366, 9426, 9258, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344248 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266068152 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531792056 weight_movement_bytes=69206016 total_nvlink_bytes=600998072 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64796 dup_rows=23986 dup_rows_remote=20626 dedup_ratio=0.243998 router_tokens_per_expert=[728, 766, 771, 772, 776, 771, 788, 765, 798, 785, 758, 760, 795, 778, 761, 749, 791, 796, 825, 735, 765, 765, 794, 765, 804, 787, 774, 731, 749, 742, 723, 752, 759, 773, 729, 748, 726, 776, 827, 800, 788, 765, 722, 782, 785, 777, 697, 799, 750, 793, 736, 721, 775, 799, 747, 746, 793, 800, 783, 766, 763, 766, 753, 789] payload_rows_per_dst=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] dup_rows_per_dst=[2960, 3044, 2826, 2886, 3088, 2740, 3082, 3360] combine_rows_per_src=[9314, 9324, 9168, 9162, 9296, 9034, 9498, 9522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64842 dup_rows=24054 dup_rows_remote=21008 dedup_ratio=0.244690 router_tokens_per_expert=[713, 743, 774, 783, 785, 764, 729, 757, 715, 748, 761, 794, 788, 776, 782, 757, 774, 734, 762, 763, 794, 809, 768, 823, 787, 761, 778, 782, 786, 795, 754, 830, 812, 757, 765, 761, 803, 782, 787, 725, 756, 732, 815, 789, 777, 757, 750, 767, 758, 768, 787, 770, 742, 793, 762, 716, 768, 764, 770, 753, 754, 720, 772, 751] payload_rows_per_dst=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] dup_rows_per_dst=[2944, 3006, 3046, 3122, 2990, 2998, 3026, 2922] combine_rows_per_src=[9152, 9236, 9408, 9424, 9394, 9288, 9166, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65112 dup_rows=23982 dup_rows_remote=21064 dedup_ratio=0.243958 router_tokens_per_expert=[760, 764, 797, 760, 750, 783, 754, 802, 773, 730, 764, 773, 744, 746, 776, 773, 803, 766, 786, 756, 749, 769, 798, 748, 768, 806, 787, 761, 754, 800, 756, 758, 725, 761, 796, 824, 761, 812, 809, 779, 776, 760, 750, 775, 744, 774, 767, 797, 775, 725, 748, 787, 743, 731, 777, 778, 753, 777, 741, 736, 767, 771, 725, 794] payload_rows_per_dst=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] dup_rows_per_dst=[3020, 2910, 3016, 3002, 3118, 3080, 2918, 2918] combine_rows_per_src=[9320, 9248, 9334, 9378, 9416, 9206, 9210, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64874 dup_rows=24134 dup_rows_remote=21188 dedup_ratio=0.245504 router_tokens_per_expert=[772, 796, 785, 775, 764, 777, 728, 761, 792, 729, 760, 749, 795, 785, 790, 749, 758, 806, 766, 772, 820, 777, 760, 836, 703, 766, 766, 738, 809, 736, 708, 802, 773, 776, 781, 786, 781, 769, 798, 770, 778, 801, 707, 785, 797, 767, 749, 737, 751, 773, 728, 731, 734, 772, 725, 761, 797, 767, 814, 747, 763, 808, 749, 747] payload_rows_per_dst=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] dup_rows_per_dst=[3020, 3014, 3126, 2896, 3214, 2946, 2834, 3084] combine_rows_per_src=[9296, 9284, 9464, 9160, 9254, 9296, 9116, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344448 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266707328 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533070208 weight_movement_bytes=69206016 total_nvlink_bytes=602276224 weight_to_token_ratio=0.129825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64968 dup_rows=24024 dup_rows_remote=20996 dedup_ratio=0.244385 router_tokens_per_expert=[750, 762, 799, 763, 699, 802, 778, 751, 764, 757, 805, 790, 750, 759, 767, 752, 730, 753, 736, 811, 770, 788, 757, 754, 813, 806, 761, 780, 760, 803, 749, 704, 787, 776, 761, 786, 784, 773, 784, 790, 767, 780, 769, 775, 794, 765, 764, 786, 724, 770, 815, 858, 727, 756, 775, 745, 769, 734, 803, 781, 760, 743, 722, 706] payload_rows_per_dst=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] dup_rows_per_dst=[2952, 3010, 2962, 3060, 3156, 3022, 3028, 2834] combine_rows_per_src=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343728 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313392 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283056 weight_movement_bytes=69206016 total_nvlink_bytes=601489072 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344448 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266707328 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533070208 weight_movement_bytes=69206016 total_nvlink_bytes=602276224 weight_to_token_ratio=0.129825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343400 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265936232 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531529064 weight_movement_bytes=103809024 total_nvlink_bytes=635338088 weight_to_token_ratio=0.195303 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343800 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266354424 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532365048 weight_movement_bytes=69206016 total_nvlink_bytes=601571064 weight_to_token_ratio=0.129997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265404416 dispatch_meta_bytes=341688 combine_payload_bytes=265404416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265746104 ingress_bytes=300007424 link_bytes=300007424 token_bytes=531150520 weight_movement_bytes=34603008 total_nvlink_bytes=565753528 weight_to_token_ratio=0.065147 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128903784e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343400 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265936232 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531529064 weight_movement_bytes=103809024 total_nvlink_bytes=635338088 weight_to_token_ratio=0.195303 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64818 dup_rows=24018 dup_rows_remote=20874 dedup_ratio=0.244324 router_tokens_per_expert=[753, 755, 797, 728, 759, 728, 759, 791, 768, 774, 770, 799, 794, 857, 788, 756, 758, 761, 749, 751, 758, 777, 769, 739, 755, 746, 816, 830, 780, 771, 789, 761, 788, 741, 776, 752, 740, 788, 767, 765, 753, 759, 781, 750, 773, 714, 771, 769, 766, 767, 771, 768, 771, 769, 728, 786, 757, 767, 734, 789, 783, 766, 792, 765] payload_rows_per_dst=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] dup_rows_per_dst=[2930, 3144, 2948, 3108, 2934, 2976, 2982, 2996] combine_rows_per_src=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344704 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043456 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742208 weight_movement_bytes=34603008 total_nvlink_bytes=568345216 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344248 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266068152 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531792056 weight_movement_bytes=69206016 total_nvlink_bytes=600998072 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64728 dup_rows=23930 dup_rows_remote=20472 dedup_ratio=0.243429 router_tokens_per_expert=[793, 777, 769, 761, 761, 797, 739, 768, 782, 759, 771, 787, 816, 783, 749, 756, 808, 769, 794, 732, 762, 711, 788, 748, 797, 793, 746, 754, 820, 779, 808, 775, 756, 744, 799, 741, 781, 781, 765, 755, 809, 744, 808, 737, 801, 785, 744, 794, 772, 786, 730, 741, 768, 798, 754, 762, 714, 730, 760, 748, 719, 786, 785, 703] payload_rows_per_dst=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] dup_rows_per_dst=[3458, 2820, 2922, 3122, 2850, 3044, 2938, 2776] combine_rows_per_src=[9646, 9088, 9316, 9422, 9118, 9386, 9284, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65174 dup_rows=23884 dup_rows_remote=21016 dedup_ratio=0.242961 router_tokens_per_expert=[776, 746, 763, 718, 806, 798, 796, 748, 739, 841, 750, 741, 802, 811, 781, 791, 725, 751, 721, 784, 785, 773, 770, 777, 771, 731, 775, 740, 764, 797, 795, 746, 799, 797, 791, 770, 796, 751, 756, 789, 782, 773, 717, 774, 731, 765, 762, 753, 801, 801, 750, 772, 745, 784, 765, 804, 736, 764, 764, 758, 745, 753, 716, 776] payload_rows_per_dst=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] dup_rows_per_dst=[2980, 3096, 2916, 2928, 3080, 2868, 3122, 2894] combine_rows_per_src=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343856 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452784 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561712 weight_movement_bytes=34603008 total_nvlink_bytes=567164720 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64902 dup_rows=24060 dup_rows_remote=20982 dedup_ratio=0.244751 router_tokens_per_expert=[825, 783, 751, 721, 777, 782, 776, 771, 767, 780, 801, 779, 776, 789, 765, 761, 799, 741, 729, 722, 765, 756, 763, 739, 773, 790, 794, 756, 763, 799, 783, 752, 740, 754, 772, 765, 799, 746, 776, 762, 768, 763, 793, 744, 756, 800, 733, 790, 760, 758, 727, 775, 763, 789, 764, 768, 759, 739, 758, 798, 773, 804, 753, 775] payload_rows_per_dst=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] dup_rows_per_dst=[3108, 3018, 2884, 3078, 2978, 3046, 2930, 3018] combine_rows_per_src=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342768 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265837296 ingress_bytes=438509568 link_bytes=438509568 token_bytes=531331824 weight_movement_bytes=173015040 total_nvlink_bytes=704346864 weight_to_token_ratio=0.325625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650076201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64818 dup_rows=24018 dup_rows_remote=20874 dedup_ratio=0.244324 router_tokens_per_expert=[753, 755, 797, 728, 759, 728, 759, 791, 768, 774, 770, 799, 794, 857, 788, 756, 758, 761, 749, 751, 758, 777, 769, 739, 755, 746, 816, 830, 780, 771, 789, 761, 788, 741, 776, 752, 740, 788, 767, 765, 753, 759, 781, 750, 773, 714, 771, 769, 766, 767, 771, 768, 771, 769, 728, 786, 757, 767, 734, 789, 783, 766, 792, 765] payload_rows_per_dst=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] dup_rows_per_dst=[2930, 3144, 2948, 3108, 2934, 2976, 2982, 2996] combine_rows_per_src=[9210, 9468, 9176, 9388, 9300, 9164, 9270, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=340800 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265466688 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530592576 weight_movement_bytes=34603008 total_nvlink_bytes=565195584 weight_to_token_ratio=0.065216 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344760 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297464 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250168 weight_movement_bytes=34603008 total_nvlink_bytes=568853176 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64898 dup_rows=24082 dup_rows_remote=20930 dedup_ratio=0.244975 router_tokens_per_expert=[760, 752, 722, 765, 757, 776, 810, 773, 739, 761, 806, 803, 807, 802, 794, 730, 773, 803, 758, 783, 743, 778, 759, 745, 732, 706, 760, 750, 753, 773, 755, 753, 740, 807, 845, 812, 779, 762, 753, 740, 790, 755, 719, 742, 753, 730, 774, 789, 736, 731, 786, 815, 809, 791, 820, 740, 785, 751, 772, 803, 778, 747, 753, 764] payload_rows_per_dst=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] dup_rows_per_dst=[3012, 3076, 3050, 2838, 3152, 2890, 3086, 2978] combine_rows_per_src=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20892 dedup_ratio=0.244059 router_tokens_per_expert=[786, 758, 770, 781, 776, 771, 762, 762, 745, 784, 787, 737, 786, 817, 766, 715, 730, 760, 827, 796, 761, 768, 742, 749, 792, 762, 795, 813, 777, 782, 759, 731, 737, 790, 743, 773, 742, 730, 763, 797, 780, 768, 745, 747, 726, 810, 788, 754, 758, 793, 745, 772, 789, 794, 813, 719, 805, 753, 821, 717, 732, 765, 764, 772] payload_rows_per_dst=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] dup_rows_per_dst=[3034, 2456, 3206, 3152, 2864, 3274, 2906, 3100] combine_rows_per_src=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343536 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182128 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532020720 weight_movement_bytes=34603008 total_nvlink_bytes=566623728 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64902 dup_rows=24060 dup_rows_remote=20982 dedup_ratio=0.244751 router_tokens_per_expert=[825, 783, 751, 721, 777, 782, 776, 771, 767, 780, 801, 779, 776, 789, 765, 761, 799, 741, 729, 722, 765, 756, 763, 739, 773, 790, 794, 756, 763, 799, 783, 752, 740, 754, 772, 765, 799, 746, 776, 762, 768, 763, 793, 744, 756, 800, 733, 790, 760, 758, 727, 775, 763, 789, 764, 768, 759, 739, 758, 798, 773, 804, 753, 775] payload_rows_per_dst=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] dup_rows_per_dst=[3108, 3018, 2884, 3078, 2978, 3046, 2930, 3018] combine_rows_per_src=[9264, 9418, 9144, 9342, 9250, 9248, 9278, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65104 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[756, 769, 711, 766, 782, 775, 764, 769, 796, 766, 791, 749, 804, 756, 758, 746, 759, 790, 742, 746, 785, 740, 740, 782, 761, 765, 758, 777, 819, 766, 823, 749, 759, 702, 802, 768, 794, 823, 743, 761, 688, 826, 784, 803, 742, 758, 817, 788, 708, 767, 788, 787, 744, 764, 760, 775, 720, 781, 759, 737, 810, 767, 786, 781] payload_rows_per_dst=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] dup_rows_per_dst=[2836, 3032, 2988, 3122, 2942, 3058, 3000, 3042] combine_rows_per_src=[9348, 9300, 9180, 9314, 9362, 9354, 9186, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343312 combine_payload_bytes=265822208 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266165520 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987728 weight_movement_bytes=34603008 total_nvlink_bytes=566590736 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64894 dup_rows=23992 dup_rows_remote=20892 dedup_ratio=0.244059 router_tokens_per_expert=[786, 758, 770, 781, 776, 771, 762, 762, 745, 784, 787, 737, 786, 817, 766, 715, 730, 760, 827, 796, 761, 768, 742, 749, 792, 762, 795, 813, 777, 782, 759, 731, 737, 790, 743, 773, 742, 730, 763, 797, 780, 768, 745, 747, 726, 810, 788, 754, 758, 793, 745, 772, 789, 794, 813, 719, 805, 753, 821, 717, 732, 765, 764, 772] payload_rows_per_dst=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] dup_rows_per_dst=[3034, 2456, 3206, 3152, 2864, 3274, 2906, 3100] combine_rows_per_src=[9298, 8768, 9610, 9468, 9248, 9264, 9238, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64898 dup_rows=24082 dup_rows_remote=20930 dedup_ratio=0.244975 router_tokens_per_expert=[760, 752, 722, 765, 757, 776, 810, 773, 739, 761, 806, 803, 807, 802, 794, 730, 773, 803, 758, 783, 743, 778, 759, 745, 732, 706, 760, 750, 753, 773, 755, 753, 740, 807, 845, 812, 779, 762, 753, 740, 790, 755, 719, 742, 753, 730, 774, 789, 736, 731, 786, 815, 809, 791, 820, 740, 785, 751, 772, 803, 778, 747, 753, 764] payload_rows_per_dst=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] dup_rows_per_dst=[3012, 3076, 3050, 2838, 3152, 2890, 3086, 2978] combine_rows_per_src=[9218, 9408, 9234, 9126, 9324, 9214, 9370, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64968 dup_rows=24024 dup_rows_remote=20996 dedup_ratio=0.244385 router_tokens_per_expert=[750, 762, 799, 763, 699, 802, 778, 751, 764, 757, 805, 790, 750, 759, 767, 752, 730, 753, 736, 811, 770, 788, 757, 754, 813, 806, 761, 780, 760, 803, 749, 704, 787, 776, 761, 786, 784, 773, 784, 790, 767, 780, 769, 775, 794, 765, 764, 786, 724, 770, 815, 858, 727, 756, 775, 745, 769, 734, 803, 781, 760, 743, 722, 706] payload_rows_per_dst=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] dup_rows_per_dst=[2952, 3010, 2962, 3060, 3156, 3022, 3028, 2834] combine_rows_per_src=[9256, 9278, 9236, 9292, 9326, 9378, 9312, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65174 dup_rows=23884 dup_rows_remote=21016 dedup_ratio=0.242961 router_tokens_per_expert=[776, 746, 763, 718, 806, 798, 796, 748, 739, 841, 750, 741, 802, 811, 781, 791, 725, 751, 721, 784, 785, 773, 770, 777, 771, 731, 775, 740, 764, 797, 795, 746, 799, 797, 791, 770, 796, 751, 756, 789, 782, 773, 717, 774, 731, 765, 762, 753, 801, 801, 750, 772, 745, 784, 765, 804, 736, 764, 764, 758, 745, 753, 716, 776] payload_rows_per_dst=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] dup_rows_per_dst=[2980, 3096, 2916, 2928, 3080, 2868, 3122, 2894] combine_rows_per_src=[9322, 9416, 9256, 9310, 9418, 9246, 9322, 9130] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343144 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148968 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954792 weight_movement_bytes=34603008 total_nvlink_bytes=566557800 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64768 dup_rows=24242 dup_rows_remote=21172 dedup_ratio=0.246602 router_tokens_per_expert=[784, 764, 811, 777, 718, 814, 751, 695, 792, 778, 835, 753, 784, 786, 773, 789, 756, 730, 758, 736, 787, 759, 770, 777, 768, 748, 773, 730, 766, 765, 703, 804, 779, 769, 789, 732, 721, 811, 771, 770, 786, 764, 793, 751, 767, 749, 786, 721, 769, 751, 800, 799, 761, 795, 771, 736, 794, 761, 768, 765, 763, 776, 798, 752] payload_rows_per_dst=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] dup_rows_per_dst=[2950, 3130, 2922, 2974, 3148, 2946, 3070, 3102] combine_rows_per_src=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342768 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=265837296 ingress_bytes=438509568 link_bytes=438509568 token_bytes=531331824 weight_movement_bytes=173015040 total_nvlink_bytes=704346864 weight_to_token_ratio=0.325625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650076201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=343536 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182128 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532020720 weight_movement_bytes=34603008 total_nvlink_bytes=566623728 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344544 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010528 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676512 weight_movement_bytes=34603008 total_nvlink_bytes=568279520 weight_to_token_ratio=0.064839 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21080 dedup_ratio=0.244324 router_tokens_per_expert=[803, 765, 770, 741, 780, 811, 787, 728, 779, 791, 752, 762, 741, 773, 750, 754, 745, 787, 739, 732, 752, 769, 811, 767, 799, 767, 803, 752, 751, 753, 715, 817, 720, 784, 752, 806, 775, 749, 784, 786, 780, 729, 775, 820, 751, 745, 797, 784, 775, 747, 781, 768, 744, 799, 741, 777, 817, 725, 736, 774, 772, 763, 750, 800] payload_rows_per_dst=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] dup_rows_per_dst=[2980, 2938, 2932, 2998, 3090, 3042, 3032, 3006] combine_rows_per_src=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343144 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266148968 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531954792 weight_movement_bytes=34603008 total_nvlink_bytes=566557800 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343312 combine_payload_bytes=265822208 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266165520 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531987728 weight_movement_bytes=34603008 total_nvlink_bytes=566590736 weight_to_token_ratio=0.065045 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343856 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452784 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561712 weight_movement_bytes=34603008 total_nvlink_bytes=567164720 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344760 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297464 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250168 weight_movement_bytes=34603008 total_nvlink_bytes=568853176 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64826 dup_rows=24100 dup_rows_remote=20962 dedup_ratio=0.245158 router_tokens_per_expert=[738, 779, 774, 741, 797, 769, 748, 775, 775, 737, 795, 719, 740, 794, 795, 786, 784, 806, 795, 753, 792, 768, 770, 815, 756, 726, 739, 770, 780, 758, 740, 739, 786, 783, 717, 719, 813, 747, 784, 735, 768, 797, 757, 773, 827, 779, 770, 763, 779, 754, 752, 779, 780, 773, 771, 795, 762, 789, 757, 788, 773, 732, 759, 738] payload_rows_per_dst=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] dup_rows_per_dst=[3138, 2802, 3140, 2948, 2958, 3198, 2988, 2928] combine_rows_per_src=[9378, 9322, 9426, 9068, 9210, 9270, 9262, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65002 dup_rows=23986 dup_rows_remote=20976 dedup_ratio=0.243998 router_tokens_per_expert=[761, 722, 775, 783, 795, 748, 757, 761, 783, 823, 799, 755, 738, 761, 791, 782, 748, 826, 786, 760, 752, 725, 821, 765, 758, 760, 778, 777, 753, 754, 760, 786, 821, 737, 728, 735, 763, 746, 728, 778, 764, 751, 791, 761, 759, 806, 767, 764, 772, 736, 761, 785, 752, 810, 771, 834, 723, 783, 757, 784, 802, 759, 737, 744] payload_rows_per_dst=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] dup_rows_per_dst=[2920, 3016, 3158, 3044, 2862, 3010, 3054, 2922] combine_rows_per_src=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343760 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265633488 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530923216 weight_movement_bytes=103809024 total_nvlink_bytes=634732240 weight_to_token_ratio=0.195525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65256 dup_rows=23858 dup_rows_remote=20994 dedup_ratio=0.242696 router_tokens_per_expert=[762, 738, 747, 770, 779, 766, 708, 762, 790, 799, 750, 714, 844, 765, 774, 769, 748, 761, 789, 757, 729, 782, 779, 777, 729, 773, 758, 739, 752, 737, 748, 791, 781, 782, 750, 740, 778, 728, 743, 773, 780, 808, 783, 794, 733, 793, 791, 782, 778, 768, 815, 709, 770, 743, 755, 751, 810, 789, 753, 830, 730, 836, 838, 752] payload_rows_per_dst=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] dup_rows_per_dst=[2886, 3020, 3010, 2864, 2974, 3058, 2920, 3126] combine_rows_per_src=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344400 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666320 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988240 weight_movement_bytes=69206016 total_nvlink_bytes=602194256 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343152 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265870448 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531397744 weight_movement_bytes=34603008 total_nvlink_bytes=566000752 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343912 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592104 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532840296 weight_movement_bytes=103809024 total_nvlink_bytes=636649320 weight_to_token_ratio=0.194822 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65020 dup_rows=24018 dup_rows_remote=21080 dedup_ratio=0.244324 router_tokens_per_expert=[803, 765, 770, 741, 780, 811, 787, 728, 779, 791, 752, 762, 741, 773, 750, 754, 745, 787, 739, 732, 752, 769, 811, 767, 799, 767, 803, 752, 751, 753, 715, 817, 720, 784, 752, 806, 775, 749, 784, 786, 780, 729, 775, 820, 751, 745, 797, 784, 775, 747, 781, 768, 744, 799, 741, 777, 817, 725, 736, 774, 772, 763, 750, 800] payload_rows_per_dst=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] dup_rows_per_dst=[2980, 2938, 2932, 2998, 3090, 3042, 3032, 3006] combine_rows_per_src=[9390, 9266, 9272, 9316, 9222, 9320, 9232, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65256 dup_rows=23858 dup_rows_remote=20994 dedup_ratio=0.242696 router_tokens_per_expert=[762, 738, 747, 770, 779, 766, 708, 762, 790, 799, 750, 714, 844, 765, 774, 769, 748, 761, 789, 757, 729, 782, 779, 777, 729, 773, 758, 739, 752, 737, 748, 791, 781, 782, 750, 740, 778, 728, 743, 773, 780, 808, 783, 794, 733, 793, 791, 782, 778, 768, 815, 709, 770, 743, 755, 751, 810, 789, 753, 830, 730, 836, 838, 752] payload_rows_per_dst=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] dup_rows_per_dst=[2886, 3020, 3010, 2864, 2974, 3058, 2920, 3126] combine_rows_per_src=[9178, 9390, 9234, 9190, 9176, 9470, 9258, 9550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65164 dup_rows=23910 dup_rows_remote=20916 dedup_ratio=0.243225 router_tokens_per_expert=[751, 731, 794, 753, 758, 748, 761, 758, 783, 731, 791, 807, 760, 775, 797, 740, 756, 777, 777, 777, 749, 747, 754, 775, 786, 722, 825, 694, 714, 820, 706, 753, 770, 772, 745, 752, 797, 761, 771, 754, 745, 806, 759, 744, 818, 775, 772, 785, 773, 773, 781, 786, 762, 800, 806, 774, 757, 805, 750, 822, 736, 785, 759, 787] payload_rows_per_dst=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] dup_rows_per_dst=[2974, 2902, 2994, 2904, 2948, 3030, 3060, 3098] combine_rows_per_src=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65244 dup_rows=23716 dup_rows_remote=20650 dedup_ratio=0.241252 router_tokens_per_expert=[787, 744, 747, 746, 762, 774, 721, 774, 755, 789, 774, 750, 765, 795, 748, 767, 732, 730, 727, 776, 747, 763, 840, 738, 771, 736, 780, 822, 791, 777, 799, 762, 814, 822, 746, 743, 778, 745, 743, 764, 773, 775, 786, 736, 800, 749, 763, 768, 759, 800, 775, 767, 792, 799, 753, 780, 774, 769, 745, 762, 811, 755, 719, 798] payload_rows_per_dst=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] dup_rows_per_dst=[2880, 2964, 2896, 3002, 2988, 2886, 3034, 3066] combine_rows_per_src=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64874 dup_rows=24082 dup_rows_remote=21154 dedup_ratio=0.244975 router_tokens_per_expert=[762, 776, 736, 757, 762, 787, 764, 784, 742, 775, 787, 781, 756, 738, 747, 767, 795, 769, 760, 743, 740, 731, 767, 795, 757, 806, 821, 734, 767, 737, 725, 776, 772, 817, 774, 781, 739, 771, 722, 762, 748, 853, 740, 786, 748, 796, 797, 755, 774, 744, 783, 800, 792, 765, 782, 773, 760, 765, 747, 758, 775, 756, 801, 772] payload_rows_per_dst=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] dup_rows_per_dst=[3016, 2966, 2914, 3036, 2928, 3046, 3156, 3020] combine_rows_per_src=[9240, 9220, 9286, 9210, 9348, 9400, 9270, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64768 dup_rows=24242 dup_rows_remote=21172 dedup_ratio=0.246602 router_tokens_per_expert=[784, 764, 811, 777, 718, 814, 751, 695, 792, 778, 835, 753, 784, 786, 773, 789, 756, 730, 758, 736, 787, 759, 770, 777, 768, 748, 773, 730, 766, 765, 703, 804, 779, 769, 789, 732, 721, 811, 771, 770, 786, 764, 793, 751, 767, 749, 786, 721, 769, 751, 800, 799, 761, 795, 771, 736, 794, 761, 768, 765, 763, 776, 798, 752] payload_rows_per_dst=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] dup_rows_per_dst=[2950, 3130, 2922, 2974, 3148, 2946, 3070, 3102] combine_rows_per_src=[9278, 9450, 9224, 9140, 9136, 9288, 9294, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74588 payload_rows_remote=65244 dup_rows=23716 dup_rows_remote=20650 dedup_ratio=0.241252 router_tokens_per_expert=[787, 744, 747, 746, 762, 774, 721, 774, 755, 789, 774, 750, 765, 795, 748, 767, 732, 730, 727, 776, 747, 763, 840, 738, 771, 736, 780, 822, 791, 777, 799, 762, 814, 822, 746, 743, 778, 745, 743, 764, 773, 775, 786, 736, 800, 749, 763, 768, 759, 800, 775, 767, 792, 799, 753, 780, 774, 769, 745, 762, 811, 755, 719, 798] payload_rows_per_dst=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] dup_rows_per_dst=[2880, 2964, 2896, 3002, 2988, 2886, 3034, 3066] combine_rows_per_src=[9230, 9162, 9210, 9474, 9322, 9324, 9522, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65002 dup_rows=23986 dup_rows_remote=20976 dedup_ratio=0.243998 router_tokens_per_expert=[761, 722, 775, 783, 795, 748, 757, 761, 783, 823, 799, 755, 738, 761, 791, 782, 748, 826, 786, 760, 752, 725, 821, 765, 758, 760, 778, 777, 753, 754, 760, 786, 821, 737, 728, 735, 763, 746, 728, 778, 764, 751, 791, 761, 759, 806, 767, 764, 772, 736, 761, 785, 752, 810, 771, 834, 723, 783, 757, 784, 802, 759, 737, 744] payload_rows_per_dst=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] dup_rows_per_dst=[2920, 3016, 3158, 3044, 2862, 3010, 3054, 2922] combine_rows_per_src=[9284, 9448, 9208, 9208, 9210, 9316, 9388, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344400 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666320 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532988240 weight_movement_bytes=69206016 total_nvlink_bytes=602194256 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267288576 dispatch_meta_bytes=345000 combine_payload_bytes=267288576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267633576 ingress_bytes=301891584 link_bytes=301891584 token_bytes=534922152 weight_movement_bytes=34603008 total_nvlink_bytes=569525160 weight_to_token_ratio=0.064688 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135993726e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65164 dup_rows=23910 dup_rows_remote=20916 dedup_ratio=0.243225 router_tokens_per_expert=[751, 731, 794, 753, 758, 748, 761, 758, 783, 731, 791, 807, 760, 775, 797, 740, 756, 777, 777, 777, 749, 747, 754, 775, 786, 722, 825, 694, 714, 820, 706, 753, 770, 772, 745, 752, 797, 761, 771, 754, 745, 806, 759, 744, 818, 775, 772, 785, 773, 773, 781, 786, 762, 800, 806, 774, 757, 805, 750, 822, 736, 785, 759, 787] payload_rows_per_dst=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] dup_rows_per_dst=[2974, 2902, 2994, 2904, 2948, 3030, 3060, 3098] combine_rows_per_src=[9134, 9466, 9230, 9136, 9296, 9378, 9450, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344112 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266068016 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531791920 weight_movement_bytes=34603008 total_nvlink_bytes=566394928 weight_to_token_ratio=0.065069 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20900 dedup_ratio=0.243510 router_tokens_per_expert=[751, 751, 798, 773, 767, 746, 743, 765, 816, 816, 743, 793, 756, 743, 775, 769, 764, 781, 761, 737, 766, 742, 740, 807, 774, 758, 740, 784, 755, 768, 742, 783, 790, 801, 782, 720, 773, 770, 754, 770, 750, 758, 748, 767, 727, 806, 799, 727, 790, 782, 806, 808, 766, 745, 740, 805, 751, 777, 751, 788, 742, 781, 770, 801] payload_rows_per_dst=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] dup_rows_per_dst=[2978, 3078, 2936, 2954, 3002, 2944, 3038, 3008] combine_rows_per_src=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=343576 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267583000 ingress_bytes=336445440 link_bytes=336445440 token_bytes=534822424 weight_movement_bytes=69206016 total_nvlink_bytes=604028440 weight_to_token_ratio=0.129400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266017105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344320 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256064 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167808 weight_movement_bytes=34603008 total_nvlink_bytes=568770816 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64990 dup_rows=23882 dup_rows_remote=20744 dedup_ratio=0.242940 router_tokens_per_expert=[735, 810, 689, 785, 761, 761, 773, 772, 792, 808, 815, 762, 810, 786, 732, 780, 828, 789, 757, 781, 754, 776, 744, 755, 802, 763, 739, 838, 736, 740, 739, 706, 819, 798, 729, 726, 797, 789, 777, 764, 804, 767, 800, 755, 755, 737, 776, 736, 752, 762, 756, 728, 745, 752, 753, 733, 805, 763, 769, 807, 768, 767, 733, 812] payload_rows_per_dst=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] dup_rows_per_dst=[2924, 3138, 3030, 2874, 2984, 3038, 2858, 3036] combine_rows_per_src=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=343576 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267583000 ingress_bytes=336445440 link_bytes=336445440 token_bytes=534822424 weight_movement_bytes=69206016 total_nvlink_bytes=604028440 weight_to_token_ratio=0.129400 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266017105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=344112 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266068016 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531791920 weight_movement_bytes=34603008 total_nvlink_bytes=566394928 weight_to_token_ratio=0.065069 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73710 payload_rows_remote=64134 dup_rows=24594 dup_rows_remote=21166 dedup_ratio=0.250183 router_tokens_per_expert=[806, 814, 756, 799, 789, 785, 775, 793, 748, 762, 772, 779, 770, 728, 757, 746, 761, 805, 734, 757, 744, 771, 757, 794, 778, 708, 751, 763, 751, 748, 776, 801, 763, 716, 744, 800, 786, 745, 737, 803, 787, 762, 757, 790, 756, 830, 761, 746, 813, 756, 755, 820, 774, 798, 708, 764, 736, 735, 790, 782, 773, 781, 769, 737] payload_rows_per_dst=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] dup_rows_per_dst=[3428, 2958, 3414, 3314, 3198, 2780, 2686, 2816] combine_rows_per_src=[9576, 9166, 9350, 9400, 9422, 9080, 8758, 8958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65140 dup_rows=23790 dup_rows_remote=20666 dedup_ratio=0.242004 router_tokens_per_expert=[762, 773, 751, 752, 767, 732, 763, 742, 739, 788, 764, 788, 765, 769, 767, 773, 788, 785, 789, 736, 751, 736, 747, 801, 760, 774, 781, 805, 767, 760, 741, 786, 719, 815, 741, 717, 813, 742, 744, 759, 779, 801, 798, 715, 817, 793, 749, 797, 717, 761, 784, 774, 779, 787, 727, 780, 778, 783, 744, 774, 746, 792, 830, 795] payload_rows_per_dst=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] dup_rows_per_dst=[2924, 3028, 2944, 2986, 2856, 3124, 2888, 3040] combine_rows_per_src=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343760 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265633488 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530923216 weight_movement_bytes=103809024 total_nvlink_bytes=634732240 weight_to_token_ratio=0.195525 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343912 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592104 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532840296 weight_movement_bytes=103809024 total_nvlink_bytes=636649320 weight_to_token_ratio=0.194822 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343280 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255600 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167920 weight_movement_bytes=103809024 total_nvlink_bytes=635976944 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64894 dup_rows=24046 dup_rows_remote=21002 dedup_ratio=0.244609 router_tokens_per_expert=[784, 791, 772, 765, 773, 727, 754, 742, 790, 793, 745, 759, 790, 812, 755, 772, 744, 804, 776, 766, 750, 755, 790, 799, 783, 765, 774, 791, 780, 763, 787, 761, 781, 724, 776, 799, 797, 772, 765, 731, 759, 755, 730, 750, 744, 755, 777, 761, 759, 763, 762, 760, 817, 822, 785, 746, 813, 803, 728, 712, 742, 730, 767, 755] payload_rows_per_dst=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] dup_rows_per_dst=[2976, 3042, 3068, 3044, 3006, 2924, 3088, 2898] combine_rows_per_src=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342936 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266541976 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532741016 weight_movement_bytes=138412032 total_nvlink_bytes=671153048 weight_to_token_ratio=0.259811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=262692864 dispatch_meta_bytes=341200 combine_payload_bytes=262692864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263034064 ingress_bytes=297295872 link_bytes=297295872 token_bytes=525726928 weight_movement_bytes=34603008 total_nvlink_bytes=560329936 weight_to_token_ratio=0.065819 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.118700432e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64990 dup_rows=23882 dup_rows_remote=20744 dedup_ratio=0.242940 router_tokens_per_expert=[735, 810, 689, 785, 761, 761, 773, 772, 792, 808, 815, 762, 810, 786, 732, 780, 828, 789, 757, 781, 754, 776, 744, 755, 802, 763, 739, 838, 736, 740, 739, 706, 819, 798, 729, 726, 797, 789, 777, 764, 804, 767, 800, 755, 755, 737, 776, 736, 752, 762, 756, 728, 745, 752, 753, 733, 805, 763, 769, 807, 768, 767, 733, 812] payload_rows_per_dst=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] dup_rows_per_dst=[2924, 3138, 3030, 2874, 2984, 3038, 2858, 3036] combine_rows_per_src=[9248, 9432, 9338, 9252, 9414, 9222, 9104, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64894 dup_rows=24046 dup_rows_remote=21002 dedup_ratio=0.244609 router_tokens_per_expert=[784, 791, 772, 765, 773, 727, 754, 742, 790, 793, 745, 759, 790, 812, 755, 772, 744, 804, 776, 766, 750, 755, 790, 799, 783, 765, 774, 791, 780, 763, 787, 761, 781, 724, 776, 799, 797, 772, 765, 731, 759, 755, 730, 750, 744, 755, 777, 761, 759, 763, 762, 760, 817, 822, 785, 746, 813, 803, 728, 712, 742, 730, 767, 755] payload_rows_per_dst=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] dup_rows_per_dst=[2976, 3042, 3068, 3044, 3006, 2924, 3088, 2898] combine_rows_per_src=[9240, 9390, 9300, 9364, 9284, 9138, 9340, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64998 dup_rows=23966 dup_rows_remote=20984 dedup_ratio=0.243795 router_tokens_per_expert=[737, 768, 778, 751, 775, 796, 768, 769, 720, 791, 796, 773, 728, 790, 801, 775, 780, 748, 782, 749, 770, 761, 812, 759, 783, 742, 722, 840, 768, 795, 794, 763, 775, 756, 786, 719, 731, 773, 804, 738, 770, 738, 719, 782, 798, 788, 744, 763, 808, 758, 791, 781, 754, 758, 740, 795, 766, 758, 746, 757, 768, 766, 785, 753] payload_rows_per_dst=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] dup_rows_per_dst=[2980, 3106, 2982, 3026, 2924, 2904, 2982, 3062] combine_rows_per_src=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64788 dup_rows=24206 dup_rows_remote=21214 dedup_ratio=0.246236 router_tokens_per_expert=[768, 732, 758, 752, 742, 765, 735, 772, 724, 758, 748, 807, 741, 741, 791, 765, 801, 800, 781, 763, 762, 766, 743, 750, 774, 789, 767, 770, 815, 722, 738, 782, 744, 777, 792, 768, 742, 780, 719, 824, 795, 774, 799, 794, 793, 791, 767, 801, 770, 779, 801, 743, 766, 772, 764, 731, 737, 814, 810, 746, 768, 745, 776, 748] payload_rows_per_dst=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] dup_rows_per_dst=[2962, 2924, 3052, 3006, 2948, 3258, 3064, 2992] combine_rows_per_src=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64920 dup_rows=24166 dup_rows_remote=21022 dedup_ratio=0.245829 router_tokens_per_expert=[742, 788, 762, 771, 780, 762, 821, 744, 739, 730, 750, 798, 797, 777, 778, 795, 755, 748, 771, 805, 781, 723, 729, 744, 780, 735, 758, 749, 767, 779, 741, 799, 761, 764, 769, 777, 803, 757, 786, 764, 787, 754, 742, 758, 807, 773, 733, 767, 818, 735, 772, 736, 773, 771, 789, 797, 765, 785, 744, 799, 776, 752, 772, 768] payload_rows_per_dst=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] dup_rows_per_dst=[3012, 3128, 2854, 3002, 3144, 2920, 3026, 3080] combine_rows_per_src=[9328, 9200, 9258, 9214, 9218, 9322, 9356, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343224 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267156664 ingress_bytes=336019456 link_bytes=336019456 token_bytes=533970104 weight_movement_bytes=69206016 total_nvlink_bytes=603176120 weight_to_token_ratio=0.129607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264414161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64788 dup_rows=24206 dup_rows_remote=21214 dedup_ratio=0.246236 router_tokens_per_expert=[768, 732, 758, 752, 742, 765, 735, 772, 724, 758, 748, 807, 741, 741, 791, 765, 801, 800, 781, 763, 762, 766, 743, 750, 774, 789, 767, 770, 815, 722, 738, 782, 744, 777, 792, 768, 742, 780, 719, 824, 795, 774, 799, 794, 793, 791, 767, 801, 770, 779, 801, 743, 766, 772, 764, 731, 737, 814, 810, 746, 768, 745, 776, 748] payload_rows_per_dst=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] dup_rows_per_dst=[2962, 2924, 3052, 3006, 2948, 3258, 3064, 2992] combine_rows_per_src=[9086, 9160, 9322, 9310, 9252, 9470, 9188, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20900 dedup_ratio=0.243510 router_tokens_per_expert=[751, 751, 798, 773, 767, 746, 743, 765, 816, 816, 743, 793, 756, 743, 775, 769, 764, 781, 761, 737, 766, 742, 740, 807, 774, 758, 740, 784, 755, 768, 742, 783, 790, 801, 782, 720, 773, 770, 754, 770, 750, 758, 748, 767, 727, 806, 799, 727, 790, 782, 806, 808, 766, 745, 740, 805, 751, 777, 751, 788, 742, 781, 770, 801] payload_rows_per_dst=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] dup_rows_per_dst=[2978, 3078, 2936, 2954, 3002, 2944, 3038, 3008] combine_rows_per_src=[9210, 9344, 9260, 9254, 9318, 9220, 9446, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65140 dup_rows=23790 dup_rows_remote=20666 dedup_ratio=0.242004 router_tokens_per_expert=[762, 773, 751, 752, 767, 732, 763, 742, 739, 788, 764, 788, 765, 769, 767, 773, 788, 785, 789, 736, 751, 736, 747, 801, 760, 774, 781, 805, 767, 760, 741, 786, 719, 815, 741, 717, 813, 742, 744, 759, 779, 801, 798, 715, 817, 793, 749, 797, 717, 761, 784, 774, 779, 787, 727, 780, 778, 783, 744, 774, 746, 792, 830, 795] payload_rows_per_dst=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] dup_rows_per_dst=[2924, 3028, 2944, 2986, 2856, 3124, 2888, 3040] combine_rows_per_src=[9160, 9278, 9322, 9362, 9244, 9374, 9330, 9444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64998 dup_rows=23966 dup_rows_remote=20984 dedup_ratio=0.243795 router_tokens_per_expert=[737, 768, 778, 751, 775, 796, 768, 769, 720, 791, 796, 773, 728, 790, 801, 775, 780, 748, 782, 749, 770, 761, 812, 759, 783, 742, 722, 840, 768, 795, 794, 763, 775, 756, 786, 719, 731, 773, 804, 738, 770, 738, 719, 782, 798, 788, 744, 763, 808, 758, 791, 781, 754, 758, 740, 795, 766, 758, 746, 757, 768, 766, 785, 753] payload_rows_per_dst=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] dup_rows_per_dst=[2980, 3106, 2982, 3026, 2924, 2904, 2982, 3062] combine_rows_per_src=[9304, 9242, 9340, 9388, 9240, 9300, 9388, 9136] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342936 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266541976 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532741016 weight_movement_bytes=138412032 total_nvlink_bytes=671153048 weight_to_token_ratio=0.259811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343584 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266149408 ingress_bytes=300408832 link_bytes=300408832 token_bytes=531955232 weight_movement_bytes=34603008 total_nvlink_bytes=566558240 weight_to_token_ratio=0.065049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130414250e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343768 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256088 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168408 weight_movement_bytes=69206016 total_nvlink_bytes=601374424 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65024 dup_rows=24054 dup_rows_remote=20984 dedup_ratio=0.244690 router_tokens_per_expert=[750, 772, 785, 754, 755, 785, 772, 750, 720, 723, 762, 738, 776, 782, 758, 819, 734, 729, 760, 798, 762, 836, 737, 778, 757, 741, 759, 756, 760, 776, 760, 767, 787, 800, 785, 749, 819, 767, 769, 783, 775, 797, 770, 713, 792, 753, 792, 799, 806, 752, 717, 775, 804, 797, 726, 771, 713, 783, 726, 816, 762, 780, 783, 780] payload_rows_per_dst=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] dup_rows_per_dst=[2940, 2894, 2950, 2990, 3118, 3056, 3070, 3036] combine_rows_per_src=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344008 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265715656 ingress_bytes=334577664 link_bytes=334577664 token_bytes=531087304 weight_movement_bytes=69206016 total_nvlink_bytes=600293320 weight_to_token_ratio=0.130310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258988814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343928 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266575736 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532807544 weight_movement_bytes=34603008 total_nvlink_bytes=567410552 weight_to_token_ratio=0.064945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344008 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265715656 ingress_bytes=334577664 link_bytes=334577664 token_bytes=531087304 weight_movement_bytes=69206016 total_nvlink_bytes=600293320 weight_to_token_ratio=0.130310 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258988814e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343768 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256088 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168408 weight_movement_bytes=69206016 total_nvlink_bytes=601374424 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64732 dup_rows=23966 dup_rows_remote=20658 dedup_ratio=0.243795 router_tokens_per_expert=[794, 805, 753, 801, 789, 806, 780, 767, 765, 757, 745, 764, 747, 801, 745, 716, 768, 764, 776, 765, 750, 751, 770, 769, 692, 766, 839, 787, 797, 736, 765, 780, 753, 768, 758, 772, 750, 820, 749, 766, 755, 759, 710, 827, 763, 786, 734, 792, 742, 750, 779, 796, 747, 780, 761, 761, 764, 798, 746, 744, 787, 766, 773, 786] payload_rows_per_dst=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] dup_rows_per_dst=[3308, 2900, 2890, 3038, 2938, 2826, 3082, 2984] combine_rows_per_src=[9606, 9180, 9336, 9286, 9334, 9102, 9314, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65152 dup_rows=24036 dup_rows_remote=21176 dedup_ratio=0.244507 router_tokens_per_expert=[719, 783, 733, 788, 781, 821, 735, 701, 741, 709, 794, 720, 781, 734, 765, 744, 761, 752, 786, 759, 772, 754, 816, 792, 753, 754, 786, 782, 824, 769, 759, 788, 793, 771, 821, 807, 768, 775, 798, 781, 737, 727, 704, 837, 811, 762, 783, 779, 751, 735, 756, 764, 749, 742, 780, 823, 784, 789, 699, 745, 779, 735, 806, 805] payload_rows_per_dst=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] dup_rows_per_dst=[2872, 2860, 3158, 3092, 3146, 2936, 3018, 2954] combine_rows_per_src=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64884 dup_rows=24144 dup_rows_remote=21100 dedup_ratio=0.245605 router_tokens_per_expert=[801, 773, 785, 755, 764, 782, 772, 764, 791, 772, 763, 810, 800, 742, 771, 726, 785, 736, 810, 773, 763, 812, 770, 750, 759, 792, 748, 811, 740, 775, 785, 752, 802, 751, 755, 748, 826, 728, 735, 818, 759, 792, 759, 796, 781, 744, 754, 775, 756, 768, 735, 782, 745, 758, 775, 752, 785, 743, 786, 731, 782, 722, 776, 701] payload_rows_per_dst=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] dup_rows_per_dst=[3074, 3042, 3076, 2944, 3100, 3044, 2918, 2946] combine_rows_per_src=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343280 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255600 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167920 weight_movement_bytes=103809024 total_nvlink_bytes=635976944 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343224 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267156664 ingress_bytes=336019456 link_bytes=336019456 token_bytes=533970104 weight_movement_bytes=69206016 total_nvlink_bytes=603176120 weight_to_token_ratio=0.129607 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264414161e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344032 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682336 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020640 weight_movement_bytes=34603008 total_nvlink_bytes=567623648 weight_to_token_ratio=0.064919 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65024 dup_rows=24042 dup_rows_remote=21056 dedup_ratio=0.244568 router_tokens_per_expert=[737, 754, 767, 764, 759, 768, 762, 780, 788, 716, 735, 776, 758, 797, 722, 759, 757, 770, 696, 758, 805, 788, 805, 751, 761, 747, 781, 747, 789, 755, 771, 761, 759, 752, 757, 750, 769, 802, 708, 793, 742, 788, 743, 819, 782, 759, 775, 788, 807, 783, 769, 753, 793, 764, 758, 737, 774, 785, 805, 814, 766, 784, 793, 797] payload_rows_per_dst=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] dup_rows_per_dst=[2960, 2940, 2908, 2986, 2996, 3032, 3040, 3180] combine_rows_per_src=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341560 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483832 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626104 weight_movement_bytes=34603008 total_nvlink_bytes=565229112 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266862592 dispatch_meta_bytes=345312 combine_payload_bytes=266862592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267207904 ingress_bytes=301465600 link_bytes=301465600 token_bytes=534070496 weight_movement_bytes=34603008 total_nvlink_bytes=568673504 weight_to_token_ratio=0.064791 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134390783e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65152 dup_rows=24036 dup_rows_remote=21176 dedup_ratio=0.244507 router_tokens_per_expert=[719, 783, 733, 788, 781, 821, 735, 701, 741, 709, 794, 720, 781, 734, 765, 744, 761, 752, 786, 759, 772, 754, 816, 792, 753, 754, 786, 782, 824, 769, 759, 788, 793, 771, 821, 807, 768, 775, 798, 781, 737, 727, 704, 837, 811, 762, 783, 779, 751, 735, 756, 764, 749, 742, 780, 823, 784, 789, 699, 745, 779, 735, 806, 805] payload_rows_per_dst=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] dup_rows_per_dst=[2872, 2860, 3158, 3092, 3146, 2936, 3018, 2954] combine_rows_per_src=[9250, 9116, 9226, 9338, 9482, 9344, 9182, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65024 dup_rows=24042 dup_rows_remote=21056 dedup_ratio=0.244568 router_tokens_per_expert=[737, 754, 767, 764, 759, 768, 762, 780, 788, 716, 735, 776, 758, 797, 722, 759, 757, 770, 696, 758, 805, 788, 805, 751, 761, 747, 781, 747, 789, 755, 771, 761, 759, 752, 757, 750, 769, 802, 708, 793, 742, 788, 743, 819, 782, 759, 775, 788, 807, 783, 769, 753, 793, 764, 758, 737, 774, 785, 805, 814, 766, 784, 793, 797] payload_rows_per_dst=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] dup_rows_per_dst=[2960, 2940, 2908, 2986, 2996, 3032, 3040, 3180] combine_rows_per_src=[9222, 9162, 9352, 9238, 9184, 9360, 9288, 9456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65080 dup_rows=23954 dup_rows_remote=20902 dedup_ratio=0.243673 router_tokens_per_expert=[764, 784, 724, 773, 793, 791, 746, 735, 799, 805, 780, 781, 779, 706, 777, 789, 752, 805, 740, 766, 778, 762, 806, 752, 743, 825, 758, 762, 796, 799, 740, 739, 789, 736, 778, 784, 764, 757, 787, 731, 773, 764, 768, 811, 750, 749, 752, 786, 796, 795, 757, 803, 744, 754, 776, 762, 746, 727, 731, 743, 736, 769, 808, 777] payload_rows_per_dst=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] dup_rows_per_dst=[3004, 3088, 3052, 3014, 2900, 3074, 3016, 2806] combine_rows_per_src=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64654 dup_rows=24284 dup_rows_remote=21076 dedup_ratio=0.247030 router_tokens_per_expert=[786, 842, 747, 786, 772, 829, 750, 792, 762, 764, 776, 821, 763, 718, 744, 753, 747, 779, 789, 776, 742, 794, 810, 762, 784, 780, 772, 732, 762, 760, 740, 806, 722, 756, 743, 801, 759, 748, 742, 744, 791, 747, 749, 821, 757, 820, 783, 746, 748, 771, 795, 726, 805, 722, 775, 761, 758, 774, 756, 763, 740, 753, 743, 793] payload_rows_per_dst=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] dup_rows_per_dst=[2916, 2938, 3078, 2786, 2930, 3198, 3230, 3208] combine_rows_per_src=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64922 dup_rows=24224 dup_rows_remote=21164 dedup_ratio=0.246419 router_tokens_per_expert=[775, 785, 807, 778, 787, 819, 752, 753, 768, 771, 778, 778, 779, 767, 783, 725, 748, 785, 794, 791, 768, 723, 761, 774, 760, 772, 724, 775, 784, 785, 792, 742, 761, 786, 763, 767, 748, 757, 756, 771, 795, 728, 803, 781, 773, 804, 751, 768, 758, 758, 741, 752, 759, 777, 756, 772, 777, 702, 760, 804, 777, 737, 740, 787] payload_rows_per_dst=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] dup_rows_per_dst=[3198, 3114, 3050, 2980, 3060, 2982, 2882, 2958] combine_rows_per_src=[9314, 9184, 9238, 9288, 9158, 9424, 9264, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=343936 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108800 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873664 weight_movement_bytes=69206016 total_nvlink_bytes=601079680 weight_to_token_ratio=0.130117 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65024 dup_rows=24054 dup_rows_remote=20984 dedup_ratio=0.244690 router_tokens_per_expert=[750, 772, 785, 754, 755, 785, 772, 750, 720, 723, 762, 738, 776, 782, 758, 819, 734, 729, 760, 798, 762, 836, 737, 778, 757, 741, 759, 756, 760, 776, 760, 767, 787, 800, 785, 749, 819, 767, 769, 783, 775, 797, 770, 713, 792, 753, 792, 799, 806, 752, 717, 775, 804, 797, 726, 771, 713, 783, 726, 816, 762, 780, 783, 780] payload_rows_per_dst=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] dup_rows_per_dst=[2940, 2894, 2950, 2990, 3118, 3056, 3070, 3036] combine_rows_per_src=[9306, 9262, 9318, 9162, 9400, 9326, 9226, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64654 dup_rows=24284 dup_rows_remote=21076 dedup_ratio=0.247030 router_tokens_per_expert=[786, 842, 747, 786, 772, 829, 750, 792, 762, 764, 776, 821, 763, 718, 744, 753, 747, 779, 789, 776, 742, 794, 810, 762, 784, 780, 772, 732, 762, 760, 740, 806, 722, 756, 743, 801, 759, 748, 742, 744, 791, 747, 749, 821, 757, 820, 783, 746, 748, 771, 795, 726, 805, 722, 775, 761, 758, 774, 756, 763, 740, 753, 743, 793] payload_rows_per_dst=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] dup_rows_per_dst=[2916, 2938, 3078, 2786, 2930, 3198, 3230, 3208] combine_rows_per_src=[9106, 9264, 9272, 8972, 9272, 9278, 9490, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64884 dup_rows=24144 dup_rows_remote=21100 dedup_ratio=0.245605 router_tokens_per_expert=[801, 773, 785, 755, 764, 782, 772, 764, 791, 772, 763, 810, 800, 742, 771, 726, 785, 736, 810, 773, 763, 812, 770, 750, 759, 792, 748, 811, 740, 775, 785, 752, 802, 751, 755, 748, 826, 728, 735, 818, 759, 792, 759, 796, 781, 744, 754, 775, 756, 768, 735, 782, 745, 758, 775, 752, 785, 743, 786, 731, 782, 722, 776, 701] payload_rows_per_dst=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] dup_rows_per_dst=[3074, 3042, 3076, 2944, 3100, 3044, 2918, 2946] combine_rows_per_src=[9318, 9308, 9322, 9380, 9226, 9276, 9224, 9106] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266862592 dispatch_meta_bytes=345312 combine_payload_bytes=266862592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267207904 ingress_bytes=301465600 link_bytes=301465600 token_bytes=534070496 weight_movement_bytes=34603008 total_nvlink_bytes=568673504 weight_to_token_ratio=0.064791 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134390783e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65080 dup_rows=23954 dup_rows_remote=20902 dedup_ratio=0.243673 router_tokens_per_expert=[764, 784, 724, 773, 793, 791, 746, 735, 799, 805, 780, 781, 779, 706, 777, 789, 752, 805, 740, 766, 778, 762, 806, 752, 743, 825, 758, 762, 796, 799, 740, 739, 789, 736, 778, 784, 764, 757, 787, 731, 773, 764, 768, 811, 750, 749, 752, 786, 796, 795, 757, 803, 744, 754, 776, 762, 746, 727, 731, 743, 736, 769, 808, 777] payload_rows_per_dst=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] dup_rows_per_dst=[3004, 3088, 3052, 3014, 2900, 3074, 3016, 2806] combine_rows_per_src=[9216, 9344, 9270, 9310, 9352, 9232, 9358, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344320 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266682624 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020928 weight_movement_bytes=69206016 total_nvlink_bytes=602226944 weight_to_token_ratio=0.129837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344344 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266264856 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532185368 weight_movement_bytes=34603008 total_nvlink_bytes=566788376 weight_to_token_ratio=0.065021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342920 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165704 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988488 weight_movement_bytes=34603008 total_nvlink_bytes=564591496 weight_to_token_ratio=0.065290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65250 dup_rows=23770 dup_rows_remote=20804 dedup_ratio=0.241801 router_tokens_per_expert=[771, 787, 769, 752, 786, 790, 774, 795, 774, 766, 757, 750, 844, 762, 813, 746, 767, 774, 732, 740, 746, 758, 774, 746, 793, 796, 730, 785, 761, 849, 762, 779, 768, 763, 766, 787, 741, 759, 754, 754, 790, 766, 737, 763, 751, 780, 738, 751, 747, 784, 766, 736, 770, 759, 785, 778, 744, 759, 775, 756, 835, 783, 749, 730] payload_rows_per_dst=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] dup_rows_per_dst=[3080, 3040, 2936, 3038, 2882, 2926, 2966, 2902] combine_rows_per_src=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=343928 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266911608 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479288 weight_movement_bytes=34603008 total_nvlink_bytes=568082296 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342920 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165704 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988488 weight_movement_bytes=34603008 total_nvlink_bytes=564591496 weight_to_token_ratio=0.065290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65046 dup_rows=23948 dup_rows_remote=20852 dedup_ratio=0.243612 router_tokens_per_expert=[780, 788, 803, 753, 735, 780, 779, 758, 757, 725, 764, 801, 783, 730, 698, 802, 725, 741, 755, 785, 794, 755, 714, 845, 770, 727, 777, 767, 816, 807, 802, 794, 804, 807, 742, 785, 756, 751, 774, 738, 756, 769, 766, 780, 753, 799, 810, 753, 719, 760, 790, 742, 761, 749, 770, 781, 791, 781, 719, 769, 788, 742, 801, 736] payload_rows_per_dst=[9310, 9522, 9252, 9332, 9294, 9106, 9302, 9238] dup_rows_per_dst=[3096, 2998, 2976, 3134, 3020, 2866, 2842, 3016] combine_rows_per_src=[9310, 9522, 9252, 9332, 9294, 9106, 9302, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65038 dup_rows=24098 dup_rows_remote=21204 dedup_ratio=0.245138 router_tokens_per_expert=[786, 763, 815, 819, 797, 737, 747, 791, 743, 753, 767, 731, 757, 800, 756, 724, 796, 771, 727, 810, 721, 751, 775, 752, 772, 801, 797, 770, 757, 800, 750, 800, 834, 722, 776, 755, 750, 767, 764, 778, 729, 733, 747, 767, 757, 774, 785, 767, 761, 796, 802, 765, 804, 784, 785, 731, 766, 715, 770, 748, 785, 763, 780, 756] payload_rows_per_dst=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] dup_rows_per_dst=[3070, 2894, 3044, 3116, 2984, 2844, 3120, 3026] combine_rows_per_src=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74692 payload_rows_remote=65534 dup_rows=23612 dup_rows_remote=20880 dedup_ratio=0.240194 router_tokens_per_expert=[804, 749, 783, 783, 764, 751, 785, 748, 742, 785, 762, 825, 772, 745, 799, 788, 787, 755, 773, 794, 767, 778, 748, 748, 774, 750, 722, 791, 776, 761, 770, 781, 770, 798, 776, 741, 776, 723, 744, 801, 718, 778, 724, 746, 760, 738, 719, 762, 745, 828, 759, 745, 785, 775, 817, 791, 742, 786, 747, 785, 780, 798, 754, 781] payload_rows_per_dst=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] dup_rows_per_dst=[2972, 3022, 2924, 2944, 3072, 2732, 3002, 2944] combine_rows_per_src=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=344344 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266264856 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532185368 weight_movement_bytes=34603008 total_nvlink_bytes=566788376 weight_to_token_ratio=0.065021 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344032 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682336 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020640 weight_movement_bytes=34603008 total_nvlink_bytes=567623648 weight_to_token_ratio=0.064919 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=343936 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266108800 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531873664 weight_movement_bytes=69206016 total_nvlink_bytes=601079680 weight_to_token_ratio=0.130117 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344216 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608216 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872216 weight_movement_bytes=34603008 total_nvlink_bytes=569475224 weight_to_token_ratio=0.064694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65174 dup_rows=23818 dup_rows_remote=20854 dedup_ratio=0.242289 router_tokens_per_expert=[761, 733, 795, 738, 784, 733, 772, 841, 756, 756, 761, 784, 750, 761, 721, 778, 789, 767, 793, 754, 748, 740, 813, 773, 760, 766, 782, 752, 755, 775, 815, 733, 786, 747, 772, 812, 725, 825, 739, 786, 766, 788, 747, 751, 760, 748, 755, 774, 779, 793, 812, 780, 793, 756, 766, 757, 761, 735, 738, 774, 802, 748, 748, 790] payload_rows_per_dst=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] dup_rows_per_dst=[3044, 2832, 2986, 2964, 3032, 2962, 3058, 2940] combine_rows_per_src=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343592 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266772008 ingress_bytes=370237440 link_bytes=370237440 token_bytes=533200424 weight_movement_bytes=103809024 total_nvlink_bytes=637009448 weight_to_token_ratio=0.194690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393173680e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344968 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740616 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533136264 weight_movement_bytes=34603008 total_nvlink_bytes=567739272 weight_to_token_ratio=0.064905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65038 dup_rows=24098 dup_rows_remote=21204 dedup_ratio=0.245138 router_tokens_per_expert=[786, 763, 815, 819, 797, 737, 747, 791, 743, 753, 767, 731, 757, 800, 756, 724, 796, 771, 727, 810, 721, 751, 775, 752, 772, 801, 797, 770, 757, 800, 750, 800, 834, 722, 776, 755, 750, 767, 764, 778, 729, 733, 747, 767, 757, 774, 785, 767, 761, 796, 802, 765, 804, 784, 785, 731, 766, 715, 770, 748, 785, 763, 780, 756] payload_rows_per_dst=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] dup_rows_per_dst=[3070, 2894, 3044, 3116, 2984, 2844, 3120, 3026] combine_rows_per_src=[9440, 9168, 9162, 9378, 9308, 9274, 9336, 9140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65038 dup_rows=23832 dup_rows_remote=20744 dedup_ratio=0.242432 router_tokens_per_expert=[810, 719, 769, 808, 757, 712, 758, 796, 744, 736, 792, 774, 764, 756, 748, 747, 773, 785, 796, 775, 802, 762, 834, 734, 797, 769, 756, 801, 781, 759, 776, 762, 807, 807, 735, 757, 750, 786, 763, 766, 846, 755, 727, 760, 753, 747, 743, 749, 718, 785, 777, 807, 774, 742, 749, 773, 740, 756, 749, 854, 757, 788, 751, 729] payload_rows_per_dst=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] dup_rows_per_dst=[2966, 2884, 3088, 2934, 3006, 2932, 3024, 2998] combine_rows_per_src=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65174 dup_rows=23818 dup_rows_remote=20854 dedup_ratio=0.242289 router_tokens_per_expert=[761, 733, 795, 738, 784, 733, 772, 841, 756, 756, 761, 784, 750, 761, 721, 778, 789, 767, 793, 754, 748, 740, 813, 773, 760, 766, 782, 752, 755, 775, 815, 733, 786, 747, 772, 812, 725, 825, 739, 786, 766, 788, 747, 751, 760, 748, 755, 774, 779, 793, 812, 780, 793, 756, 766, 757, 761, 735, 738, 774, 802, 748, 748, 790] payload_rows_per_dst=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] dup_rows_per_dst=[3044, 2832, 2986, 2964, 3032, 2962, 3058, 2940] combine_rows_per_src=[9270, 9302, 9368, 9312, 9352, 9216, 9414, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64846 dup_rows=24142 dup_rows_remote=21084 dedup_ratio=0.245585 router_tokens_per_expert=[739, 800, 792, 768, 747, 767, 760, 787, 775, 734, 798, 778, 752, 768, 786, 758, 800, 806, 761, 787, 754, 755, 776, 761, 765, 734, 735, 722, 784, 729, 788, 785, 787, 784, 718, 759, 749, 774, 795, 764, 792, 764, 768, 774, 738, 750, 771, 743, 794, 814, 796, 783, 766, 763, 776, 826, 755, 764, 751, 754, 764, 764, 726, 775] payload_rows_per_dst=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] dup_rows_per_dst=[3074, 2978, 3086, 2764, 3032, 2998, 3152, 3058] combine_rows_per_src=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74534 payload_rows_remote=65250 dup_rows=23770 dup_rows_remote=20804 dedup_ratio=0.241801 router_tokens_per_expert=[771, 787, 769, 752, 786, 790, 774, 795, 774, 766, 757, 750, 844, 762, 813, 746, 767, 774, 732, 740, 746, 758, 774, 746, 793, 796, 730, 785, 761, 849, 762, 779, 768, 763, 766, 787, 741, 759, 754, 754, 790, 766, 737, 763, 751, 780, 738, 751, 747, 784, 766, 736, 770, 759, 785, 778, 744, 759, 775, 756, 835, 783, 749, 730] payload_rows_per_dst=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] dup_rows_per_dst=[3080, 3040, 2936, 3038, 2882, 2926, 2966, 2902] combine_rows_per_src=[9368, 9384, 9138, 9472, 9302, 9226, 9284, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64986 dup_rows=24134 dup_rows_remote=21096 dedup_ratio=0.245504 router_tokens_per_expert=[770, 752, 749, 788, 768, 724, 780, 781, 741, 770, 732, 777, 713, 748, 776, 797, 791, 747, 784, 813, 765, 839, 797, 726, 739, 767, 749, 750, 771, 739, 800, 787, 738, 777, 770, 743, 768, 768, 755, 792, 818, 805, 787, 815, 791, 783, 733, 748, 776, 774, 749, 735, 793, 753, 750, 716, 778, 751, 808, 754, 777, 790, 776, 751] payload_rows_per_dst=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] dup_rows_per_dst=[3004, 2954, 3030, 2948, 3038, 3172, 2856, 3132] combine_rows_per_src=[9220, 9154, 9494, 9256, 9184, 9388, 9236, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=268427264 dispatch_meta_bytes=345656 combine_payload_bytes=268427264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268772920 ingress_bytes=303030272 link_bytes=303030272 token_bytes=537200184 weight_movement_bytes=34603008 total_nvlink_bytes=571803192 weight_to_token_ratio=0.064414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140278518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64846 dup_rows=24142 dup_rows_remote=21084 dedup_ratio=0.245585 router_tokens_per_expert=[739, 800, 792, 768, 747, 767, 760, 787, 775, 734, 798, 778, 752, 768, 786, 758, 800, 806, 761, 787, 754, 755, 776, 761, 765, 734, 735, 722, 784, 729, 788, 785, 787, 784, 718, 759, 749, 774, 795, 764, 792, 764, 768, 774, 738, 750, 771, 743, 794, 814, 796, 783, 766, 763, 776, 826, 755, 764, 751, 754, 764, 764, 726, 775] payload_rows_per_dst=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] dup_rows_per_dst=[3074, 2978, 3086, 2764, 3032, 2998, 3152, 3058] combine_rows_per_src=[9246, 9320, 9314, 9052, 9166, 9202, 9546, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74692 payload_rows_remote=65534 dup_rows=23612 dup_rows_remote=20880 dedup_ratio=0.240194 router_tokens_per_expert=[804, 749, 783, 783, 764, 751, 785, 748, 742, 785, 762, 825, 772, 745, 799, 788, 787, 755, 773, 794, 767, 778, 748, 748, 774, 750, 722, 791, 776, 761, 770, 781, 770, 798, 776, 741, 776, 723, 744, 801, 718, 778, 724, 746, 760, 738, 719, 762, 745, 828, 759, 745, 785, 775, 817, 791, 742, 786, 747, 785, 780, 798, 754, 781] payload_rows_per_dst=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] dup_rows_per_dst=[2972, 3022, 2924, 2944, 3072, 2732, 3002, 2944] combine_rows_per_src=[9362, 9414, 9376, 9306, 9186, 9158, 9488, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344968 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740616 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533136264 weight_movement_bytes=34603008 total_nvlink_bytes=567739272 weight_to_token_ratio=0.064905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343128 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738776 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134424 weight_movement_bytes=69206016 total_nvlink_bytes=602340440 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267296816 ingress_bytes=370761728 link_bytes=370761728 token_bytes=534249520 weight_movement_bytes=103809024 total_nvlink_bytes=638058544 weight_to_token_ratio=0.194308 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395146534e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65038 dup_rows=23832 dup_rows_remote=20744 dedup_ratio=0.242432 router_tokens_per_expert=[810, 719, 769, 808, 757, 712, 758, 796, 744, 736, 792, 774, 764, 756, 748, 747, 773, 785, 796, 775, 802, 762, 834, 734, 797, 769, 756, 801, 781, 759, 776, 762, 807, 807, 735, 757, 750, 786, 763, 766, 846, 755, 727, 760, 753, 747, 743, 749, 718, 785, 777, 807, 774, 742, 749, 773, 740, 756, 749, 854, 757, 788, 751, 729] payload_rows_per_dst=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] dup_rows_per_dst=[2966, 2884, 3088, 2934, 3006, 2932, 3024, 2998] combine_rows_per_src=[9292, 9238, 9434, 9468, 9336, 9228, 9226, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344328 combine_payload_bytes=266182656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266526984 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709640 weight_movement_bytes=103809024 total_nvlink_bytes=636518664 weight_to_token_ratio=0.194870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343720 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952936 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531562152 weight_movement_bytes=34603008 total_nvlink_bytes=566165160 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65104 dup_rows=23882 dup_rows_remote=20976 dedup_ratio=0.242940 router_tokens_per_expert=[743, 721, 772, 773, 732, 789, 738, 779, 787, 783, 771, 757, 787, 806, 785, 804, 769, 748, 751, 767, 811, 777, 761, 720, 781, 808, 782, 783, 803, 774, 755, 774, 736, 771, 747, 796, 735, 795, 755, 733, 771, 780, 763, 741, 760, 767, 772, 783, 779, 765, 752, 764, 780, 769, 721, 782, 757, 789, 748, 759, 747, 776, 817, 751] payload_rows_per_dst=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] dup_rows_per_dst=[2948, 3108, 2998, 3066, 2886, 2914, 2906, 3056] combine_rows_per_src=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343128 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266738776 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533134424 weight_movement_bytes=69206016 total_nvlink_bytes=602340440 weight_to_token_ratio=0.129810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65180 dup_rows=23730 dup_rows_remote=20874 dedup_ratio=0.241394 router_tokens_per_expert=[783, 759, 749, 733, 802, 739, 780, 778, 738, 772, 779, 760, 774, 759, 759, 784, 778, 705, 780, 739, 739, 780, 787, 746, 772, 747, 746, 800, 761, 820, 753, 778, 789, 768, 795, 776, 767, 773, 758, 814, 795, 709, 766, 769, 765, 813, 812, 724, 794, 757, 771, 782, 753, 737, 741, 785, 788, 764, 754, 747, 783, 785, 786, 753] payload_rows_per_dst=[9394, 9488, 9016, 9476, 9444, 9582, 8880, 9294] dup_rows_per_dst=[2856, 3096, 2774, 2866, 3028, 3342, 2742, 3026] combine_rows_per_src=[9394, 9488, 9016, 9476, 9444, 9582, 8880, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64854 dup_rows=24092 dup_rows_remote=21208 dedup_ratio=0.245076 router_tokens_per_expert=[748, 747, 743, 783, 783, 747, 798, 737, 706, 768, 778, 794, 827, 777, 740, 731, 779, 781, 758, 809, 746, 813, 771, 790, 736, 781, 756, 790, 717, 770, 761, 738, 773, 798, 773, 757, 767, 773, 749, 769, 774, 781, 746, 819, 739, 769, 729, 762, 795, 791, 741, 772, 771, 761, 735, 793, 720, 834, 722, 821, 764, 751, 784, 816] payload_rows_per_dst=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] dup_rows_per_dst=[3004, 2884, 3208, 2868, 3024, 3078, 3016, 3010] combine_rows_per_src=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64998 dup_rows=24014 dup_rows_remote=20916 dedup_ratio=0.244283 router_tokens_per_expert=[756, 797, 732, 769, 747, 807, 796, 800, 744, 792, 768, 739, 749, 741, 772, 751, 745, 758, 753, 796, 742, 771, 788, 763, 776, 750, 770, 766, 760, 749, 804, 814, 770, 738, 763, 748, 747, 800, 737, 818, 781, 772, 825, 734, 804, 768, 775, 736, 818, 786, 731, 754, 780, 771, 796, 801, 764, 727, 792, 711, 795, 733, 764, 748] payload_rows_per_dst=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] dup_rows_per_dst=[3040, 2922, 2928, 3054, 3010, 3098, 3008, 2954] combine_rows_per_src=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344216 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608216 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872216 weight_movement_bytes=34603008 total_nvlink_bytes=569475224 weight_to_token_ratio=0.064694 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344328 combine_payload_bytes=266182656 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266526984 ingress_bytes=369991680 link_bytes=369991680 token_bytes=532709640 weight_movement_bytes=103809024 total_nvlink_bytes=636518664 weight_to_token_ratio=0.194870 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392248905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=268427264 dispatch_meta_bytes=345656 combine_payload_bytes=268427264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268772920 ingress_bytes=303030272 link_bytes=303030272 token_bytes=537200184 weight_movement_bytes=34603008 total_nvlink_bytes=571803192 weight_to_token_ratio=0.064414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.140278518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344320 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010304 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676288 weight_movement_bytes=69206016 total_nvlink_bytes=602882304 weight_to_token_ratio=0.129678 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65096 dup_rows=23918 dup_rows_remote=20994 dedup_ratio=0.243306 router_tokens_per_expert=[797, 792, 770, 803, 760, 795, 767, 766, 765, 770, 767, 739, 799, 776, 727, 747, 725, 744, 766, 795, 729, 732, 803, 759, 735, 793, 766, 779, 766, 765, 766, 737, 799, 794, 778, 796, 784, 757, 837, 752, 717, 756, 781, 754, 716, 724, 788, 751, 786, 763, 742, 759, 788, 765, 755, 772, 728, 821, 806, 799, 769, 780, 762, 773] payload_rows_per_dst=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] dup_rows_per_dst=[3050, 2978, 2946, 2924, 3142, 2754, 2990, 3134] combine_rows_per_src=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344216 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321496 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298776 weight_movement_bytes=34603008 total_nvlink_bytes=568901784 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65412 dup_rows=23726 dup_rows_remote=20744 dedup_ratio=0.241353 router_tokens_per_expert=[807, 816, 746, 739, 752, 751, 806, 776, 750, 784, 808, 815, 776, 799, 778, 761, 764, 737, 743, 828, 790, 720, 817, 752, 804, 759, 785, 751, 803, 752, 732, 764, 741, 759, 735, 775, 768, 752, 762, 782, 701, 768, 788, 752, 817, 775, 750, 782, 751, 783, 754, 745, 749, 821, 754, 754, 725, 747, 771, 745, 786, 769, 786, 740] payload_rows_per_dst=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] dup_rows_per_dst=[3070, 3084, 2986, 2896, 2982, 2924, 2920, 2864] combine_rows_per_src=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344248 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986232 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628216 weight_movement_bytes=34603008 total_nvlink_bytes=566231224 weight_to_token_ratio=0.065089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343656 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266575464 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807272 weight_movement_bytes=69206016 total_nvlink_bytes=602013288 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64854 dup_rows=24092 dup_rows_remote=21208 dedup_ratio=0.245076 router_tokens_per_expert=[748, 747, 743, 783, 783, 747, 798, 737, 706, 768, 778, 794, 827, 777, 740, 731, 779, 781, 758, 809, 746, 813, 771, 790, 736, 781, 756, 790, 717, 770, 761, 738, 773, 798, 773, 757, 767, 773, 749, 769, 774, 781, 746, 819, 739, 769, 729, 762, 795, 791, 741, 772, 771, 761, 735, 793, 720, 834, 722, 821, 764, 751, 784, 816] payload_rows_per_dst=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] dup_rows_per_dst=[3004, 2884, 3208, 2868, 3024, 3078, 3016, 3010] combine_rows_per_src=[9168, 9358, 9286, 9230, 9294, 9160, 9302, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65164 dup_rows=23730 dup_rows_remote=20822 dedup_ratio=0.241394 router_tokens_per_expert=[800, 799, 766, 751, 780, 742, 728, 798, 771, 794, 757, 767, 723, 751, 773, 727, 777, 757, 759, 730, 744, 779, 796, 817, 792, 799, 799, 747, 746, 745, 769, 735, 754, 760, 722, 772, 771, 785, 710, 829, 756, 749, 831, 740, 776, 765, 818, 787, 772, 775, 802, 729, 787, 763, 787, 783, 780, 793, 731, 768, 696, 775, 770, 798] payload_rows_per_dst=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] dup_rows_per_dst=[2960, 2864, 2908, 2930, 2974, 3062, 3006, 3026] combine_rows_per_src=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73948 payload_rows_remote=64602 dup_rows=24356 dup_rows_remote=21034 dedup_ratio=0.247762 router_tokens_per_expert=[735, 745, 782, 811, 762, 820, 794, 805, 784, 734, 744, 733, 806, 771, 754, 704, 797, 768, 770, 782, 725, 765, 728, 763, 771, 810, 740, 715, 753, 726, 762, 741, 755, 747, 767, 783, 777, 721, 811, 765, 717, 791, 785, 860, 767, 782, 776, 767, 762, 778, 753, 848, 737, 768, 781, 769, 775, 757, 739, 771, 774, 786, 773, 810] payload_rows_per_dst=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] dup_rows_per_dst=[2924, 2964, 2842, 3128, 2980, 3126, 3070, 3322] combine_rows_per_src=[9034, 9096, 9144, 9336, 9272, 9366, 9354, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65104 dup_rows=23882 dup_rows_remote=20976 dedup_ratio=0.242940 router_tokens_per_expert=[743, 721, 772, 773, 732, 789, 738, 779, 787, 783, 771, 757, 787, 806, 785, 804, 769, 748, 751, 767, 811, 777, 761, 720, 781, 808, 782, 783, 803, 774, 755, 774, 736, 771, 747, 796, 735, 795, 755, 733, 771, 780, 763, 741, 760, 767, 772, 783, 779, 765, 752, 764, 780, 769, 721, 782, 757, 789, 748, 759, 747, 776, 817, 751] payload_rows_per_dst=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] dup_rows_per_dst=[2948, 3108, 2998, 3066, 2886, 2914, 2906, 3056] combine_rows_per_src=[9146, 9452, 9210, 9454, 9250, 9360, 9318, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65096 dup_rows=23918 dup_rows_remote=20994 dedup_ratio=0.243306 router_tokens_per_expert=[797, 792, 770, 803, 760, 795, 767, 766, 765, 770, 767, 739, 799, 776, 727, 747, 725, 744, 766, 795, 729, 732, 803, 759, 735, 793, 766, 779, 766, 765, 766, 737, 799, 794, 778, 796, 784, 757, 837, 752, 717, 756, 781, 754, 716, 724, 788, 751, 786, 763, 742, 759, 788, 765, 755, 772, 728, 821, 806, 799, 769, 780, 762, 773] payload_rows_per_dst=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] dup_rows_per_dst=[3050, 2978, 2946, 2924, 3142, 2754, 2990, 3134] combine_rows_per_src=[9450, 9202, 9160, 9290, 9452, 9220, 9270, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74578 payload_rows_remote=65412 dup_rows=23726 dup_rows_remote=20744 dedup_ratio=0.241353 router_tokens_per_expert=[807, 816, 746, 739, 752, 751, 806, 776, 750, 784, 808, 815, 776, 799, 778, 761, 764, 737, 743, 828, 790, 720, 817, 752, 804, 759, 785, 751, 803, 752, 732, 764, 741, 759, 735, 775, 768, 752, 762, 782, 701, 768, 788, 752, 817, 775, 750, 782, 751, 783, 754, 745, 749, 821, 754, 754, 725, 747, 771, 745, 786, 769, 786, 740] payload_rows_per_dst=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] dup_rows_per_dst=[3070, 3084, 2986, 2896, 2982, 2924, 2920, 2864] combine_rows_per_src=[9316, 9458, 9316, 9404, 9166, 9342, 9302, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64998 dup_rows=24014 dup_rows_remote=20916 dedup_ratio=0.244283 router_tokens_per_expert=[756, 797, 732, 769, 747, 807, 796, 800, 744, 792, 768, 739, 749, 741, 772, 751, 745, 758, 753, 796, 742, 771, 788, 763, 776, 750, 770, 766, 760, 749, 804, 814, 770, 738, 763, 748, 747, 800, 737, 818, 781, 772, 825, 734, 804, 768, 775, 736, 818, 786, 731, 754, 780, 771, 796, 801, 764, 727, 792, 711, 795, 733, 764, 748] payload_rows_per_dst=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] dup_rows_per_dst=[3040, 2922, 2928, 3054, 3010, 3098, 3008, 2954] combine_rows_per_src=[9368, 9190, 9304, 9324, 9232, 9292, 9466, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344248 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986232 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628216 weight_movement_bytes=34603008 total_nvlink_bytes=566231224 weight_to_token_ratio=0.065089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65164 dup_rows=23730 dup_rows_remote=20822 dedup_ratio=0.241394 router_tokens_per_expert=[800, 799, 766, 751, 780, 742, 728, 798, 771, 794, 757, 767, 723, 751, 773, 727, 777, 757, 759, 730, 744, 779, 796, 817, 792, 799, 799, 747, 746, 745, 769, 735, 754, 760, 722, 772, 771, 785, 710, 829, 756, 749, 831, 740, 776, 765, 818, 787, 772, 775, 802, 729, 787, 763, 787, 783, 780, 793, 731, 768, 696, 775, 770, 798] payload_rows_per_dst=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] dup_rows_per_dst=[2960, 2864, 2908, 2930, 2974, 3062, 3006, 3026] combine_rows_per_src=[9368, 9262, 9410, 9334, 9232, 9382, 9390, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343944 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255688 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167432 weight_movement_bytes=69206016 total_nvlink_bytes=603373448 weight_to_token_ratio=0.129559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64996 dup_rows=24010 dup_rows_remote=21078 dedup_ratio=0.244242 router_tokens_per_expert=[783, 786, 757, 747, 765, 735, 778, 764, 785, 760, 762, 775, 758, 770, 785, 774, 752, 770, 760, 810, 766, 760, 770, 811, 769, 764, 779, 774, 723, 836, 738, 781, 789, 744, 773, 687, 792, 747, 808, 737, 754, 802, 777, 780, 768, 758, 824, 795, 725, 757, 776, 773, 767, 776, 738, 774, 757, 781, 730, 722, 788, 755, 790, 761] payload_rows_per_dst=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] dup_rows_per_dst=[2932, 3222, 3054, 2888, 3094, 2974, 2862, 2984] combine_rows_per_src=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267927552 dispatch_meta_bytes=344624 combine_payload_bytes=267927552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268272176 ingress_bytes=302530560 link_bytes=302530560 token_bytes=536199728 weight_movement_bytes=34603008 total_nvlink_bytes=570802736 weight_to_token_ratio=0.064534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138398142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65200 dup_rows=23918 dup_rows_remote=20896 dedup_ratio=0.243306 router_tokens_per_expert=[739, 759, 747, 776, 752, 830, 764, 794, 794, 772, 761, 772, 760, 795, 758, 748, 773, 803, 801, 766, 757, 782, 750, 802, 767, 747, 778, 736, 747, 782, 808, 764, 773, 747, 778, 726, 787, 777, 751, 771, 771, 807, 755, 745, 760, 765, 799, 782, 767, 762, 754, 777, 777, 770, 729, 768, 762, 773, 745, 752, 761, 741, 747, 789] payload_rows_per_dst=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] dup_rows_per_dst=[2996, 3038, 3050, 2948, 2882, 3086, 3022, 2896] combine_rows_per_src=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344320 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267010304 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533676288 weight_movement_bytes=69206016 total_nvlink_bytes=602882304 weight_to_token_ratio=0.129678 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264609792 dispatch_meta_bytes=342544 combine_payload_bytes=264609792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264952336 ingress_bytes=299212800 link_bytes=299212800 token_bytes=529562128 weight_movement_bytes=34603008 total_nvlink_bytes=564165136 weight_to_token_ratio=0.065343 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125913678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64996 dup_rows=24010 dup_rows_remote=21078 dedup_ratio=0.244242 router_tokens_per_expert=[783, 786, 757, 747, 765, 735, 778, 764, 785, 760, 762, 775, 758, 770, 785, 774, 752, 770, 760, 810, 766, 760, 770, 811, 769, 764, 779, 774, 723, 836, 738, 781, 789, 744, 773, 687, 792, 747, 808, 737, 754, 802, 777, 780, 768, 758, 824, 795, 725, 757, 776, 773, 767, 776, 738, 774, 757, 781, 730, 722, 788, 755, 790, 761] payload_rows_per_dst=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] dup_rows_per_dst=[2932, 3222, 3054, 2888, 3094, 2974, 2862, 2984] combine_rows_per_src=[9298, 9500, 9344, 9440, 9230, 9158, 9140, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343944 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255688 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167432 weight_movement_bytes=69206016 total_nvlink_bytes=603373448 weight_to_token_ratio=0.129559 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65414 dup_rows=23754 dup_rows_remote=20830 dedup_ratio=0.241638 router_tokens_per_expert=[739, 783, 773, 779, 766, 788, 720, 790, 784, 732, 767, 718, 777, 741, 752, 759, 763, 747, 804, 821, 757, 722, 797, 728, 765, 794, 753, 778, 724, 774, 757, 765, 751, 736, 799, 724, 769, 788, 764, 784, 767, 768, 747, 793, 763, 803, 777, 824, 791, 776, 730, 755, 818, 761, 788, 768, 817, 742, 742, 782, 743, 741, 822, 802] payload_rows_per_dst=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] dup_rows_per_dst=[2984, 2924, 2850, 2942, 2958, 3088, 2994, 3014] combine_rows_per_src=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74022 payload_rows_remote=64720 dup_rows=24282 dup_rows_remote=21260 dedup_ratio=0.247009 router_tokens_per_expert=[805, 734, 768, 750, 770, 751, 759, 810, 752, 793, 761, 794, 766, 728, 743, 789, 790, 773, 784, 778, 777, 754, 807, 733, 717, 783, 747, 717, 749, 800, 768, 768, 775, 726, 798, 797, 768, 762, 810, 771, 774, 796, 770, 746, 730, 800, 791, 755, 746, 787, 755, 748, 731, 756, 756, 804, 769, 748, 743, 727, 810, 822, 786, 777] payload_rows_per_dst=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] dup_rows_per_dst=[2978, 3008, 3128, 2940, 3168, 3022, 2986, 3052] combine_rows_per_src=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267927552 dispatch_meta_bytes=344624 combine_payload_bytes=267927552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268272176 ingress_bytes=302530560 link_bytes=302530560 token_bytes=536199728 weight_movement_bytes=34603008 total_nvlink_bytes=570802736 weight_to_token_ratio=0.064534 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138398142e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344384 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267403584 ingress_bytes=336265216 link_bytes=336265216 token_bytes=534462784 weight_movement_bytes=69206016 total_nvlink_bytes=603668800 weight_to_token_ratio=0.129487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343656 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266575464 ingress_bytes=335437824 link_bytes=335437824 token_bytes=532807272 weight_movement_bytes=69206016 total_nvlink_bytes=602013288 weight_to_token_ratio=0.129889 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262225527e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344296 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266567912 ingress_bytes=404635648 link_bytes=404635648 token_bytes=532791528 weight_movement_bytes=138412032 total_nvlink_bytes=671203560 weight_to_token_ratio=0.259786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522611368e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65414 dup_rows=23754 dup_rows_remote=20830 dedup_ratio=0.241638 router_tokens_per_expert=[739, 783, 773, 779, 766, 788, 720, 790, 784, 732, 767, 718, 777, 741, 752, 759, 763, 747, 804, 821, 757, 722, 797, 728, 765, 794, 753, 778, 724, 774, 757, 765, 751, 736, 799, 724, 769, 788, 764, 784, 767, 768, 747, 793, 763, 803, 777, 824, 791, 776, 730, 755, 818, 761, 788, 768, 817, 742, 742, 782, 743, 741, 822, 802] payload_rows_per_dst=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] dup_rows_per_dst=[2984, 2924, 2850, 2942, 2958, 3088, 2994, 3014] combine_rows_per_src=[9292, 9136, 9428, 9278, 9272, 9396, 9380, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65130 dup_rows=23988 dup_rows_remote=21058 dedup_ratio=0.244019 router_tokens_per_expert=[785, 753, 761, 776, 759, 783, 752, 789, 740, 825, 795, 820, 780, 761, 781, 794, 751, 674, 782, 803, 746, 768, 774, 760, 805, 741, 745, 722, 733, 801, 769, 776, 789, 781, 784, 775, 741, 782, 776, 809, 777, 781, 746, 777, 737, 760, 750, 794, 743, 747, 784, 770, 767, 732, 736, 752, 778, 782, 788, 773, 765, 796, 722, 754] payload_rows_per_dst=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] dup_rows_per_dst=[2980, 3132, 2930, 3030, 3134, 2890, 2876, 3016] combine_rows_per_src=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65040 dup_rows=23876 dup_rows_remote=20880 dedup_ratio=0.242879 router_tokens_per_expert=[795, 756, 748, 763, 770, 798, 758, 781, 711, 783, 756, 742, 772, 778, 793, 743, 787, 771, 794, 731, 792, 762, 715, 791, 767, 767, 785, 854, 753, 714, 788, 764, 731, 796, 807, 820, 802, 791, 733, 764, 743, 791, 764, 762, 734, 793, 725, 774, 770, 794, 811, 767, 753, 793, 715, 758, 754, 780, 785, 751, 754, 756, 758, 741] payload_rows_per_dst=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] dup_rows_per_dst=[3074, 2886, 2986, 2996, 3088, 2938, 3012, 2896] combine_rows_per_src=[9264, 9270, 9300, 9388, 9400, 9234, 9310, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=344976 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268280720 ingress_bytes=371744768 link_bytes=371744768 token_bytes=536216464 weight_movement_bytes=103809024 total_nvlink_bytes=640025488 weight_to_token_ratio=0.193595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398845634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64896 dup_rows=24026 dup_rows_remote=21030 dedup_ratio=0.244405 router_tokens_per_expert=[792, 802, 740, 778, 748, 776, 762, 730, 740, 806, 763, 787, 787, 768, 747, 792, 774, 785, 779, 798, 768, 781, 758, 753, 751, 719, 725, 773, 727, 776, 746, 820, 740, 811, 761, 759, 757, 726, 838, 740, 781, 767, 751, 765, 762, 762, 757, 745, 776, 725, 809, 747, 833, 790, 738, 774, 763, 794, 764, 731, 801, 737, 793, 804] payload_rows_per_dst=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] dup_rows_per_dst=[2954, 2874, 3168, 2958, 2994, 3054, 3028, 2996] combine_rows_per_src=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65200 dup_rows=23918 dup_rows_remote=20896 dedup_ratio=0.243306 router_tokens_per_expert=[739, 759, 747, 776, 752, 830, 764, 794, 794, 772, 761, 772, 760, 795, 758, 748, 773, 803, 801, 766, 757, 782, 750, 802, 767, 747, 778, 736, 747, 782, 808, 764, 773, 747, 778, 726, 787, 777, 751, 771, 771, 807, 755, 745, 760, 765, 799, 782, 767, 762, 754, 777, 777, 770, 729, 768, 762, 773, 745, 752, 761, 741, 747, 789] payload_rows_per_dst=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] dup_rows_per_dst=[2996, 3038, 3050, 2948, 2882, 3086, 3022, 2896] combine_rows_per_src=[9326, 9282, 9418, 9310, 9338, 9282, 9186, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64940 dup_rows=24116 dup_rows_remote=21120 dedup_ratio=0.245321 router_tokens_per_expert=[735, 733, 756, 772, 817, 773, 779, 773, 750, 759, 781, 778, 792, 765, 737, 748, 741, 734, 787, 764, 791, 767, 771, 793, 723, 822, 780, 778, 732, 793, 797, 825, 797, 754, 754, 713, 732, 762, 813, 797, 729, 780, 794, 749, 767, 816, 748, 717, 768, 774, 786, 797, 715, 763, 741, 805, 789, 712, 772, 746, 755, 748, 823, 790] payload_rows_per_dst=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] dup_rows_per_dst=[3004, 2876, 3032, 3168, 2996, 3062, 3024, 2954] combine_rows_per_src=[9272, 9344, 9264, 9332, 9248, 9138, 9274, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343920 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437040 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530160 weight_movement_bytes=34603008 total_nvlink_bytes=565133168 weight_to_token_ratio=0.065223 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64896 dup_rows=24026 dup_rows_remote=21030 dedup_ratio=0.244405 router_tokens_per_expert=[792, 802, 740, 778, 748, 776, 762, 730, 740, 806, 763, 787, 787, 768, 747, 792, 774, 785, 779, 798, 768, 781, 758, 753, 751, 719, 725, 773, 727, 776, 746, 820, 740, 811, 761, 759, 757, 726, 838, 740, 781, 767, 751, 765, 762, 762, 757, 745, 776, 725, 809, 747, 833, 790, 738, 774, 763, 794, 764, 731, 801, 737, 793, 804] payload_rows_per_dst=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] dup_rows_per_dst=[2954, 2874, 3168, 2958, 2994, 3054, 3028, 2996] combine_rows_per_src=[9302, 9218, 9466, 9116, 9266, 9172, 9356, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343680 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747520 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151360 weight_movement_bytes=34603008 total_nvlink_bytes=567754368 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65130 dup_rows=23988 dup_rows_remote=21058 dedup_ratio=0.244019 router_tokens_per_expert=[785, 753, 761, 776, 759, 783, 752, 789, 740, 825, 795, 820, 780, 761, 781, 794, 751, 674, 782, 803, 746, 768, 774, 760, 805, 741, 745, 722, 733, 801, 769, 776, 789, 781, 784, 775, 741, 782, 776, 809, 777, 781, 746, 777, 737, 760, 750, 794, 743, 747, 784, 770, 767, 732, 736, 752, 778, 782, 788, 773, 765, 796, 722, 754] payload_rows_per_dst=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] dup_rows_per_dst=[2980, 3132, 2930, 3030, 3134, 2890, 2876, 3016] combine_rows_per_src=[9336, 9460, 9186, 9154, 9340, 9354, 9186, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267935744 dispatch_meta_bytes=344976 combine_payload_bytes=267935744 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=268280720 ingress_bytes=371744768 link_bytes=371744768 token_bytes=536216464 weight_movement_bytes=103809024 total_nvlink_bytes=640025488 weight_to_token_ratio=0.193595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.398845634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74022 payload_rows_remote=64720 dup_rows=24282 dup_rows_remote=21260 dedup_ratio=0.247009 router_tokens_per_expert=[805, 734, 768, 750, 770, 751, 759, 810, 752, 793, 761, 794, 766, 728, 743, 789, 790, 773, 784, 778, 777, 754, 807, 733, 717, 783, 747, 717, 749, 800, 768, 768, 775, 726, 798, 797, 768, 762, 810, 771, 774, 796, 770, 746, 730, 800, 791, 755, 746, 787, 755, 748, 731, 756, 756, 804, 769, 748, 743, 727, 810, 822, 786, 777] payload_rows_per_dst=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] dup_rows_per_dst=[2978, 3008, 3128, 2940, 3168, 3022, 2986, 3052] combine_rows_per_src=[9316, 9244, 9264, 9158, 9246, 9302, 9180, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344240 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338480 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332720 weight_movement_bytes=103809024 total_nvlink_bytes=636141744 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65016 dup_rows=24036 dup_rows_remote=21112 dedup_ratio=0.244507 router_tokens_per_expert=[730, 735, 768, 760, 762, 746, 763, 726, 741, 754, 795, 782, 800, 779, 785, 761, 751, 800, 714, 780, 739, 758, 748, 768, 766, 803, 752, 773, 745, 780, 751, 790, 767, 762, 749, 786, 751, 801, 783, 773, 770, 784, 755, 750, 801, 775, 790, 760, 778, 729, 791, 773, 760, 752, 755, 750, 813, 807, 734, 787, 771, 782, 782, 826] payload_rows_per_dst=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] dup_rows_per_dst=[2874, 3080, 2914, 3028, 3022, 2990, 2924, 3204] combine_rows_per_src=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64728 dup_rows=24088 dup_rows_remote=21008 dedup_ratio=0.245036 router_tokens_per_expert=[784, 768, 768, 769, 758, 774, 758, 768, 816, 745, 750, 780, 783, 789, 790, 806, 755, 764, 755, 770, 775, 760, 769, 745, 727, 768, 747, 749, 724, 789, 742, 746, 811, 782, 788, 765, 804, 790, 778, 814, 805, 725, 717, 717, 780, 823, 798, 777, 762, 777, 784, 733, 825, 723, 770, 757, 767, 769, 728, 788, 753, 726, 771, 754] payload_rows_per_dst=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] dup_rows_per_dst=[3080, 3088, 2886, 3204, 2814, 3026, 3162, 2828] combine_rows_per_src=[9488, 9450, 9300, 9320, 9036, 9258, 9360, 9004] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343704 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157720 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971736 weight_movement_bytes=103809024 total_nvlink_bytes=635780760 weight_to_token_ratio=0.195140 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344752 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117232 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889712 weight_movement_bytes=34603008 total_nvlink_bytes=568492720 weight_to_token_ratio=0.064813 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65096 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[786, 775, 764, 754, 785, 741, 789, 774, 779, 729, 719, 791, 795, 781, 745, 768, 764, 755, 770, 744, 708, 807, 779, 803, 750, 785, 780, 832, 769, 791, 780, 779, 741, 787, 728, 769, 737, 752, 783, 763, 789, 738, 802, 734, 778, 758, 782, 834, 756, 805, 750, 790, 774, 760, 755, 785, 777, 758, 724, 758, 780, 780, 717, 737] payload_rows_per_dst=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] dup_rows_per_dst=[3046, 2914, 3028, 3096, 2926, 3034, 2966, 2898] combine_rows_per_src=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343680 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747520 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151360 weight_movement_bytes=34603008 total_nvlink_bytes=567754368 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343704 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157720 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971736 weight_movement_bytes=103809024 total_nvlink_bytes=635780760 weight_to_token_ratio=0.195140 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65110 dup_rows=23836 dup_rows_remote=20810 dedup_ratio=0.242472 router_tokens_per_expert=[766, 769, 760, 799, 730, 747, 770, 727, 754, 768, 765, 779, 738, 727, 741, 752, 788, 773, 735, 758, 767, 736, 789, 721, 780, 779, 755, 764, 755, 777, 739, 772, 764, 786, 790, 781, 792, 771, 761, 735, 830, 779, 775, 789, 738, 771, 789, 721, 722, 793, 771, 818, 797, 801, 808, 757, 757, 787, 793, 825, 792, 761, 763, 755] payload_rows_per_dst=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] dup_rows_per_dst=[2898, 2780, 2896, 2962, 3016, 3026, 3168, 3090] combine_rows_per_src=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344512 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266650048 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955584 weight_movement_bytes=34603008 total_nvlink_bytes=567558592 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267059200 dispatch_meta_bytes=344384 combine_payload_bytes=267059200 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267403584 ingress_bytes=336265216 link_bytes=336265216 token_bytes=534462784 weight_movement_bytes=69206016 total_nvlink_bytes=603668800 weight_to_token_ratio=0.129487 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265338936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344240 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338480 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332720 weight_movement_bytes=103809024 total_nvlink_bytes=636141744 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343920 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437040 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530160 weight_movement_bytes=34603008 total_nvlink_bytes=565133168 weight_to_token_ratio=0.065223 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342944 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468832 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594720 weight_movement_bytes=34603008 total_nvlink_bytes=565197728 weight_to_token_ratio=0.065216 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65096 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[786, 775, 764, 754, 785, 741, 789, 774, 779, 729, 719, 791, 795, 781, 745, 768, 764, 755, 770, 744, 708, 807, 779, 803, 750, 785, 780, 832, 769, 791, 780, 779, 741, 787, 728, 769, 737, 752, 783, 763, 789, 738, 802, 734, 778, 758, 782, 834, 756, 805, 750, 790, 774, 760, 755, 785, 777, 758, 724, 758, 780, 780, 717, 737] payload_rows_per_dst=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] dup_rows_per_dst=[3046, 2914, 3028, 3096, 2926, 3034, 2966, 2898] combine_rows_per_src=[9290, 9300, 9232, 9436, 9194, 9396, 9384, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64926 dup_rows=23982 dup_rows_remote=20916 dedup_ratio=0.243958 router_tokens_per_expert=[747, 785, 766, 778, 754, 812, 802, 768, 755, 773, 773, 773, 788, 778, 763, 762, 764, 785, 800, 803, 777, 776, 713, 813, 773, 727, 710, 740, 766, 813, 787, 786, 765, 751, 751, 748, 780, 747, 750, 778, 735, 770, 826, 755, 738, 753, 749, 707, 706, 788, 813, 755, 785, 785, 790, 781, 759, 773, 804, 752, 797, 736, 740, 775] payload_rows_per_dst=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] dup_rows_per_dst=[3004, 3024, 3066, 2938, 3000, 2904, 3052, 2994] combine_rows_per_src=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65070 dup_rows=23794 dup_rows_remote=20888 dedup_ratio=0.242045 router_tokens_per_expert=[770, 757, 731, 781, 754, 785, 774, 760, 785, 740, 784, 762, 751, 786, 756, 749, 776, 782, 774, 774, 717, 756, 765, 763, 768, 769, 788, 810, 742, 759, 779, 758, 766, 750, 759, 782, 772, 750, 784, 729, 788, 756, 801, 740, 762, 822, 766, 774, 772, 799, 766, 770, 758, 786, 778, 777, 776, 777, 782, 765, 743, 764, 753, 780] payload_rows_per_dst=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] dup_rows_per_dst=[2932, 2944, 2976, 2906, 3024, 3018, 3034, 2960] combine_rows_per_src=[9292, 9282, 9238, 9440, 9160, 9400, 9378, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65032 dup_rows=24000 dup_rows_remote=21046 dedup_ratio=0.244141 router_tokens_per_expert=[785, 790, 734, 783, 748, 777, 728, 755, 754, 801, 762, 752, 773, 810, 729, 740, 768, 789, 817, 783, 748, 789, 776, 812, 772, 745, 724, 747, 817, 780, 758, 825, 777, 789, 759, 778, 766, 712, 771, 761, 738, 763, 763, 806, 763, 837, 799, 731, 728, 778, 774, 761, 795, 739, 769, 728, 809, 762, 793, 726, 755, 748, 764, 739] payload_rows_per_dst=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] dup_rows_per_dst=[2914, 2948, 3082, 3050, 2954, 3100, 2920, 3032] combine_rows_per_src=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343680 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267034240 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724800 weight_movement_bytes=34603008 total_nvlink_bytes=568327808 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64704 dup_rows=24212 dup_rows_remote=21142 dedup_ratio=0.246297 router_tokens_per_expert=[803, 758, 762, 753, 767, 755, 749, 793, 766, 774, 778, 762, 704, 781, 768, 821, 780, 762, 853, 735, 719, 802, 802, 784, 767, 831, 755, 765, 751, 746, 785, 760, 755, 749, 773, 719, 737, 735, 750, 761, 719, 718, 794, 768, 795, 741, 798, 757, 807, 789, 696, 782, 792, 745, 748, 805, 769, 763, 785, 764, 781, 782, 801, 783] payload_rows_per_dst=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] dup_rows_per_dst=[3042, 3066, 2704, 3094, 3234, 3038, 2964, 3070] combine_rows_per_src=[9238, 9242, 8904, 9226, 9590, 9142, 9362, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343832 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870552 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397272 weight_movement_bytes=69206016 total_nvlink_bytes=602603288 weight_to_token_ratio=0.129746 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64926 dup_rows=23982 dup_rows_remote=20916 dedup_ratio=0.243958 router_tokens_per_expert=[747, 785, 766, 778, 754, 812, 802, 768, 755, 773, 773, 773, 788, 778, 763, 762, 764, 785, 800, 803, 777, 776, 713, 813, 773, 727, 710, 740, 766, 813, 787, 786, 765, 751, 751, 748, 780, 747, 750, 778, 735, 770, 826, 755, 738, 753, 749, 707, 706, 788, 813, 755, 785, 785, 790, 781, 759, 773, 804, 752, 797, 736, 740, 775] payload_rows_per_dst=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] dup_rows_per_dst=[3004, 3024, 3066, 2938, 3000, 2904, 3052, 2994] combine_rows_per_src=[9420, 9306, 9396, 9266, 9140, 9162, 9354, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344312 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715384 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086456 weight_movement_bytes=34603008 total_nvlink_bytes=567689464 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65050 dup_rows=23864 dup_rows_remote=20758 dedup_ratio=0.242757 router_tokens_per_expert=[771, 790, 753, 817, 747, 736, 723, 789, 767, 756, 813, 800, 780, 771, 792, 779, 785, 754, 786, 756, 824, 768, 766, 843, 753, 768, 801, 773, 709, 791, 721, 751, 765, 759, 753, 741, 733, 757, 744, 750, 792, 725, 784, 800, 770, 751, 721, 744, 797, 755, 773, 796, 811, 725, 786, 805, 739, 781, 722, 771, 763, 760, 780, 766] payload_rows_per_dst=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] dup_rows_per_dst=[3062, 3036, 3116, 2938, 2784, 2954, 3106, 2868] combine_rows_per_src=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65016 dup_rows=24036 dup_rows_remote=21112 dedup_ratio=0.244507 router_tokens_per_expert=[730, 735, 768, 760, 762, 746, 763, 726, 741, 754, 795, 782, 800, 779, 785, 761, 751, 800, 714, 780, 739, 758, 748, 768, 766, 803, 752, 773, 745, 780, 751, 790, 767, 762, 749, 786, 751, 801, 783, 773, 770, 784, 755, 750, 801, 775, 790, 760, 778, 729, 791, 773, 760, 752, 755, 750, 813, 807, 734, 787, 771, 782, 782, 826] payload_rows_per_dst=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] dup_rows_per_dst=[2874, 3080, 2914, 3028, 3022, 2990, 2924, 3204] combine_rows_per_src=[9106, 9314, 9202, 9292, 9322, 9380, 9252, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65110 dup_rows=23836 dup_rows_remote=20810 dedup_ratio=0.242472 router_tokens_per_expert=[766, 769, 760, 799, 730, 747, 770, 727, 754, 768, 765, 779, 738, 727, 741, 752, 788, 773, 735, 758, 767, 736, 789, 721, 780, 779, 755, 764, 755, 777, 739, 772, 764, 786, 790, 781, 792, 771, 761, 735, 830, 779, 775, 789, 738, 771, 789, 721, 722, 793, 771, 818, 797, 801, 808, 757, 757, 787, 793, 825, 792, 761, 763, 755] payload_rows_per_dst=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] dup_rows_per_dst=[2898, 2780, 2896, 2962, 3016, 3026, 3168, 3090] combine_rows_per_src=[9238, 9268, 9238, 9280, 9344, 9358, 9366, 9376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65032 dup_rows=24000 dup_rows_remote=21046 dedup_ratio=0.244141 router_tokens_per_expert=[785, 790, 734, 783, 748, 777, 728, 755, 754, 801, 762, 752, 773, 810, 729, 740, 768, 789, 817, 783, 748, 789, 776, 812, 772, 745, 724, 747, 817, 780, 758, 825, 777, 789, 759, 778, 766, 712, 771, 761, 738, 763, 763, 806, 763, 837, 799, 731, 728, 778, 774, 761, 795, 739, 769, 728, 809, 762, 793, 726, 755, 748, 764, 739] payload_rows_per_dst=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] dup_rows_per_dst=[2914, 2948, 3082, 3050, 2954, 3100, 2920, 3032] combine_rows_per_src=[9286, 9294, 9482, 9286, 9272, 9300, 9224, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64940 dup_rows=23930 dup_rows_remote=20912 dedup_ratio=0.243429 router_tokens_per_expert=[757, 810, 765, 792, 772, 765, 771, 755, 752, 783, 783, 769, 793, 718, 780, 759, 745, 799, 755, 776, 735, 799, 740, 789, 784, 761, 795, 739, 772, 763, 813, 775, 785, 817, 766, 795, 748, 767, 770, 775, 745, 728, 726, 700, 769, 774, 745, 794, 758, 737, 739, 743, 780, 766, 743, 774, 758, 775, 737, 840, 822, 792, 779, 741] payload_rows_per_dst=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] dup_rows_per_dst=[3018, 3028, 3068, 2958, 3080, 3012, 2728, 3038] combine_rows_per_src=[9434, 9392, 9430, 9232, 9366, 9240, 8908, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343384 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265370968 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398552 weight_movement_bytes=69206016 total_nvlink_bytes=599604568 weight_to_token_ratio=0.130479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=344360 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266977576 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533610792 weight_movement_bytes=34603008 total_nvlink_bytes=568213800 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343368 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280264 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217160 weight_movement_bytes=69206016 total_nvlink_bytes=601423176 weight_to_token_ratio=0.130033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64822 dup_rows=24188 dup_rows_remote=21140 dedup_ratio=0.246053 router_tokens_per_expert=[769, 784, 745, 766, 747, 767, 793, 794, 797, 773, 802, 778, 738, 807, 723, 753, 784, 747, 746, 751, 778, 767, 762, 700, 764, 764, 716, 780, 766, 802, 749, 763, 794, 803, 785, 714, 772, 799, 790, 757, 729, 773, 758, 807, 758, 738, 784, 758, 779, 751, 797, 759, 747, 752, 810, 762, 777, 802, 773, 770, 752, 799, 732, 796] payload_rows_per_dst=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] dup_rows_per_dst=[2986, 3048, 2870, 2992, 3166, 2954, 3092, 3080] combine_rows_per_src=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343232 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788032 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533232832 weight_movement_bytes=138412032 total_nvlink_bytes=671644864 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20780 dedup_ratio=0.242635 router_tokens_per_expert=[744, 738, 757, 775, 776, 734, 804, 785, 772, 728, 758, 747, 792, 722, 777, 761, 740, 781, 793, 761, 726, 773, 773, 765, 751, 768, 802, 734, 734, 776, 769, 773, 764, 774, 755, 774, 790, 816, 797, 766, 788, 768, 752, 741, 815, 778, 797, 774, 765, 816, 766, 791, 777, 850, 744, 787, 762, 763, 780, 753, 757, 730, 708, 765] payload_rows_per_dst=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] dup_rows_per_dst=[2992, 2964, 2972, 2946, 2954, 3072, 3102, 2850] combine_rows_per_src=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=343832 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266870552 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533397272 weight_movement_bytes=69206016 total_nvlink_bytes=602603288 weight_to_token_ratio=0.129746 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343384 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265370968 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398552 weight_movement_bytes=69206016 total_nvlink_bytes=599604568 weight_to_token_ratio=0.130479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344512 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266650048 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955584 weight_movement_bytes=34603008 total_nvlink_bytes=567558592 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=343680 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267034240 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533724800 weight_movement_bytes=34603008 total_nvlink_bytes=568327808 weight_to_token_ratio=0.064833 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344312 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715384 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086456 weight_movement_bytes=34603008 total_nvlink_bytes=567689464 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343408 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337648 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331888 weight_movement_bytes=34603008 total_nvlink_bytes=566934896 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64822 dup_rows=24188 dup_rows_remote=21140 dedup_ratio=0.246053 router_tokens_per_expert=[769, 784, 745, 766, 747, 767, 793, 794, 797, 773, 802, 778, 738, 807, 723, 753, 784, 747, 746, 751, 778, 767, 762, 700, 764, 764, 716, 780, 766, 802, 749, 763, 794, 803, 785, 714, 772, 799, 790, 757, 729, 773, 758, 807, 758, 738, 784, 758, 779, 751, 797, 759, 747, 752, 810, 762, 777, 802, 773, 770, 752, 799, 732, 796] payload_rows_per_dst=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] dup_rows_per_dst=[2986, 3048, 2870, 2992, 3166, 2954, 3092, 3080] combine_rows_per_src=[9344, 9294, 9200, 9216, 9262, 9256, 9222, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65104 dup_rows=23784 dup_rows_remote=20716 dedup_ratio=0.241943 router_tokens_per_expert=[800, 731, 774, 837, 798, 763, 792, 736, 740, 733, 753, 738, 718, 775, 786, 742, 767, 772, 757, 743, 805, 769, 825, 804, 713, 779, 789, 786, 769, 808, 752, 764, 797, 815, 744, 745, 771, 778, 764, 723, 797, 717, 775, 775, 796, 721, 747, 791, 795, 778, 777, 778, 749, 746, 740, 791, 768, 769, 730, 777, 768, 783, 779, 750] payload_rows_per_dst=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] dup_rows_per_dst=[3022, 2764, 3068, 3076, 2976, 2966, 2942, 2970] combine_rows_per_src=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64808 dup_rows=24242 dup_rows_remote=21376 dedup_ratio=0.246602 router_tokens_per_expert=[774, 774, 755, 806, 737, 703, 785, 732, 803, 828, 765, 803, 799, 762, 794, 750, 775, 758, 741, 741, 791, 779, 757, 779, 750, 746, 759, 756, 792, 771, 709, 777, 825, 776, 759, 787, 796, 761, 767, 762, 812, 746, 781, 757, 792, 794, 745, 716, 788, 800, 746, 774, 791, 747, 754, 778, 725, 751, 739, 766, 747, 765, 787, 767] payload_rows_per_dst=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] dup_rows_per_dst=[3042, 3094, 2982, 2866, 3158, 3066, 3054, 2980] combine_rows_per_src=[9090, 9514, 9260, 9254, 9308, 9220, 9302, 9114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64622 dup_rows=23902 dup_rows_remote=20404 dedup_ratio=0.243144 router_tokens_per_expert=[793, 752, 750, 787, 744, 723, 776, 739, 780, 778, 758, 801, 745, 747, 762, 750, 777, 793, 776, 776, 705, 789, 797, 774, 737, 788, 771, 792, 739, 768, 792, 735, 815, 735, 698, 793, 795, 783, 778, 778, 756, 789, 767, 820, 754, 799, 760, 743, 742, 786, 782, 779, 796, 748, 788, 755, 786, 799, 737, 791, 729, 766, 790, 721] payload_rows_per_dst=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] dup_rows_per_dst=[2868, 3032, 3008, 2942, 2532, 2970, 3052, 3498] combine_rows_per_src=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20780 dedup_ratio=0.242635 router_tokens_per_expert=[744, 738, 757, 775, 776, 734, 804, 785, 772, 728, 758, 747, 792, 722, 777, 761, 740, 781, 793, 761, 726, 773, 773, 765, 751, 768, 802, 734, 734, 776, 769, 773, 764, 774, 755, 774, 790, 816, 797, 766, 788, 768, 752, 741, 815, 778, 797, 774, 765, 816, 766, 791, 777, 850, 744, 787, 762, 763, 780, 753, 757, 730, 708, 765] payload_rows_per_dst=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] dup_rows_per_dst=[2992, 2964, 2972, 2946, 2954, 3072, 3102, 2850] combine_rows_per_src=[9234, 9150, 9252, 9268, 9518, 9354, 9490, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65050 dup_rows=23864 dup_rows_remote=20758 dedup_ratio=0.242757 router_tokens_per_expert=[771, 790, 753, 817, 747, 736, 723, 789, 767, 756, 813, 800, 780, 771, 792, 779, 785, 754, 786, 756, 824, 768, 766, 843, 753, 768, 801, 773, 709, 791, 721, 751, 765, 759, 753, 741, 733, 757, 744, 750, 792, 725, 784, 800, 770, 751, 721, 744, 797, 755, 773, 796, 811, 725, 786, 805, 739, 781, 722, 771, 763, 760, 780, 766] payload_rows_per_dst=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] dup_rows_per_dst=[3062, 3036, 3116, 2938, 2784, 2954, 3106, 2868] combine_rows_per_src=[9190, 9480, 9448, 9196, 9220, 9220, 9390, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64824 dup_rows=24064 dup_rows_remote=21038 dedup_ratio=0.244792 router_tokens_per_expert=[808, 771, 789, 761, 776, 747, 690, 749, 763, 768, 794, 704, 790, 798, 728, 791, 761, 711, 793, 758, 807, 832, 731, 736, 753, 789, 800, 758, 758, 782, 779, 751, 782, 798, 795, 793, 752, 821, 760, 789, 780, 785, 833, 742, 752, 822, 754, 761, 795, 787, 737, 778, 757, 742, 741, 763, 749, 702, 744, 755, 761, 750, 759, 787] payload_rows_per_dst=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] dup_rows_per_dst=[3026, 3018, 3044, 3052, 3318, 2972, 2740, 2894] combine_rows_per_src=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=343280 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267009264 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533675248 weight_movement_bytes=34603008 total_nvlink_bytes=568278256 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20734 dedup_ratio=0.241821 router_tokens_per_expert=[788, 748, 751, 783, 735, 759, 791, 771, 727, 764, 766, 778, 782, 750, 765, 832, 777, 733, 796, 792, 793, 787, 778, 742, 749, 748, 748, 802, 762, 761, 759, 821, 790, 768, 774, 785, 778, 754, 782, 772, 772, 762, 721, 732, 776, 758, 838, 738, 760, 784, 766, 750, 791, 773, 770, 772, 776, 729, 754, 724, 778, 757, 772, 758] payload_rows_per_dst=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] dup_rows_per_dst=[2934, 2934, 3014, 3002, 3038, 2884, 3090, 2876] combine_rows_per_src=[9318, 9394, 9382, 9298, 9368, 9310, 9242, 9220] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343848 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265854760 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531365672 weight_movement_bytes=34603008 total_nvlink_bytes=565968680 weight_to_token_ratio=0.065121 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343512 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266984920 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533626328 weight_movement_bytes=103809024 total_nvlink_bytes=637435352 weight_to_token_ratio=0.194535 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64622 dup_rows=23902 dup_rows_remote=20404 dedup_ratio=0.243144 router_tokens_per_expert=[793, 752, 750, 787, 744, 723, 776, 739, 780, 778, 758, 801, 745, 747, 762, 750, 777, 793, 776, 776, 705, 789, 797, 774, 737, 788, 771, 792, 739, 768, 792, 735, 815, 735, 698, 793, 795, 783, 778, 778, 756, 789, 767, 820, 754, 799, 760, 743, 742, 786, 782, 779, 796, 748, 788, 755, 786, 799, 737, 791, 729, 766, 790, 721] payload_rows_per_dst=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] dup_rows_per_dst=[2868, 3032, 3008, 2942, 2532, 2970, 3052, 3498] combine_rows_per_src=[9260, 9210, 9366, 9284, 8778, 9406, 9318, 9780] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=344736 combine_payload_bytes=265453568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265798304 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531251872 weight_movement_bytes=34603008 total_nvlink_bytes=565854880 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74520 payload_rows_remote=65104 dup_rows=23784 dup_rows_remote=20716 dedup_ratio=0.241943 router_tokens_per_expert=[800, 731, 774, 837, 798, 763, 792, 736, 740, 733, 753, 738, 718, 775, 786, 742, 767, 772, 757, 743, 805, 769, 825, 804, 713, 779, 789, 786, 769, 808, 752, 764, 797, 815, 744, 745, 771, 778, 764, 723, 797, 717, 775, 775, 796, 721, 747, 791, 795, 778, 777, 778, 749, 746, 740, 791, 768, 769, 730, 777, 768, 783, 779, 750] payload_rows_per_dst=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] dup_rows_per_dst=[3022, 2764, 3068, 3076, 2976, 2966, 2942, 2970] combine_rows_per_src=[9440, 9206, 9416, 9244, 9298, 9272, 9366, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343592 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255336 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167080 weight_movement_bytes=69206016 total_nvlink_bytes=603373096 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65122 dup_rows=23866 dup_rows_remote=20864 dedup_ratio=0.242778 router_tokens_per_expert=[733, 817, 796, 763, 715, 811, 748, 783, 754, 747, 802, 766, 747, 774, 769, 814, 771, 719, 752, 758, 759, 737, 811, 769, 795, 772, 798, 761, 785, 782, 796, 799, 689, 753, 742, 743, 732, 813, 776, 767, 759, 741, 823, 776, 809, 784, 743, 772, 733, 753, 816, 782, 714, 768, 767, 826, 730, 766, 732, 785, 743, 783, 780, 749] payload_rows_per_dst=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] dup_rows_per_dst=[2950, 3006, 2922, 3150, 2888, 3030, 3002, 2918] combine_rows_per_src=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64824 dup_rows=24064 dup_rows_remote=21038 dedup_ratio=0.244792 router_tokens_per_expert=[808, 771, 789, 761, 776, 747, 690, 749, 763, 768, 794, 704, 790, 798, 728, 791, 761, 711, 793, 758, 807, 832, 731, 736, 753, 789, 800, 758, 758, 782, 779, 751, 782, 798, 795, 793, 752, 821, 760, 789, 780, 785, 833, 742, 752, 822, 754, 761, 795, 787, 737, 778, 757, 742, 741, 763, 749, 702, 744, 755, 761, 750, 759, 787] payload_rows_per_dst=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] dup_rows_per_dst=[3026, 3018, 3044, 3052, 3318, 2972, 2740, 2894] combine_rows_per_src=[9416, 9336, 9318, 9288, 9600, 9188, 8974, 9120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264691712 dispatch_meta_bytes=340104 combine_payload_bytes=264691712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265031816 ingress_bytes=299294720 link_bytes=299294720 token_bytes=529723528 weight_movement_bytes=34603008 total_nvlink_bytes=564326536 weight_to_token_ratio=0.065323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126221936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=343280 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267009264 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533675248 weight_movement_bytes=34603008 total_nvlink_bytes=568278256 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=344736 combine_payload_bytes=265453568 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265798304 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531251872 weight_movement_bytes=34603008 total_nvlink_bytes=565854880 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264691712 dispatch_meta_bytes=340104 combine_payload_bytes=264691712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265031816 ingress_bytes=299294720 link_bytes=299294720 token_bytes=529723528 weight_movement_bytes=34603008 total_nvlink_bytes=564326536 weight_to_token_ratio=0.065323 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126221936e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343512 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266984920 ingress_bytes=370450432 link_bytes=370450432 token_bytes=533626328 weight_movement_bytes=103809024 total_nvlink_bytes=637435352 weight_to_token_ratio=0.194535 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393975152e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343232 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266788032 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533232832 weight_movement_bytes=138412032 total_nvlink_bytes=671644864 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343448 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862552 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381656 weight_movement_bytes=34603008 total_nvlink_bytes=565984664 weight_to_token_ratio=0.065119 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343592 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267255336 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167080 weight_movement_bytes=69206016 total_nvlink_bytes=603373096 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65042 dup_rows=24008 dup_rows_remote=21038 dedup_ratio=0.244222 router_tokens_per_expert=[732, 711, 756, 751, 747, 782, 794, 743, 763, 784, 809, 802, 784, 717, 743, 745, 767, 759, 778, 762, 762, 752, 793, 739, 778, 783, 781, 819, 765, 787, 808, 784, 696, 794, 796, 780, 730, 748, 758, 752, 767, 739, 770, 808, 826, 805, 716, 780, 764, 779, 743, 726, 774, 790, 739, 778, 787, 760, 800, 787, 741, 774, 804, 761] payload_rows_per_dst=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] dup_rows_per_dst=[2908, 2932, 2970, 3140, 2930, 3104, 2906, 3118] combine_rows_per_src=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65090 dup_rows=23860 dup_rows_remote=20910 dedup_ratio=0.242716 router_tokens_per_expert=[756, 744, 778, 766, 741, 809, 813, 782, 760, 744, 741, 782, 807, 738, 811, 769, 738, 744, 771, 763, 764, 748, 794, 740, 709, 806, 734, 739, 765, 735, 749, 778, 768, 785, 772, 799, 767, 772, 756, 769, 781, 756, 786, 815, 768, 704, 789, 797, 769, 770, 777, 762, 765, 780, 838, 739, 793, 722, 744, 759, 793, 830, 800, 709] payload_rows_per_dst=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] dup_rows_per_dst=[2998, 2950, 2926, 2854, 3010, 3050, 3058, 3014] combine_rows_per_src=[9380, 9354, 9198, 9176, 9366, 9342, 9342, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74094 payload_rows_remote=64808 dup_rows=24210 dup_rows_remote=21110 dedup_ratio=0.246277 router_tokens_per_expert=[771, 791, 732, 780, 789, 786, 765, 771, 764, 743, 769, 784, 758, 718, 772, 790, 772, 772, 767, 751, 743, 731, 715, 762, 792, 798, 797, 819, 765, 742, 776, 704, 759, 800, 766, 755, 781, 824, 814, 757, 763, 789, 747, 789, 812, 763, 731, 768, 770, 781, 828, 755, 733, 798, 764, 769, 784, 748, 731, 790, 772, 743, 712, 767] payload_rows_per_dst=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] dup_rows_per_dst=[3058, 3020, 2916, 3100, 3084, 3020, 3082, 2930] combine_rows_per_src=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65098 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[802, 791, 772, 765, 758, 740, 767, 761, 821, 776, 778, 765, 798, 765, 798, 730, 760, 726, 822, 743, 740, 777, 731, 726, 765, 778, 762, 747, 773, 786, 778, 789, 721, 774, 763, 732, 766, 785, 739, 744, 786, 750, 762, 821, 787, 781, 770, 799, 769, 804, 797, 751, 689, 745, 780, 816, 738, 775, 793, 791, 745, 767, 765, 757] payload_rows_per_dst=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] dup_rows_per_dst=[2966, 3076, 2816, 3058, 2774, 3172, 3004, 3060] combine_rows_per_src=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344320 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756352 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168384 weight_movement_bytes=34603008 total_nvlink_bytes=567771392 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64998 dup_rows=23978 dup_rows_remote=20802 dedup_ratio=0.243917 router_tokens_per_expert=[744, 745, 773, 819, 769, 768, 789, 792, 784, 733, 766, 743, 752, 807, 713, 738, 729, 798, 756, 771, 747, 765, 808, 752, 726, 773, 780, 773, 805, 693, 773, 804, 825, 765, 780, 801, 790, 746, 797, 744, 781, 763, 768, 812, 791, 767, 797, 773, 792, 763, 737, 766, 785, 756, 778, 735, 766, 728, 736, 690, 776, 754, 797, 805] payload_rows_per_dst=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] dup_rows_per_dst=[2976, 2954, 2956, 2918, 3098, 3176, 3000, 2900] combine_rows_per_src=[9422, 9118, 9296, 9336, 9398, 9328, 9224, 9204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343944 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267083656 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533823368 weight_movement_bytes=138412032 total_nvlink_bytes=672235400 weight_to_token_ratio=0.259284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343448 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862552 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381656 weight_movement_bytes=34603008 total_nvlink_bytes=565984664 weight_to_token_ratio=0.065119 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64968 dup_rows=23978 dup_rows_remote=21012 dedup_ratio=0.243917 router_tokens_per_expert=[764, 752, 757, 745, 786, 762, 719, 795, 708, 749, 778, 759, 779, 807, 761, 753, 757, 737, 800, 839, 758, 729, 746, 735, 768, 767, 779, 714, 758, 765, 736, 779, 758, 792, 790, 785, 758, 781, 774, 784, 742, 814, 748, 760, 763, 724, 803, 815, 779, 742, 766, 821, 791, 774, 766, 768, 786, 768, 810, 752, 764, 772, 766, 795] payload_rows_per_dst=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] dup_rows_per_dst=[2966, 3072, 3254, 3110, 2928, 2606, 2950, 3092] combine_rows_per_src=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65122 dup_rows=23866 dup_rows_remote=20864 dedup_ratio=0.242778 router_tokens_per_expert=[733, 817, 796, 763, 715, 811, 748, 783, 754, 747, 802, 766, 747, 774, 769, 814, 771, 719, 752, 758, 759, 737, 811, 769, 795, 772, 798, 761, 785, 782, 796, 799, 689, 753, 742, 743, 732, 813, 776, 767, 759, 741, 823, 776, 809, 784, 743, 772, 733, 753, 816, 782, 714, 768, 767, 826, 730, 766, 732, 785, 743, 783, 780, 749] payload_rows_per_dst=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] dup_rows_per_dst=[2950, 3006, 2922, 3150, 2888, 3030, 3002, 2918] combine_rows_per_src=[9382, 9340, 9230, 9426, 9142, 9384, 9316, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74094 payload_rows_remote=64808 dup_rows=24210 dup_rows_remote=21110 dedup_ratio=0.246277 router_tokens_per_expert=[771, 791, 732, 780, 789, 786, 765, 771, 764, 743, 769, 784, 758, 718, 772, 790, 772, 772, 767, 751, 743, 731, 715, 762, 792, 798, 797, 819, 765, 742, 776, 704, 759, 800, 766, 755, 781, 824, 814, 757, 763, 789, 747, 789, 812, 763, 731, 768, 770, 781, 828, 755, 733, 798, 764, 769, 784, 748, 731, 790, 772, 743, 712, 767] payload_rows_per_dst=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] dup_rows_per_dst=[3058, 3020, 2916, 3100, 3084, 3020, 3082, 2930] combine_rows_per_src=[9312, 9176, 9110, 9286, 9428, 9304, 9314, 9164] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64814 dup_rows=24038 dup_rows_remote=20870 dedup_ratio=0.244527 router_tokens_per_expert=[746, 729, 767, 771, 805, 780, 771, 693, 777, 755, 724, 782, 759, 746, 766, 778, 769, 787, 809, 809, 783, 750, 750, 768, 759, 775, 743, 766, 785, 742, 730, 741, 819, 789, 781, 779, 792, 772, 770, 808, 736, 790, 767, 800, 752, 761, 800, 757, 795, 768, 782, 777, 732, 786, 740, 761, 734, 758, 770, 777, 800, 794, 765, 725] payload_rows_per_dst=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] dup_rows_per_dst=[2920, 2996, 2996, 2924, 3168, 3042, 3046, 2946] combine_rows_per_src=[9204, 9178, 9454, 9158, 9452, 9284, 9236, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344000 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952640 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561280 weight_movement_bytes=34603008 total_nvlink_bytes=568164288 weight_to_token_ratio=0.064853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343672 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265797240 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531250808 weight_movement_bytes=34603008 total_nvlink_bytes=565853816 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343856 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985264 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626672 weight_movement_bytes=34603008 total_nvlink_bytes=568229680 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=343200 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266575008 ingress_bytes=370040832 link_bytes=370040832 token_bytes=532806816 weight_movement_bytes=103809024 total_nvlink_bytes=636615840 weight_to_token_ratio=0.194834 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392433860e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65098 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[802, 791, 772, 765, 758, 740, 767, 761, 821, 776, 778, 765, 798, 765, 798, 730, 760, 726, 822, 743, 740, 777, 731, 726, 765, 778, 762, 747, 773, 786, 778, 789, 721, 774, 763, 732, 766, 785, 739, 744, 786, 750, 762, 821, 787, 781, 770, 799, 769, 804, 797, 751, 689, 745, 780, 816, 738, 775, 793, 791, 745, 767, 765, 757] payload_rows_per_dst=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] dup_rows_per_dst=[2966, 3076, 2816, 3058, 2774, 3172, 3004, 3060] combine_rows_per_src=[9346, 9386, 9234, 9298, 9188, 9340, 9306, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343672 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265797240 ingress_bytes=300056576 link_bytes=300056576 token_bytes=531250808 weight_movement_bytes=34603008 total_nvlink_bytes=565853816 weight_to_token_ratio=0.065135 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129088739e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343920 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452848 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561776 weight_movement_bytes=34603008 total_nvlink_bytes=567164784 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343944 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267083656 ingress_bytes=405151744 link_bytes=405151744 token_bytes=533823368 weight_movement_bytes=138412032 total_nvlink_bytes=672235400 weight_to_token_ratio=0.259284 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524553395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342736 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265820880 ingress_bytes=369287168 link_bytes=369287168 token_bytes=531299024 weight_movement_bytes=103809024 total_nvlink_bytes=635108048 weight_to_token_ratio=0.195387 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389597883e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65042 dup_rows=24008 dup_rows_remote=21038 dedup_ratio=0.244222 router_tokens_per_expert=[732, 711, 756, 751, 747, 782, 794, 743, 763, 784, 809, 802, 784, 717, 743, 745, 767, 759, 778, 762, 762, 752, 793, 739, 778, 783, 781, 819, 765, 787, 808, 784, 696, 794, 796, 780, 730, 748, 758, 752, 767, 739, 770, 808, 826, 805, 716, 780, 764, 779, 743, 726, 774, 790, 739, 778, 787, 760, 800, 787, 741, 774, 804, 761] payload_rows_per_dst=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] dup_rows_per_dst=[2908, 2932, 2970, 3140, 2930, 3104, 2906, 3118] combine_rows_per_src=[9124, 9362, 9254, 9470, 9178, 9318, 9280, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64968 dup_rows=23978 dup_rows_remote=21012 dedup_ratio=0.243917 router_tokens_per_expert=[764, 752, 757, 745, 786, 762, 719, 795, 708, 749, 778, 759, 779, 807, 761, 753, 757, 737, 800, 839, 758, 729, 746, 735, 768, 767, 779, 714, 758, 765, 736, 779, 758, 792, 790, 785, 758, 781, 774, 784, 742, 814, 748, 760, 763, 724, 803, 815, 779, 742, 766, 821, 791, 774, 766, 768, 786, 768, 810, 752, 764, 772, 766, 795] payload_rows_per_dst=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] dup_rows_per_dst=[2966, 3072, 3254, 3110, 2928, 2606, 2950, 3092] combine_rows_per_src=[9358, 9330, 9652, 9430, 9108, 8882, 9232, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64896 dup_rows=24070 dup_rows_remote=21084 dedup_ratio=0.244853 router_tokens_per_expert=[798, 792, 789, 764, 760, 730, 761, 789, 805, 803, 746, 779, 791, 782, 725, 769, 754, 812, 738, 760, 773, 749, 761, 789, 747, 736, 761, 743, 755, 743, 751, 787, 778, 743, 772, 759, 745, 779, 764, 757, 785, 776, 802, 738, 727, 725, 819, 796, 788, 785, 786, 818, 719, 723, 758, 785, 809, 808, 781, 744, 775, 763, 724, 779] payload_rows_per_dst=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] dup_rows_per_dst=[3136, 2976, 3060, 2874, 2956, 3022, 2986, 3060] combine_rows_per_src=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343856 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985264 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533626672 weight_movement_bytes=34603008 total_nvlink_bytes=568229680 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344320 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756352 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168384 weight_movement_bytes=34603008 total_nvlink_bytes=567771392 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64964 dup_rows=24018 dup_rows_remote=20960 dedup_ratio=0.244324 router_tokens_per_expert=[787, 759, 693, 729, 793, 717, 794, 753, 784, 822, 774, 750, 747, 761, 785, 799, 755, 788, 751, 767, 759, 800, 783, 787, 727, 800, 788, 768, 764, 772, 785, 738, 733, 732, 740, 782, 805, 819, 785, 756, 800, 820, 777, 765, 784, 758, 748, 732, 749, 790, 799, 761, 770, 781, 734, 796, 739, 765, 747, 774, 742, 803, 783, 704] payload_rows_per_dst=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] dup_rows_per_dst=[2920, 3060, 3058, 2986, 3028, 3066, 3000, 2900] combine_rows_per_src=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=64990 dup_rows=23830 dup_rows_remote=20704 dedup_ratio=0.242411 router_tokens_per_expert=[773, 739, 743, 776, 822, 769, 728, 785, 762, 795, 767, 829, 797, 786, 795, 774, 790, 747, 731, 753, 784, 776, 789, 782, 748, 749, 736, 764, 750, 723, 814, 724, 728, 784, 800, 727, 770, 764, 792, 735, 741, 714, 759, 753, 756, 824, 754, 744, 747, 754, 739, 766, 754, 807, 824, 803, 806, 784, 780, 792, 726, 716, 786, 823] payload_rows_per_dst=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] dup_rows_per_dst=[2998, 3126, 3020, 2900, 2942, 2814, 3028, 3002] combine_rows_per_src=[9272, 9484, 9284, 9116, 9258, 9276, 9360, 9424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343920 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452848 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561776 weight_movement_bytes=34603008 total_nvlink_bytes=567164784 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343920 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157936 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971952 weight_movement_bytes=103809024 total_nvlink_bytes=635780976 weight_to_token_ratio=0.195140 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436240 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528784 weight_movement_bytes=34603008 total_nvlink_bytes=567131792 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65038 dup_rows=23938 dup_rows_remote=20990 dedup_ratio=0.243510 router_tokens_per_expert=[744, 763, 795, 793, 751, 784, 809, 769, 760, 774, 758, 755, 762, 815, 762, 800, 778, 767, 791, 819, 782, 750, 755, 783, 716, 778, 790, 802, 715, 768, 734, 766, 743, 779, 806, 744, 759, 714, 742, 757, 769, 791, 762, 726, 804, 784, 783, 810, 793, 736, 741, 750, 805, 725, 716, 788, 769, 753, 771, 764, 787, 774, 752, 767] payload_rows_per_dst=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] dup_rows_per_dst=[2684, 2732, 3084, 2886, 3314, 3298, 2992, 2948] combine_rows_per_src=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65100 dup_rows=23924 dup_rows_remote=20876 dedup_ratio=0.243368 router_tokens_per_expert=[815, 773, 781, 734, 772, 745, 769, 803, 765, 731, 736, 797, 760, 800, 754, 784, 803, 749, 760, 781, 797, 738, 733, 751, 757, 738, 736, 780, 795, 781, 766, 811, 767, 742, 754, 738, 751, 740, 802, 766, 764, 794, 781, 754, 772, 796, 733, 766, 780, 745, 806, 770, 740, 775, 763, 780, 803, 792, 754, 761, 755, 750, 779, 784] payload_rows_per_dst=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] dup_rows_per_dst=[3018, 2988, 2928, 3048, 2890, 2992, 3032, 3028] combine_rows_per_src=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65168 dup_rows=23932 dup_rows_remote=20984 dedup_ratio=0.243449 router_tokens_per_expert=[783, 780, 768, 824, 789, 786, 770, 778, 795, 755, 767, 775, 778, 730, 798, 765, 713, 783, 794, 724, 776, 749, 782, 732, 771, 799, 784, 796, 745, 788, 776, 760, 816, 775, 762, 716, 753, 792, 829, 741, 765, 762, 787, 742, 795, 754, 733, 738, 733, 777, 762, 782, 732, 721, 766, 755, 808, 790, 807, 724, 779, 765, 750, 728] payload_rows_per_dst=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] dup_rows_per_dst=[3038, 2980, 2928, 2988, 3122, 2948, 2916, 3012] combine_rows_per_src=[9518, 9346, 9178, 9450, 9246, 9204, 9140, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64972 dup_rows=24042 dup_rows_remote=21110 dedup_ratio=0.244568 router_tokens_per_expert=[691, 773, 766, 782, 804, 800, 765, 730, 757, 743, 809, 743, 791, 784, 801, 740, 825, 794, 738, 762, 790, 795, 779, 777, 744, 737, 757, 787, 773, 793, 793, 767, 767, 749, 765, 745, 776, 764, 778, 741, 772, 735, 748, 747, 786, 776, 748, 796, 773, 743, 742, 783, 783, 773, 781, 731, 781, 757, 795, 742, 732, 831, 745, 777] payload_rows_per_dst=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] dup_rows_per_dst=[2932, 2994, 3132, 3018, 2992, 3112, 2990, 2872] combine_rows_per_src=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65100 dup_rows=23924 dup_rows_remote=20876 dedup_ratio=0.243368 router_tokens_per_expert=[815, 773, 781, 734, 772, 745, 769, 803, 765, 731, 736, 797, 760, 800, 754, 784, 803, 749, 760, 781, 797, 738, 733, 751, 757, 738, 736, 780, 795, 781, 766, 811, 767, 742, 754, 738, 751, 740, 802, 766, 764, 794, 781, 754, 772, 796, 733, 766, 780, 745, 806, 770, 740, 775, 763, 780, 803, 792, 754, 761, 755, 750, 779, 784] payload_rows_per_dst=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] dup_rows_per_dst=[3018, 2988, 2928, 3048, 2890, 2992, 3032, 3028] combine_rows_per_src=[9366, 9266, 9296, 9280, 9230, 9328, 9286, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64896 dup_rows=24070 dup_rows_remote=21084 dedup_ratio=0.244853 router_tokens_per_expert=[798, 792, 789, 764, 760, 730, 761, 789, 805, 803, 746, 779, 791, 782, 725, 769, 754, 812, 738, 760, 773, 749, 761, 789, 747, 736, 761, 743, 755, 743, 751, 787, 778, 743, 772, 759, 745, 779, 764, 757, 785, 776, 802, 738, 727, 725, 819, 796, 788, 785, 786, 818, 719, 723, 758, 785, 809, 808, 781, 744, 775, 763, 724, 779] payload_rows_per_dst=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] dup_rows_per_dst=[3136, 2976, 3060, 2874, 2956, 3022, 2986, 3060] combine_rows_per_src=[9230, 9424, 9212, 9172, 9238, 9314, 9338, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65004 dup_rows=24158 dup_rows_remote=21158 dedup_ratio=0.245748 router_tokens_per_expert=[746, 752, 808, 785, 817, 788, 770, 797, 787, 742, 750, 778, 750, 811, 765, 746, 754, 751, 816, 746, 764, 725, 738, 755, 729, 776, 750, 796, 799, 797, 783, 754, 762, 739, 713, 764, 782, 774, 763, 774, 763, 747, 767, 731, 718, 763, 787, 760, 800, 821, 796, 792, 731, 792, 714, 792, 793, 799, 770, 805, 760, 750, 767, 738] payload_rows_per_dst=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] dup_rows_per_dst=[3088, 3024, 2936, 3070, 3000, 2936, 3084, 3020] combine_rows_per_src=[9438, 9234, 9162, 9298, 9142, 9136, 9392, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=342776 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266541816 ingress_bytes=404611072 link_bytes=404611072 token_bytes=532740856 weight_movement_bytes=138412032 total_nvlink_bytes=671152888 weight_to_token_ratio=0.259811 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522518890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344112 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266739760 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135408 weight_movement_bytes=103809024 total_nvlink_bytes=636944432 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343904 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993504 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643104 weight_movement_bytes=34603008 total_nvlink_bytes=568246112 weight_to_token_ratio=0.064843 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344608 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272736 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200864 weight_movement_bytes=34603008 total_nvlink_bytes=568803872 weight_to_token_ratio=0.064775 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65038 dup_rows=23938 dup_rows_remote=20990 dedup_ratio=0.243510 router_tokens_per_expert=[744, 763, 795, 793, 751, 784, 809, 769, 760, 774, 758, 755, 762, 815, 762, 800, 778, 767, 791, 819, 782, 750, 755, 783, 716, 778, 790, 802, 715, 768, 734, 766, 743, 779, 806, 744, 759, 714, 742, 757, 769, 791, 762, 726, 804, 784, 783, 810, 793, 736, 741, 750, 805, 725, 716, 788, 769, 753, 771, 764, 787, 774, 752, 767] payload_rows_per_dst=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] dup_rows_per_dst=[2684, 2732, 3084, 2886, 3314, 3298, 2992, 2948] combine_rows_per_src=[9002, 9002, 9490, 9252, 9502, 9496, 9294, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343904 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993504 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643104 weight_movement_bytes=34603008 total_nvlink_bytes=568246112 weight_to_token_ratio=0.064843 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344328 combine_payload_bytes=266125312 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266469640 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532594952 weight_movement_bytes=69206016 total_nvlink_bytes=601800968 weight_to_token_ratio=0.129941 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343920 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266157936 ingress_bytes=369623040 link_bytes=369623040 token_bytes=531971952 weight_movement_bytes=103809024 total_nvlink_bytes=635780976 weight_to_token_ratio=0.195140 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390861742e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=344648 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266601032 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532857416 weight_movement_bytes=34603008 total_nvlink_bytes=567460424 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64964 dup_rows=24018 dup_rows_remote=20960 dedup_ratio=0.244324 router_tokens_per_expert=[787, 759, 693, 729, 793, 717, 794, 753, 784, 822, 774, 750, 747, 761, 785, 799, 755, 788, 751, 767, 759, 800, 783, 787, 727, 800, 788, 768, 764, 772, 785, 738, 733, 732, 740, 782, 805, 819, 785, 756, 800, 820, 777, 765, 784, 758, 748, 732, 749, 790, 799, 761, 770, 781, 734, 796, 739, 765, 747, 774, 742, 803, 783, 704] payload_rows_per_dst=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] dup_rows_per_dst=[2920, 3060, 3058, 2986, 3028, 3066, 3000, 2900] combine_rows_per_src=[9130, 9384, 9322, 9298, 9276, 9302, 9360, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64972 dup_rows=24042 dup_rows_remote=21110 dedup_ratio=0.244568 router_tokens_per_expert=[691, 773, 766, 782, 804, 800, 765, 730, 757, 743, 809, 743, 791, 784, 801, 740, 825, 794, 738, 762, 790, 795, 779, 777, 744, 737, 757, 787, 773, 793, 793, 767, 767, 749, 765, 745, 776, 764, 778, 741, 772, 735, 748, 747, 786, 776, 748, 796, 773, 743, 742, 783, 783, 773, 781, 731, 781, 757, 795, 742, 732, 831, 745, 777] payload_rows_per_dst=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] dup_rows_per_dst=[2932, 2994, 3132, 3018, 2992, 3112, 2990, 2872] combine_rows_per_src=[9290, 9364, 9388, 9458, 9178, 9104, 9228, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65168 dup_rows=23802 dup_rows_remote=20934 dedup_ratio=0.242126 router_tokens_per_expert=[779, 801, 786, 814, 781, 768, 790, 786, 770, 748, 775, 790, 775, 771, 769, 762, 738, 741, 720, 780, 733, 709, 832, 796, 777, 803, 723, 770, 783, 742, 775, 755, 776, 786, 728, 785, 767, 738, 762, 750, 808, 770, 782, 727, 793, 770, 771, 725, 766, 747, 787, 707, 768, 781, 776, 769, 788, 740, 801, 777, 785, 785, 737, 758] payload_rows_per_dst=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] dup_rows_per_dst=[3130, 3016, 2860, 2958, 2990, 2980, 2868, 3000] combine_rows_per_src=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344112 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266739760 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135408 weight_movement_bytes=103809024 total_nvlink_bytes=636944432 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=65016 dup_rows=24080 dup_rows_remote=21076 dedup_ratio=0.244954 router_tokens_per_expert=[833, 801, 791, 778, 769, 738, 789, 773, 777, 802, 808, 755, 773, 830, 780, 750, 758, 768, 772, 768, 763, 760, 732, 785, 759, 726, 772, 742, 761, 766, 800, 755, 796, 802, 716, 718, 791, 741, 733, 808, 784, 734, 780, 760, 780, 764, 804, 758, 787, 760, 752, 732, 772, 747, 778, 734, 768, 739, 736, 820, 709, 774, 762, 779] payload_rows_per_dst=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] dup_rows_per_dst=[3140, 3080, 3004, 2964, 2958, 3088, 2954, 2892] combine_rows_per_src=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344368 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649904 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955440 weight_movement_bytes=69206016 total_nvlink_bytes=602161456 weight_to_token_ratio=0.129853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65298 dup_rows=23738 dup_rows_remote=20832 dedup_ratio=0.241475 router_tokens_per_expert=[770, 799, 781, 750, 773, 755, 764, 797, 795, 754, 751, 740, 777, 711, 789, 770, 771, 783, 733, 765, 761, 806, 796, 770, 762, 744, 780, 745, 776, 757, 789, 770, 818, 754, 784, 777, 748, 769, 772, 733, 727, 742, 753, 699, 740, 772, 751, 777, 794, 800, 772, 749, 811, 768, 781, 765, 738, 769, 742, 795, 817, 785, 778, 788] payload_rows_per_dst=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] dup_rows_per_dst=[2984, 2906, 3018, 2958, 2878, 2826, 3134, 3034] combine_rows_per_src=[9394, 9268, 9352, 9288, 9432, 9096, 9346, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344328 combine_payload_bytes=266125312 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266469640 ingress_bytes=335331328 link_bytes=335331328 token_bytes=532594952 weight_movement_bytes=69206016 total_nvlink_bytes=601800968 weight_to_token_ratio=0.129941 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261824791e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343696 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266436240 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532528784 weight_movement_bytes=34603008 total_nvlink_bytes=567131792 weight_to_token_ratio=0.064979 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344408 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272536 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200664 weight_movement_bytes=34603008 total_nvlink_bytes=568803672 weight_to_token_ratio=0.064775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64962 dup_rows=23942 dup_rows_remote=20872 dedup_ratio=0.243551 router_tokens_per_expert=[771, 755, 790, 720, 826, 752, 784, 763, 799, 781, 795, 766, 773, 788, 759, 772, 750, 781, 827, 724, 778, 847, 787, 820, 749, 781, 785, 775, 774, 780, 713, 771, 743, 736, 743, 735, 769, 785, 799, 777, 780, 768, 789, 738, 709, 755, 705, 752, 745, 763, 759, 763, 767, 825, 806, 766, 755, 782, 737, 764, 757, 750, 756, 738] payload_rows_per_dst=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] dup_rows_per_dst=[2940, 2916, 3050, 3134, 3052, 2990, 2790, 3070] combine_rows_per_src=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65008 dup_rows=23926 dup_rows_remote=21044 dedup_ratio=0.243388 router_tokens_per_expert=[741, 800, 750, 824, 716, 755, 761, 720, 764, 761, 734, 791, 786, 765, 805, 770, 759, 808, 780, 755, 769, 711, 798, 828, 787, 772, 786, 767, 763, 746, 735, 770, 695, 824, 755, 766, 786, 738, 792, 771, 749, 804, 725, 815, 780, 769, 781, 741, 753, 767, 760, 783, 811, 751, 775, 771, 754, 746, 728, 802, 753, 799, 765, 766] payload_rows_per_dst=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] dup_rows_per_dst=[2992, 3114, 3062, 2882, 3006, 3016, 2956, 2898] combine_rows_per_src=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65016 dup_rows=23974 dup_rows_remote=20872 dedup_ratio=0.243876 router_tokens_per_expert=[770, 750, 736, 768, 755, 766, 765, 750, 758, 770, 734, 832, 732, 819, 757, 740, 748, 747, 733, 795, 768, 743, 773, 748, 742, 769, 797, 800, 731, 784, 734, 770, 745, 790, 757, 802, 765, 765, 775, 797, 783, 762, 739, 806, 781, 779, 788, 770, 755, 771, 823, 762, 794, 775, 733, 808, 754, 741, 782, 776, 747, 811, 793, 739] payload_rows_per_dst=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] dup_rows_per_dst=[2878, 2938, 2858, 2932, 3056, 3102, 3174, 3036] combine_rows_per_src=[9242, 9346, 9252, 9322, 9336, 9314, 9268, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64890 dup_rows=24104 dup_rows_remote=21162 dedup_ratio=0.245199 router_tokens_per_expert=[739, 786, 747, 729, 751, 779, 799, 760, 728, 777, 820, 760, 760, 771, 745, 740, 716, 770, 729, 792, 732, 792, 815, 794, 759, 771, 750, 739, 797, 756, 776, 758, 784, 791, 778, 740, 747, 761, 777, 793, 735, 776, 790, 818, 764, 771, 777, 776, 772, 764, 821, 770, 740, 767, 738, 804, 774, 765, 771, 767, 796, 785, 735, 768] payload_rows_per_dst=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] dup_rows_per_dst=[2942, 2962, 3054, 2962, 3046, 3098, 3028, 3012] combine_rows_per_src=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64970 dup_rows=24024 dup_rows_remote=21056 dedup_ratio=0.244385 router_tokens_per_expert=[770, 755, 740, 776, 814, 791, 805, 785, 759, 714, 771, 780, 766, 780, 785, 781, 765, 759, 780, 731, 775, 734, 748, 772, 762, 758, 847, 736, 768, 754, 734, 762, 767, 722, 826, 721, 775, 763, 754, 811, 746, 725, 791, 800, 766, 763, 786, 799, 720, 759, 846, 768, 717, 770, 797, 778, 755, 779, 772, 797, 747, 783, 763, 729] payload_rows_per_dst=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] dup_rows_per_dst=[3114, 2996, 3006, 2958, 2968, 3040, 2994, 2948] combine_rows_per_src=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65008 dup_rows=23926 dup_rows_remote=21044 dedup_ratio=0.243388 router_tokens_per_expert=[741, 800, 750, 824, 716, 755, 761, 720, 764, 761, 734, 791, 786, 765, 805, 770, 759, 808, 780, 755, 769, 711, 798, 828, 787, 772, 786, 767, 763, 746, 735, 770, 695, 824, 755, 766, 786, 738, 792, 771, 749, 804, 725, 815, 780, 769, 781, 741, 753, 767, 760, 783, 811, 751, 775, 771, 754, 746, 728, 802, 753, 799, 765, 766] payload_rows_per_dst=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] dup_rows_per_dst=[2992, 3114, 3062, 2882, 3006, 3016, 2956, 2898] combine_rows_per_src=[9142, 9238, 9354, 9370, 9248, 9312, 9386, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65168 dup_rows=23802 dup_rows_remote=20934 dedup_ratio=0.242126 router_tokens_per_expert=[779, 801, 786, 814, 781, 768, 790, 786, 770, 748, 775, 790, 775, 771, 769, 762, 738, 741, 720, 780, 733, 709, 832, 796, 777, 803, 723, 770, 783, 742, 775, 755, 776, 786, 728, 785, 767, 738, 762, 750, 808, 770, 782, 727, 793, 770, 771, 725, 766, 747, 787, 707, 768, 781, 776, 769, 788, 740, 801, 777, 785, 785, 737, 758] payload_rows_per_dst=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] dup_rows_per_dst=[3130, 3016, 2860, 2958, 2990, 2980, 2868, 3000] combine_rows_per_src=[9480, 9304, 9238, 9298, 9194, 9312, 9334, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=64970 dup_rows=24024 dup_rows_remote=21056 dedup_ratio=0.244385 router_tokens_per_expert=[770, 755, 740, 776, 814, 791, 805, 785, 759, 714, 771, 780, 766, 780, 785, 781, 765, 759, 780, 731, 775, 734, 748, 772, 762, 758, 847, 736, 768, 754, 734, 762, 767, 722, 826, 721, 775, 763, 754, 811, 746, 725, 791, 800, 766, 763, 786, 799, 720, 759, 846, 768, 717, 770, 797, 778, 755, 779, 772, 797, 747, 783, 763, 729] payload_rows_per_dst=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] dup_rows_per_dst=[3114, 2996, 3006, 2958, 2968, 3040, 2994, 2948] combine_rows_per_src=[9358, 9276, 9122, 9284, 9310, 9312, 9316, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344520 combine_payload_bytes=267460608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267805128 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535265736 weight_movement_bytes=103809024 total_nvlink_bytes=639074760 weight_to_token_ratio=0.193939 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343336 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266427688 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512040 weight_movement_bytes=34603008 total_nvlink_bytes=567115048 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344208 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616976 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889744 weight_movement_bytes=34603008 total_nvlink_bytes=567492752 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343552 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649088 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532954624 weight_movement_bytes=34603008 total_nvlink_bytes=567557632 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64962 dup_rows=23942 dup_rows_remote=20872 dedup_ratio=0.243551 router_tokens_per_expert=[771, 755, 790, 720, 826, 752, 784, 763, 799, 781, 795, 766, 773, 788, 759, 772, 750, 781, 827, 724, 778, 847, 787, 820, 749, 781, 785, 775, 774, 780, 713, 771, 743, 736, 743, 735, 769, 785, 799, 777, 780, 768, 789, 738, 709, 755, 705, 752, 745, 763, 759, 763, 767, 825, 806, 766, 755, 782, 737, 764, 757, 750, 756, 738] payload_rows_per_dst=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] dup_rows_per_dst=[2940, 2916, 3050, 3134, 3052, 2990, 2790, 3070] combine_rows_per_src=[9098, 9210, 9398, 9454, 9302, 9286, 9214, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923088 weight_movement_bytes=69206016 total_nvlink_bytes=601129104 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64944 dup_rows=24166 dup_rows_remote=21158 dedup_ratio=0.245829 router_tokens_per_expert=[759, 750, 760, 793, 766, 790, 750, 750, 831, 743, 796, 764, 786, 752, 741, 748, 775, 774, 767, 760, 753, 729, 770, 773, 755, 772, 758, 811, 764, 763, 827, 772, 794, 774, 781, 739, 782, 783, 770, 765, 749, 767, 737, 797, 777, 783, 778, 765, 793, 788, 762, 781, 715, 752, 757, 772, 755, 748, 776, 794, 753, 803, 708, 752] payload_rows_per_dst=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] dup_rows_per_dst=[3046, 3008, 3008, 3034, 3104, 3028, 2970, 2968] combine_rows_per_src=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=344408 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267272536 ingress_bytes=301531136 link_bytes=301531136 token_bytes=534200664 weight_movement_bytes=34603008 total_nvlink_bytes=568803672 weight_to_token_ratio=0.064775 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134637390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344104 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266461224 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578344 weight_movement_bytes=34603008 total_nvlink_bytes=567181352 weight_to_token_ratio=0.064973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344208 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266616976 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889744 weight_movement_bytes=34603008 total_nvlink_bytes=567492752 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64890 dup_rows=24104 dup_rows_remote=21162 dedup_ratio=0.245199 router_tokens_per_expert=[739, 786, 747, 729, 751, 779, 799, 760, 728, 777, 820, 760, 760, 771, 745, 740, 716, 770, 729, 792, 732, 792, 815, 794, 759, 771, 750, 739, 797, 756, 776, 758, 784, 791, 778, 740, 747, 761, 777, 793, 735, 776, 790, 818, 764, 771, 777, 776, 772, 764, 821, 770, 740, 767, 738, 804, 774, 765, 771, 767, 796, 785, 735, 768] payload_rows_per_dst=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] dup_rows_per_dst=[2942, 2962, 3054, 2962, 3046, 3098, 3028, 3012] combine_rows_per_src=[9310, 9168, 9268, 9250, 9296, 9316, 9282, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=65016 dup_rows=24080 dup_rows_remote=21076 dedup_ratio=0.244954 router_tokens_per_expert=[833, 801, 791, 778, 769, 738, 789, 773, 777, 802, 808, 755, 773, 830, 780, 750, 758, 768, 772, 768, 763, 760, 732, 785, 759, 726, 772, 742, 761, 766, 800, 755, 796, 802, 716, 718, 791, 741, 733, 808, 784, 734, 780, 760, 780, 764, 804, 758, 787, 760, 752, 732, 772, 747, 778, 734, 768, 739, 736, 820, 709, 774, 762, 779] payload_rows_per_dst=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] dup_rows_per_dst=[3140, 3080, 3004, 2964, 2958, 3088, 2954, 2892] combine_rows_per_src=[9404, 9470, 9208, 9198, 9252, 9240, 9170, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65054 dup_rows=23932 dup_rows_remote=20958 dedup_ratio=0.243449 router_tokens_per_expert=[787, 752, 776, 802, 752, 765, 754, 795, 772, 766, 810, 746, 770, 762, 756, 826, 728, 778, 776, 731, 764, 765, 818, 789, 799, 779, 792, 768, 754, 767, 735, 780, 728, 785, 796, 715, 747, 755, 767, 789, 769, 817, 741, 741, 763, 792, 743, 768, 785, 805, 726, 793, 789, 795, 689, 764, 786, 751, 760, 750, 770, 734, 775, 750] payload_rows_per_dst=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] dup_rows_per_dst=[3028, 2994, 2998, 3058, 2950, 3000, 2974, 2930] combine_rows_per_src=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343336 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266427688 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512040 weight_movement_bytes=34603008 total_nvlink_bytes=567115048 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344408 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355032 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365656 weight_movement_bytes=34603008 total_nvlink_bytes=566968664 weight_to_token_ratio=0.064999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64808 dup_rows=24270 dup_rows_remote=21076 dedup_ratio=0.246887 router_tokens_per_expert=[758, 794, 742, 762, 729, 774, 817, 796, 799, 755, 797, 786, 782, 728, 792, 771, 795, 757, 752, 740, 778, 804, 803, 780, 815, 766, 716, 743, 703, 755, 796, 771, 774, 794, 785, 749, 768, 769, 785, 739, 746, 783, 799, 761, 778, 785, 732, 730, 751, 790, 729, 760, 724, 756, 759, 803, 771, 818, 782, 736, 741, 765, 800, 734] payload_rows_per_dst=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] dup_rows_per_dst=[3076, 3194, 3044, 2938, 2984, 3114, 2894, 3026] combine_rows_per_src=[9268, 9226, 9374, 9192, 9342, 9114, 9250, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266133648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923088 weight_movement_bytes=69206016 total_nvlink_bytes=601129104 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344368 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649904 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532955440 weight_movement_bytes=69206016 total_nvlink_bytes=602161456 weight_to_token_ratio=0.129853 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344048 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266805232 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266416 weight_movement_bytes=103809024 total_nvlink_bytes=637075440 weight_to_token_ratio=0.194666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65180 dup_rows=23798 dup_rows_remote=20758 dedup_ratio=0.242086 router_tokens_per_expert=[761, 760, 750, 775, 797, 748, 772, 796, 792, 783, 738, 739, 798, 791, 779, 776, 754, 764, 758, 807, 750, 776, 725, 753, 736, 818, 736, 750, 752, 778, 784, 809, 758, 765, 749, 785, 734, 750, 775, 742, 788, 813, 716, 789, 786, 791, 758, 742, 759, 737, 781, 815, 779, 730, 770, 782, 753, 763, 797, 777, 723, 775, 801, 764] payload_rows_per_dst=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] dup_rows_per_dst=[3000, 3000, 2938, 3024, 2828, 3040, 2970, 2998] combine_rows_per_src=[9318, 9392, 9236, 9302, 9288, 9326, 9336, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65032 dup_rows=23996 dup_rows_remote=21142 dedup_ratio=0.244100 router_tokens_per_expert=[765, 754, 773, 744, 741, 806, 779, 766, 817, 774, 730, 775, 765, 787, 756, 746, 779, 726, 728, 735, 786, 769, 778, 752, 750, 777, 750, 770, 806, 785, 801, 807, 745, 717, 752, 816, 745, 717, 791, 782, 773, 684, 839, 805, 750, 769, 765, 743, 757, 765, 734, 777, 755, 776, 809, 764, 739, 790, 771, 768, 806, 816, 792, 763] payload_rows_per_dst=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] dup_rows_per_dst=[3002, 3048, 2898, 3128, 2854, 2990, 3010, 3066] combine_rows_per_src=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64708 dup_rows=24334 dup_rows_remote=21230 dedup_ratio=0.247538 router_tokens_per_expert=[733, 750, 749, 793, 853, 730, 770, 803, 761, 748, 771, 746, 742, 759, 758, 791, 768, 754, 772, 749, 800, 765, 791, 796, 806, 729, 766, 784, 734, 795, 772, 775, 766, 759, 725, 787, 760, 717, 791, 770, 764, 809, 746, 788, 760, 817, 786, 797, 808, 771, 790, 757, 745, 771, 758, 733, 728, 774, 757, 804, 728, 760, 786, 727] payload_rows_per_dst=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] dup_rows_per_dst=[3044, 2894, 3174, 3078, 3010, 3006, 3024, 3104] combine_rows_per_src=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74648 payload_rows_remote=65446 dup_rows=23656 dup_rows_remote=20740 dedup_ratio=0.240641 router_tokens_per_expert=[759, 797, 751, 776, 790, 751, 788, 789, 773, 767, 764, 787, 813, 756, 797, 750, 773, 744, 793, 729, 734, 803, 744, 746, 800, 773, 754, 738, 772, 757, 728, 737, 817, 753, 734, 716, 794, 769, 777, 779, 788, 778, 763, 785, 769, 771, 743, 749, 737, 740, 772, 773, 777, 781, 795, 807, 735, 771, 760, 785, 772, 778, 758, 793] payload_rows_per_dst=[9340, 9466, 9266, 9202, 9342, 9288, 9390, 9354] dup_rows_per_dst=[3062, 2948, 2866, 2916, 2936, 3004, 2974, 2950] combine_rows_per_src=[9340, 9466, 9266, 9202, 9342, 9288, 9390, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65146 dup_rows=23780 dup_rows_remote=20818 dedup_ratio=0.241903 router_tokens_per_expert=[781, 780, 781, 780, 797, 760, 710, 774, 753, 766, 772, 795, 828, 751, 775, 787, 725, 773, 744, 784, 773, 794, 756, 793, 756, 754, 749, 743, 779, 781, 788, 794, 718, 804, 787, 785, 763, 753, 794, 750, 765, 794, 734, 791, 798, 717, 768, 756, 777, 765, 781, 739, 714, 843, 775, 739, 798, 726, 746, 743, 717, 788, 793, 755] payload_rows_per_dst=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] dup_rows_per_dst=[2962, 3068, 2936, 2980, 2986, 2936, 2930, 2982] combine_rows_per_src=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65054 dup_rows=23932 dup_rows_remote=20958 dedup_ratio=0.243449 router_tokens_per_expert=[787, 752, 776, 802, 752, 765, 754, 795, 772, 766, 810, 746, 770, 762, 756, 826, 728, 778, 776, 731, 764, 765, 818, 789, 799, 779, 792, 768, 754, 767, 735, 780, 728, 785, 796, 715, 747, 755, 767, 789, 769, 817, 741, 741, 763, 792, 743, 768, 785, 805, 726, 793, 789, 795, 689, 764, 786, 751, 760, 750, 770, 734, 775, 750] payload_rows_per_dst=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] dup_rows_per_dst=[3028, 2994, 2998, 3058, 2950, 3000, 2974, 2930] combine_rows_per_src=[9338, 9422, 9300, 9290, 9214, 9268, 9318, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65032 dup_rows=23996 dup_rows_remote=21142 dedup_ratio=0.244100 router_tokens_per_expert=[765, 754, 773, 744, 741, 806, 779, 766, 817, 774, 730, 775, 765, 787, 756, 746, 779, 726, 728, 735, 786, 769, 778, 752, 750, 777, 750, 770, 806, 785, 801, 807, 745, 717, 752, 816, 745, 717, 791, 782, 773, 684, 839, 805, 750, 769, 765, 743, 757, 765, 734, 777, 755, 776, 809, 764, 739, 790, 771, 768, 806, 816, 792, 763] payload_rows_per_dst=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] dup_rows_per_dst=[3002, 3048, 2898, 3128, 2854, 2990, 3010, 3066] combine_rows_per_src=[9254, 9252, 9208, 9364, 9276, 9266, 9264, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268066816 dispatch_meta_bytes=344744 combine_payload_bytes=268066816 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=268411560 ingress_bytes=302669824 link_bytes=302669824 token_bytes=536478376 weight_movement_bytes=34603008 total_nvlink_bytes=571081384 weight_to_token_ratio=0.064500 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138922181e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65272 dup_rows=23786 dup_rows_remote=20840 dedup_ratio=0.241964 router_tokens_per_expert=[774, 760, 792, 759, 825, 771, 788, 797, 758, 768, 767, 796, 832, 778, 736, 786, 768, 743, 791, 727, 763, 790, 752, 762, 772, 759, 765, 842, 695, 736, 784, 756, 726, 774, 731, 770, 764, 760, 771, 798, 739, 781, 805, 727, 723, 732, 768, 761, 784, 800, 770, 776, 752, 799, 754, 776, 806, 728, 790, 803, 790, 766, 683, 753] payload_rows_per_dst=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] dup_rows_per_dst=[3110, 3042, 2946, 2864, 2954, 2926, 3010, 2934] combine_rows_per_src=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343536 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797104 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531250672 weight_movement_bytes=69206016 total_nvlink_bytes=600456688 weight_to_token_ratio=0.130270 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=343752 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267321032 ingress_bytes=336183296 link_bytes=336183296 token_bytes=534298312 weight_movement_bytes=69206016 total_nvlink_bytes=603504328 weight_to_token_ratio=0.129527 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265030678e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64708 dup_rows=24334 dup_rows_remote=21230 dedup_ratio=0.247538 router_tokens_per_expert=[733, 750, 749, 793, 853, 730, 770, 803, 761, 748, 771, 746, 742, 759, 758, 791, 768, 754, 772, 749, 800, 765, 791, 796, 806, 729, 766, 784, 734, 795, 772, 775, 766, 759, 725, 787, 760, 717, 791, 770, 764, 809, 746, 788, 760, 817, 786, 797, 808, 771, 790, 757, 745, 771, 758, 733, 728, 774, 757, 804, 728, 760, 786, 727] payload_rows_per_dst=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] dup_rows_per_dst=[3044, 2894, 3174, 3078, 3010, 3006, 3024, 3104] combine_rows_per_src=[9222, 8992, 9482, 9242, 9202, 9324, 9244, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344696 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715768 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086840 weight_movement_bytes=34603008 total_nvlink_bytes=567689848 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=268066816 dispatch_meta_bytes=344744 combine_payload_bytes=268066816 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=268411560 ingress_bytes=302669824 link_bytes=302669824 token_bytes=536478376 weight_movement_bytes=34603008 total_nvlink_bytes=571081384 weight_to_token_ratio=0.064500 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138922181e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65146 dup_rows=23780 dup_rows_remote=20818 dedup_ratio=0.241903 router_tokens_per_expert=[781, 780, 781, 780, 797, 760, 710, 774, 753, 766, 772, 795, 828, 751, 775, 787, 725, 773, 744, 784, 773, 794, 756, 793, 756, 754, 749, 743, 779, 781, 788, 794, 718, 804, 787, 785, 763, 753, 794, 750, 765, 794, 734, 791, 798, 717, 768, 756, 777, 765, 781, 739, 714, 843, 775, 739, 798, 726, 746, 743, 717, 788, 793, 755] payload_rows_per_dst=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] dup_rows_per_dst=[2962, 3068, 2936, 2980, 2986, 2936, 2930, 2982] combine_rows_per_src=[9378, 9424, 9348, 9308, 9322, 9310, 9284, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64944 dup_rows=24166 dup_rows_remote=21158 dedup_ratio=0.245829 router_tokens_per_expert=[759, 750, 760, 793, 766, 790, 750, 750, 831, 743, 796, 764, 786, 752, 741, 748, 775, 774, 767, 760, 753, 729, 770, 773, 755, 772, 758, 811, 764, 763, 827, 772, 794, 774, 781, 739, 782, 783, 770, 765, 749, 767, 737, 797, 777, 783, 778, 765, 793, 788, 762, 781, 715, 752, 757, 772, 755, 748, 776, 794, 753, 803, 708, 752] payload_rows_per_dst=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] dup_rows_per_dst=[3046, 3008, 3008, 3034, 3104, 3028, 2970, 2968] combine_rows_per_src=[9190, 9314, 9194, 9410, 9272, 9278, 9270, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65106 dup_rows=23754 dup_rows_remote=20704 dedup_ratio=0.241638 router_tokens_per_expert=[763, 735, 762, 781, 790, 748, 760, 732, 767, 746, 762, 772, 746, 765, 740, 741, 746, 734, 811, 802, 760, 762, 781, 770, 726, 796, 737, 790, 786, 707, 778, 773, 792, 745, 754, 779, 841, 817, 790, 761, 823, 767, 772, 783, 759, 754, 754, 728, 770, 803, 767, 772, 826, 759, 764, 786, 792, 768, 733, 769, 775, 780, 744, 756] payload_rows_per_dst=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] dup_rows_per_dst=[2910, 2910, 2956, 2846, 3118, 2990, 3050, 2974] combine_rows_per_src=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343752 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265387720 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530431688 weight_movement_bytes=34603008 total_nvlink_bytes=565034696 weight_to_token_ratio=0.065236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64862 dup_rows=24286 dup_rows_remote=21358 dedup_ratio=0.247050 router_tokens_per_expert=[760, 730, 827, 817, 694, 745, 787, 818, 747, 771, 715, 801, 753, 709, 759, 787, 755, 759, 782, 780, 768, 765, 744, 727, 811, 783, 771, 822, 760, 818, 779, 763, 764, 782, 782, 794, 742, 721, 809, 772, 761, 809, 762, 776, 742, 752, 774, 772, 816, 763, 737, 749, 782, 822, 756, 751, 765, 761, 734, 758, 762, 733, 788, 754] payload_rows_per_dst=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] dup_rows_per_dst=[3034, 2928, 3002, 3156, 3120, 3058, 3044, 2944] combine_rows_per_src=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343856 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267181872 ingress_bytes=370647040 link_bytes=370647040 token_bytes=534019888 weight_movement_bytes=103809024 total_nvlink_bytes=637828912 weight_to_token_ratio=0.194392 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394714972e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344408 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355032 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365656 weight_movement_bytes=34603008 total_nvlink_bytes=566968664 weight_to_token_ratio=0.064999 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343240 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267017416 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533691592 weight_movement_bytes=103809024 total_nvlink_bytes=637500616 weight_to_token_ratio=0.194511 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65088 dup_rows=24024 dup_rows_remote=21064 dedup_ratio=0.244385 router_tokens_per_expert=[768, 761, 760, 754, 775, 775, 752, 797, 797, 756, 772, 774, 736, 760, 755, 816, 777, 750, 697, 795, 807, 781, 800, 754, 778, 764, 765, 751, 701, 752, 745, 776, 766, 746, 756, 737, 782, 831, 727, 742, 761, 750, 785, 756, 765, 766, 759, 734, 783, 787, 816, 754, 814, 783, 770, 780, 745, 796, 767, 763, 780, 745, 790, 815] payload_rows_per_dst=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] dup_rows_per_dst=[3052, 2978, 3012, 2920, 2868, 2960, 3150, 3084] combine_rows_per_src=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65132 dup_rows=23886 dup_rows_remote=20886 dedup_ratio=0.242981 router_tokens_per_expert=[740, 782, 760, 755, 813, 809, 790, 758, 746, 738, 768, 726, 819, 755, 742, 786, 763, 736, 772, 783, 793, 811, 773, 793, 750, 797, 722, 758, 794, 736, 758, 746, 761, 777, 755, 774, 805, 757, 755, 759, 813, 728, 781, 753, 750, 816, 752, 833, 800, 729, 796, 747, 795, 733, 815, 765, 750, 793, 749, 770, 743, 692, 744, 790] payload_rows_per_dst=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] dup_rows_per_dst=[3074, 2878, 3002, 2968, 3000, 3046, 3034, 2884] combine_rows_per_src=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344696 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715768 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086840 weight_movement_bytes=34603008 total_nvlink_bytes=567689848 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344048 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266805232 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266416 weight_movement_bytes=103809024 total_nvlink_bytes=637075440 weight_to_token_ratio=0.194666 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=344448 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267698560 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535052672 weight_movement_bytes=34603008 total_nvlink_bytes=569655680 weight_to_token_ratio=0.064672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65094 dup_rows=23924 dup_rows_remote=20990 dedup_ratio=0.243368 router_tokens_per_expert=[759, 793, 793, 770, 803, 801, 746, 755, 749, 801, 737, 777, 740, 763, 794, 758, 795, 743, 742, 751, 760, 752, 747, 750, 735, 770, 742, 800, 776, 778, 753, 756, 772, 800, 797, 797, 737, 757, 790, 742, 769, 753, 817, 781, 787, 772, 771, 780, 778, 737, 763, 761, 795, 733, 724, 764, 735, 790, 812, 798, 755, 734, 759, 803] payload_rows_per_dst=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] dup_rows_per_dst=[3070, 2968, 2868, 2934, 3058, 3120, 2902, 3004] combine_rows_per_src=[9370, 9270, 9212, 9286, 9326, 9340, 9208, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344880 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266019632 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531694384 weight_movement_bytes=34603008 total_nvlink_bytes=566297392 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344608 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266945056 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545504 weight_movement_bytes=69206016 total_nvlink_bytes=602751520 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64862 dup_rows=24286 dup_rows_remote=21358 dedup_ratio=0.247050 router_tokens_per_expert=[760, 730, 827, 817, 694, 745, 787, 818, 747, 771, 715, 801, 753, 709, 759, 787, 755, 759, 782, 780, 768, 765, 744, 727, 811, 783, 771, 822, 760, 818, 779, 763, 764, 782, 782, 794, 742, 721, 809, 772, 761, 809, 762, 776, 742, 752, 774, 772, 816, 763, 737, 749, 782, 822, 756, 751, 765, 761, 734, 758, 762, 733, 788, 754] payload_rows_per_dst=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] dup_rows_per_dst=[3034, 2928, 3002, 3156, 3120, 3058, 3044, 2944] combine_rows_per_src=[9322, 9156, 9158, 9458, 9212, 9238, 9308, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65088 dup_rows=24024 dup_rows_remote=21064 dedup_ratio=0.244385 router_tokens_per_expert=[768, 761, 760, 754, 775, 775, 752, 797, 797, 756, 772, 774, 736, 760, 755, 816, 777, 750, 697, 795, 807, 781, 800, 754, 778, 764, 765, 751, 701, 752, 745, 776, 766, 746, 756, 737, 782, 831, 727, 742, 761, 750, 785, 756, 765, 766, 759, 734, 783, 787, 816, 754, 814, 783, 770, 780, 745, 796, 767, 763, 780, 745, 790, 815] payload_rows_per_dst=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] dup_rows_per_dst=[3052, 2978, 3012, 2920, 2868, 2960, 3150, 3084] combine_rows_per_src=[9232, 9354, 9310, 9144, 9306, 9192, 9424, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64812 dup_rows=24072 dup_rows_remote=20946 dedup_ratio=0.244873 router_tokens_per_expert=[757, 766, 794, 736, 764, 766, 779, 830, 820, 746, 787, 757, 756, 772, 756, 722, 721, 749, 779, 785, 804, 802, 759, 817, 747, 779, 752, 741, 785, 783, 794, 743, 775, 787, 762, 761, 766, 754, 738, 773, 773, 766, 833, 807, 802, 728, 728, 728, 792, 761, 767, 771, 749, 770, 788, 732, 801, 786, 738, 730, 769, 757, 744, 768] payload_rows_per_dst=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] dup_rows_per_dst=[2590, 3140, 3050, 2918, 3174, 3008, 3066, 3126] combine_rows_per_src=[8710, 9410, 9382, 9330, 9464, 9322, 9194, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344072 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267124744 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905416 weight_movement_bytes=34603008 total_nvlink_bytes=568508424 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344336 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969360 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594384 weight_movement_bytes=34603008 total_nvlink_bytes=568197392 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73862 payload_rows_remote=64468 dup_rows=24442 dup_rows_remote=21236 dedup_ratio=0.248637 router_tokens_per_expert=[814, 801, 796, 805, 776, 776, 774, 752, 767, 775, 758, 700, 767, 793, 753, 784, 820, 735, 739, 770, 759, 754, 793, 725, 761, 766, 796, 732, 752, 811, 745, 729, 766, 768, 796, 726, 791, 811, 755, 755, 780, 767, 739, 789, 760, 753, 770, 784, 781, 696, 785, 764, 805, 769, 774, 777, 773, 756, 788, 742, 735, 802, 727, 790] payload_rows_per_dst=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] dup_rows_per_dst=[3206, 2992, 3168, 3106, 3030, 2926, 2970, 3044] combine_rows_per_src=[9394, 9202, 9214, 9270, 9306, 9198, 9096, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65132 dup_rows=23886 dup_rows_remote=20886 dedup_ratio=0.242981 router_tokens_per_expert=[740, 782, 760, 755, 813, 809, 790, 758, 746, 738, 768, 726, 819, 755, 742, 786, 763, 736, 772, 783, 793, 811, 773, 793, 750, 797, 722, 758, 794, 736, 758, 746, 761, 777, 755, 774, 805, 757, 755, 759, 813, 728, 781, 753, 750, 816, 752, 833, 800, 729, 796, 747, 795, 733, 815, 765, 750, 793, 749, 770, 743, 692, 744, 790] payload_rows_per_dst=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] dup_rows_per_dst=[3074, 2878, 3002, 2968, 3000, 3046, 3034, 2884] combine_rows_per_src=[9340, 9282, 9446, 9154, 9286, 9406, 9326, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65030 dup_rows=24056 dup_rows_remote=21114 dedup_ratio=0.244710 router_tokens_per_expert=[771, 744, 798, 740, 794, 764, 780, 798, 814, 783, 753, 777, 750, 750, 772, 753, 739, 773, 779, 724, 786, 778, 784, 781, 772, 761, 789, 802, 824, 744, 780, 749, 771, 723, 748, 748, 759, 745, 750, 739, 747, 796, 747, 797, 787, 792, 747, 796, 761, 762, 779, 782, 786, 747, 709, 754, 757, 772, 794, 785, 812, 730, 769, 755] payload_rows_per_dst=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] dup_rows_per_dst=[3040, 3026, 3006, 3078, 2878, 3092, 2942, 2994] combine_rows_per_src=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65272 dup_rows=23786 dup_rows_remote=20840 dedup_ratio=0.241964 router_tokens_per_expert=[774, 760, 792, 759, 825, 771, 788, 797, 758, 768, 767, 796, 832, 778, 736, 786, 768, 743, 791, 727, 763, 790, 752, 762, 772, 759, 765, 842, 695, 736, 784, 756, 726, 774, 731, 770, 764, 760, 771, 798, 739, 781, 805, 727, 723, 732, 768, 761, 784, 800, 770, 776, 752, 799, 754, 776, 806, 728, 790, 803, 790, 766, 683, 753] payload_rows_per_dst=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] dup_rows_per_dst=[3110, 3042, 2946, 2864, 2954, 2926, 3010, 2934] combine_rows_per_src=[9422, 9400, 9246, 9354, 9234, 9146, 9412, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65106 dup_rows=23754 dup_rows_remote=20704 dedup_ratio=0.241638 router_tokens_per_expert=[763, 735, 762, 781, 790, 748, 760, 732, 767, 746, 762, 772, 746, 765, 740, 741, 746, 734, 811, 802, 760, 762, 781, 770, 726, 796, 737, 790, 786, 707, 778, 773, 792, 745, 754, 779, 841, 817, 790, 761, 823, 767, 772, 783, 759, 754, 754, 728, 770, 803, 767, 772, 826, 759, 764, 786, 792, 768, 733, 769, 775, 780, 744, 756] payload_rows_per_dst=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] dup_rows_per_dst=[2910, 2910, 2956, 2846, 3118, 2990, 3050, 2974] combine_rows_per_src=[9232, 9168, 9376, 9340, 9440, 9290, 9444, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344336 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969360 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594384 weight_movement_bytes=34603008 total_nvlink_bytes=568197392 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64926 dup_rows=23962 dup_rows_remote=20956 dedup_ratio=0.243754 router_tokens_per_expert=[766, 744, 786, 751, 789, 782, 754, 795, 751, 724, 782, 820, 743, 776, 787, 761, 818, 749, 769, 765, 777, 771, 801, 761, 783, 779, 805, 745, 761, 763, 752, 827, 808, 793, 823, 758, 772, 793, 778, 734, 735, 791, 777, 790, 760, 746, 792, 768, 749, 759, 758, 734, 763, 745, 737, 695, 768, 755, 774, 744, 747, 772, 757, 740] payload_rows_per_dst=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] dup_rows_per_dst=[3078, 2946, 3006, 3010, 3186, 3056, 2740, 2940] combine_rows_per_src=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344880 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266019632 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531694384 weight_movement_bytes=34603008 total_nvlink_bytes=566297392 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344608 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266945056 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533545504 weight_movement_bytes=69206016 total_nvlink_bytes=602751520 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343032 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265812984 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531282936 weight_movement_bytes=34603008 total_nvlink_bytes=565885944 weight_to_token_ratio=0.065131 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264060928 dispatch_meta_bytes=342816 combine_payload_bytes=264060928 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=264403744 ingress_bytes=402472960 link_bytes=402472960 token_bytes=528464672 weight_movement_bytes=138412032 total_nvlink_bytes=666876704 weight_to_token_ratio=0.261914 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.514473347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344072 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267124744 ingress_bytes=301383680 link_bytes=301383680 token_bytes=533905416 weight_movement_bytes=34603008 total_nvlink_bytes=568508424 weight_to_token_ratio=0.064811 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134082524e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343528 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280424 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217320 weight_movement_bytes=69206016 total_nvlink_bytes=601423336 weight_to_token_ratio=0.130033 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343240 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267017416 ingress_bytes=370483200 link_bytes=370483200 token_bytes=533691592 weight_movement_bytes=103809024 total_nvlink_bytes=637500616 weight_to_token_ratio=0.194511 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394098455e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64932 dup_rows=24186 dup_rows_remote=21340 dedup_ratio=0.246033 router_tokens_per_expert=[804, 772, 843, 779, 829, 762, 765, 766, 819, 776, 734, 801, 815, 746, 807, 771, 768, 743, 813, 775, 776, 742, 692, 729, 785, 748, 738, 749, 725, 719, 790, 762, 749, 789, 769, 781, 785, 824, 805, 801, 803, 764, 771, 763, 724, 742, 708, 787, 742, 797, 777, 749, 786, 774, 746, 744, 774, 771, 789, 744, 755, 720, 722, 754] payload_rows_per_dst=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] dup_rows_per_dst=[3176, 3168, 2878, 2846, 3210, 2932, 3056, 2920] combine_rows_per_src=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64768 dup_rows=24198 dup_rows_remote=21048 dedup_ratio=0.246155 router_tokens_per_expert=[793, 803, 774, 795, 750, 743, 758, 738, 737, 758, 773, 801, 779, 819, 804, 773, 745, 734, 745, 828, 779, 757, 755, 739, 759, 727, 770, 747, 753, 730, 795, 728, 776, 785, 758, 787, 743, 777, 703, 801, 755, 796, 772, 757, 756, 793, 795, 792, 775, 782, 778, 783, 788, 773, 772, 744, 795, 739, 795, 748, 725, 795, 769, 756] payload_rows_per_dst=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] dup_rows_per_dst=[3024, 3150, 3006, 2826, 3046, 3164, 3004, 2978] combine_rows_per_src=[9284, 9338, 9158, 9192, 9214, 9268, 9386, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344576 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707456 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070336 weight_movement_bytes=103809024 total_nvlink_bytes=636879360 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64946 dup_rows=24114 dup_rows_remote=21078 dedup_ratio=0.245300 router_tokens_per_expert=[791, 773, 764, 773, 772, 739, 756, 768, 767, 773, 757, 776, 728, 749, 760, 734, 782, 772, 745, 739, 801, 786, 782, 773, 781, 742, 806, 787, 779, 748, 766, 733, 776, 747, 770, 787, 742, 735, 793, 749, 769, 725, 783, 759, 796, 810, 757, 741, 769, 776, 779, 770, 834, 764, 756, 801, 730, 737, 810, 796, 772, 779, 774, 764] payload_rows_per_dst=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] dup_rows_per_dst=[2952, 2968, 3060, 3010, 2996, 3036, 3132, 2960] combine_rows_per_src=[9320, 9120, 9300, 9274, 9202, 9244, 9366, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267354112 dispatch_meta_bytes=344448 combine_payload_bytes=267354112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267698560 ingress_bytes=301957120 link_bytes=301957120 token_bytes=535052672 weight_movement_bytes=34603008 total_nvlink_bytes=569655680 weight_to_token_ratio=0.064672 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136240333e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65148 dup_rows=23930 dup_rows_remote=21066 dedup_ratio=0.243429 router_tokens_per_expert=[813, 752, 746, 723, 749, 814, 755, 758, 753, 801, 729, 788, 771, 725, 790, 793, 777, 779, 796, 775, 780, 782, 775, 745, 756, 815, 799, 708, 831, 759, 732, 764, 784, 810, 723, 748, 734, 759, 735, 752, 736, 758, 791, 742, 757, 792, 771, 764, 768, 751, 755, 743, 816, 771, 802, 791, 751, 743, 750, 771, 769, 793, 822, 767] payload_rows_per_dst=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] dup_rows_per_dst=[2926, 2966, 3088, 2996, 2864, 2996, 3104, 2990] combine_rows_per_src=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64932 dup_rows=24186 dup_rows_remote=21340 dedup_ratio=0.246033 router_tokens_per_expert=[804, 772, 843, 779, 829, 762, 765, 766, 819, 776, 734, 801, 815, 746, 807, 771, 768, 743, 813, 775, 776, 742, 692, 729, 785, 748, 738, 749, 725, 719, 790, 762, 749, 789, 769, 781, 785, 824, 805, 801, 803, 764, 771, 763, 724, 742, 708, 787, 742, 797, 777, 749, 786, 774, 746, 744, 774, 771, 789, 744, 755, 720, 722, 754] payload_rows_per_dst=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] dup_rows_per_dst=[3176, 3168, 2878, 2846, 3210, 2932, 3056, 2920] combine_rows_per_src=[9464, 9370, 9198, 9186, 9396, 9192, 9174, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343264 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632992 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922720 weight_movement_bytes=103809024 total_nvlink_bytes=634731744 weight_to_token_ratio=0.195526 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344096 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362912 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381728 weight_movement_bytes=34603008 total_nvlink_bytes=566984736 weight_to_token_ratio=0.064997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64654 dup_rows=24132 dup_rows_remote=21028 dedup_ratio=0.245483 router_tokens_per_expert=[791, 794, 746, 812, 752, 740, 769, 753, 791, 782, 814, 761, 769, 754, 703, 815, 759, 770, 763, 766, 789, 786, 772, 810, 726, 798, 788, 791, 793, 747, 756, 772, 747, 773, 723, 769, 791, 754, 746, 784, 775, 750, 758, 773, 749, 733, 752, 782, 805, 791, 725, 779, 771, 786, 812, 731, 810, 775, 755, 705, 749, 759, 757, 751] payload_rows_per_dst=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] dup_rows_per_dst=[2792, 2726, 2980, 3296, 2958, 3172, 3104, 3104] combine_rows_per_src=[8952, 8864, 9224, 9636, 9184, 9466, 9328, 9518] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344856 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191064 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037272 weight_movement_bytes=69206016 total_nvlink_bytes=603243288 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=345088 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266306560 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532268032 weight_movement_bytes=34603008 total_nvlink_bytes=566871040 weight_to_token_ratio=0.065010 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64790 dup_rows=24084 dup_rows_remote=20994 dedup_ratio=0.244995 router_tokens_per_expert=[780, 806, 755, 755, 779, 817, 768, 797, 755, 757, 820, 770, 793, 797, 766, 796, 756, 775, 790, 764, 805, 757, 809, 744, 749, 734, 738, 791, 754, 751, 795, 771, 757, 721, 759, 816, 751, 759, 784, 731, 758, 759, 726, 751, 784, 765, 736, 735, 751, 773, 781, 745, 772, 801, 811, 714, 769, 779, 763, 742, 760, 777, 750, 778] payload_rows_per_dst=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] dup_rows_per_dst=[3090, 3152, 3012, 2928, 2952, 2906, 3030, 3014] combine_rows_per_src=[9430, 9356, 9388, 9238, 9198, 9122, 9266, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65148 dup_rows=23930 dup_rows_remote=21066 dedup_ratio=0.243429 router_tokens_per_expert=[813, 752, 746, 723, 749, 814, 755, 758, 753, 801, 729, 788, 771, 725, 790, 793, 777, 779, 796, 775, 780, 782, 775, 745, 756, 815, 799, 708, 831, 759, 732, 764, 784, 810, 723, 748, 734, 759, 735, 752, 736, 758, 791, 742, 757, 792, 771, 764, 768, 751, 755, 743, 816, 771, 802, 791, 751, 743, 750, 771, 769, 793, 822, 767] payload_rows_per_dst=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] dup_rows_per_dst=[2926, 2966, 3088, 2996, 2864, 2996, 3104, 2990] combine_rows_per_src=[9294, 9334, 9330, 9332, 9226, 9226, 9290, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65104 dup_rows=23972 dup_rows_remote=21016 dedup_ratio=0.243856 router_tokens_per_expert=[792, 765, 738, 794, 750, 732, 760, 802, 775, 754, 838, 800, 768, 774, 789, 754, 775, 732, 721, 772, 763, 796, 800, 771, 730, 769, 761, 726, 738, 767, 785, 781, 812, 790, 757, 761, 759, 796, 750, 745, 771, 779, 794, 753, 752, 774, 757, 751, 768, 759, 752, 755, 744, 779, 758, 777, 798, 782, 752, 747, 773, 766, 782, 787] payload_rows_per_dst=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] dup_rows_per_dst=[2884, 3186, 2976, 2840, 3112, 2984, 2956, 3034] combine_rows_per_src=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65072 dup_rows=23934 dup_rows_remote=20918 dedup_ratio=0.243469 router_tokens_per_expert=[783, 780, 784, 754, 820, 784, 789, 747, 756, 819, 763, 775, 694, 743, 777, 735, 778, 744, 781, 775, 752, 777, 749, 801, 794, 807, 778, 780, 770, 757, 776, 792, 792, 756, 748, 758, 746, 772, 789, 757, 780, 738, 762, 771, 761, 723, 766, 778, 776, 809, 737, 774, 767, 775, 816, 832, 753, 704, 759, 739, 728, 736, 732, 804] payload_rows_per_dst=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] dup_rows_per_dst=[3068, 2938, 3016, 3096, 2914, 2968, 3106, 2828] combine_rows_per_src=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=345088 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266306560 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532268032 weight_movement_bytes=34603008 total_nvlink_bytes=566871040 weight_to_token_ratio=0.065010 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64926 dup_rows=23962 dup_rows_remote=20956 dedup_ratio=0.243754 router_tokens_per_expert=[766, 744, 786, 751, 789, 782, 754, 795, 751, 724, 782, 820, 743, 776, 787, 761, 818, 749, 769, 765, 777, 771, 801, 761, 783, 779, 805, 745, 761, 763, 752, 827, 808, 793, 823, 758, 772, 793, 778, 734, 735, 791, 777, 790, 760, 746, 792, 768, 749, 759, 758, 734, 763, 745, 737, 695, 768, 755, 774, 744, 747, 772, 757, 740] payload_rows_per_dst=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] dup_rows_per_dst=[3078, 2946, 3006, 3010, 3186, 3056, 2740, 2940] combine_rows_per_src=[9256, 9342, 9416, 9420, 9332, 9262, 9140, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65030 dup_rows=24056 dup_rows_remote=21114 dedup_ratio=0.244710 router_tokens_per_expert=[771, 744, 798, 740, 794, 764, 780, 798, 814, 783, 753, 777, 750, 750, 772, 753, 739, 773, 779, 724, 786, 778, 784, 781, 772, 761, 789, 802, 824, 744, 780, 749, 771, 723, 748, 748, 759, 745, 750, 739, 747, 796, 747, 797, 787, 792, 747, 796, 761, 762, 779, 782, 786, 747, 709, 754, 757, 772, 794, 785, 812, 730, 769, 755] payload_rows_per_dst=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] dup_rows_per_dst=[3040, 3026, 3006, 3078, 2878, 3092, 2942, 2994] combine_rows_per_src=[9338, 9278, 9282, 9364, 9088, 9326, 9218, 9354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343264 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632992 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922720 weight_movement_bytes=103809024 total_nvlink_bytes=634731744 weight_to_token_ratio=0.195526 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344096 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362912 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381728 weight_movement_bytes=34603008 total_nvlink_bytes=566984736 weight_to_token_ratio=0.064997 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264822784 dispatch_meta_bytes=342728 combine_payload_bytes=264822784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265165512 ingress_bytes=299425792 link_bytes=299425792 token_bytes=529988296 weight_movement_bytes=34603008 total_nvlink_bytes=564591304 weight_to_token_ratio=0.065290 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126715150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343136 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265722976 ingress_bytes=369188864 link_bytes=369188864 token_bytes=531102816 weight_movement_bytes=103809024 total_nvlink_bytes=634911840 weight_to_token_ratio=0.195459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389227973e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266878872 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533413784 weight_movement_bytes=34603008 total_nvlink_bytes=568016792 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344856 combine_payload_bytes=266846208 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267191064 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534037272 weight_movement_bytes=69206016 total_nvlink_bytes=603243288 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344576 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707456 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070336 weight_movement_bytes=103809024 total_nvlink_bytes=636879360 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261994256 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6134951696 ingress_bytes=7476559872 link_bytes=7476559872 total_nvlink_bytes=13611511568 weight_to_token_ratio=0.110057 prefetch_share_of_ingress=0.101820 dedup_ratio=0.243594 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223204039e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65150 dup_rows=23898 dup_rows_remote=20794 dedup_ratio=0.243103 router_tokens_per_expert=[743, 760, 781, 709, 758, 719, 783, 767, 730, 803, 751, 717, 780, 779, 783, 812, 729, 786, 754, 758, 746, 778, 722, 760, 750, 777, 759, 795, 796, 764, 753, 786, 784, 765, 774, 776, 788, 762, 790, 755, 786, 808, 787, 739, 800, 790, 827, 808, 748, 742, 725, 774, 780, 820, 771, 770, 745, 776, 771, 744, 798, 773, 755, 733] payload_rows_per_dst=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] dup_rows_per_dst=[2776, 3098, 2818, 3104, 3036, 3180, 2926, 2960] combine_rows_per_src=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64984 dup_rows=24058 dup_rows_remote=20994 dedup_ratio=0.244731 router_tokens_per_expert=[774, 773, 740, 750, 777, 778, 764, 788, 745, 740, 791, 777, 758, 827, 778, 747, 754, 748, 765, 735, 771, 790, 762, 757, 755, 817, 749, 769, 760, 719, 789, 760, 768, 815, 750, 786, 789, 771, 785, 747, 821, 803, 783, 768, 763, 744, 807, 701, 741, 730, 737, 761, 806, 807, 842, 779, 728, 778, 767, 743, 766, 804, 715, 740] payload_rows_per_dst=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] dup_rows_per_dst=[2984, 3064, 3000, 3012, 2968, 2960, 3154, 2916] combine_rows_per_src=[9304, 9262, 9164, 9224, 9454, 9420, 9252, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010464 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676448 weight_movement_bytes=34603008 total_nvlink_bytes=568279456 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65094 dup_rows=23828 dup_rows_remote=20862 dedup_ratio=0.242391 router_tokens_per_expert=[751, 797, 740, 743, 805, 752, 745, 739, 752, 747, 733, 796, 760, 752, 782, 751, 810, 807, 768, 766, 778, 758, 824, 746, 800, 751, 757, 781, 800, 713, 723, 749, 737, 799, 759, 759, 796, 763, 771, 752, 785, 752, 743, 804, 770, 731, 793, 796, 761, 700, 765, 810, 727, 807, 764, 798, 771, 777, 762, 817, 786, 747, 769, 805] payload_rows_per_dst=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] dup_rows_per_dst=[2930, 2940, 3058, 2936, 3006, 2966, 2980, 3012] combine_rows_per_src=[9214, 9206, 9456, 9212, 9266, 9382, 9284, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=343528 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266280424 ingress_bytes=335142912 link_bytes=335142912 token_bytes=532217320 weight_movement_bytes=69206016 total_nvlink_bytes=601423336 weight_to_token_ratio=0.130033 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261115797e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64480 dup_rows=24066 dup_rows_remote=20682 dedup_ratio=0.244812 router_tokens_per_expert=[737, 824, 780, 752, 785, 834, 769, 761, 756, 770, 766, 786, 757, 733, 845, 747, 723, 766, 768, 770, 759, 759, 790, 759, 778, 770, 754, 722, 777, 817, 761, 748, 777, 777, 755, 783, 781, 802, 764, 809, 735, 769, 748, 780, 744, 744, 703, 803, 784, 764, 741, 751, 787, 756, 750, 769, 796, 757, 751, 769, 729, 764, 792, 795] payload_rows_per_dst=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] dup_rows_per_dst=[3202, 2958, 2988, 2906, 3082, 2576, 2970, 3384] combine_rows_per_src=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64928 dup_rows=23920 dup_rows_remote=20836 dedup_ratio=0.243327 router_tokens_per_expert=[761, 776, 788, 756, 752, 756, 800, 798, 772, 755, 777, 772, 803, 793, 797, 766, 708, 780, 773, 760, 726, 764, 741, 764, 758, 783, 760, 812, 737, 773, 835, 817, 772, 744, 757, 729, 728, 786, 748, 760, 805, 771, 767, 771, 779, 749, 824, 776, 743, 781, 766, 809, 824, 736, 746, 754, 774, 766, 737, 759, 713, 761, 733, 771] payload_rows_per_dst=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] dup_rows_per_dst=[3084, 3072, 2862, 3098, 2800, 3122, 2998, 2884] combine_rows_per_src=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64904 dup_rows=24114 dup_rows_remote=21160 dedup_ratio=0.245300 router_tokens_per_expert=[767, 764, 775, 773, 791, 733, 765, 713, 796, 806, 782, 723, 770, 775, 776, 746, 798, 751, 772, 785, 753, 748, 829, 821, 752, 753, 795, 778, 804, 810, 767, 760, 730, 736, 764, 789, 790, 789, 733, 789, 757, 772, 786, 796, 763, 752, 795, 805, 731, 779, 743, 734, 760, 754, 788, 769, 732, 773, 721, 750, 751, 763, 753, 774] payload_rows_per_dst=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] dup_rows_per_dst=[2950, 2998, 3214, 3132, 2954, 3064, 2958, 2844] combine_rows_per_src=[9212, 9350, 9300, 9306, 9286, 9388, 9158, 9190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65150 dup_rows=23898 dup_rows_remote=20794 dedup_ratio=0.243103 router_tokens_per_expert=[743, 760, 781, 709, 758, 719, 783, 767, 730, 803, 751, 717, 780, 779, 783, 812, 729, 786, 754, 758, 746, 778, 722, 760, 750, 777, 759, 795, 796, 764, 753, 786, 784, 765, 774, 776, 788, 762, 790, 755, 786, 808, 787, 739, 800, 790, 827, 808, 748, 742, 725, 774, 780, 820, 771, 770, 745, 776, 771, 744, 798, 773, 755, 733] payload_rows_per_dst=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] dup_rows_per_dst=[2776, 3098, 2818, 3104, 3036, 3180, 2926, 2960] combine_rows_per_src=[9264, 9212, 9248, 9256, 9352, 9510, 9334, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65104 dup_rows=23972 dup_rows_remote=21016 dedup_ratio=0.243856 router_tokens_per_expert=[792, 765, 738, 794, 750, 732, 760, 802, 775, 754, 838, 800, 768, 774, 789, 754, 775, 732, 721, 772, 763, 796, 800, 771, 730, 769, 761, 726, 738, 767, 785, 781, 812, 790, 757, 761, 759, 796, 750, 745, 771, 779, 794, 753, 752, 774, 757, 751, 768, 759, 752, 755, 744, 779, 758, 777, 798, 782, 752, 747, 773, 766, 782, 787] payload_rows_per_dst=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] dup_rows_per_dst=[2884, 3186, 2976, 2840, 3112, 2984, 2956, 3034] combine_rows_per_src=[9382, 9318, 9284, 9274, 9228, 9278, 9228, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343776 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267198176 ingress_bytes=405266432 link_bytes=405266432 token_bytes=534052576 weight_movement_bytes=138412032 total_nvlink_bytes=672464608 weight_to_token_ratio=0.259173 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524984957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343912 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518376 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692840 weight_movement_bytes=34603008 total_nvlink_bytes=567295848 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343824 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968848 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593872 weight_movement_bytes=69206016 total_nvlink_bytes=602799888 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64480 dup_rows=24066 dup_rows_remote=20682 dedup_ratio=0.244812 router_tokens_per_expert=[737, 824, 780, 752, 785, 834, 769, 761, 756, 770, 766, 786, 757, 733, 845, 747, 723, 766, 768, 770, 759, 759, 790, 759, 778, 770, 754, 722, 777, 817, 761, 748, 777, 777, 755, 783, 781, 802, 764, 809, 735, 769, 748, 780, 744, 744, 703, 803, 784, 764, 741, 751, 787, 756, 750, 769, 796, 757, 751, 769, 729, 764, 792, 795] payload_rows_per_dst=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] dup_rows_per_dst=[3202, 2958, 2988, 2906, 3082, 2576, 2970, 3384] combine_rows_per_src=[9282, 9362, 9200, 9162, 9444, 8796, 9234, 9758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=64928 dup_rows=23920 dup_rows_remote=20836 dedup_ratio=0.243327 router_tokens_per_expert=[761, 776, 788, 756, 752, 756, 800, 798, 772, 755, 777, 772, 803, 793, 797, 766, 708, 780, 773, 760, 726, 764, 741, 764, 758, 783, 760, 812, 737, 773, 835, 817, 772, 744, 757, 729, 728, 786, 748, 760, 805, 771, 767, 771, 779, 749, 824, 776, 743, 781, 766, 809, 824, 736, 746, 754, 774, 766, 737, 759, 713, 761, 733, 771] payload_rows_per_dst=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] dup_rows_per_dst=[3084, 3072, 2862, 3098, 2800, 3122, 2998, 2884] combine_rows_per_src=[9456, 9492, 9264, 9192, 9248, 9362, 9226, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343056 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288144 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233232 weight_movement_bytes=34603008 total_nvlink_bytes=566836240 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264110080 dispatch_meta_bytes=340648 combine_payload_bytes=264110080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264450728 ingress_bytes=298713088 link_bytes=298713088 token_bytes=528560808 weight_movement_bytes=34603008 total_nvlink_bytes=563163816 weight_to_token_ratio=0.065466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124033302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344256 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191040 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037824 weight_movement_bytes=34603008 total_nvlink_bytes=566640832 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261855048 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6134886216 ingress_bytes=7268868096 link_bytes=7268868096 total_nvlink_bytes=13403754312 weight_to_token_ratio=0.093126 prefetch_share_of_ingress=0.090448 dedup_ratio=0.243975 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189224585e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=344480 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267010464 ingress_bytes=301268992 link_bytes=301268992 token_bytes=533676448 weight_movement_bytes=34603008 total_nvlink_bytes=568279456 weight_to_token_ratio=0.064839 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133650963e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259328616 prefetch_bytes=657457152 reduce_bytes=1038090240 weight_movement_bytes=1695547392 egress_bytes=6133621352 ingress_bytes=7821254656 link_bytes=7821254656 total_nvlink_bytes=13954876008 weight_to_token_ratio=0.138307 prefetch_share_of_ingress=0.084060 dedup_ratio=0.244174 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=30 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.279597896e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343776 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267198176 ingress_bytes=405266432 link_bytes=405266432 token_bytes=534052576 weight_movement_bytes=138412032 total_nvlink_bytes=672464608 weight_to_token_ratio=0.259173 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524984957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=20934 dedup_ratio=0.243530 router_tokens_per_expert=[755, 789, 740, 802, 745, 733, 774, 744, 727, 730, 774, 759, 757, 790, 755, 791, 763, 805, 802, 739, 766, 774, 770, 790, 731, 790, 725, 741, 805, 795, 751, 790, 781, 741, 760, 766, 777, 723, 781, 757, 794, 791, 796, 783, 782, 734, 781, 749, 755, 759, 776, 820, 716, 775, 766, 762, 740, 769, 807, 757, 774, 769, 780, 829] payload_rows_per_dst=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] dup_rows_per_dst=[2912, 2948, 3064, 2956, 2924, 3076, 3006, 3054] combine_rows_per_src=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266699520 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6137305856 ingress_bytes=7478910976 link_bytes=7478910976 total_nvlink_bytes=13616216832 weight_to_token_ratio=0.110015 prefetch_share_of_ingress=0.101788 dedup_ratio=0.243613 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027826 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223588691e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65072 dup_rows=23934 dup_rows_remote=20918 dedup_ratio=0.243469 router_tokens_per_expert=[783, 780, 784, 754, 820, 784, 789, 747, 756, 819, 763, 775, 694, 743, 777, 735, 778, 744, 781, 775, 752, 777, 749, 801, 794, 807, 778, 780, 770, 757, 776, 792, 792, 756, 748, 758, 746, 772, 789, 757, 780, 738, 762, 771, 761, 723, 766, 778, 776, 809, 737, 774, 767, 775, 816, 832, 753, 704, 759, 739, 728, 736, 732, 804] payload_rows_per_dst=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] dup_rows_per_dst=[3068, 2938, 3016, 3096, 2914, 2968, 3106, 2828] combine_rows_per_src=[9414, 9186, 9298, 9412, 9322, 9190, 9466, 9082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264110080 dispatch_meta_bytes=340648 combine_payload_bytes=264110080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264450728 ingress_bytes=298713088 link_bytes=298713088 token_bytes=528560808 weight_movement_bytes=34603008 total_nvlink_bytes=563163816 weight_to_token_ratio=0.065466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124033302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343056 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266288144 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532233232 weight_movement_bytes=34603008 total_nvlink_bytes=566836240 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12232651112 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6120272232 ingress_bytes=7461896192 link_bytes=7461896192 total_nvlink_bytes=13582168424 weight_to_token_ratio=0.110321 prefetch_share_of_ingress=0.092746 dedup_ratio=0.244256 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220804984e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12217453680 prefetch_bytes=795869184 reduce_bytes=242221056 weight_movement_bytes=1038090240 egress_bytes=6112668784 ingress_bytes=7142875136 link_bytes=7142875136 total_nvlink_bytes=13255543920 weight_to_token_ratio=0.084968 prefetch_share_of_ingress=0.111421 dedup_ratio=0.244770 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=7 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.168611482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256003456 prefetch_bytes=519045120 reduce_bytes=795869184 weight_movement_bytes=1314914304 egress_bytes=6131959168 ingress_bytes=7438958592 link_bytes=7438958592 total_nvlink_bytes=13570917760 weight_to_token_ratio=0.107287 prefetch_share_of_ingress=0.069774 dedup_ratio=0.244439 prefetch_slots=30 unused_prefetch_slots=338 slot_utilization=0.081522 reduce_slots=23 remote_demand=15 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28194 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.217052274e-04 prefetch_bytes_per_gemm_flop=8.491847826e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261855048 prefetch_bytes=657457152 reduce_bytes=484442112 weight_movement_bytes=1141899264 egress_bytes=6134886216 ingress_bytes=7268868096 link_bytes=7268868096 total_nvlink_bytes=13403754312 weight_to_token_ratio=0.093126 prefetch_share_of_ingress=0.090448 dedup_ratio=0.243975 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=14 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.189224585e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65112 dup_rows=23940 dup_rows_remote=20934 dedup_ratio=0.243530 router_tokens_per_expert=[755, 789, 740, 802, 745, 733, 774, 744, 727, 730, 774, 759, 757, 790, 755, 791, 763, 805, 802, 739, 766, 774, 770, 790, 731, 790, 725, 741, 805, 795, 751, 790, 781, 741, 760, 766, 777, 723, 781, 757, 794, 791, 796, 783, 782, 734, 781, 749, 755, 759, 776, 820, 716, 775, 766, 762, 740, 769, 807, 757, 774, 769, 780, 829] payload_rows_per_dst=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] dup_rows_per_dst=[2912, 2948, 3064, 2956, 2924, 3076, 3006, 3054] combine_rows_per_src=[9252, 9218, 9354, 9300, 9248, 9344, 9252, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344184 combine_payload_bytes=266698752 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267042936 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533741688 weight_movement_bytes=34603008 total_nvlink_bytes=568344696 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266878872 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533413784 weight_movement_bytes=34603008 total_nvlink_bytes=568016792 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12217453680 prefetch_bytes=795869184 reduce_bytes=242221056 weight_movement_bytes=1038090240 egress_bytes=6112668784 ingress_bytes=7142875136 link_bytes=7142875136 total_nvlink_bytes=13255543920 weight_to_token_ratio=0.084968 prefetch_share_of_ingress=0.111421 dedup_ratio=0.244770 prefetch_slots=46 unused_prefetch_slots=322 slot_utilization=0.125000 reduce_slots=7 remote_demand=23 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29210 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.168611482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12232651112 prefetch_bytes=692060160 reduce_bytes=657457152 weight_movement_bytes=1349517312 egress_bytes=6120272232 ingress_bytes=7461896192 link_bytes=7461896192 total_nvlink_bytes=13582168424 weight_to_token_ratio=0.110321 prefetch_share_of_ingress=0.092746 dedup_ratio=0.244256 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=19 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.220804984e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12268323528 prefetch_bytes=622854144 reduce_bytes=1072693248 weight_movement_bytes=1695547392 egress_bytes=6138118856 ingress_bytes=7825752064 link_bytes=7825752064 total_nvlink_bytes=13963870920 weight_to_token_ratio=0.138205 prefetch_share_of_ingress=0.079590 dedup_ratio=0.243605 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=31 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.280333695e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261994256 prefetch_bytes=761266176 reduce_bytes=588251136 weight_movement_bytes=1349517312 egress_bytes=6134951696 ingress_bytes=7476559872 link_bytes=7476559872 total_nvlink_bytes=13611511568 weight_to_token_ratio=0.110057 prefetch_share_of_ingress=0.101820 dedup_ratio=0.243594 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=17 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.223204039e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none - [2026-08-26 16:56:37] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 790.8 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.388426E+00 | loss scale: 1.0 | grad norm: 5.302 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64804 dup_rows=24168 dup_rows_remote=21056 dedup_ratio=0.245850 router_tokens_per_expert=[786, 762, 843, 766, 781, 763, 773, 743, 785, 780, 755, 767, 769, 811, 761, 776, 760, 756, 779, 751, 783, 777, 818, 762, 761, 744, 811, 750, 748, 763, 800, 729, 737, 814, 714, 812, 776, 800, 817, 752, 751, 774, 767, 753, 773, 808, 749, 775, 738, 764, 768, 756, 745, 773, 705, 788, 771, 774, 769, 736, 744, 747, 766, 723] payload_rows_per_dst=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] dup_rows_per_dst=[3090, 3062, 3018, 2964, 3112, 3050, 2978, 2894] combine_rows_per_src=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] + [2026-08-26 18:08:43] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 789.1 | throughput per GPU (TFLOP/s/GPU): 161.7 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 9.388426E+00 | loss scale: 1.0 | grad norm: 5.302 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64900 dup_rows=24096 dup_rows_remote=21086 dedup_ratio=0.245117 router_tokens_per_expert=[719, 768, 773, 771, 790, 751, 743, 736, 734, 717, 750, 727, 811, 796, 767, 770, 714, 797, 745, 745, 765, 804, 747, 758, 779, 784, 798, 776, 785, 741, 753, 743, 766, 841, 801, 811, 798, 776, 746, 759, 814, 766, 800, 767, 802, 740, 813, 714, 733, 743, 775, 741, 757, 810, 796, 795, 751, 749, 762, 772, 777, 780, 746, 794] payload_rows_per_dst=[9174, 9280, 9202, 9308, 9400, 9366, 9244, 9234] dup_rows_per_dst=[2928, 2864, 2948, 3010, 3196, 3066, 3056, 3028] combine_rows_per_src=[9174, 9280, 9202, 9308, 9400, 9366, 9244, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20912 dedup_ratio=0.243083 router_tokens_per_expert=[800, 743, 741, 746, 748, 760, 725, 721, 785, 764, 737, 771, 767, 799, 773, 762, 787, 751, 748, 742, 776, 786, 747, 752, 760, 760, 779, 786, 776, 822, 786, 757, 814, 752, 755, 805, 778, 755, 772, 775, 814, 775, 760, 778, 776, 730, 750, 804, 800, 750, 746, 779, 738, 814, 785, 797, 752, 796, 751, 777, 740, 753, 757, 767] payload_rows_per_dst=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] dup_rows_per_dst=[2834, 3048, 2950, 3092, 2988, 2984, 3022, 2978] combine_rows_per_src=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64822 dup_rows=24134 dup_rows_remote=20972 dedup_ratio=0.245504 router_tokens_per_expert=[750, 745, 761, 769, 787, 772, 742, 775, 819, 794, 743, 744, 743, 770, 740, 743, 786, 769, 761, 775, 743, 735, 798, 757, 810, 723, 794, 787, 809, 726, 744, 772, 781, 802, 753, 742, 806, 749, 775, 785, 733, 786, 724, 758, 749, 767, 743, 788, 742, 749, 772, 773, 772, 821, 769, 772, 791, 782, 754, 769, 792, 786, 790, 791] payload_rows_per_dst=[9190, 9192, 9224, 9324, 9422, 9140, 9330, 9348] dup_rows_per_dst=[3012, 3000, 3024, 3006, 2964, 2956, 3010, 3162] combine_rows_per_src=[9190, 9192, 9224, 9324, 9422, 9140, 9330, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65018 dup_rows=23896 dup_rows_remote=20912 dedup_ratio=0.243083 router_tokens_per_expert=[800, 743, 741, 746, 748, 760, 725, 721, 785, 764, 737, 771, 767, 799, 773, 762, 787, 751, 748, 742, 776, 786, 747, 752, 760, 760, 779, 786, 776, 822, 786, 757, 814, 752, 755, 805, 778, 755, 772, 775, 814, 775, 760, 778, 776, 730, 750, 804, 800, 750, 746, 779, 738, 814, 785, 797, 752, 796, 751, 777, 740, 753, 757, 767] payload_rows_per_dst=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] dup_rows_per_dst=[2834, 3048, 2950, 3092, 2988, 2984, 3022, 2978] combine_rows_per_src=[9134, 9268, 9228, 9360, 9424, 9390, 9396, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64842 dup_rows=24050 dup_rows_remote=20954 dedup_ratio=0.244649 router_tokens_per_expert=[810, 719, 756, 761, 762, 784, 730, 745, 763, 805, 777, 763, 761, 696, 790, 748, 777, 776, 788, 752, 803, 768, 823, 767, 763, 717, 811, 807, 753, 718, 728, 758, 807, 755, 767, 784, 760, 760, 750, 768, 753, 776, 813, 786, 791, 797, 762, 743, 762, 761, 771, 720, 796, 711, 821, 738, 786, 780, 814, 804, 762, 746, 739, 790] payload_rows_per_dst=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] dup_rows_per_dst=[2922, 2964, 3096, 2950, 2990, 3110, 2942, 3076] combine_rows_per_src=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65024 dup_rows=23926 dup_rows_remote=20970 dedup_ratio=0.243388 router_tokens_per_expert=[753, 776, 760, 787, 796, 757, 783, 752, 781, 723, 726, 761, 810, 776, 823, 755, 728, 733, 743, 764, 741, 773, 749, 793, 785, 729, 814, 792, 744, 743, 741, 769, 789, 755, 766, 731, 777, 806, 778, 789, 792, 823, 769, 741, 778, 744, 779, 774, 776, 768, 754, 748, 773, 752, 805, 769, 759, 785, 789, 742, 785, 749, 733, 814] payload_rows_per_dst=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] dup_rows_per_dst=[3068, 2956, 2890, 2994, 3058, 2992, 2954, 3014] combine_rows_per_src=[9260, 9354, 9158, 9240, 9324, 9408, 9336, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65206 dup_rows=23798 dup_rows_remote=20718 dedup_ratio=0.242086 router_tokens_per_expert=[801, 758, 781, 746, 782, 708, 760, 818, 754, 746, 755, 765, 760, 766, 804, 778, 759, 732, 759, 749, 804, 738, 784, 747, 740, 754, 764, 780, 762, 774, 754, 802, 796, 741, 755, 737, 777, 799, 766, 762, 760, 749, 763, 849, 768, 758, 795, 755, 741, 756, 761, 781, 782, 793, 801, 775, 802, 786, 796, 763, 766, 792, 715, 728] payload_rows_per_dst=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] dup_rows_per_dst=[2964, 2916, 2876, 3000, 2944, 3000, 3080, 3018] combine_rows_per_src=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64804 dup_rows=24168 dup_rows_remote=21056 dedup_ratio=0.245850 router_tokens_per_expert=[786, 762, 843, 766, 781, 763, 773, 743, 785, 780, 755, 767, 769, 811, 761, 776, 760, 756, 779, 751, 783, 777, 818, 762, 761, 744, 811, 750, 748, 763, 800, 729, 737, 814, 714, 812, 776, 800, 817, 752, 751, 774, 767, 753, 773, 808, 749, 775, 738, 764, 768, 756, 745, 773, 705, 788, 771, 774, 769, 736, 744, 747, 766, 723] payload_rows_per_dst=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] dup_rows_per_dst=[3090, 3062, 3018, 2964, 3112, 3050, 2978, 2894] combine_rows_per_src=[9344, 9346, 9354, 9248, 9332, 9250, 9096, 9166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343440 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265780624 ingress_bytes=369246208 link_bytes=369246208 token_bytes=531217808 weight_movement_bytes=103809024 total_nvlink_bytes=635026832 weight_to_token_ratio=0.195417 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389443754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343944 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174344 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004744 weight_movement_bytes=34603008 total_nvlink_bytes=566607752 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343720 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266657448 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971176 weight_movement_bytes=103809024 total_nvlink_bytes=636780200 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64842 dup_rows=24050 dup_rows_remote=20954 dedup_ratio=0.244649 router_tokens_per_expert=[810, 719, 756, 761, 762, 784, 730, 745, 763, 805, 777, 763, 761, 696, 790, 748, 777, 776, 788, 752, 803, 768, 823, 767, 763, 717, 811, 807, 753, 718, 728, 758, 807, 755, 767, 784, 760, 760, 750, 768, 753, 776, 813, 786, 791, 797, 762, 743, 762, 761, 771, 720, 796, 711, 821, 738, 786, 780, 814, 804, 762, 746, 739, 790] payload_rows_per_dst=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] dup_rows_per_dst=[2922, 2964, 3096, 2950, 2990, 3110, 2942, 3076] combine_rows_per_src=[9212, 9242, 9412, 9160, 9312, 9332, 9218, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343944 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174344 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004744 weight_movement_bytes=34603008 total_nvlink_bytes=566607752 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343176 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854088 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365000 weight_movement_bytes=103809024 total_nvlink_bytes=635174024 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64632 dup_rows=24148 dup_rows_remote=20844 dedup_ratio=0.245646 router_tokens_per_expert=[764, 785, 752, 731, 770, 770, 737, 780, 773, 775, 762, 756, 760, 744, 781, 765, 785, 790, 761, 783, 748, 784, 700, 833, 738, 787, 693, 726, 789, 772, 788, 773, 738, 774, 769, 811, 775, 799, 765, 748, 787, 792, 728, 792, 742, 800, 792, 756, 804, 781, 783, 759, 762, 759, 756, 743, 783, 778, 810, 787, 723, 763, 775, 763] payload_rows_per_dst=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] dup_rows_per_dst=[3304, 3092, 3104, 3082, 2622, 2890, 3034, 3020] combine_rows_per_src=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343696 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427472 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511248 weight_movement_bytes=34603008 total_nvlink_bytes=569114256 weight_to_token_ratio=0.064738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343720 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266657448 ingress_bytes=370122752 link_bytes=370122752 token_bytes=532971176 weight_movement_bytes=103809024 total_nvlink_bytes=636780200 weight_to_token_ratio=0.194774 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392742118e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343184 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936016 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528848 weight_movement_bytes=34603008 total_nvlink_bytes=566131856 weight_to_token_ratio=0.065101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343976 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682280 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020584 weight_movement_bytes=34603008 total_nvlink_bytes=567623592 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343440 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265780624 ingress_bytes=369246208 link_bytes=369246208 token_bytes=531217808 weight_movement_bytes=103809024 total_nvlink_bytes=635026832 weight_to_token_ratio=0.195417 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389443754e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65206 dup_rows=23798 dup_rows_remote=20718 dedup_ratio=0.242086 router_tokens_per_expert=[801, 758, 781, 746, 782, 708, 760, 818, 754, 746, 755, 765, 760, 766, 804, 778, 759, 732, 759, 749, 804, 738, 784, 747, 740, 754, 764, 780, 762, 774, 754, 802, 796, 741, 755, 737, 777, 799, 766, 762, 760, 749, 763, 849, 768, 758, 795, 755, 741, 756, 761, 781, 782, 793, 801, 775, 802, 786, 796, 763, 766, 792, 715, 728] payload_rows_per_dst=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] dup_rows_per_dst=[2964, 2916, 2876, 3000, 2944, 3000, 3080, 3018] combine_rows_per_src=[9344, 9340, 9268, 9260, 9322, 9394, 9300, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64632 dup_rows=24148 dup_rows_remote=20844 dedup_ratio=0.245646 router_tokens_per_expert=[764, 785, 752, 731, 770, 770, 737, 780, 773, 775, 762, 756, 760, 744, 781, 765, 785, 790, 761, 783, 748, 784, 700, 833, 738, 787, 693, 726, 789, 772, 788, 773, 738, 774, 769, 811, 775, 799, 765, 748, 787, 792, 728, 792, 742, 800, 792, 756, 804, 781, 783, 759, 762, 759, 756, 743, 783, 778, 810, 787, 723, 763, 775, 763] payload_rows_per_dst=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] dup_rows_per_dst=[3304, 3092, 3104, 3082, 2622, 2890, 3034, 3020] combine_rows_per_src=[9524, 9380, 9552, 9306, 8894, 8976, 9308, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343184 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936016 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531528848 weight_movement_bytes=34603008 total_nvlink_bytes=566131856 weight_to_token_ratio=0.065101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64780 dup_rows=24172 dup_rows_remote=21010 dedup_ratio=0.245890 router_tokens_per_expert=[785, 752, 788, 785, 746, 763, 769, 753, 769, 766, 793, 777, 716, 769, 755, 759, 781, 824, 779, 789, 760, 793, 768, 796, 798, 729, 752, 752, 779, 799, 720, 777, 766, 773, 790, 796, 767, 807, 790, 768, 748, 752, 793, 800, 765, 749, 731, 827, 746, 752, 767, 787, 794, 717, 774, 728, 791, 754, 763, 734, 747, 707, 731, 797] payload_rows_per_dst=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] dup_rows_per_dst=[3010, 2990, 3148, 2998, 3162, 2984, 2970, 2910] combine_rows_per_src=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74016 payload_rows_remote=64720 dup_rows=24288 dup_rows_remote=21258 dedup_ratio=0.247070 router_tokens_per_expert=[831, 814, 811, 755, 796, 753, 756, 810, 710, 753, 765, 784, 717, 744, 764, 803, 792, 723, 770, 732, 743, 768, 721, 797, 820, 789, 787, 765, 752, 723, 770, 757, 737, 745, 708, 795, 790, 777, 788, 752, 758, 769, 751, 763, 743, 777, 789, 807, 733, 739, 769, 814, 753, 785, 807, 774, 771, 724, 770, 763, 797, 782, 771, 776] payload_rows_per_dst=[9534, 9102, 9190, 9296, 9198, 9200, 9296, 9200] dup_rows_per_dst=[3118, 2978, 2902, 3030, 2986, 3114, 3052, 3108] combine_rows_per_src=[9534, 9102, 9190, 9296, 9198, 9200, 9296, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65090 dup_rows=23944 dup_rows_remote=20876 dedup_ratio=0.243571 router_tokens_per_expert=[720, 777, 809, 789, 769, 845, 781, 767, 769, 763, 740, 807, 821, 705, 786, 727, 709, 754, 766, 769, 770, 762, 768, 771, 767, 785, 787, 802, 740, 779, 750, 734, 717, 767, 807, 745, 771, 757, 741, 832, 770, 784, 744, 785, 797, 806, 747, 771, 772, 756, 760, 754, 776, 726, 780, 730, 763, 786, 728, 780, 752, 753, 792, 815] payload_rows_per_dst=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] dup_rows_per_dst=[2852, 3028, 2878, 3194, 3008, 2990, 2926, 3068] combine_rows_per_src=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65072 dup_rows=24010 dup_rows_remote=21056 dedup_ratio=0.244242 router_tokens_per_expert=[784, 782, 790, 714, 746, 784, 788, 752, 734, 810, 774, 744, 731, 788, 808, 761, 776, 770, 806, 776, 769, 816, 746, 760, 750, 773, 750, 771, 751, 762, 774, 726, 763, 757, 761, 770, 728, 789, 777, 759, 764, 772, 771, 752, 758, 738, 764, 769, 756, 779, 781, 785, 785, 809, 782, 768, 775, 755, 757, 765, 807, 795, 737, 758] payload_rows_per_dst=[9350, 9304, 9444, 9132, 9234, 9222, 9316, 9292] dup_rows_per_dst=[2930, 2996, 2994, 2982, 2974, 2954, 3174, 3006] combine_rows_per_src=[9350, 9304, 9444, 9132, 9234, 9222, 9316, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=341904 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265074576 ingress_bytes=299335680 link_bytes=299335680 token_bytes=529807248 weight_movement_bytes=34603008 total_nvlink_bytes=564410256 weight_to_token_ratio=0.065312 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126376065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65008 dup_rows=24016 dup_rows_remote=21098 dedup_ratio=0.244303 router_tokens_per_expert=[710, 762, 792, 797, 823, 833, 767, 737, 787, 788, 811, 784, 793, 781, 761, 765, 806, 747, 761, 735, 731, 730, 820, 769, 757, 770, 719, 769, 743, 733, 739, 749, 766, 812, 780, 755, 749, 748, 800, 783, 754, 774, 745, 807, 711, 753, 759, 751, 796, 727, 749, 777, 768, 798, 725, 776, 782, 763, 799, 766, 825, 770, 746, 769] payload_rows_per_dst=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] dup_rows_per_dst=[3090, 3114, 2918, 2924, 2996, 2982, 2968, 3024] combine_rows_per_src=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343696 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427472 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511248 weight_movement_bytes=34603008 total_nvlink_bytes=569114256 weight_to_token_ratio=0.064738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65090 dup_rows=23944 dup_rows_remote=20876 dedup_ratio=0.243571 router_tokens_per_expert=[720, 777, 809, 789, 769, 845, 781, 767, 769, 763, 740, 807, 821, 705, 786, 727, 709, 754, 766, 769, 770, 762, 768, 771, 767, 785, 787, 802, 740, 779, 750, 734, 717, 767, 807, 745, 771, 757, 741, 832, 770, 784, 744, 785, 797, 806, 747, 771, 772, 756, 760, 754, 776, 726, 780, 730, 763, 786, 728, 780, 752, 753, 792, 815] payload_rows_per_dst=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] dup_rows_per_dst=[2852, 3028, 2878, 3194, 3008, 2990, 2926, 3068] combine_rows_per_src=[9208, 9208, 9260, 9548, 9266, 9418, 9182, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=343160 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265682040 ingress_bytes=299941888 link_bytes=299941888 token_bytes=531020920 weight_movement_bytes=34603008 total_nvlink_bytes=565623928 weight_to_token_ratio=0.065163 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128657177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64948 dup_rows=24084 dup_rows_remote=21232 dedup_ratio=0.244995 router_tokens_per_expert=[799, 776, 737, 761, 781, 774, 774, 775, 771, 747, 797, 748, 806, 758, 810, 728, 804, 776, 736, 783, 729, 774, 777, 747, 771, 803, 746, 735, 786, 744, 782, 739, 743, 749, 812, 736, 783, 740, 770, 754, 760, 779, 763, 780, 731, 778, 816, 758, 757, 761, 771, 748, 741, 795, 756, 733, 784, 758, 780, 837, 827, 742, 754, 782] payload_rows_per_dst=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] dup_rows_per_dst=[3064, 3052, 3066, 2868, 3000, 3074, 2852, 3108] combine_rows_per_src=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343912 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437032 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530152 weight_movement_bytes=34603008 total_nvlink_bytes=565133160 weight_to_token_ratio=0.065223 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344512 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266879424 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533414336 weight_movement_bytes=34603008 total_nvlink_bytes=568017344 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264732672 dispatch_meta_bytes=341904 combine_payload_bytes=264732672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265074576 ingress_bytes=299335680 link_bytes=299335680 token_bytes=529807248 weight_movement_bytes=34603008 total_nvlink_bytes=564410256 weight_to_token_ratio=0.065312 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126376065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64958 dup_rows=24238 dup_rows_remote=21330 dedup_ratio=0.246562 router_tokens_per_expert=[772, 736, 753, 761, 712, 770, 775, 706, 787, 754, 740, 757, 724, 741, 734, 771, 764, 792, 769, 781, 776, 804, 786, 773, 719, 744, 819, 769, 765, 758, 771, 772, 849, 800, 763, 804, 756, 737, 783, 764, 764, 739, 713, 815, 754, 780, 786, 762, 831, 765, 751, 751, 790, 812, 784, 768, 779, 764, 756, 754, 810, 756, 785, 772] payload_rows_per_dst=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] dup_rows_per_dst=[2918, 2908, 3082, 3030, 3104, 3030, 3164, 3002] combine_rows_per_src=[9052, 9108, 9408, 9204, 9408, 9196, 9340, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64780 dup_rows=24172 dup_rows_remote=21010 dedup_ratio=0.245890 router_tokens_per_expert=[785, 752, 788, 785, 746, 763, 769, 753, 769, 766, 793, 777, 716, 769, 755, 759, 781, 824, 779, 789, 760, 793, 768, 796, 798, 729, 752, 752, 779, 799, 720, 777, 766, 773, 790, 796, 767, 807, 790, 768, 748, 752, 793, 800, 765, 749, 731, 827, 746, 752, 767, 787, 794, 717, 774, 728, 791, 754, 763, 734, 747, 707, 731, 797] payload_rows_per_dst=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] dup_rows_per_dst=[3010, 2990, 3148, 2998, 3162, 2984, 2970, 2910] combine_rows_per_src=[9272, 9218, 9432, 9214, 9352, 9346, 9160, 9138] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266952504 ingress_bytes=405020672 link_bytes=405020672 token_bytes=533561144 weight_movement_bytes=138412032 total_nvlink_bytes=671973176 weight_to_token_ratio=0.259412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524060182e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344720 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371728 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398736 weight_movement_bytes=34603008 total_nvlink_bytes=567001744 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265093120 dispatch_meta_bytes=343912 combine_payload_bytes=265093120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265437032 ingress_bytes=299696128 link_bytes=299696128 token_bytes=530530152 weight_movement_bytes=34603008 total_nvlink_bytes=565133160 weight_to_token_ratio=0.065223 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127732402e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344512 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266879424 ingress_bytes=301137920 link_bytes=301137920 token_bytes=533414336 weight_movement_bytes=34603008 total_nvlink_bytes=568017344 weight_to_token_ratio=0.064871 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133157749e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344424 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617192 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532889960 weight_movement_bytes=103809024 total_nvlink_bytes=636698984 weight_to_token_ratio=0.194804 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65008 dup_rows=24016 dup_rows_remote=21098 dedup_ratio=0.244303 router_tokens_per_expert=[710, 762, 792, 797, 823, 833, 767, 737, 787, 788, 811, 784, 793, 781, 761, 765, 806, 747, 761, 735, 731, 730, 820, 769, 757, 770, 719, 769, 743, 733, 739, 749, 766, 812, 780, 755, 749, 748, 800, 783, 754, 774, 745, 807, 711, 753, 759, 751, 796, 727, 749, 777, 768, 798, 725, 776, 782, 763, 799, 766, 825, 770, 746, 769] payload_rows_per_dst=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] dup_rows_per_dst=[3090, 3114, 2918, 2924, 2996, 2982, 2968, 3024] combine_rows_per_src=[9352, 9426, 9280, 9034, 9390, 9126, 9264, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=345152 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266413120 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532481088 weight_movement_bytes=34603008 total_nvlink_bytes=567084096 weight_to_token_ratio=0.064984 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265338880 dispatch_meta_bytes=343160 combine_payload_bytes=265338880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265682040 ingress_bytes=299941888 link_bytes=299941888 token_bytes=531020920 weight_movement_bytes=34603008 total_nvlink_bytes=565623928 weight_to_token_ratio=0.065163 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128657177e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64948 dup_rows=24084 dup_rows_remote=21232 dedup_ratio=0.244995 router_tokens_per_expert=[799, 776, 737, 761, 781, 774, 774, 775, 771, 747, 797, 748, 806, 758, 810, 728, 804, 776, 736, 783, 729, 774, 777, 747, 771, 803, 746, 735, 786, 744, 782, 739, 743, 749, 812, 736, 783, 740, 770, 754, 760, 779, 763, 780, 731, 778, 816, 758, 757, 761, 771, 748, 741, 795, 756, 733, 784, 758, 780, 837, 827, 742, 754, 782] payload_rows_per_dst=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] dup_rows_per_dst=[3064, 3052, 3066, 2868, 3000, 3074, 2852, 3108] combine_rows_per_src=[9290, 9278, 9186, 9344, 9174, 9256, 9272, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64956 dup_rows=24114 dup_rows_remote=21056 dedup_ratio=0.245300 router_tokens_per_expert=[762, 812, 745, 812, 793, 783, 757, 749, 805, 789, 708, 762, 801, 719, 801, 772, 756, 752, 773, 781, 708, 747, 839, 760, 811, 766, 779, 775, 806, 782, 717, 755, 793, 800, 732, 796, 751, 819, 698, 798, 792, 764, 787, 740, 808, 786, 745, 755, 743, 756, 786, 766, 746, 764, 707, 741, 752, 740, 783, 776, 742, 778, 771, 760] payload_rows_per_dst=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] dup_rows_per_dst=[2932, 2992, 2872, 2858, 3138, 3154, 3110, 3058] combine_rows_per_src=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=64886 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[746, 818, 749, 758, 778, 743, 747, 768, 820, 725, 761, 766, 742, 769, 784, 761, 751, 766, 800, 726, 770, 771, 748, 738, 786, 788, 747, 771, 775, 786, 812, 811, 775, 794, 780, 696, 727, 767, 747, 791, 732, 767, 741, 774, 847, 752, 807, 766, 727, 740, 788, 801, 789, 846, 693, 809, 767, 795, 771, 789, 743, 764, 751, 735] payload_rows_per_dst=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] dup_rows_per_dst=[2968, 3038, 2860, 3134, 2898, 3002, 3054, 3046] combine_rows_per_src=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64854 dup_rows=24278 dup_rows_remote=21228 dedup_ratio=0.246969 router_tokens_per_expert=[744, 727, 765, 751, 737, 769, 767, 743, 742, 738, 763, 721, 794, 792, 736, 808, 775, 767, 751, 782, 730, 771, 743, 792, 801, 789, 774, 727, 720, 768, 760, 807, 815, 765, 797, 805, 750, 749, 787, 747, 787, 770, 716, 776, 780, 775, 837, 759, 773, 758, 784, 749, 751, 794, 780, 740, 725, 770, 800, 772, 823, 792, 789, 783] payload_rows_per_dst=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] dup_rows_per_dst=[2930, 2964, 3050, 2998, 3128, 3110, 2994, 3104] combine_rows_per_src=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64886 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[803, 741, 795, 788, 719, 772, 743, 732, 791, 810, 760, 737, 809, 787, 787, 736, 782, 789, 762, 756, 740, 788, 805, 753, 805, 774, 743, 800, 739, 768, 793, 831, 741, 719, 774, 745, 768, 717, 749, 802, 751, 794, 714, 776, 768, 790, 785, 728, 765, 783, 773, 713, 768, 789, 782, 769, 764, 744, 799, 735, 781, 785, 789, 754] payload_rows_per_dst=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] dup_rows_per_dst=[2982, 3110, 2894, 3170, 2970, 3020, 3034, 2956] combine_rows_per_src=[9282, 9484, 9222, 9336, 9082, 9392, 9334, 9036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64972 dup_rows=24154 dup_rows_remote=21298 dedup_ratio=0.245707 router_tokens_per_expert=[792, 786, 811, 778, 749, 768, 733, 783, 761, 774, 754, 711, 747, 811, 796, 766, 774, 785, 740, 801, 714, 760, 816, 810, 778, 743, 780, 735, 777, 775, 783, 766, 768, 737, 765, 746, 734, 752, 758, 757, 730, 747, 750, 784, 782, 762, 769, 768, 772, 762, 747, 830, 817, 820, 763, 749, 774, 793, 757, 838, 738, 741, 757, 728] payload_rows_per_dst=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] dup_rows_per_dst=[3020, 3054, 3072, 3042, 2856, 2920, 3116, 3074] combine_rows_per_src=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65064 dup_rows=24072 dup_rows_remote=21094 dedup_ratio=0.244873 router_tokens_per_expert=[735, 725, 752, 796, 785, 767, 778, 766, 775, 775, 817, 846, 748, 755, 723, 775, 767, 781, 846, 761, 766, 783, 769, 798, 767, 786, 743, 766, 799, 807, 775, 753, 822, 749, 812, 785, 748, 760, 740, 756, 775, 732, 755, 764, 738, 778, 773, 758, 775, 719, 743, 777, 751, 766, 770, 787, 774, 752, 724, 739, 721, 729, 758, 837] payload_rows_per_dst=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] dup_rows_per_dst=[2962, 3090, 3104, 3012, 3014, 2978, 2944, 2968] combine_rows_per_src=[9246, 9338, 9438, 9380, 9330, 9168, 9232, 9100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=64886 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[746, 818, 749, 758, 778, 743, 747, 768, 820, 725, 761, 766, 742, 769, 784, 761, 751, 766, 800, 726, 770, 771, 748, 738, 786, 788, 747, 771, 775, 786, 812, 811, 775, 794, 780, 696, 727, 767, 747, 791, 732, 767, 741, 774, 847, 752, 807, 766, 727, 740, 788, 801, 789, 846, 693, 809, 767, 795, 771, 789, 743, 764, 751, 735] payload_rows_per_dst=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] dup_rows_per_dst=[2968, 3038, 2860, 3134, 2898, 3002, 3054, 3046] combine_rows_per_src=[9246, 9218, 9280, 9418, 9256, 9370, 9332, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344424 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617192 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532889960 weight_movement_bytes=103809024 total_nvlink_bytes=636698984 weight_to_token_ratio=0.194804 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344160 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266117216 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531890272 weight_movement_bytes=69206016 total_nvlink_bytes=601096288 weight_to_token_ratio=0.130113 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=345080 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266470392 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595704 weight_movement_bytes=34603008 total_nvlink_bytes=567198712 weight_to_token_ratio=0.064970 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64956 dup_rows=24114 dup_rows_remote=21056 dedup_ratio=0.245300 router_tokens_per_expert=[762, 812, 745, 812, 793, 783, 757, 749, 805, 789, 708, 762, 801, 719, 801, 772, 756, 752, 773, 781, 708, 747, 839, 760, 811, 766, 779, 775, 806, 782, 717, 755, 793, 800, 732, 796, 751, 819, 698, 798, 792, 764, 787, 740, 808, 786, 745, 755, 743, 756, 786, 766, 746, 764, 707, 741, 752, 740, 783, 776, 742, 778, 771, 760] payload_rows_per_dst=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] dup_rows_per_dst=[2932, 2992, 2872, 2858, 3138, 3154, 3110, 3058] combine_rows_per_src=[9260, 9328, 9360, 9050, 9242, 9428, 9288, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64998 dup_rows=24122 dup_rows_remote=21104 dedup_ratio=0.245382 router_tokens_per_expert=[779, 756, 785, 791, 795, 765, 757, 799, 794, 779, 758, 714, 780, 782, 792, 801, 796, 736, 808, 781, 733, 770, 793, 744, 800, 801, 786, 750, 775, 772, 772, 768, 769, 709, 753, 749, 772, 771, 764, 748, 745, 779, 749, 765, 820, 757, 762, 736, 745, 753, 766, 771, 774, 772, 768, 752, 734, 739, 787, 786, 770, 749, 766, 760] payload_rows_per_dst=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] dup_rows_per_dst=[3104, 3090, 3002, 3020, 2874, 2996, 3018, 3018] combine_rows_per_src=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344632 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846776 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348920 weight_movement_bytes=34603008 total_nvlink_bytes=567951928 weight_to_token_ratio=0.064879 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343008 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116064 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889120 weight_movement_bytes=138412032 total_nvlink_bytes=670301152 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65012 dup_rows=24016 dup_rows_remote=21048 dedup_ratio=0.244303 router_tokens_per_expert=[749, 778, 791, 804, 833, 788, 759, 794, 762, 754, 746, 781, 776, 775, 774, 754, 799, 730, 743, 766, 769, 776, 742, 760, 768, 835, 723, 742, 778, 750, 747, 774, 757, 780, 780, 774, 817, 805, 752, 774, 786, 790, 756, 719, 807, 780, 727, 783, 748, 756, 763, 753, 760, 738, 765, 759, 783, 747, 741, 759, 757, 808, 747, 761] payload_rows_per_dst=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] dup_rows_per_dst=[3156, 2968, 2912, 2982, 3092, 3048, 2864, 2994] combine_rows_per_src=[9436, 9276, 9258, 9252, 9386, 9248, 9220, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64972 dup_rows=24154 dup_rows_remote=21298 dedup_ratio=0.245707 router_tokens_per_expert=[792, 786, 811, 778, 749, 768, 733, 783, 761, 774, 754, 711, 747, 811, 796, 766, 774, 785, 740, 801, 714, 760, 816, 810, 778, 743, 780, 735, 777, 775, 783, 766, 768, 737, 765, 746, 734, 752, 758, 757, 730, 747, 750, 784, 782, 762, 769, 768, 772, 762, 747, 830, 817, 820, 763, 749, 774, 793, 757, 838, 738, 741, 757, 728] payload_rows_per_dst=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] dup_rows_per_dst=[3020, 3054, 3072, 3042, 2856, 2920, 3116, 3074] combine_rows_per_src=[9380, 9186, 9328, 9232, 9178, 9264, 9404, 9178] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344048 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403824 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463600 weight_movement_bytes=34603008 total_nvlink_bytes=567066608 weight_to_token_ratio=0.064987 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344408 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576216 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808024 weight_movement_bytes=34603008 total_nvlink_bytes=567411032 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344720 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371728 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398736 weight_movement_bytes=34603008 total_nvlink_bytes=567001744 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343008 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116064 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889120 weight_movement_bytes=138412032 total_nvlink_bytes=670301152 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344328 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986312 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628296 weight_movement_bytes=34603008 total_nvlink_bytes=566231304 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344632 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846776 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348920 weight_movement_bytes=34603008 total_nvlink_bytes=567951928 weight_to_token_ratio=0.064879 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344160 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266117216 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531890272 weight_movement_bytes=69206016 total_nvlink_bytes=601096288 weight_to_token_ratio=0.130113 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344240 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633392 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922544 weight_movement_bytes=34603008 total_nvlink_bytes=567525552 weight_to_token_ratio=0.064931 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=345080 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266470392 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532595704 weight_movement_bytes=34603008 total_nvlink_bytes=567198712 weight_to_token_ratio=0.064970 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64516 dup_rows=24148 dup_rows_remote=20772 dedup_ratio=0.245646 router_tokens_per_expert=[773, 746, 763, 833, 759, 772, 768, 814, 726, 750, 728, 769, 781, 764, 738, 784, 774, 775, 779, 738, 768, 798, 762, 800, 788, 762, 794, 790, 797, 779, 761, 760, 777, 750, 746, 779, 773, 729, 786, 759, 760, 797, 734, 761, 746, 736, 811, 804, 757, 776, 747, 774, 709, 762, 793, 714, 772, 784, 763, 767, 773, 725, 797, 798] payload_rows_per_dst=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] dup_rows_per_dst=[2682, 3008, 3074, 2668, 3184, 3054, 3102, 3376] combine_rows_per_src=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64964 dup_rows=23944 dup_rows_remote=20770 dedup_ratio=0.243571 router_tokens_per_expert=[757, 829, 786, 741, 815, 775, 773, 769, 760, 771, 811, 747, 748, 744, 730, 764, 811, 681, 747, 783, 836, 770, 817, 753, 739, 764, 770, 793, 758, 830, 833, 798, 738, 741, 711, 794, 814, 747, 726, 819, 798, 756, 787, 763, 749, 787, 808, 782, 746, 729, 746, 701, 786, 709, 827, 777, 759, 745, 731, 716, 761, 771, 764, 761] payload_rows_per_dst=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] dup_rows_per_dst=[3080, 2886, 3058, 3174, 2886, 3090, 2904, 2866] combine_rows_per_src=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64926 dup_rows=24086 dup_rows_remote=21140 dedup_ratio=0.245015 router_tokens_per_expert=[765, 782, 827, 775, 780, 721, 778, 771, 749, 745, 801, 773, 781, 752, 737, 829, 766, 772, 731, 732, 755, 804, 790, 769, 762, 769, 773, 765, 827, 808, 749, 833, 780, 765, 787, 730, 767, 785, 720, 762, 782, 786, 717, 763, 791, 803, 743, 772, 797, 790, 763, 763, 794, 697, 746, 720, 786, 752, 794, 754, 723, 729, 746, 774] payload_rows_per_dst=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] dup_rows_per_dst=[3048, 3022, 2946, 3186, 3014, 3024, 2962, 2884] combine_rows_per_src=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64998 dup_rows=24122 dup_rows_remote=21104 dedup_ratio=0.245382 router_tokens_per_expert=[779, 756, 785, 791, 795, 765, 757, 799, 794, 779, 758, 714, 780, 782, 792, 801, 796, 736, 808, 781, 733, 770, 793, 744, 800, 801, 786, 750, 775, 772, 772, 768, 769, 709, 753, 749, 772, 771, 764, 748, 745, 779, 749, 765, 820, 757, 762, 736, 745, 753, 766, 771, 774, 772, 768, 752, 734, 739, 787, 786, 770, 749, 766, 760] payload_rows_per_dst=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] dup_rows_per_dst=[3104, 3090, 3002, 3020, 2874, 2996, 3018, 3018] combine_rows_per_src=[9350, 9310, 9320, 9428, 9196, 9230, 9184, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65322 dup_rows=23886 dup_rows_remote=21082 dedup_ratio=0.242981 router_tokens_per_expert=[762, 761, 815, 802, 788, 773, 778, 720, 804, 797, 768, 796, 759, 771, 748, 752, 818, 780, 735, 800, 733, 753, 758, 751, 789, 838, 781, 773, 809, 771, 755, 774, 790, 776, 743, 717, 685, 778, 805, 741, 766, 700, 738, 746, 747, 779, 752, 722, 796, 768, 794, 748, 743, 781, 785, 758, 757, 790, 795, 762, 756, 801, 741, 780] payload_rows_per_dst=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] dup_rows_per_dst=[2982, 3032, 2902, 3138, 2878, 2804, 3046, 3104] combine_rows_per_src=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65274 dup_rows=23798 dup_rows_remote=20792 dedup_ratio=0.242086 router_tokens_per_expert=[752, 770, 768, 723, 780, 789, 757, 780, 757, 776, 770, 784, 773, 739, 765, 750, 761, 791, 736, 750, 795, 795, 733, 773, 753, 735, 776, 785, 749, 742, 778, 788, 765, 783, 813, 771, 750, 763, 760, 802, 811, 769, 732, 790, 756, 713, 733, 745, 731, 782, 742, 771, 727, 807, 795, 785, 775, 837, 785, 733, 803, 742, 824, 784] payload_rows_per_dst=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] dup_rows_per_dst=[3006, 2958, 2968, 2962, 2980, 2916, 2878, 3130] combine_rows_per_src=[9232, 9270, 9308, 9242, 9434, 9182, 9402, 9436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64854 dup_rows=24278 dup_rows_remote=21228 dedup_ratio=0.246969 router_tokens_per_expert=[744, 727, 765, 751, 737, 769, 767, 743, 742, 738, 763, 721, 794, 792, 736, 808, 775, 767, 751, 782, 730, 771, 743, 792, 801, 789, 774, 727, 720, 768, 760, 807, 815, 765, 797, 805, 750, 749, 787, 747, 787, 770, 716, 776, 780, 775, 837, 759, 773, 758, 784, 749, 751, 794, 780, 740, 725, 770, 800, 772, 823, 792, 789, 783] payload_rows_per_dst=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] dup_rows_per_dst=[2930, 2964, 3050, 2998, 3128, 3110, 2994, 3104] combine_rows_per_src=[9076, 9224, 9172, 9294, 9302, 9290, 9264, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64922 dup_rows=24016 dup_rows_remote=20900 dedup_ratio=0.244303 router_tokens_per_expert=[774, 818, 745, 779, 767, 764, 815, 774, 731, 789, 781, 801, 705, 828, 808, 798, 724, 777, 709, 759, 704, 731, 791, 731, 765, 759, 747, 791, 812, 776, 787, 741, 766, 812, 766, 767, 719, 758, 824, 757, 759, 759, 770, 791, 731, 775, 724, 767, 794, 787, 747, 778, 810, 794, 792, 765, 762, 756, 760, 744, 721, 782, 764, 770] payload_rows_per_dst=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] dup_rows_per_dst=[3100, 3116, 2752, 3066, 3054, 2966, 3052, 2910] combine_rows_per_src=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=341152 combine_payload_bytes=264257536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264598688 ingress_bytes=298860544 link_bytes=298860544 token_bytes=528856224 weight_movement_bytes=34603008 total_nvlink_bytes=563459232 weight_to_token_ratio=0.065430 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124588167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65050 dup_rows=24056 dup_rows_remote=21072 dedup_ratio=0.244710 router_tokens_per_expert=[820, 815, 759, 767, 783, 770, 742, 763, 749, 771, 782, 788, 748, 766, 750, 753, 803, 794, 778, 790, 739, 797, 745, 780, 826, 729, 765, 793, 749, 774, 792, 781, 756, 753, 759, 767, 742, 780, 758, 776, 785, 743, 772, 790, 805, 756, 722, 769, 706, 758, 799, 770, 759, 766, 730, 769, 779, 832, 785, 751, 740, 731, 736, 747] payload_rows_per_dst=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] dup_rows_per_dst=[3044, 3040, 2994, 3058, 2984, 3044, 2944, 2948] combine_rows_per_src=[9394, 9174, 9458, 9360, 9198, 9240, 9170, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65322 dup_rows=23886 dup_rows_remote=21082 dedup_ratio=0.242981 router_tokens_per_expert=[762, 761, 815, 802, 788, 773, 778, 720, 804, 797, 768, 796, 759, 771, 748, 752, 818, 780, 735, 800, 733, 753, 758, 751, 789, 838, 781, 773, 809, 771, 755, 774, 790, 776, 743, 717, 685, 778, 805, 741, 766, 700, 738, 746, 747, 779, 752, 722, 796, 768, 794, 748, 743, 781, 785, 758, 757, 790, 795, 762, 756, 801, 741, 780] payload_rows_per_dst=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] dup_rows_per_dst=[2982, 3032, 2902, 3138, 2878, 2804, 3046, 3104] combine_rows_per_src=[9416, 9358, 9354, 9442, 9192, 9096, 9300, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64964 dup_rows=23944 dup_rows_remote=20770 dedup_ratio=0.243571 router_tokens_per_expert=[757, 829, 786, 741, 815, 775, 773, 769, 760, 771, 811, 747, 748, 744, 730, 764, 811, 681, 747, 783, 836, 770, 817, 753, 739, 764, 770, 793, 758, 830, 833, 798, 738, 741, 711, 794, 814, 747, 726, 819, 798, 756, 787, 763, 749, 787, 808, 782, 746, 729, 746, 701, 786, 709, 827, 777, 759, 745, 731, 716, 761, 771, 764, 761] payload_rows_per_dst=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] dup_rows_per_dst=[3080, 2886, 3058, 3174, 2886, 3090, 2904, 2866] combine_rows_per_src=[9410, 9264, 9338, 9396, 9294, 9370, 9138, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266435480 ingress_bytes=404504576 link_bytes=404504576 token_bytes=532528024 weight_movement_bytes=138412032 total_nvlink_bytes=670940056 weight_to_token_ratio=0.259915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522118154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=344264 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266281160 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532218056 weight_movement_bytes=34603008 total_nvlink_bytes=566821064 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267558912 dispatch_meta_bytes=345616 combine_payload_bytes=267558912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267904528 ingress_bytes=302161920 link_bytes=302161920 token_bytes=535463440 weight_movement_bytes=34603008 total_nvlink_bytes=570066448 weight_to_token_ratio=0.064623 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137010979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344408 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576216 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808024 weight_movement_bytes=34603008 total_nvlink_bytes=567411032 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344264 combine_payload_bytes=267362304 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267706568 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068872 weight_movement_bytes=69206016 total_nvlink_bytes=604274888 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344328 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986312 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628296 weight_movement_bytes=34603008 total_nvlink_bytes=566231304 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64516 dup_rows=24148 dup_rows_remote=20772 dedup_ratio=0.245646 router_tokens_per_expert=[773, 746, 763, 833, 759, 772, 768, 814, 726, 750, 728, 769, 781, 764, 738, 784, 774, 775, 779, 738, 768, 798, 762, 800, 788, 762, 794, 790, 797, 779, 761, 760, 777, 750, 746, 779, 773, 729, 786, 759, 760, 797, 734, 761, 746, 736, 811, 804, 757, 776, 747, 774, 709, 762, 793, 714, 772, 784, 763, 767, 773, 725, 797, 798] payload_rows_per_dst=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] dup_rows_per_dst=[2682, 3008, 3074, 2668, 3184, 3054, 3102, 3376] combine_rows_per_src=[8978, 9072, 9368, 8756, 9602, 9384, 9356, 9640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343288 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266263800 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532184312 weight_movement_bytes=34603008 total_nvlink_bytes=566787320 weight_to_token_ratio=0.065021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344488 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789288 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234088 weight_movement_bytes=34603008 total_nvlink_bytes=567837096 weight_to_token_ratio=0.064893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64820 dup_rows=24192 dup_rows_remote=21140 dedup_ratio=0.246094 router_tokens_per_expert=[736, 723, 764, 753, 779, 811, 730, 740, 754, 777, 790, 808, 739, 727, 776, 726, 754, 782, 772, 814, 736, 744, 756, 771, 749, 769, 735, 830, 767, 782, 756, 772, 785, 766, 785, 780, 758, 788, 788, 763, 746, 780, 742, 752, 816, 772, 748, 790, 820, 802, 765, 813, 722, 772, 742, 765, 791, 805, 755, 775, 748, 747, 785, 764] payload_rows_per_dst=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] dup_rows_per_dst=[2842, 3068, 3032, 2852, 3136, 3014, 3196, 3052] combine_rows_per_src=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65020 dup_rows=23962 dup_rows_remote=21076 dedup_ratio=0.243754 router_tokens_per_expert=[746, 829, 805, 754, 778, 784, 770, 746, 779, 756, 766, 774, 762, 727, 769, 756, 753, 761, 779, 721, 788, 759, 764, 779, 786, 760, 750, 800, 779, 792, 788, 782, 780, 779, 760, 772, 780, 758, 741, 723, 731, 752, 808, 753, 761, 736, 828, 761, 765, 728, 804, 761, 756, 757, 801, 759, 791, 777, 781, 743, 721, 754, 807, 782] payload_rows_per_dst=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] dup_rows_per_dst=[3036, 2952, 2886, 3100, 2978, 2958, 3072, 2980] combine_rows_per_src=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=65052 dup_rows=24128 dup_rows_remote=21160 dedup_ratio=0.245443 router_tokens_per_expert=[775, 751, 770, 774, 843, 756, 776, 793, 787, 762, 806, 790, 784, 741, 814, 734, 776, 747, 768, 786, 767, 761, 737, 759, 760, 710, 773, 785, 737, 750, 780, 751, 813, 806, 745, 724, 751, 741, 721, 782, 754, 789, 806, 755, 744, 770, 757, 764, 798, 760, 739, 753, 785, 779, 794, 757, 775, 780, 794, 761, 780, 761, 765, 746] payload_rows_per_dst=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] dup_rows_per_dst=[3182, 3088, 2986, 2968, 2948, 2984, 2996, 2976] combine_rows_per_src=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64712 dup_rows=24188 dup_rows_remote=21164 dedup_ratio=0.246053 router_tokens_per_expert=[758, 789, 776, 729, 760, 751, 747, 740, 793, 767, 770, 783, 797, 765, 701, 815, 759, 769, 776, 736, 752, 749, 736, 774, 773, 776, 783, 779, 757, 748, 782, 763, 783, 795, 772, 781, 734, 760, 727, 746, 738, 805, 738, 780, 796, 792, 792, 773, 765, 765, 746, 799, 758, 769, 749, 795, 805, 718, 788, 781, 797, 766, 808, 778] payload_rows_per_dst=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] dup_rows_per_dst=[2968, 3102, 2946, 3028, 2992, 3024, 2998, 3130] combine_rows_per_src=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64992 dup_rows=23972 dup_rows_remote=20958 dedup_ratio=0.243856 router_tokens_per_expert=[743, 737, 717, 801, 774, 760, 739, 773, 750, 754, 772, 793, 795, 783, 742, 764, 787, 744, 806, 739, 778, 758, 723, 761, 766, 755, 724, 759, 792, 787, 814, 764, 772, 753, 753, 725, 792, 796, 801, 754, 785, 765, 820, 789, 771, 806, 744, 802, 786, 794, 769, 720, 754, 781, 790, 783, 791, 736, 761, 741, 751, 816, 745, 752] payload_rows_per_dst=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] dup_rows_per_dst=[2876, 3004, 2972, 2984, 2966, 3168, 3014, 2988] combine_rows_per_src=[9212, 9302, 9220, 9338, 9326, 9396, 9340, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267558912 dispatch_meta_bytes=345616 combine_payload_bytes=267558912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267904528 ingress_bytes=302161920 link_bytes=302161920 token_bytes=535463440 weight_movement_bytes=34603008 total_nvlink_bytes=570066448 weight_to_token_ratio=0.064623 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137010979e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266435480 ingress_bytes=404504576 link_bytes=404504576 token_bytes=532528024 weight_movement_bytes=138412032 total_nvlink_bytes=670940056 weight_to_token_ratio=0.259915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522118154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64922 dup_rows=24016 dup_rows_remote=20900 dedup_ratio=0.244303 router_tokens_per_expert=[774, 818, 745, 779, 767, 764, 815, 774, 731, 789, 781, 801, 705, 828, 808, 798, 724, 777, 709, 759, 704, 731, 791, 731, 765, 759, 747, 791, 812, 776, 787, 741, 766, 812, 766, 767, 719, 758, 824, 757, 759, 759, 770, 791, 731, 775, 724, 767, 794, 787, 747, 778, 810, 794, 792, 765, 762, 756, 760, 744, 721, 782, 764, 770] payload_rows_per_dst=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] dup_rows_per_dst=[3100, 3116, 2752, 3066, 3054, 2966, 3052, 2910] combine_rows_per_src=[9372, 9366, 9100, 9290, 9284, 9186, 9482, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264257536 dispatch_meta_bytes=341152 combine_payload_bytes=264257536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264598688 ingress_bytes=298860544 link_bytes=298860544 token_bytes=528856224 weight_movement_bytes=34603008 total_nvlink_bytes=563459232 weight_to_token_ratio=0.065430 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124588167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343800 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551032 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758264 weight_movement_bytes=34603008 total_nvlink_bytes=567361272 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65172 dup_rows=23968 dup_rows_remote=21128 dedup_ratio=0.243815 router_tokens_per_expert=[758, 741, 789, 792, 742, 834, 750, 778, 775, 765, 753, 752, 738, 723, 772, 724, 755, 819, 774, 738, 799, 735, 767, 762, 820, 825, 750, 782, 769, 806, 741, 741, 747, 788, 781, 759, 765, 792, 750, 712, 765, 797, 762, 778, 774, 767, 727, 765, 758, 749, 763, 790, 806, 799, 802, 769, 726, 785, 755, 775, 735, 828, 722, 792] payload_rows_per_dst=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] dup_rows_per_dst=[3082, 2840, 2994, 3098, 2894, 2910, 3104, 3046] combine_rows_per_src=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343840 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846560 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349280 weight_movement_bytes=69206016 total_nvlink_bytes=600555296 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65130 dup_rows=23840 dup_rows_remote=20814 dedup_ratio=0.242513 router_tokens_per_expert=[783, 770, 803, 751, 779, 742, 783, 769, 771, 728, 786, 755, 765, 785, 749, 779, 760, 757, 788, 746, 768, 847, 763, 809, 782, 719, 774, 807, 765, 753, 728, 759, 723, 700, 775, 801, 801, 768, 757, 807, 751, 769, 747, 776, 728, 794, 779, 752, 780, 785, 755, 757, 763, 747, 797, 774, 777, 777, 760, 806, 783, 739, 758, 743] payload_rows_per_dst=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] dup_rows_per_dst=[3026, 3002, 3088, 2942, 3006, 2884, 2932, 2960] combine_rows_per_src=[9334, 9346, 9388, 9120, 9258, 9308, 9384, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343288 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266263800 ingress_bytes=300523520 link_bytes=300523520 token_bytes=532184312 weight_movement_bytes=34603008 total_nvlink_bytes=566787320 weight_to_token_ratio=0.065021 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130845812e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64926 dup_rows=24086 dup_rows_remote=21140 dedup_ratio=0.245015 router_tokens_per_expert=[765, 782, 827, 775, 780, 721, 778, 771, 749, 745, 801, 773, 781, 752, 737, 829, 766, 772, 731, 732, 755, 804, 790, 769, 762, 769, 773, 765, 827, 808, 749, 833, 780, 765, 787, 730, 767, 785, 720, 762, 782, 786, 717, 763, 791, 803, 743, 772, 797, 790, 763, 763, 794, 697, 746, 720, 786, 752, 794, 754, 723, 729, 746, 774] payload_rows_per_dst=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] dup_rows_per_dst=[3048, 3022, 2946, 3186, 3014, 3024, 2962, 2884] combine_rows_per_src=[9350, 9312, 9292, 9386, 9178, 9290, 9178, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64734 dup_rows=24168 dup_rows_remote=21074 dedup_ratio=0.245850 router_tokens_per_expert=[762, 767, 804, 773, 773, 814, 750, 720, 747, 801, 816, 776, 755, 745, 750, 790, 754, 745, 794, 765, 719, 772, 766, 750, 774, 784, 806, 746, 806, 772, 770, 767, 799, 802, 758, 822, 754, 792, 773, 748, 783, 777, 747, 792, 746, 715, 783, 773, 744, 777, 733, 739, 724, 751, 820, 734, 709, 745, 790, 795, 767, 772, 755, 800] payload_rows_per_dst=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] dup_rows_per_dst=[3086, 3032, 2912, 3084, 3094, 2950, 3002, 3008] combine_rows_per_src=[9240, 9328, 9218, 9366, 9402, 9282, 9042, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74116 payload_rows_remote=64712 dup_rows=24188 dup_rows_remote=21164 dedup_ratio=0.246053 router_tokens_per_expert=[758, 789, 776, 729, 760, 751, 747, 740, 793, 767, 770, 783, 797, 765, 701, 815, 759, 769, 776, 736, 752, 749, 736, 774, 773, 776, 783, 779, 757, 748, 782, 763, 783, 795, 772, 781, 734, 760, 727, 746, 738, 805, 738, 780, 796, 792, 792, 773, 765, 765, 746, 799, 758, 769, 749, 795, 805, 718, 788, 781, 797, 766, 808, 778] payload_rows_per_dst=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] dup_rows_per_dst=[2968, 3102, 2946, 3028, 2992, 3024, 2998, 3130] combine_rows_per_src=[9132, 9280, 9156, 9294, 9204, 9404, 9294, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=65052 dup_rows=24128 dup_rows_remote=21160 dedup_ratio=0.245443 router_tokens_per_expert=[775, 751, 770, 774, 843, 756, 776, 793, 787, 762, 806, 790, 784, 741, 814, 734, 776, 747, 768, 786, 767, 761, 737, 759, 760, 710, 773, 785, 737, 750, 780, 751, 813, 806, 745, 724, 751, 741, 721, 782, 754, 789, 806, 755, 744, 770, 757, 764, 798, 760, 739, 753, 785, 779, 794, 757, 775, 780, 794, 761, 780, 761, 765, 746] payload_rows_per_dst=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] dup_rows_per_dst=[3182, 3088, 2986, 2968, 2948, 2984, 2996, 2976] combine_rows_per_src=[9294, 9348, 9216, 9124, 9218, 9294, 9334, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344848 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797840 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250832 weight_movement_bytes=69206016 total_nvlink_bytes=602456848 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344384 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266666304 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532988224 weight_movement_bytes=34603008 total_nvlink_bytes=567591232 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343504 combine_payload_bytes=265060352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265403856 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530464208 weight_movement_bytes=138412032 total_nvlink_bytes=668876240 weight_to_token_ratio=0.260926 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343800 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551032 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758264 weight_movement_bytes=34603008 total_nvlink_bytes=567361272 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=345200 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289712 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534234224 weight_movement_bytes=34603008 total_nvlink_bytes=568837232 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343776 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267116256 ingress_bytes=405184512 link_bytes=405184512 token_bytes=533888736 weight_movement_bytes=138412032 total_nvlink_bytes=672300768 weight_to_token_ratio=0.259253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524676699e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64904 dup_rows=24124 dup_rows_remote=21170 dedup_ratio=0.245402 router_tokens_per_expert=[760, 743, 755, 798, 803, 775, 765, 802, 781, 785, 775, 756, 774, 755, 768, 757, 793, 776, 785, 773, 761, 786, 813, 749, 790, 729, 767, 813, 711, 758, 764, 789, 757, 773, 783, 754, 730, 756, 732, 769, 779, 744, 777, 750, 778, 787, 813, 769, 812, 764, 797, 775, 756, 730, 756, 725, 766, 766, 705, 710, 842, 748, 762, 778] payload_rows_per_dst=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] dup_rows_per_dst=[3030, 2966, 3160, 3044, 2982, 3000, 2954, 2988] combine_rows_per_src=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64820 dup_rows=24192 dup_rows_remote=21140 dedup_ratio=0.246094 router_tokens_per_expert=[736, 723, 764, 753, 779, 811, 730, 740, 754, 777, 790, 808, 739, 727, 776, 726, 754, 782, 772, 814, 736, 744, 756, 771, 749, 769, 735, 830, 767, 782, 756, 772, 785, 766, 785, 780, 758, 788, 788, 763, 746, 780, 742, 752, 816, 772, 748, 790, 820, 802, 765, 813, 722, 772, 742, 765, 791, 805, 755, 775, 748, 747, 785, 764] payload_rows_per_dst=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] dup_rows_per_dst=[2842, 3068, 3032, 2852, 3136, 3014, 3196, 3052] combine_rows_per_src=[9096, 9126, 9226, 9216, 9542, 9274, 9340, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343232 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265493696 ingress_bytes=299753472 link_bytes=299753472 token_bytes=530644160 weight_movement_bytes=34603008 total_nvlink_bytes=565247168 weight_to_token_ratio=0.065209 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127948183e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265936896 dispatch_meta_bytes=344264 combine_payload_bytes=265936896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266281160 ingress_bytes=300539904 link_bytes=300539904 token_bytes=532218056 weight_movement_bytes=34603008 total_nvlink_bytes=566821064 weight_to_token_ratio=0.065017 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130907463e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344848 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797840 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250832 weight_movement_bytes=69206016 total_nvlink_bytes=602456848 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265060352 dispatch_meta_bytes=343504 combine_payload_bytes=265060352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265403856 ingress_bytes=403472384 link_bytes=403472384 token_bytes=530464208 weight_movement_bytes=138412032 total_nvlink_bytes=668876240 weight_to_token_ratio=0.260926 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518234099e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65172 dup_rows=23968 dup_rows_remote=21128 dedup_ratio=0.243815 router_tokens_per_expert=[758, 741, 789, 792, 742, 834, 750, 778, 775, 765, 753, 752, 738, 723, 772, 724, 755, 819, 774, 738, 799, 735, 767, 762, 820, 825, 750, 782, 769, 806, 741, 741, 747, 788, 781, 759, 765, 792, 750, 712, 765, 797, 762, 778, 774, 767, 727, 765, 758, 749, 763, 790, 806, 799, 802, 769, 726, 785, 755, 775, 735, 828, 722, 792] payload_rows_per_dst=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] dup_rows_per_dst=[3082, 2840, 2994, 3098, 2894, 2910, 3104, 3046] combine_rows_per_src=[9286, 9164, 9304, 9370, 9294, 9360, 9368, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344296 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191080 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037864 weight_movement_bytes=34603008 total_nvlink_bytes=566640872 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343840 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846560 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349280 weight_movement_bytes=69206016 total_nvlink_bytes=600555296 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64912 dup_rows=24186 dup_rows_remote=21122 dedup_ratio=0.246033 router_tokens_per_expert=[738, 775, 783, 774, 789, 758, 773, 735, 742, 790, 796, 774, 758, 771, 759, 792, 785, 759, 785, 739, 805, 758, 781, 810, 756, 786, 751, 732, 727, 809, 712, 817, 749, 756, 768, 749, 769, 775, 762, 815, 780, 766, 769, 766, 784, 773, 773, 775, 769, 756, 753, 724, 791, 716, 801, 759, 799, 737, 775, 723, 749, 829, 743, 780] payload_rows_per_dst=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] dup_rows_per_dst=[3064, 3102, 3232, 2982, 2982, 2956, 2900, 2968] combine_rows_per_src=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=345200 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289712 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534234224 weight_movement_bytes=34603008 total_nvlink_bytes=568837232 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65184 dup_rows=23926 dup_rows_remote=20918 dedup_ratio=0.243388 router_tokens_per_expert=[791, 738, 773, 755, 770, 738, 744, 743, 808, 736, 809, 789, 770, 761, 763, 781, 768, 745, 755, 770, 763, 758, 788, 793, 776, 816, 769, 782, 792, 766, 735, 778, 765, 760, 768, 749, 800, 786, 723, 750, 813, 728, 814, 790, 719, 738, 751, 777, 806, 798, 809, 767, 754, 772, 709, 734, 758, 773, 788, 771, 757, 751, 769, 782] payload_rows_per_dst=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] dup_rows_per_dst=[2888, 3106, 3006, 3078, 3008, 2952, 2928, 2960] combine_rows_per_src=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64618 dup_rows=24284 dup_rows_remote=21240 dedup_ratio=0.247030 router_tokens_per_expert=[800, 726, 758, 704, 769, 743, 754, 716, 824, 754, 745, 756, 785, 796, 772, 777, 756, 716, 800, 803, 753, 811, 741, 736, 790, 706, 755, 729, 786, 820, 834, 769, 739, 766, 795, 806, 772, 755, 799, 780, 732, 794, 726, 801, 768, 789, 785, 767, 750, 786, 781, 783, 774, 782, 712, 763, 775, 734, 772, 787, 754, 783, 792, 766] payload_rows_per_dst=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] dup_rows_per_dst=[2818, 2778, 3050, 2890, 3282, 3080, 3342, 3044] combine_rows_per_src=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64800 dup_rows=24194 dup_rows_remote=21206 dedup_ratio=0.246114 router_tokens_per_expert=[744, 747, 744, 779, 762, 786, 731, 802, 707, 720, 829, 774, 798, 776, 797, 730, 776, 742, 743, 812, 768, 780, 798, 770, 805, 785, 757, 757, 744, 782, 739, 780, 752, 752, 789, 766, 782, 766, 771, 785, 775, 809, 767, 749, 797, 785, 758, 745, 765, 757, 801, 767, 770, 780, 762, 746, 768, 768, 739, 771, 732, 769, 768, 777] payload_rows_per_dst=[9206, 9198, 9374, 9310, 9236, 9304, 9210, 9272] dup_rows_per_dst=[2984, 3064, 3004, 2988, 3090, 3066, 3086, 2912] combine_rows_per_src=[9206, 9198, 9374, 9310, 9236, 9304, 9210, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64818 dup_rows=24056 dup_rows_remote=20916 dedup_ratio=0.244710 router_tokens_per_expert=[751, 739, 774, 765, 711, 743, 765, 790, 743, 771, 795, 779, 811, 767, 754, 732, 782, 813, 745, 806, 784, 780, 810, 765, 790, 779, 793, 763, 751, 797, 778, 743, 742, 767, 746, 757, 738, 791, 783, 748, 821, 799, 778, 777, 749, 747, 764, 824, 735, 737, 758, 732, 721, 772, 738, 786, 771, 761, 733, 782, 779, 782, 775, 790] payload_rows_per_dst=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] dup_rows_per_dst=[2900, 3046, 3140, 3060, 2990, 3174, 2752, 2994] combine_rows_per_src=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64948 dup_rows=23940 dup_rows_remote=20878 dedup_ratio=0.243530 router_tokens_per_expert=[758, 746, 716, 742, 780, 713, 750, 792, 792, 791, 771, 777, 775, 768, 786, 757, 740, 798, 758, 765, 717, 787, 756, 766, 742, 780, 796, 745, 752, 726, 739, 774, 740, 782, 760, 769, 809, 738, 843, 740, 780, 778, 762, 768, 755, 814, 810, 772, 712, 781, 780, 782, 793, 798, 810, 816, 704, 776, 774, 766, 743, 754, 809, 779] payload_rows_per_dst=[9134, 9352, 9220, 9196, 9382, 9416, 9408, 9256] dup_rows_per_dst=[2860, 3082, 2954, 2912, 2980, 3062, 3136, 2954] combine_rows_per_src=[9134, 9352, 9220, 9196, 9382, 9416, 9408, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65020 dup_rows=23962 dup_rows_remote=21076 dedup_ratio=0.243754 router_tokens_per_expert=[746, 829, 805, 754, 778, 784, 770, 746, 779, 756, 766, 774, 762, 727, 769, 756, 753, 761, 779, 721, 788, 759, 764, 779, 786, 760, 750, 800, 779, 792, 788, 782, 780, 779, 760, 772, 780, 758, 741, 723, 731, 752, 808, 753, 761, 736, 828, 761, 765, 728, 804, 761, 756, 757, 801, 759, 791, 777, 781, 743, 721, 754, 807, 782] payload_rows_per_dst=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] dup_rows_per_dst=[3036, 2952, 2886, 3100, 2978, 2958, 3072, 2980] combine_rows_per_src=[9388, 9226, 9322, 9374, 9208, 9302, 9190, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344136 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223688 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103240 weight_movement_bytes=69206016 total_nvlink_bytes=601309256 weight_to_token_ratio=0.130061 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343432 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018760 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694088 weight_movement_bytes=69206016 total_nvlink_bytes=598900104 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64924 dup_rows=24018 dup_rows_remote=20978 dedup_ratio=0.244324 router_tokens_per_expert=[776, 737, 760, 741, 739, 745, 773, 799, 800, 749, 755, 787, 782, 761, 766, 801, 749, 747, 796, 768, 822, 739, 780, 740, 779, 775, 765, 800, 765, 790, 758, 842, 768, 729, 785, 776, 754, 772, 767, 767, 804, 765, 793, 778, 776, 840, 767, 732, 717, 736, 800, 685, 773, 773, 713, 767, 780, 770, 766, 768, 760, 749, 801, 735] payload_rows_per_dst=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] dup_rows_per_dst=[2996, 3040, 3044, 3050, 2950, 3120, 2844, 2974] combine_rows_per_src=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65262 dup_rows=23836 dup_rows_remote=20938 dedup_ratio=0.242472 router_tokens_per_expert=[808, 810, 810, 745, 771, 721, 721, 752, 760, 804, 745, 758, 788, 800, 755, 754, 757, 750, 769, 763, 770, 826, 795, 741, 767, 823, 737, 756, 755, 810, 731, 721, 766, 780, 717, 759, 693, 788, 821, 750, 780, 791, 797, 791, 776, 727, 771, 797, 746, 715, 743, 748, 780, 735, 804, 781, 781, 759, 764, 791, 757, 777, 776, 818] payload_rows_per_dst=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] dup_rows_per_dst=[3032, 3018, 3014, 2870, 2920, 3084, 2898, 3000] combine_rows_per_src=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64904 dup_rows=24124 dup_rows_remote=21170 dedup_ratio=0.245402 router_tokens_per_expert=[760, 743, 755, 798, 803, 775, 765, 802, 781, 785, 775, 756, 774, 755, 768, 757, 793, 776, 785, 773, 761, 786, 813, 749, 790, 729, 767, 813, 711, 758, 764, 789, 757, 773, 783, 754, 730, 756, 732, 769, 779, 744, 777, 750, 778, 787, 813, 769, 812, 764, 797, 775, 756, 730, 756, 725, 766, 766, 705, 710, 842, 748, 762, 778] payload_rows_per_dst=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] dup_rows_per_dst=[3030, 2966, 3160, 3044, 2982, 3000, 2954, 2988] combine_rows_per_src=[9372, 9336, 9312, 9198, 9126, 9394, 9276, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64912 dup_rows=24186 dup_rows_remote=21122 dedup_ratio=0.246033 router_tokens_per_expert=[738, 775, 783, 774, 789, 758, 773, 735, 742, 790, 796, 774, 758, 771, 759, 792, 785, 759, 785, 739, 805, 758, 781, 810, 756, 786, 751, 732, 727, 809, 712, 817, 749, 756, 768, 749, 769, 775, 762, 815, 780, 766, 769, 766, 784, 773, 773, 775, 769, 756, 753, 724, 791, 716, 801, 759, 799, 737, 775, 723, 749, 829, 743, 780] payload_rows_per_dst=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] dup_rows_per_dst=[3064, 3102, 3232, 2982, 2982, 2956, 2900, 2968] combine_rows_per_src=[9206, 9346, 9402, 9206, 9304, 9198, 9238, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65184 dup_rows=23926 dup_rows_remote=20918 dedup_ratio=0.243388 router_tokens_per_expert=[791, 738, 773, 755, 770, 738, 744, 743, 808, 736, 809, 789, 770, 761, 763, 781, 768, 745, 755, 770, 763, 758, 788, 793, 776, 816, 769, 782, 792, 766, 735, 778, 765, 760, 768, 749, 800, 786, 723, 750, 813, 728, 814, 790, 719, 738, 751, 777, 806, 798, 809, 767, 754, 772, 709, 734, 758, 773, 788, 771, 757, 751, 769, 782] payload_rows_per_dst=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] dup_rows_per_dst=[2888, 3106, 3006, 3078, 3008, 2952, 2928, 2960] combine_rows_per_src=[9216, 9328, 9274, 9350, 9194, 9308, 9370, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64618 dup_rows=24284 dup_rows_remote=21240 dedup_ratio=0.247030 router_tokens_per_expert=[800, 726, 758, 704, 769, 743, 754, 716, 824, 754, 745, 756, 785, 796, 772, 777, 756, 716, 800, 803, 753, 811, 741, 736, 790, 706, 755, 729, 786, 820, 834, 769, 739, 766, 795, 806, 772, 755, 799, 780, 732, 794, 726, 801, 768, 789, 785, 767, 750, 786, 781, 783, 774, 782, 712, 763, 775, 734, 772, 787, 754, 783, 792, 766] payload_rows_per_dst=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] dup_rows_per_dst=[2818, 2778, 3050, 2890, 3282, 3080, 3342, 3044] combine_rows_per_src=[9122, 8930, 9152, 9082, 9430, 9272, 9630, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344408 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338072 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331736 weight_movement_bytes=34603008 total_nvlink_bytes=568934744 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=344024 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265764824 ingress_bytes=369229824 link_bytes=369229824 token_bytes=531185624 weight_movement_bytes=103809024 total_nvlink_bytes=634994648 weight_to_token_ratio=0.195429 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389382102e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342936 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265837464 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531331992 weight_movement_bytes=69206016 total_nvlink_bytes=600538008 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343304 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266370312 ingress_bytes=335233024 link_bytes=335233024 token_bytes=532397320 weight_movement_bytes=69206016 total_nvlink_bytes=601603336 weight_to_token_ratio=0.129989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261454881e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344384 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266666304 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532988224 weight_movement_bytes=34603008 total_nvlink_bytes=567591232 weight_to_token_ratio=0.064923 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64924 dup_rows=24018 dup_rows_remote=20978 dedup_ratio=0.244324 router_tokens_per_expert=[776, 737, 760, 741, 739, 745, 773, 799, 800, 749, 755, 787, 782, 761, 766, 801, 749, 747, 796, 768, 822, 739, 780, 740, 779, 775, 765, 800, 765, 790, 758, 842, 768, 729, 785, 776, 754, 772, 767, 767, 804, 765, 793, 778, 776, 840, 767, 732, 717, 736, 800, 685, 773, 773, 713, 767, 780, 770, 766, 768, 760, 749, 801, 735] payload_rows_per_dst=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] dup_rows_per_dst=[2996, 3040, 3044, 3050, 2950, 3120, 2844, 2974] combine_rows_per_src=[9144, 9362, 9238, 9498, 9286, 9390, 9084, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344800 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657952 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534971104 weight_movement_bytes=34603008 total_nvlink_bytes=569574112 weight_to_token_ratio=0.064682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343432 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018760 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694088 weight_movement_bytes=69206016 total_nvlink_bytes=598900104 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64216 dup_rows=24148 dup_rows_remote=20504 dedup_ratio=0.245646 router_tokens_per_expert=[800, 796, 774, 775, 806, 767, 724, 798, 772, 739, 733, 766, 750, 743, 747, 838, 745, 806, 689, 742, 774, 801, 742, 739, 724, 748, 818, 796, 792, 748, 788, 767, 756, 754, 721, 761, 784, 774, 752, 785, 781, 766, 790, 788, 766, 797, 743, 799, 729, 763, 775, 789, 712, 747, 798, 789, 773, 792, 802, 734, 737, 743, 802, 803] payload_rows_per_dst=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] dup_rows_per_dst=[3644, 3068, 2862, 3058, 3002, 2612, 2988, 2914] combine_rows_per_src=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343608 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266272312 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201016 weight_movement_bytes=69206016 total_nvlink_bytes=601407032 weight_to_token_ratio=0.130037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65052 dup_rows=24052 dup_rows_remote=21072 dedup_ratio=0.244670 router_tokens_per_expert=[770, 783, 761, 785, 705, 753, 719, 739, 735, 810, 752, 741, 814, 848, 706, 746, 751, 785, 742, 799, 741, 778, 774, 748, 789, 805, 750, 775, 807, 759, 799, 762, 779, 756, 728, 766, 808, 784, 758, 711, 775, 791, 805, 766, 821, 782, 792, 788, 797, 746, 751, 751, 782, 780, 736, 710, 746, 794, 735, 764, 785, 818, 768, 748] payload_rows_per_dst=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] dup_rows_per_dst=[2954, 2946, 3014, 3052, 2980, 3174, 2892, 3040] combine_rows_per_src=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344296 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266191080 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037864 weight_movement_bytes=34603008 total_nvlink_bytes=566640872 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344136 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223688 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103240 weight_movement_bytes=69206016 total_nvlink_bytes=601309256 weight_to_token_ratio=0.130061 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344408 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338072 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534331736 weight_movement_bytes=34603008 total_nvlink_bytes=568934744 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64776 dup_rows=24176 dup_rows_remote=21036 dedup_ratio=0.245931 router_tokens_per_expert=[799, 783, 796, 757, 818, 763, 775, 709, 754, 771, 691, 757, 803, 777, 708, 775, 770, 754, 751, 791, 760, 756, 758, 779, 803, 772, 806, 732, 711, 755, 784, 804, 727, 811, 797, 750, 790, 777, 789, 757, 789, 780, 764, 810, 753, 783, 757, 757, 791, 747, 757, 796, 753, 791, 739, 741, 726, 840, 720, 756, 750, 765, 830, 737] payload_rows_per_dst=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] dup_rows_per_dst=[2898, 2896, 2988, 3098, 2860, 3124, 3172, 3140] combine_rows_per_src=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343248 combine_payload_bytes=265322496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265665744 ingress_bytes=299925504 link_bytes=299925504 token_bytes=530988240 weight_movement_bytes=34603008 total_nvlink_bytes=565591248 weight_to_token_ratio=0.065167 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128595526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65072 dup_rows=23968 dup_rows_remote=20988 dedup_ratio=0.243815 router_tokens_per_expert=[751, 809, 704, 765, 781, 768, 788, 746, 860, 775, 762, 750, 767, 780, 779, 832, 790, 747, 748, 782, 770, 795, 801, 756, 788, 790, 759, 748, 742, 766, 783, 746, 775, 755, 774, 768, 715, 730, 780, 732, 822, 778, 784, 771, 722, 795, 771, 798, 723, 778, 800, 749, 756, 751, 735, 748, 769, 765, 730, 758, 792, 789, 767, 744] payload_rows_per_dst=[9210, 9404, 9308, 9264, 9182, 9486, 9192, 9290] dup_rows_per_dst=[3014, 3206, 3070, 2980, 2876, 2996, 2888, 2938] combine_rows_per_src=[9210, 9404, 9308, 9264, 9182, 9486, 9192, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65044 dup_rows=24054 dup_rows_remote=21062 dedup_ratio=0.244690 router_tokens_per_expert=[738, 788, 781, 755, 793, 760, 765, 762, 785, 785, 760, 722, 767, 812, 736, 755, 764, 781, 774, 758, 805, 781, 766, 742, 773, 764, 776, 757, 811, 754, 739, 732, 822, 776, 781, 787, 782, 753, 829, 805, 731, 754, 762, 729, 776, 756, 803, 788, 735, 754, 818, 791, 746, 742, 738, 805, 749, 751, 696, 775, 787, 767, 762, 761] payload_rows_per_dst=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] dup_rows_per_dst=[3018, 3058, 3038, 2968, 3196, 2992, 2918, 2866] combine_rows_per_src=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=338880 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263367616 ingress_bytes=297631744 link_bytes=297631744 token_bytes=526396352 weight_movement_bytes=34603008 total_nvlink_bytes=560999360 weight_to_token_ratio=0.065736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119964291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65010 dup_rows=23988 dup_rows_remote=20906 dedup_ratio=0.244019 router_tokens_per_expert=[789, 725, 764, 755, 764, 743, 815, 742, 764, 774, 783, 779, 710, 795, 748, 815, 728, 776, 750, 776, 812, 765, 795, 792, 785, 717, 756, 793, 793, 734, 758, 790, 777, 786, 789, 803, 762, 764, 773, 751, 762, 763, 741, 745, 720, 775, 762, 735, 767, 734, 791, 785, 802, 745, 765, 809, 786, 773, 769, 734, 794, 813, 772, 720] payload_rows_per_dst=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] dup_rows_per_dst=[2978, 3094, 3082, 2932, 3040, 2808, 3004, 3050] combine_rows_per_src=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64818 dup_rows=24056 dup_rows_remote=20916 dedup_ratio=0.244710 router_tokens_per_expert=[751, 739, 774, 765, 711, 743, 765, 790, 743, 771, 795, 779, 811, 767, 754, 732, 782, 813, 745, 806, 784, 780, 810, 765, 790, 779, 793, 763, 751, 797, 778, 743, 742, 767, 746, 757, 738, 791, 783, 748, 821, 799, 778, 777, 749, 747, 764, 824, 735, 737, 758, 732, 721, 772, 738, 786, 771, 761, 733, 782, 779, 782, 775, 790] payload_rows_per_dst=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] dup_rows_per_dst=[2900, 3046, 3140, 3060, 2990, 3174, 2752, 2994] combine_rows_per_src=[9176, 9258, 9430, 9328, 9154, 9344, 9206, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=65086 dup_rows=24094 dup_rows_remote=21080 dedup_ratio=0.245097 router_tokens_per_expert=[797, 787, 808, 790, 792, 755, 793, 751, 766, 747, 739, 775, 795, 750, 814, 750, 747, 769, 782, 748, 729, 717, 753, 767, 758, 783, 806, 763, 763, 763, 757, 787, 804, 819, 784, 784, 807, 753, 725, 781, 737, 776, 764, 771, 699, 772, 767, 820, 750, 820, 719, 742, 769, 759, 749, 761, 777, 808, 781, 745, 728, 772, 753, 755] payload_rows_per_dst=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] dup_rows_per_dst=[3046, 3038, 2850, 3060, 3096, 2980, 3014, 3010] combine_rows_per_src=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64776 dup_rows=24176 dup_rows_remote=21036 dedup_ratio=0.245931 router_tokens_per_expert=[799, 783, 796, 757, 818, 763, 775, 709, 754, 771, 691, 757, 803, 777, 708, 775, 770, 754, 751, 791, 760, 756, 758, 779, 803, 772, 806, 732, 711, 755, 784, 804, 727, 811, 797, 750, 790, 777, 789, 757, 789, 780, 764, 810, 753, 783, 757, 757, 791, 747, 757, 796, 753, 791, 739, 741, 726, 840, 720, 756, 750, 765, 830, 737] payload_rows_per_dst=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] dup_rows_per_dst=[2898, 2896, 2988, 3098, 2860, 3124, 3172, 3140] combine_rows_per_src=[9222, 9176, 9250, 9272, 9082, 9262, 9512, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344240 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879152 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414064 weight_movement_bytes=69206016 total_nvlink_bytes=602620080 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344496 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797488 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250480 weight_movement_bytes=69206016 total_nvlink_bytes=602456496 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344424 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266764648 ingress_bytes=370229248 link_bytes=370229248 token_bytes=533184872 weight_movement_bytes=103809024 total_nvlink_bytes=636993896 weight_to_token_ratio=0.194696 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393142854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65078 dup_rows=24000 dup_rows_remote=21128 dedup_ratio=0.244141 router_tokens_per_expert=[751, 753, 764, 754, 831, 789, 789, 809, 763, 755, 736, 759, 791, 774, 731, 740, 755, 803, 724, 811, 750, 727, 770, 759, 778, 764, 790, 783, 759, 755, 737, 820, 802, 732, 774, 731, 751, 779, 724, 759, 762, 757, 727, 797, 797, 808, 755, 786, 753, 767, 776, 776, 793, 795, 754, 803, 779, 738, 762, 754, 840, 730, 769, 748] payload_rows_per_dst=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] dup_rows_per_dst=[3086, 2872, 2982, 3046, 2882, 3024, 3130, 2978] combine_rows_per_src=[9394, 9226, 9216, 9326, 9222, 9354, 9304, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=342936 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265837464 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531331992 weight_movement_bytes=69206016 total_nvlink_bytes=600538008 weight_to_token_ratio=0.130250 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529176 weight_movement_bytes=34603008 total_nvlink_bytes=568132184 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65262 dup_rows=23836 dup_rows_remote=20938 dedup_ratio=0.242472 router_tokens_per_expert=[808, 810, 810, 745, 771, 721, 721, 752, 760, 804, 745, 758, 788, 800, 755, 754, 757, 750, 769, 763, 770, 826, 795, 741, 767, 823, 737, 756, 755, 810, 731, 721, 766, 780, 717, 759, 693, 788, 821, 750, 780, 791, 797, 791, 776, 727, 771, 797, 746, 715, 743, 748, 780, 735, 804, 781, 781, 759, 764, 791, 757, 777, 776, 818] payload_rows_per_dst=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] dup_rows_per_dst=[3032, 3018, 3014, 2870, 2920, 3084, 2898, 3000] combine_rows_per_src=[9244, 9310, 9328, 9330, 9228, 9376, 9206, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65044 dup_rows=24054 dup_rows_remote=21062 dedup_ratio=0.244690 router_tokens_per_expert=[738, 788, 781, 755, 793, 760, 765, 762, 785, 785, 760, 722, 767, 812, 736, 755, 764, 781, 774, 758, 805, 781, 766, 742, 773, 764, 776, 757, 811, 754, 739, 732, 822, 776, 781, 787, 782, 753, 829, 805, 731, 754, 762, 729, 776, 756, 803, 788, 735, 754, 818, 791, 746, 742, 738, 805, 749, 751, 696, 775, 787, 767, 762, 761] payload_rows_per_dst=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] dup_rows_per_dst=[3018, 3058, 3038, 2968, 3196, 2992, 2918, 2866] combine_rows_per_src=[9266, 9186, 9304, 9244, 9474, 9206, 9340, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64216 dup_rows=24148 dup_rows_remote=20504 dedup_ratio=0.245646 router_tokens_per_expert=[800, 796, 774, 775, 806, 767, 724, 798, 772, 739, 733, 766, 750, 743, 747, 838, 745, 806, 689, 742, 774, 801, 742, 739, 724, 748, 818, 796, 792, 748, 788, 767, 756, 754, 721, 761, 784, 774, 752, 785, 781, 766, 790, 788, 766, 797, 743, 799, 729, 763, 775, 789, 712, 747, 798, 789, 773, 792, 802, 734, 737, 743, 802, 803] payload_rows_per_dst=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] dup_rows_per_dst=[3644, 3068, 2862, 3058, 3002, 2612, 2988, 2914] combine_rows_per_src=[9940, 9364, 9214, 9304, 9172, 8744, 9216, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344240 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879152 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414064 weight_movement_bytes=69206016 total_nvlink_bytes=602620080 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343664 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266624624 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532905584 weight_movement_bytes=34603008 total_nvlink_bytes=567508592 weight_to_token_ratio=0.064933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65052 dup_rows=24052 dup_rows_remote=21072 dedup_ratio=0.244670 router_tokens_per_expert=[770, 783, 761, 785, 705, 753, 719, 739, 735, 810, 752, 741, 814, 848, 706, 746, 751, 785, 742, 799, 741, 778, 774, 748, 789, 805, 750, 775, 807, 759, 799, 762, 779, 756, 728, 766, 808, 784, 758, 711, 775, 791, 805, 766, 821, 782, 792, 788, 797, 746, 751, 751, 782, 780, 736, 710, 746, 794, 735, 764, 785, 818, 768, 748] payload_rows_per_dst=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] dup_rows_per_dst=[2954, 2946, 3014, 3052, 2980, 3174, 2892, 3040] combine_rows_per_src=[9076, 9358, 9222, 9440, 9200, 9466, 9214, 9276] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343248 combine_payload_bytes=265322496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265665744 ingress_bytes=299925504 link_bytes=299925504 token_bytes=530988240 weight_movement_bytes=34603008 total_nvlink_bytes=565591248 weight_to_token_ratio=0.065167 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128595526e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344824 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266904312 ingress_bytes=301162496 link_bytes=301162496 token_bytes=533463800 weight_movement_bytes=34603008 total_nvlink_bytes=568066808 weight_to_token_ratio=0.064865 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133250227e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64948 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[755, 782, 785, 766, 792, 786, 788, 757, 782, 743, 768, 708, 770, 771, 793, 768, 719, 773, 756, 752, 760, 752, 763, 810, 760, 773, 761, 781, 761, 736, 792, 802, 802, 772, 809, 724, 759, 782, 763, 777, 793, 735, 768, 776, 811, 753, 732, 741, 764, 777, 770, 765, 734, 761, 772, 761, 744, 736, 784, 749, 800, 797, 800, 776] payload_rows_per_dst=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] dup_rows_per_dst=[2966, 3090, 3102, 2838, 3082, 2858, 3034, 2926] combine_rows_per_src=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344800 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267657952 ingress_bytes=301916160 link_bytes=301916160 token_bytes=534971104 weight_movement_bytes=34603008 total_nvlink_bytes=569574112 weight_to_token_ratio=0.064682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136086204e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344424 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266764648 ingress_bytes=370229248 link_bytes=370229248 token_bytes=533184872 weight_movement_bytes=103809024 total_nvlink_bytes=636993896 weight_to_token_ratio=0.194696 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393142854e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64670 dup_rows=24072 dup_rows_remote=20880 dedup_ratio=0.244873 router_tokens_per_expert=[748, 737, 799, 755, 781, 712, 772, 815, 796, 728, 763, 816, 772, 742, 736, 744, 786, 716, 746, 758, 778, 794, 750, 745, 725, 780, 761, 804, 720, 763, 802, 765, 787, 790, 737, 764, 767, 795, 781, 776, 755, 745, 783, 704, 764, 785, 765, 750, 802, 759, 768, 755, 798, 787, 754, 854, 768, 781, 800, 765, 764, 804, 780, 756] payload_rows_per_dst=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] dup_rows_per_dst=[2830, 3000, 2764, 2946, 3180, 2902, 3258, 3192] combine_rows_per_src=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=263028736 dispatch_meta_bytes=338880 combine_payload_bytes=263028736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263367616 ingress_bytes=297631744 link_bytes=297631744 token_bytes=526396352 weight_movement_bytes=34603008 total_nvlink_bytes=560999360 weight_to_token_ratio=0.065736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.119964291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=344496 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266797488 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533250480 weight_movement_bytes=69206016 total_nvlink_bytes=602456496 weight_to_token_ratio=0.129781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=342200 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265230520 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530118840 weight_movement_bytes=34603008 total_nvlink_bytes=564721848 weight_to_token_ratio=0.065274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65404 dup_rows=23660 dup_rows_remote=20690 dedup_ratio=0.240682 router_tokens_per_expert=[796, 826, 778, 801, 756, 823, 809, 748, 727, 771, 767, 782, 818, 725, 764, 772, 799, 730, 749, 790, 778, 737, 761, 729, 769, 757, 757, 756, 755, 776, 776, 759, 731, 704, 823, 776, 792, 726, 752, 760, 755, 758, 769, 796, 805, 829, 734, 733, 804, 748, 782, 735, 747, 777, 780, 794, 747, 788, 729, 786, 713, 778, 773, 787] payload_rows_per_dst=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] dup_rows_per_dst=[3136, 2902, 2928, 2970, 2906, 3034, 2934, 2850] combine_rows_per_src=[9538, 9350, 9218, 9240, 9222, 9324, 9400, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64918 dup_rows=23952 dup_rows_remote=20970 dedup_ratio=0.243652 router_tokens_per_expert=[768, 782, 770, 794, 772, 768, 730, 710, 761, 770, 757, 763, 776, 786, 794, 752, 760, 753, 766, 765, 738, 786, 745, 761, 797, 733, 802, 798, 764, 820, 782, 781, 775, 835, 768, 728, 760, 782, 746, 814, 805, 763, 784, 790, 734, 729, 752, 747, 781, 789, 774, 757, 746, 719, 749, 746, 783, 803, 724, 796, 788, 774, 745, 762] payload_rows_per_dst=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] dup_rows_per_dst=[2932, 2994, 3000, 3114, 2982, 2934, 2986, 3010] combine_rows_per_src=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=65432 dup_rows=23574 dup_rows_remote=20612 dedup_ratio=0.239807 router_tokens_per_expert=[782, 731, 772, 839, 754, 728, 791, 739, 762, 773, 773, 746, 731, 766, 747, 739, 783, 795, 759, 777, 773, 787, 810, 777, 767, 730, 775, 797, 771, 735, 765, 814, 729, 715, 751, 789, 764, 757, 789, 769, 757, 843, 760, 710, 761, 748, 781, 770, 784, 777, 719, 792, 774, 777, 804, 770, 791, 766, 779, 776, 747, 818, 746, 751] payload_rows_per_dst=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] dup_rows_per_dst=[2936, 2890, 3030, 2994, 2842, 2962, 3018, 2902] combine_rows_per_src=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64890 dup_rows=24052 dup_rows_remote=20950 dedup_ratio=0.244670 router_tokens_per_expert=[755, 725, 804, 778, 755, 746, 799, 754, 789, 782, 752, 717, 764, 747, 829, 782, 764, 778, 786, 793, 779, 797, 744, 791, 765, 746, 742, 736, 752, 762, 763, 733, 773, 749, 760, 758, 795, 743, 776, 749, 762, 749, 764, 843, 762, 761, 792, 790, 735, 776, 791, 766, 782, 784, 786, 753, 759, 783, 712, 758, 792, 788, 793, 759] payload_rows_per_dst=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] dup_rows_per_dst=[2982, 2992, 3102, 2874, 2990, 3140, 2930, 3042] combine_rows_per_src=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343512 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266370520 ingress_bytes=369836032 link_bytes=369836032 token_bytes=532397528 weight_movement_bytes=103809024 total_nvlink_bytes=636206552 weight_to_token_ratio=0.194984 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391663214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64992 dup_rows=24068 dup_rows_remote=21018 dedup_ratio=0.244832 router_tokens_per_expert=[802, 757, 773, 764, 746, 738, 765, 776, 800, 764, 795, 755, 740, 789, 761, 770, 747, 789, 740, 759, 744, 798, 747, 771, 713, 745, 774, 775, 739, 722, 808, 756, 763, 843, 812, 788, 786, 793, 738, 806, 772, 692, 756, 824, 794, 788, 781, 707, 778, 774, 762, 743, 857, 717, 791, 725, 754, 780, 746, 792, 762, 781, 756, 769] payload_rows_per_dst=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] dup_rows_per_dst=[2982, 3030, 2932, 2854, 3196, 3000, 3050, 3024] combine_rows_per_src=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65010 dup_rows=23988 dup_rows_remote=20906 dedup_ratio=0.244019 router_tokens_per_expert=[789, 725, 764, 755, 764, 743, 815, 742, 764, 774, 783, 779, 710, 795, 748, 815, 728, 776, 750, 776, 812, 765, 795, 792, 785, 717, 756, 793, 793, 734, 758, 790, 777, 786, 789, 803, 762, 764, 773, 751, 762, 763, 741, 745, 720, 775, 762, 735, 767, 734, 791, 785, 802, 745, 765, 809, 786, 773, 769, 734, 794, 813, 772, 720] payload_rows_per_dst=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] dup_rows_per_dst=[2978, 3094, 3082, 2932, 3040, 2808, 3004, 3050] combine_rows_per_src=[9216, 9242, 9306, 9320, 9370, 9198, 9392, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64670 dup_rows=24072 dup_rows_remote=20880 dedup_ratio=0.244873 router_tokens_per_expert=[748, 737, 799, 755, 781, 712, 772, 815, 796, 728, 763, 816, 772, 742, 736, 744, 786, 716, 746, 758, 778, 794, 750, 745, 725, 780, 761, 804, 720, 763, 802, 765, 787, 790, 737, 764, 767, 795, 781, 776, 755, 745, 783, 704, 764, 785, 765, 750, 802, 759, 768, 755, 798, 787, 754, 854, 768, 781, 800, 765, 764, 804, 780, 756] payload_rows_per_dst=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] dup_rows_per_dst=[2830, 3000, 2764, 2946, 3180, 2902, 3258, 3192] combine_rows_per_src=[9098, 9262, 8982, 9294, 9456, 9272, 9306, 9562] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=65086 dup_rows=24094 dup_rows_remote=21080 dedup_ratio=0.245097 router_tokens_per_expert=[797, 787, 808, 790, 792, 755, 793, 751, 766, 747, 739, 775, 795, 750, 814, 750, 747, 769, 782, 748, 729, 717, 753, 767, 758, 783, 806, 763, 763, 763, 757, 787, 804, 819, 784, 784, 807, 753, 725, 781, 737, 776, 764, 771, 699, 772, 767, 820, 750, 820, 719, 742, 769, 759, 749, 761, 777, 808, 781, 745, 728, 772, 753, 755] payload_rows_per_dst=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] dup_rows_per_dst=[3046, 3038, 2850, 3060, 3096, 2980, 3014, 3010] combine_rows_per_src=[9500, 9234, 9174, 9300, 9418, 9232, 9124, 9228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267894784 dispatch_meta_bytes=344376 combine_payload_bytes=267894784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268239160 ingress_bytes=302497792 link_bytes=302497792 token_bytes=536133944 weight_movement_bytes=34603008 total_nvlink_bytes=570736952 weight_to_token_ratio=0.064542 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138274838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343552 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247680 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151808 weight_movement_bytes=34603008 total_nvlink_bytes=566754816 weight_to_token_ratio=0.065025 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=344176 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268353648 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536363120 weight_movement_bytes=34603008 total_nvlink_bytes=570966128 weight_to_token_ratio=0.064514 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74730 payload_rows_remote=65432 dup_rows=23574 dup_rows_remote=20612 dedup_ratio=0.239807 router_tokens_per_expert=[782, 731, 772, 839, 754, 728, 791, 739, 762, 773, 773, 746, 731, 766, 747, 739, 783, 795, 759, 777, 773, 787, 810, 777, 767, 730, 775, 797, 771, 735, 765, 814, 729, 715, 751, 789, 764, 757, 789, 769, 757, 843, 760, 710, 761, 748, 781, 770, 784, 777, 719, 792, 774, 777, 804, 770, 791, 766, 779, 776, 747, 818, 746, 751] payload_rows_per_dst=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] dup_rows_per_dst=[2936, 2890, 3030, 2994, 2842, 2962, 3018, 2902] combine_rows_per_src=[9336, 9184, 9492, 9314, 9284, 9298, 9376, 9446] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65116 dup_rows=23890 dup_rows_remote=20800 dedup_ratio=0.243022 router_tokens_per_expert=[835, 709, 726, 769, 801, 762, 750, 769, 811, 768, 750, 776, 753, 770, 745, 821, 853, 736, 826, 830, 783, 809, 743, 729, 767, 771, 754, 801, 740, 769, 788, 762, 767, 739, 723, 757, 732, 763, 734, 785, 730, 758, 810, 763, 771, 753, 787, 781, 774, 796, 784, 765, 776, 807, 781, 798, 716, 766, 758, 766, 754, 739, 725, 718] payload_rows_per_dst=[9330, 9298, 9510, 9314, 9134, 9298, 9452, 9078] dup_rows_per_dst=[2912, 3090, 3108, 2990, 2866, 3008, 3110, 2806] combine_rows_per_src=[9330, 9298, 9510, 9314, 9134, 9298, 9452, 9078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344040 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266551272 ingress_bytes=370016256 link_bytes=370016256 token_bytes=532758504 weight_movement_bytes=103809024 total_nvlink_bytes=636567528 weight_to_token_ratio=0.194852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392341383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=343664 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266624624 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532905584 weight_movement_bytes=34603008 total_nvlink_bytes=567508592 weight_to_token_ratio=0.064933 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=342200 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265230520 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530118840 weight_movement_bytes=34603008 total_nvlink_bytes=564721848 weight_to_token_ratio=0.065274 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343360 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266132800 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922240 weight_movement_bytes=103809024 total_nvlink_bytes=635731264 weight_to_token_ratio=0.195158 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=64948 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[755, 782, 785, 766, 792, 786, 788, 757, 782, 743, 768, 708, 770, 771, 793, 768, 719, 773, 756, 752, 760, 752, 763, 810, 760, 773, 761, 781, 761, 736, 792, 802, 802, 772, 809, 724, 759, 782, 763, 777, 793, 735, 768, 776, 811, 753, 732, 741, 764, 777, 770, 765, 734, 761, 772, 761, 744, 736, 784, 749, 800, 797, 800, 776] payload_rows_per_dst=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] dup_rows_per_dst=[2966, 3090, 3102, 2838, 3082, 2858, 3034, 2926] combine_rows_per_src=[9460, 9278, 9398, 9164, 9344, 9144, 9174, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64918 dup_rows=23952 dup_rows_remote=20970 dedup_ratio=0.243652 router_tokens_per_expert=[768, 782, 770, 794, 772, 768, 730, 710, 761, 770, 757, 763, 776, 786, 794, 752, 760, 753, 766, 765, 738, 786, 745, 761, 797, 733, 802, 798, 764, 820, 782, 781, 775, 835, 768, 728, 760, 782, 746, 814, 805, 763, 784, 790, 734, 729, 752, 747, 781, 789, 774, 757, 746, 719, 749, 746, 783, 803, 724, 796, 788, 774, 745, 762] payload_rows_per_dst=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] dup_rows_per_dst=[2932, 2994, 3000, 3114, 2982, 2934, 2986, 3010] combine_rows_per_src=[9256, 9324, 9148, 9440, 9434, 9274, 9136, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64966 dup_rows=23966 dup_rows_remote=20812 dedup_ratio=0.243795 router_tokens_per_expert=[787, 743, 789, 791, 806, 754, 798, 779, 770, 773, 787, 786, 786, 769, 743, 755, 765, 775, 772, 766, 760, 819, 832, 746, 731, 764, 713, 761, 773, 788, 750, 738, 749, 767, 764, 758, 711, 762, 790, 766, 798, 748, 771, 755, 769, 770, 776, 775, 711, 784, 767, 775, 802, 759, 798, 759, 736, 725, 774, 814, 747, 796, 761, 746] payload_rows_per_dst=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] dup_rows_per_dst=[3154, 2972, 3014, 3112, 2948, 3040, 2818, 2908] combine_rows_per_src=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344664 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936920 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533529176 weight_movement_bytes=34603008 total_nvlink_bytes=568132184 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64906 dup_rows=23936 dup_rows_remote=20892 dedup_ratio=0.243490 router_tokens_per_expert=[790, 786, 764, 752, 826, 769, 717, 744, 812, 750, 727, 764, 741, 759, 788, 751, 765, 796, 752, 741, 778, 762, 781, 768, 742, 765, 790, 786, 745, 783, 736, 767, 771, 727, 752, 804, 735, 756, 756, 750, 807, 754, 762, 781, 778, 783, 776, 790, 803, 778, 744, 762, 806, 751, 812, 788, 780, 764, 706, 754, 788, 776, 796, 765] payload_rows_per_dst=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] dup_rows_per_dst=[3032, 2538, 3486, 2906, 2850, 3130, 2950, 3044] combine_rows_per_src=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=343664 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267058800 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533773936 weight_movement_bytes=34603008 total_nvlink_bytes=568376944 weight_to_token_ratio=0.064827 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=268009472 dispatch_meta_bytes=344176 combine_payload_bytes=268009472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268353648 ingress_bytes=302612480 link_bytes=302612480 token_bytes=536363120 weight_movement_bytes=34603008 total_nvlink_bytes=570966128 weight_to_token_ratio=0.064514 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138706400e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343552 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247680 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151808 weight_movement_bytes=34603008 total_nvlink_bytes=566754816 weight_to_token_ratio=0.065025 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343512 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266370520 ingress_bytes=369836032 link_bytes=369836032 token_bytes=532397528 weight_movement_bytes=103809024 total_nvlink_bytes=636206552 weight_to_token_ratio=0.194984 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391663214e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64794 dup_rows=24226 dup_rows_remote=21038 dedup_ratio=0.246440 router_tokens_per_expert=[753, 742, 748, 771, 763, 771, 743, 751, 802, 814, 740, 782, 786, 760, 783, 759, 754, 773, 822, 711, 752, 758, 722, 759, 772, 763, 810, 749, 741, 819, 807, 775, 770, 725, 760, 771, 769, 800, 758, 760, 752, 802, 753, 761, 732, 713, 760, 804, 780, 750, 806, 813, 788, 745, 783, 773, 750, 784, 751, 793, 766, 775, 793, 757] payload_rows_per_dst=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] dup_rows_per_dst=[2882, 3106, 2930, 3188, 2906, 2956, 3168, 3090] combine_rows_per_src=[9202, 9346, 9172, 9284, 9320, 9198, 9308, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64964 dup_rows=24138 dup_rows_remote=21270 dedup_ratio=0.245544 router_tokens_per_expert=[766, 815, 741, 750, 801, 814, 721, 769, 753, 761, 776, 786, 792, 793, 815, 779, 779, 774, 782, 801, 789, 690, 746, 742, 779, 797, 747, 771, 765, 783, 770, 744, 715, 786, 741, 691, 782, 727, 803, 790, 754, 721, 786, 792, 745, 789, 775, 824, 810, 806, 751, 814, 745, 733, 733, 764, 757, 759, 767, 733, 792, 795, 734, 747] payload_rows_per_dst=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] dup_rows_per_dst=[3130, 3082, 2958, 3044, 2868, 3054, 3022, 2980] combine_rows_per_src=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343112 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266443848 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532544584 weight_movement_bytes=103809024 total_nvlink_bytes=636353608 weight_to_token_ratio=0.194930 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343192 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198168 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053144 weight_movement_bytes=34603008 total_nvlink_bytes=566656152 weight_to_token_ratio=0.065037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74000 payload_rows_remote=64806 dup_rows=24304 dup_rows_remote=21308 dedup_ratio=0.247233 router_tokens_per_expert=[747, 791, 759, 733, 799, 775, 741, 757, 804, 757, 765, 808, 791, 799, 745, 800, 791, 758, 725, 727, 767, 749, 793, 785, 786, 773, 748, 778, 771, 799, 786, 711, 766, 800, 772, 810, 790, 761, 769, 705, 772, 731, 761, 755, 826, 722, 760, 781, 822, 744, 761, 746, 802, 774, 793, 783, 768, 746, 739, 742, 721, 727, 798, 787] payload_rows_per_dst=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] dup_rows_per_dst=[3052, 3094, 2996, 3024, 3006, 3068, 3056, 3008] combine_rows_per_src=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64900 dup_rows=24038 dup_rows_remote=20974 dedup_ratio=0.244527 router_tokens_per_expert=[771, 728, 789, 798, 795, 732, 782, 747, 794, 771, 742, 805, 751, 765, 793, 804, 774, 751, 763, 702, 762, 743, 786, 783, 759, 744, 777, 760, 806, 768, 772, 778, 767, 736, 766, 775, 731, 764, 738, 782, 768, 781, 775, 759, 764, 776, 793, 799, 761, 752, 731, 794, 811, 753, 779, 754, 781, 742, 796, 726, 767, 738, 807, 791] payload_rows_per_dst=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] dup_rows_per_dst=[2990, 3118, 2892, 3100, 2902, 3064, 3000, 2972] combine_rows_per_src=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64950 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[745, 783, 765, 743, 751, 780, 756, 741, 729, 799, 738, 724, 805, 783, 799, 756, 807, 766, 703, 802, 734, 753, 732, 739, 776, 744, 761, 792, 761, 738, 784, 808, 863, 764, 792, 818, 786, 766, 765, 765, 741, 737, 752, 778, 762, 749, 774, 768, 823, 805, 754, 771, 759, 731, 786, 783, 789, 757, 754, 762, 744, 752, 826, 779] payload_rows_per_dst=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] dup_rows_per_dst=[2900, 2974, 2910, 3070, 3218, 2890, 3068, 2968] combine_rows_per_src=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64890 dup_rows=24052 dup_rows_remote=20950 dedup_ratio=0.244670 router_tokens_per_expert=[755, 725, 804, 778, 755, 746, 799, 754, 789, 782, 752, 717, 764, 747, 829, 782, 764, 778, 786, 793, 779, 797, 744, 791, 765, 746, 742, 736, 752, 762, 763, 733, 773, 749, 760, 758, 795, 743, 776, 749, 762, 749, 764, 843, 762, 761, 792, 790, 735, 776, 791, 766, 782, 784, 786, 753, 759, 783, 712, 758, 792, 788, 793, 759] payload_rows_per_dst=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] dup_rows_per_dst=[2982, 2992, 3102, 2874, 2990, 3140, 2930, 3042] combine_rows_per_src=[9250, 9332, 9362, 9124, 9216, 9306, 9416, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343328 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265739552 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135776 weight_movement_bytes=69206016 total_nvlink_bytes=600341792 weight_to_token_ratio=0.130298 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64906 dup_rows=23936 dup_rows_remote=20892 dedup_ratio=0.243490 router_tokens_per_expert=[790, 786, 764, 752, 826, 769, 717, 744, 812, 750, 727, 764, 741, 759, 788, 751, 765, 796, 752, 741, 778, 762, 781, 768, 742, 765, 790, 786, 745, 783, 736, 767, 771, 727, 752, 804, 735, 756, 756, 750, 807, 754, 762, 781, 778, 783, 776, 790, 803, 778, 744, 762, 806, 751, 812, 788, 780, 764, 706, 754, 788, 776, 796, 765] payload_rows_per_dst=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] dup_rows_per_dst=[3032, 2538, 3486, 2906, 2850, 3130, 2950, 3044] combine_rows_per_src=[9264, 8776, 9670, 9290, 9252, 9364, 9290, 9462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437480 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532530024 weight_movement_bytes=34603008 total_nvlink_bytes=567133032 weight_to_token_ratio=0.064979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64992 dup_rows=24068 dup_rows_remote=21018 dedup_ratio=0.244832 router_tokens_per_expert=[802, 757, 773, 764, 746, 738, 765, 776, 800, 764, 795, 755, 740, 789, 761, 770, 747, 789, 740, 759, 744, 798, 747, 771, 713, 745, 774, 775, 739, 722, 808, 756, 763, 843, 812, 788, 786, 793, 738, 806, 772, 692, 756, 824, 794, 788, 781, 707, 778, 774, 762, 743, 857, 717, 791, 725, 754, 780, 746, 792, 762, 781, 756, 769] payload_rows_per_dst=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] dup_rows_per_dst=[2982, 3030, 2932, 2854, 3196, 3000, 3050, 3024] combine_rows_per_src=[9260, 9318, 9258, 9210, 9462, 9228, 9244, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=344456 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265789832 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531235208 weight_movement_bytes=34603008 total_nvlink_bytes=565838216 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65064 dup_rows=24024 dup_rows_remote=21060 dedup_ratio=0.244385 router_tokens_per_expert=[775, 763, 715, 771, 791, 758, 761, 793, 769, 722, 802, 739, 793, 778, 731, 756, 754, 780, 801, 749, 774, 781, 752, 803, 791, 762, 786, 768, 719, 744, 745, 788, 798, 759, 788, 740, 779, 789, 763, 784, 752, 772, 766, 787, 742, 823, 778, 769, 738, 801, 740, 788, 782, 742, 771, 729, 781, 779, 794, 763, 759, 736, 793, 753] payload_rows_per_dst=[9264, 9216, 9320, 9206, 9390, 9252, 9278, 9354] dup_rows_per_dst=[2990, 2964, 3068, 3000, 3010, 3126, 2904, 2962] combine_rows_per_src=[9264, 9216, 9320, 9206, 9390, 9252, 9278, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343496 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173896 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004296 weight_movement_bytes=69206016 total_nvlink_bytes=601210312 weight_to_token_ratio=0.130085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343520 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266378720 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532413920 weight_movement_bytes=138412032 total_nvlink_bytes=670825952 weight_to_token_ratio=0.259971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343360 combine_payload_bytes=265789440 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266132800 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922240 weight_movement_bytes=103809024 total_nvlink_bytes=635731264 weight_to_token_ratio=0.195158 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64966 dup_rows=23966 dup_rows_remote=20812 dedup_ratio=0.243795 router_tokens_per_expert=[787, 743, 789, 791, 806, 754, 798, 779, 770, 773, 787, 786, 786, 769, 743, 755, 765, 775, 772, 766, 760, 819, 832, 746, 731, 764, 713, 761, 773, 788, 750, 738, 749, 767, 764, 758, 711, 762, 790, 766, 798, 748, 771, 755, 769, 770, 776, 775, 711, 784, 767, 775, 802, 759, 798, 759, 736, 725, 774, 814, 747, 796, 761, 746] payload_rows_per_dst=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] dup_rows_per_dst=[3154, 2972, 3014, 3112, 2948, 3040, 2818, 2908] combine_rows_per_src=[9372, 9366, 9456, 9514, 9186, 9284, 8870, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64964 dup_rows=24138 dup_rows_remote=21270 dedup_ratio=0.245544 router_tokens_per_expert=[766, 815, 741, 750, 801, 814, 721, 769, 753, 761, 776, 786, 792, 793, 815, 779, 779, 774, 782, 801, 789, 690, 746, 742, 779, 797, 747, 771, 765, 783, 770, 744, 715, 786, 741, 691, 782, 727, 803, 790, 754, 721, 786, 792, 745, 789, 775, 824, 810, 806, 751, 814, 745, 733, 733, 764, 757, 759, 767, 733, 792, 795, 734, 747] payload_rows_per_dst=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] dup_rows_per_dst=[3130, 3082, 2958, 3044, 2868, 3054, 3022, 2980] combine_rows_per_src=[9224, 9428, 9248, 9268, 9202, 9318, 9290, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64900 dup_rows=24038 dup_rows_remote=20974 dedup_ratio=0.244527 router_tokens_per_expert=[771, 728, 789, 798, 795, 732, 782, 747, 794, 771, 742, 805, 751, 765, 793, 804, 774, 751, 763, 702, 762, 743, 786, 783, 759, 744, 777, 760, 806, 768, 772, 778, 767, 736, 766, 775, 731, 764, 738, 782, 768, 781, 775, 759, 764, 776, 793, 799, 761, 752, 731, 794, 811, 753, 779, 754, 781, 742, 796, 726, 767, 738, 807, 791] payload_rows_per_dst=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] dup_rows_per_dst=[2990, 3118, 2892, 3100, 2902, 3064, 3000, 2972] combine_rows_per_src=[9294, 9332, 9236, 9228, 9216, 9366, 9270, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343192 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198168 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053144 weight_movement_bytes=34603008 total_nvlink_bytes=566656152 weight_to_token_ratio=0.065037 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64836 dup_rows=23958 dup_rows_remote=20768 dedup_ratio=0.243713 router_tokens_per_expert=[792, 755, 772, 782, 800, 796, 749, 784, 824, 779, 798, 770, 769, 737, 815, 755, 777, 763, 765, 725, 796, 722, 741, 742, 725, 773, 782, 739, 776, 747, 759, 771, 798, 787, 737, 764, 737, 778, 774, 715, 767, 799, 814, 759, 790, 729, 782, 741, 788, 755, 781, 803, 709, 733, 785, 798, 770, 758, 773, 741, 788, 758, 758, 803] payload_rows_per_dst=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] dup_rows_per_dst=[3190, 3092, 2896, 2980, 3074, 2894, 2834, 2998] combine_rows_per_src=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344040 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266551272 ingress_bytes=370016256 link_bytes=370016256 token_bytes=532758504 weight_movement_bytes=103809024 total_nvlink_bytes=636567528 weight_to_token_ratio=0.194852 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392341383e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21022 dedup_ratio=0.244873 router_tokens_per_expert=[699, 776, 777, 812, 746, 750, 774, 748, 751, 759, 762, 788, 705, 788, 765, 822, 764, 763, 779, 774, 793, 768, 781, 821, 761, 759, 737, 742, 715, 774, 716, 755, 762, 754, 722, 792, 812, 744, 762, 776, 770, 766, 843, 765, 768, 764, 741, 766, 784, 800, 763, 782, 770, 801, 722, 831, 755, 791, 773, 766, 780, 766, 770, 767] payload_rows_per_dst=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] dup_rows_per_dst=[2902, 2974, 2914, 3046, 2814, 3244, 3128, 3050] combine_rows_per_src=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344496 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266846640 ingress_bytes=335708160 link_bytes=335708160 token_bytes=533348784 weight_movement_bytes=69206016 total_nvlink_bytes=602554800 weight_to_token_ratio=0.129758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263242779e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343112 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266443848 ingress_bytes=369909760 link_bytes=369909760 token_bytes=532544584 weight_movement_bytes=103809024 total_nvlink_bytes=636353608 weight_to_token_ratio=0.194930 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391940647e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64886 dup_rows=24096 dup_rows_remote=21094 dedup_ratio=0.245117 router_tokens_per_expert=[800, 728, 760, 757, 801, 738, 744, 760, 761, 733, 772, 810, 753, 789, 747, 738, 753, 805, 764, 746, 811, 802, 782, 776, 720, 793, 746, 757, 775, 820, 779, 772, 789, 742, 769, 779, 760, 750, 818, 757, 720, 766, 793, 774, 768, 748, 803, 776, 784, 750, 756, 743, 789, 776, 764, 770, 795, 744, 785, 761, 743, 810, 715, 763] payload_rows_per_dst=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] dup_rows_per_dst=[2922, 3034, 3074, 3002, 3034, 3036, 3004, 2990] combine_rows_per_src=[9254, 9172, 9404, 9322, 9294, 9260, 9260, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64900 dup_rows=23998 dup_rows_remote=20886 dedup_ratio=0.244120 router_tokens_per_expert=[781, 767, 730, 770, 817, 753, 760, 768, 795, 767, 776, 807, 767, 766, 746, 759, 750, 762, 759, 743, 722, 768, 755, 755, 787, 750, 724, 777, 744, 768, 754, 800, 822, 769, 778, 800, 731, 778, 792, 789, 776, 787, 790, 744, 711, 782, 779, 761, 763, 761, 762, 747, 800, 779, 771, 762, 764, 843, 808, 757, 766, 750, 740, 743] payload_rows_per_dst=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] dup_rows_per_dst=[3034, 3038, 2878, 2942, 3112, 2958, 3014, 3022] combine_rows_per_src=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=342416 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265910672 ingress_bytes=300171264 link_bytes=300171264 token_bytes=531478928 weight_movement_bytes=34603008 total_nvlink_bytes=566081936 weight_to_token_ratio=0.065107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129520301e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=344936 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266437480 ingress_bytes=300695552 link_bytes=300695552 token_bytes=532530024 weight_movement_bytes=34603008 total_nvlink_bytes=567133032 weight_to_token_ratio=0.064979 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131493154e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343496 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173896 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532004296 weight_movement_bytes=69206016 total_nvlink_bytes=601210312 weight_to_token_ratio=0.130085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65074 dup_rows=24000 dup_rows_remote=20968 dedup_ratio=0.244141 router_tokens_per_expert=[731, 758, 747, 719, 782, 799, 765, 764, 784, 753, 797, 779, 748, 760, 748, 741, 784, 792, 762, 792, 768, 799, 767, 740, 765, 785, 801, 795, 721, 763, 748, 811, 753, 795, 781, 765, 794, 815, 752, 755, 817, 788, 775, 745, 786, 769, 713, 716, 751, 768, 755, 773, 795, 748, 756, 785, 758, 756, 747, 817, 763, 770, 727, 796] payload_rows_per_dst=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] dup_rows_per_dst=[2878, 2970, 3024, 3022, 3098, 2934, 3032, 3042] combine_rows_per_src=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343760 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247888 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152016 weight_movement_bytes=69206016 total_nvlink_bytes=601358032 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65118 dup_rows=23872 dup_rows_remote=20786 dedup_ratio=0.242839 router_tokens_per_expert=[816, 759, 780, 736, 806, 748, 699, 765, 725, 760, 789, 783, 738, 767, 765, 793, 778, 737, 771, 756, 768, 808, 822, 760, 810, 780, 776, 793, 777, 753, 766, 785, 784, 769, 761, 757, 733, 734, 794, 750, 711, 767, 811, 751, 775, 723, 825, 706, 784, 763, 791, 747, 714, 745, 793, 755, 769, 768, 779, 777, 770, 798, 789, 790] payload_rows_per_dst=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] dup_rows_per_dst=[2912, 2992, 3086, 3052, 2906, 2930, 2954, 3040] combine_rows_per_src=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343920 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116976 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890032 weight_movement_bytes=34603008 total_nvlink_bytes=566493040 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65030 dup_rows=24064 dup_rows_remote=21048 dedup_ratio=0.244792 router_tokens_per_expert=[760, 808, 792, 771, 796, 752, 786, 766, 752, 742, 744, 782, 789, 811, 806, 824, 727, 721, 763, 812, 769, 806, 746, 778, 744, 735, 780, 787, 776, 756, 749, 780, 791, 742, 801, 745, 727, 791, 754, 779, 763, 764, 758, 826, 745, 780, 758, 719, 722, 728, 762, 762, 791, 819, 805, 707, 793, 735, 767, 805, 722, 781, 761, 739] payload_rows_per_dst=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] dup_rows_per_dst=[3096, 3096, 3008, 3046, 2916, 3016, 2960, 2926] combine_rows_per_src=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74000 payload_rows_remote=64806 dup_rows=24304 dup_rows_remote=21308 dedup_ratio=0.247233 router_tokens_per_expert=[747, 791, 759, 733, 799, 775, 741, 757, 804, 757, 765, 808, 791, 799, 745, 800, 791, 758, 725, 727, 767, 749, 793, 785, 786, 773, 748, 778, 771, 799, 786, 711, 766, 800, 772, 810, 790, 761, 769, 705, 772, 731, 761, 755, 826, 722, 760, 781, 822, 744, 761, 746, 802, 774, 793, 783, 768, 746, 739, 742, 721, 727, 798, 787] payload_rows_per_dst=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] dup_rows_per_dst=[3052, 3094, 2996, 3024, 3006, 3068, 3056, 3008] combine_rows_per_src=[9152, 9444, 9194, 9280, 9340, 9148, 9394, 9048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21022 dedup_ratio=0.244873 router_tokens_per_expert=[699, 776, 777, 812, 746, 750, 774, 748, 751, 759, 762, 788, 705, 788, 765, 822, 764, 763, 779, 774, 793, 768, 781, 821, 761, 759, 737, 742, 715, 774, 716, 755, 762, 754, 722, 792, 812, 744, 762, 776, 770, 766, 843, 765, 768, 764, 741, 766, 784, 800, 763, 782, 770, 801, 722, 831, 755, 791, 773, 766, 780, 766, 770, 767] payload_rows_per_dst=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] dup_rows_per_dst=[2902, 2974, 2914, 3046, 2814, 3244, 3128, 3050] combine_rows_per_src=[9262, 9306, 9114, 9302, 9040, 9516, 9378, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343144 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173544 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532003944 weight_movement_bytes=69206016 total_nvlink_bytes=601209960 weight_to_token_ratio=0.130086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64950 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[745, 783, 765, 743, 751, 780, 756, 741, 729, 799, 738, 724, 805, 783, 799, 756, 807, 766, 703, 802, 734, 753, 732, 739, 776, 744, 761, 792, 761, 738, 784, 808, 863, 764, 792, 818, 786, 766, 765, 765, 741, 737, 752, 778, 762, 749, 774, 768, 823, 805, 754, 771, 759, 731, 786, 783, 789, 757, 754, 762, 744, 752, 826, 779] payload_rows_per_dst=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] dup_rows_per_dst=[2900, 2974, 2910, 3070, 3218, 2890, 3068, 2968] combine_rows_per_src=[9228, 9292, 9162, 9258, 9420, 9232, 9356, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343616 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267066944 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790272 weight_movement_bytes=138412032 total_nvlink_bytes=672202304 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65046 dup_rows=23902 dup_rows_remote=20918 dedup_ratio=0.243144 router_tokens_per_expert=[810, 792, 799, 796, 765, 734, 802, 770, 764, 777, 761, 767, 763, 759, 790, 789, 780, 793, 800, 808, 740, 777, 755, 744, 759, 795, 734, 771, 747, 727, 778, 744, 779, 785, 743, 699, 737, 801, 778, 781, 767, 806, 719, 794, 754, 722, 739, 768, 738, 749, 759, 788, 774, 784, 785, 778, 800, 742, 753, 755, 746, 788, 784, 767] payload_rows_per_dst=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] dup_rows_per_dst=[3132, 2984, 3110, 2824, 2986, 2828, 3010, 3028] combine_rows_per_src=[9404, 9356, 9284, 9286, 9220, 9310, 9300, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344168 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266887272 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430376 weight_movement_bytes=69206016 total_nvlink_bytes=602636392 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344312 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707192 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070072 weight_movement_bytes=34603008 total_nvlink_bytes=567673080 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=344456 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265789832 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531235208 weight_movement_bytes=34603008 total_nvlink_bytes=565838216 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64988 dup_rows=24014 dup_rows_remote=20990 dedup_ratio=0.244283 router_tokens_per_expert=[742, 743, 777, 754, 793, 755, 780, 799, 743, 774, 744, 753, 746, 734, 755, 746, 788, 776, 712, 778, 737, 808, 801, 779, 756, 766, 753, 758, 777, 783, 763, 765, 778, 792, 776, 819, 785, 755, 784, 736, 776, 730, 774, 785, 755, 779, 755, 780, 763, 815, 816, 746, 743, 788, 756, 776, 758, 761, 755, 763, 791, 793, 788, 743] payload_rows_per_dst=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] dup_rows_per_dst=[3024, 2982, 3082, 2992, 3150, 2858, 3078, 2848] combine_rows_per_src=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=64836 dup_rows=23958 dup_rows_remote=20768 dedup_ratio=0.243713 router_tokens_per_expert=[792, 755, 772, 782, 800, 796, 749, 784, 824, 779, 798, 770, 769, 737, 815, 755, 777, 763, 765, 725, 796, 722, 741, 742, 725, 773, 782, 739, 776, 747, 759, 771, 798, 787, 737, 764, 737, 778, 774, 715, 767, 799, 814, 759, 790, 729, 782, 741, 788, 755, 781, 803, 709, 733, 785, 798, 770, 758, 773, 741, 788, 758, 758, 803] payload_rows_per_dst=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] dup_rows_per_dst=[3190, 3092, 2896, 2980, 3074, 2894, 2834, 2998] combine_rows_per_src=[9510, 9502, 9166, 9178, 9330, 9244, 9116, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343760 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247888 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152016 weight_movement_bytes=69206016 total_nvlink_bytes=601358032 weight_to_token_ratio=0.130049 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64900 dup_rows=23998 dup_rows_remote=20886 dedup_ratio=0.244120 router_tokens_per_expert=[781, 767, 730, 770, 817, 753, 760, 768, 795, 767, 776, 807, 767, 766, 746, 759, 750, 762, 759, 743, 722, 768, 755, 755, 787, 750, 724, 777, 744, 768, 754, 800, 822, 769, 778, 800, 731, 778, 792, 789, 776, 787, 790, 744, 711, 782, 779, 761, 763, 761, 762, 747, 800, 779, 771, 762, 764, 843, 808, 757, 766, 750, 740, 743] payload_rows_per_dst=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] dup_rows_per_dst=[3034, 3038, 2878, 2942, 3112, 2958, 3014, 3022] combine_rows_per_src=[9258, 9328, 9150, 9266, 9406, 9302, 9276, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65030 dup_rows=24064 dup_rows_remote=21048 dedup_ratio=0.244792 router_tokens_per_expert=[760, 808, 792, 771, 796, 752, 786, 766, 752, 742, 744, 782, 789, 811, 806, 824, 727, 721, 763, 812, 769, 806, 746, 778, 744, 735, 780, 787, 776, 756, 749, 780, 791, 742, 801, 745, 727, 791, 754, 779, 763, 764, 758, 826, 745, 780, 758, 719, 722, 728, 762, 762, 791, 819, 805, 707, 793, 735, 767, 805, 722, 781, 761, 739] payload_rows_per_dst=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] dup_rows_per_dst=[3096, 3096, 3008, 3046, 2916, 3016, 2960, 2926] combine_rows_per_src=[9366, 9404, 9236, 9168, 9344, 9210, 9232, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343520 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266378720 ingress_bytes=404447232 link_bytes=404447232 token_bytes=532413920 weight_movement_bytes=138412032 total_nvlink_bytes=670825952 weight_to_token_ratio=0.259971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521902373e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65148 dup_rows=23900 dup_rows_remote=20856 dedup_ratio=0.243123 router_tokens_per_expert=[823, 790, 784, 718, 817, 778, 753, 748, 772, 816, 772, 739, 738, 740, 757, 770, 747, 763, 783, 768, 773, 771, 786, 790, 754, 773, 779, 724, 763, 808, 783, 850, 756, 747, 754, 736, 709, 745, 742, 754, 768, 804, 764, 766, 811, 740, 793, 788, 746, 782, 802, 725, 757, 774, 763, 756, 814, 815, 728, 782, 770, 709, 747, 775] payload_rows_per_dst=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] dup_rows_per_dst=[3034, 3038, 3026, 3026, 2732, 3046, 2954, 3044] combine_rows_per_src=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343856 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772272 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533200688 weight_movement_bytes=34603008 total_nvlink_bytes=567803696 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=342416 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265910672 ingress_bytes=300171264 link_bytes=300171264 token_bytes=531478928 weight_movement_bytes=34603008 total_nvlink_bytes=566081936 weight_to_token_ratio=0.065107 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129520301e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65088 dup_rows=23876 dup_rows_remote=20934 dedup_ratio=0.242879 router_tokens_per_expert=[773, 766, 764, 774, 765, 719, 808, 709, 742, 717, 764, 764, 794, 768, 759, 771, 736, 751, 794, 800, 754, 763, 803, 735, 767, 745, 784, 754, 804, 758, 776, 753, 795, 796, 772, 797, 775, 790, 781, 776, 796, 717, 754, 752, 731, 760, 755, 799, 769, 733, 864, 765, 773, 775, 774, 774, 772, 746, 776, 764, 800, 752, 772, 763] payload_rows_per_dst=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] dup_rows_per_dst=[2910, 2876, 2978, 2942, 3170, 2872, 3092, 3036] combine_rows_per_src=[9246, 9282, 9294, 9340, 9394, 9256, 9362, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343912 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534760 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532725608 weight_movement_bytes=34603008 total_nvlink_bytes=567328616 weight_to_token_ratio=0.064955 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64920 dup_rows=23894 dup_rows_remote=20788 dedup_ratio=0.243062 router_tokens_per_expert=[816, 756, 748, 781, 780, 768, 731, 716, 774, 735, 771, 774, 797, 752, 760, 729, 746, 757, 744, 806, 747, 805, 800, 742, 724, 807, 763, 764, 740, 730, 810, 789, 786, 794, 785, 782, 783, 818, 750, 800, 750, 771, 800, 738, 758, 741, 742, 785, 770, 763, 757, 769, 764, 819, 750, 782, 775, 738, 766, 756, 803, 753, 786, 756] payload_rows_per_dst=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] dup_rows_per_dst=[2964, 2988, 2930, 2998, 3106, 2966, 3020, 2922] combine_rows_per_src=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64954 dup_rows=23974 dup_rows_remote=20990 dedup_ratio=0.243876 router_tokens_per_expert=[767, 813, 762, 761, 771, 750, 770, 772, 781, 770, 766, 708, 795, 765, 740, 770, 817, 757, 780, 796, 756, 762, 776, 736, 792, 787, 750, 779, 762, 786, 776, 747, 766, 776, 759, 749, 768, 791, 778, 791, 758, 805, 767, 774, 753, 737, 763, 760, 775, 789, 779, 745, 764, 738, 770, 732, 797, 770, 822, 752, 742, 734, 799, 729] payload_rows_per_dst=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] dup_rows_per_dst=[3044, 2856, 2984, 3126, 3064, 2930, 2966, 3004] combine_rows_per_src=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343144 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266173544 ingress_bytes=335036416 link_bytes=335036416 token_bytes=532003944 weight_movement_bytes=69206016 total_nvlink_bytes=601209960 weight_to_token_ratio=0.130086 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260715061e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344312 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707192 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070072 weight_movement_bytes=34603008 total_nvlink_bytes=567673080 weight_to_token_ratio=0.064913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64912 dup_rows=23898 dup_rows_remote=20732 dedup_ratio=0.243103 router_tokens_per_expert=[728, 793, 767, 740, 779, 763, 715, 774, 769, 757, 752, 768, 741, 791, 769, 761, 805, 762, 784, 813, 746, 759, 777, 716, 776, 738, 752, 780, 788, 718, 731, 684, 793, 751, 748, 803, 766, 736, 774, 756, 790, 748, 788, 764, 756, 744, 740, 737, 820, 808, 774, 794, 737, 760, 840, 797, 810, 822, 788, 799, 764, 815, 779, 755] payload_rows_per_dst=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] dup_rows_per_dst=[2848, 2924, 3042, 2844, 2956, 2948, 3166, 3170] combine_rows_per_src=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344016 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036432 weight_movement_bytes=69206016 total_nvlink_bytes=603242448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344088 combine_payload_bytes=266600448 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266944536 ingress_bytes=370409472 link_bytes=370409472 token_bytes=533544984 weight_movement_bytes=103809024 total_nvlink_bytes=637354008 weight_to_token_ratio=0.194565 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393821023e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65050 dup_rows=24042 dup_rows_remote=21080 dedup_ratio=0.244568 router_tokens_per_expert=[804, 760, 759, 725, 736, 756, 769, 766, 794, 773, 782, 784, 743, 710, 796, 773, 761, 772, 775, 758, 743, 744, 782, 755, 789, 786, 733, 771, 776, 768, 869, 768, 764, 753, 787, 719, 753, 749, 789, 786, 759, 738, 739, 764, 799, 751, 771, 766, 762, 769, 763, 822, 783, 751, 796, 785, 784, 742, 802, 775, 770, 766, 733, 782] payload_rows_per_dst=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] dup_rows_per_dst=[3014, 2960, 2986, 3102, 2934, 2962, 3088, 2996] combine_rows_per_src=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342832 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255152 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167472 weight_movement_bytes=103809024 total_nvlink_bytes=635976496 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65118 dup_rows=23872 dup_rows_remote=20786 dedup_ratio=0.242839 router_tokens_per_expert=[816, 759, 780, 736, 806, 748, 699, 765, 725, 760, 789, 783, 738, 767, 765, 793, 778, 737, 771, 756, 768, 808, 822, 760, 810, 780, 776, 793, 777, 753, 766, 785, 784, 769, 761, 757, 733, 734, 794, 750, 711, 767, 811, 751, 775, 723, 825, 706, 784, 763, 791, 747, 714, 745, 793, 755, 769, 768, 779, 777, 770, 798, 789, 790] payload_rows_per_dst=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] dup_rows_per_dst=[2912, 2992, 3086, 3052, 2906, 2930, 2954, 3040] combine_rows_per_src=[9306, 9248, 9314, 9428, 9258, 9208, 9230, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65148 dup_rows=23900 dup_rows_remote=20856 dedup_ratio=0.243123 router_tokens_per_expert=[823, 790, 784, 718, 817, 778, 753, 748, 772, 816, 772, 739, 738, 740, 757, 770, 747, 763, 783, 768, 773, 771, 786, 790, 754, 773, 779, 724, 763, 808, 783, 850, 756, 747, 754, 736, 709, 745, 742, 754, 768, 804, 764, 766, 811, 740, 793, 788, 746, 782, 802, 725, 757, 774, 763, 756, 814, 815, 728, 782, 770, 709, 747, 775] payload_rows_per_dst=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] dup_rows_per_dst=[3034, 3038, 3026, 3026, 2732, 3046, 2954, 3044] combine_rows_per_src=[9388, 9170, 9336, 9398, 9072, 9422, 9362, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=342576 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266222128 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532101680 weight_movement_bytes=69206016 total_nvlink_bytes=601307696 weight_to_token_ratio=0.130062 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343776 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395360 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446944 weight_movement_bytes=34603008 total_nvlink_bytes=567049952 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65074 dup_rows=24000 dup_rows_remote=20968 dedup_ratio=0.244141 router_tokens_per_expert=[731, 758, 747, 719, 782, 799, 765, 764, 784, 753, 797, 779, 748, 760, 748, 741, 784, 792, 762, 792, 768, 799, 767, 740, 765, 785, 801, 795, 721, 763, 748, 811, 753, 795, 781, 765, 794, 815, 752, 755, 817, 788, 775, 745, 786, 769, 713, 716, 751, 768, 755, 773, 795, 748, 756, 785, 758, 756, 747, 817, 763, 770, 727, 796] payload_rows_per_dst=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] dup_rows_per_dst=[2878, 2970, 3024, 3022, 3098, 2934, 3032, 3042] combine_rows_per_src=[9252, 9250, 9384, 9356, 9322, 9284, 9230, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64858 dup_rows=24194 dup_rows_remote=21128 dedup_ratio=0.246114 router_tokens_per_expert=[754, 771, 774, 738, 778, 800, 791, 735, 784, 764, 693, 792, 731, 778, 827, 790, 781, 773, 778, 765, 735, 722, 770, 781, 812, 777, 791, 745, 755, 785, 786, 760, 790, 795, 752, 752, 761, 820, 772, 792, 772, 794, 770, 770, 771, 727, 761, 753, 732, 728, 734, 756, 745, 759, 772, 739, 722, 786, 831, 771, 764, 813, 762, 770] payload_rows_per_dst=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] dup_rows_per_dst=[3028, 3066, 2962, 3022, 3092, 2990, 2976, 3058] combine_rows_per_src=[9254, 9252, 9248, 9400, 9376, 9246, 8954, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344520 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789320 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234120 weight_movement_bytes=34603008 total_nvlink_bytes=567837128 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64880 dup_rows=23938 dup_rows_remote=20772 dedup_ratio=0.243510 router_tokens_per_expert=[774, 819, 762, 763, 787, 795, 810, 741, 798, 733, 741, 751, 771, 757, 750, 734, 747, 745, 747, 719, 786, 733, 790, 785, 771, 759, 761, 755, 792, 763, 725, 804, 761, 730, 763, 770, 768, 774, 788, 776, 745, 785, 752, 767, 740, 807, 771, 746, 800, 756, 810, 798, 791, 743, 758, 786, 760, 775, 807, 777, 797, 792, 731, 760] payload_rows_per_dst=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] dup_rows_per_dst=[3166, 3186, 2982, 2856, 2952, 3208, 2800, 2788] combine_rows_per_src=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64988 dup_rows=24014 dup_rows_remote=20990 dedup_ratio=0.244283 router_tokens_per_expert=[742, 743, 777, 754, 793, 755, 780, 799, 743, 774, 744, 753, 746, 734, 755, 746, 788, 776, 712, 778, 737, 808, 801, 779, 756, 766, 753, 758, 777, 783, 763, 765, 778, 792, 776, 819, 785, 755, 784, 736, 776, 730, 774, 785, 755, 779, 755, 780, 763, 815, 816, 746, 743, 788, 756, 776, 758, 761, 755, 763, 791, 793, 788, 743] payload_rows_per_dst=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] dup_rows_per_dst=[3024, 2982, 3082, 2992, 3150, 2858, 3078, 2848] combine_rows_per_src=[9302, 9170, 9242, 9250, 9404, 9410, 9218, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343616 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267066944 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790272 weight_movement_bytes=138412032 total_nvlink_bytes=672202304 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344016 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190224 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036432 weight_movement_bytes=69206016 total_nvlink_bytes=603242448 weight_to_token_ratio=0.129590 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344168 combine_payload_bytes=266543104 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266887272 ingress_bytes=335749120 link_bytes=335749120 token_bytes=533430376 weight_movement_bytes=69206016 total_nvlink_bytes=602636392 weight_to_token_ratio=0.129738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263396909e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65050 dup_rows=24042 dup_rows_remote=21080 dedup_ratio=0.244568 router_tokens_per_expert=[804, 760, 759, 725, 736, 756, 769, 766, 794, 773, 782, 784, 743, 710, 796, 773, 761, 772, 775, 758, 743, 744, 782, 755, 789, 786, 733, 771, 776, 768, 869, 768, 764, 753, 787, 719, 753, 749, 789, 786, 759, 738, 739, 764, 799, 751, 771, 766, 762, 769, 763, 822, 783, 751, 796, 785, 784, 742, 802, 775, 770, 766, 733, 782] payload_rows_per_dst=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] dup_rows_per_dst=[3014, 2960, 2986, 3102, 2934, 2962, 3088, 2996] combine_rows_per_src=[9136, 9350, 9194, 9418, 9266, 9212, 9374, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65030 dup_rows=23966 dup_rows_remote=20904 dedup_ratio=0.243795 router_tokens_per_expert=[736, 726, 734, 772, 719, 769, 766, 774, 759, 748, 742, 788, 749, 786, 833, 800, 767, 756, 775, 766, 787, 793, 757, 725, 750, 765, 762, 791, 788, 776, 776, 807, 738, 723, 745, 813, 744, 797, 738, 789, 726, 779, 762, 764, 733, 789, 769, 750, 783, 784, 812, 799, 790, 781, 771, 751, 802, 773, 742, 764, 758, 790, 825, 726] payload_rows_per_dst=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] dup_rows_per_dst=[2912, 3036, 2902, 3042, 2782, 3028, 3202, 3062] combine_rows_per_src=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64920 dup_rows=23894 dup_rows_remote=20788 dedup_ratio=0.243062 router_tokens_per_expert=[816, 756, 748, 781, 780, 768, 731, 716, 774, 735, 771, 774, 797, 752, 760, 729, 746, 757, 744, 806, 747, 805, 800, 742, 724, 807, 763, 764, 740, 730, 810, 789, 786, 794, 785, 782, 783, 818, 750, 800, 750, 771, 800, 738, 758, 741, 742, 785, 770, 763, 757, 769, 764, 819, 750, 782, 775, 738, 766, 756, 803, 753, 786, 756] payload_rows_per_dst=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] dup_rows_per_dst=[2964, 2988, 2930, 2998, 3106, 2966, 3020, 2922] combine_rows_per_src=[9228, 9196, 9364, 9256, 9490, 9204, 9328, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343944 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266002312 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660680 weight_movement_bytes=34603008 total_nvlink_bytes=566263688 weight_to_token_ratio=0.065085 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65062 dup_rows=23892 dup_rows_remote=20886 dedup_ratio=0.243042 router_tokens_per_expert=[767, 741, 744, 776, 755, 793, 751, 771, 785, 755, 728, 758, 764, 782, 735, 782, 725, 746, 792, 763, 782, 767, 826, 779, 760, 801, 731, 718, 819, 759, 781, 809, 760, 765, 720, 749, 774, 788, 759, 758, 748, 798, 775, 819, 755, 791, 771, 773, 774, 737, 799, 787, 715, 756, 773, 722, 804, 750, 822, 765, 787, 788, 721, 804] payload_rows_per_dst=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] dup_rows_per_dst=[2932, 2928, 2998, 3006, 2972, 3060, 2932, 3064] combine_rows_per_src=[9264, 9250, 9362, 9350, 9174, 9400, 9194, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64850 dup_rows=23998 dup_rows_remote=20842 dedup_ratio=0.244120 router_tokens_per_expert=[766, 781, 813, 758, 792, 759, 744, 733, 770, 748, 738, 770, 818, 734, 740, 783, 733, 759, 873, 746, 788, 776, 751, 746, 739, 772, 765, 734, 755, 756, 748, 827, 783, 809, 799, 819, 780, 776, 790, 750, 731, 788, 765, 728, 766, 768, 775, 745, 730, 747, 719, 808, 766, 793, 724, 766, 790, 772, 780, 804, 760, 768, 744, 794] payload_rows_per_dst=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] dup_rows_per_dst=[2902, 2990, 2978, 2958, 3156, 2978, 2924, 3112] combine_rows_per_src=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=342608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091088 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531839568 weight_movement_bytes=34603008 total_nvlink_bytes=566442576 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343912 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534760 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532725608 weight_movement_bytes=34603008 total_nvlink_bytes=567328616 weight_to_token_ratio=0.064955 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65042 dup_rows=23944 dup_rows_remote=21008 dedup_ratio=0.243571 router_tokens_per_expert=[781, 718, 777, 793, 745, 767, 779, 806, 776, 797, 714, 757, 735, 735, 785, 779, 741, 772, 768, 729, 738, 826, 779, 774, 782, 765, 796, 776, 750, 791, 739, 804, 776, 741, 753, 791, 766, 798, 724, 764, 774, 778, 763, 800, 727, 790, 761, 751, 789, 781, 798, 729, 742, 772, 725, 818, 777, 802, 816, 757, 756, 752, 748, 759] payload_rows_per_dst=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] dup_rows_per_dst=[3034, 2930, 2936, 3122, 3002, 2970, 2974, 2976] combine_rows_per_src=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344520 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266789320 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533234120 weight_movement_bytes=34603008 total_nvlink_bytes=567837128 weight_to_token_ratio=0.064893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65122 dup_rows=23950 dup_rows_remote=21034 dedup_ratio=0.243632 router_tokens_per_expert=[728, 715, 775, 786, 804, 762, 759, 718, 800, 768, 779, 792, 751, 749, 778, 770, 789, 798, 735, 792, 762, 768, 788, 783, 764, 824, 813, 804, 747, 757, 770, 773, 729, 804, 747, 770, 755, 776, 779, 766, 767, 779, 782, 751, 783, 754, 776, 782, 790, 742, 752, 766, 757, 764, 752, 751, 710, 789, 710, 746, 803, 764, 752, 803] payload_rows_per_dst=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] dup_rows_per_dst=[2854, 3058, 3066, 3082, 2994, 3016, 2916, 2964] combine_rows_per_src=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343736 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706616 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069496 weight_movement_bytes=69206016 total_nvlink_bytes=602275512 weight_to_token_ratio=0.129826 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342832 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266255152 ingress_bytes=369721344 link_bytes=369721344 token_bytes=532167472 weight_movement_bytes=103809024 total_nvlink_bytes=635976496 weight_to_token_ratio=0.195068 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391231652e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=343792 combine_payload_bytes=266493952 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266837744 ingress_bytes=335699968 link_bytes=335699968 token_bytes=533331696 weight_movement_bytes=69206016 total_nvlink_bytes=602537712 weight_to_token_ratio=0.129762 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263211954e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=342768 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265968368 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531593968 weight_movement_bytes=69206016 total_nvlink_bytes=600799984 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64928 dup_rows=24108 dup_rows_remote=21084 dedup_ratio=0.245239 router_tokens_per_expert=[764, 769, 786, 761, 766, 777, 764, 744, 768, 771, 747, 829, 787, 788, 762, 779, 776, 758, 791, 753, 754, 756, 752, 781, 765, 762, 797, 778, 729, 778, 762, 761, 755, 734, 778, 784, 772, 762, 721, 766, 794, 805, 758, 747, 777, 718, 787, 760, 783, 766, 748, 778, 789, 797, 784, 819, 766, 760, 738, 747, 750, 765, 767, 762] payload_rows_per_dst=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] dup_rows_per_dst=[3014, 3060, 2990, 2924, 3032, 3024, 3122, 2942] combine_rows_per_src=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64954 dup_rows=23974 dup_rows_remote=20990 dedup_ratio=0.243876 router_tokens_per_expert=[767, 813, 762, 761, 771, 750, 770, 772, 781, 770, 766, 708, 795, 765, 740, 770, 817, 757, 780, 796, 756, 762, 776, 736, 792, 787, 750, 779, 762, 786, 776, 747, 766, 776, 759, 749, 768, 791, 778, 791, 758, 805, 767, 774, 753, 737, 763, 760, 775, 789, 779, 745, 764, 738, 770, 732, 797, 770, 822, 752, 742, 734, 799, 729] payload_rows_per_dst=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] dup_rows_per_dst=[3044, 2856, 2984, 3126, 3064, 2930, 2966, 3004] combine_rows_per_src=[9288, 9334, 9376, 9232, 9292, 9304, 9218, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65030 dup_rows=23966 dup_rows_remote=20904 dedup_ratio=0.243795 router_tokens_per_expert=[736, 726, 734, 772, 719, 769, 766, 774, 759, 748, 742, 788, 749, 786, 833, 800, 767, 756, 775, 766, 787, 793, 757, 725, 750, 765, 762, 791, 788, 776, 776, 807, 738, 723, 745, 813, 744, 797, 738, 789, 726, 779, 762, 764, 733, 789, 769, 750, 783, 784, 812, 799, 790, 781, 771, 751, 802, 773, 742, 764, 758, 790, 825, 726] payload_rows_per_dst=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] dup_rows_per_dst=[2912, 3036, 2902, 3042, 2782, 3028, 3202, 3062] combine_rows_per_src=[9080, 9374, 9340, 9376, 9184, 9336, 9340, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344624 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267084336 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533824048 weight_movement_bytes=34603008 total_nvlink_bytes=568427056 weight_to_token_ratio=0.064821 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344200 combine_payload_bytes=266412032 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266756232 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533168264 weight_movement_bytes=69206016 total_nvlink_bytes=602374280 weight_to_token_ratio=0.129801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64912 dup_rows=23898 dup_rows_remote=20732 dedup_ratio=0.243103 router_tokens_per_expert=[728, 793, 767, 740, 779, 763, 715, 774, 769, 757, 752, 768, 741, 791, 769, 761, 805, 762, 784, 813, 746, 759, 777, 716, 776, 738, 752, 780, 788, 718, 731, 684, 793, 751, 748, 803, 766, 736, 774, 756, 790, 748, 788, 764, 756, 744, 740, 737, 820, 808, 774, 794, 737, 760, 840, 797, 810, 822, 788, 799, 764, 815, 779, 755] payload_rows_per_dst=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] dup_rows_per_dst=[2848, 2924, 3042, 2844, 2956, 2948, 3166, 3170] combine_rows_per_src=[9270, 9292, 9282, 9090, 9298, 9186, 9494, 9494] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64914 dup_rows=24082 dup_rows_remote=21024 dedup_ratio=0.244975 router_tokens_per_expert=[755, 775, 709, 756, 779, 735, 772, 770, 779, 766, 776, 803, 729, 761, 788, 781, 710, 765, 806, 765, 828, 724, 770, 775, 793, 798, 745, 781, 780, 794, 764, 751, 786, 787, 746, 749, 785, 817, 759, 793, 784, 763, 770, 765, 754, 782, 704, 737, 786, 720, 792, 761, 734, 758, 736, 792, 792, 788, 784, 783, 774, 711, 790, 787] payload_rows_per_dst=[9158, 9308, 9278, 9308, 9334, 9258, 9238, 9340] dup_rows_per_dst=[2944, 3058, 3008, 3104, 3110, 2860, 2920, 3078] combine_rows_per_src=[9158, 9308, 9278, 9308, 9334, 9258, 9238, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64592 dup_rows=24028 dup_rows_remote=20678 dedup_ratio=0.244425 router_tokens_per_expert=[766, 787, 737, 765, 737, 792, 800, 782, 722, 768, 741, 732, 775, 757, 790, 778, 748, 764, 799, 770, 778, 765, 750, 714, 806, 742, 774, 798, 784, 769, 787, 767, 788, 793, 775, 779, 780, 786, 756, 800, 795, 850, 725, 773, 756, 757, 790, 705, 735, 776, 777, 796, 760, 776, 741, 759, 790, 762, 755, 729, 806, 742, 722, 774] payload_rows_per_dst=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] dup_rows_per_dst=[3350, 2972, 2908, 2784, 3094, 3002, 3064, 2854] combine_rows_per_src=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266289136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532234224 weight_movement_bytes=34603008 total_nvlink_bytes=566837232 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64880 dup_rows=23938 dup_rows_remote=20772 dedup_ratio=0.243510 router_tokens_per_expert=[774, 819, 762, 763, 787, 795, 810, 741, 798, 733, 741, 751, 771, 757, 750, 734, 747, 745, 747, 719, 786, 733, 790, 785, 771, 759, 761, 755, 792, 763, 725, 804, 761, 730, 763, 770, 768, 774, 788, 776, 745, 785, 752, 767, 740, 807, 771, 746, 800, 756, 810, 798, 791, 743, 758, 786, 760, 775, 807, 777, 797, 792, 731, 760] payload_rows_per_dst=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] dup_rows_per_dst=[3166, 3186, 2982, 2856, 2952, 3208, 2800, 2788] combine_rows_per_src=[9486, 9534, 9320, 9254, 9110, 9552, 9150, 8960] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343776 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395360 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446944 weight_movement_bytes=34603008 total_nvlink_bytes=567049952 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343736 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706616 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069496 weight_movement_bytes=69206016 total_nvlink_bytes=602275512 weight_to_token_ratio=0.129826 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64818 dup_rows=24132 dup_rows_remote=21156 dedup_ratio=0.245483 router_tokens_per_expert=[712, 785, 757, 726, 798, 765, 794, 782, 743, 808, 810, 732, 799, 753, 746, 723, 766, 757, 796, 785, 784, 788, 773, 812, 745, 794, 786, 749, 781, 809, 747, 754, 741, 741, 760, 755, 761, 779, 737, 762, 809, 779, 775, 747, 765, 786, 774, 729, 763, 730, 778, 776, 764, 750, 758, 787, 741, 761, 769, 814, 781, 753, 808, 760] payload_rows_per_dst=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] dup_rows_per_dst=[3010, 3068, 3116, 2976, 2888, 3078, 2992, 3004] combine_rows_per_src=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65128 dup_rows=23930 dup_rows_remote=21094 dedup_ratio=0.243429 router_tokens_per_expert=[792, 740, 848, 760, 779, 846, 772, 786, 779, 730, 792, 785, 803, 765, 738, 788, 751, 796, 727, 798, 755, 760, 730, 781, 739, 791, 783, 737, 778, 772, 779, 814, 751, 754, 755, 766, 738, 762, 767, 748, 710, 738, 791, 740, 757, 759, 733, 809, 742, 770, 801, 730, 763, 813, 768, 745, 779, 764, 749, 807, 745, 778, 742, 784] payload_rows_per_dst=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] dup_rows_per_dst=[3260, 3058, 2928, 2934, 2836, 2940, 3042, 2932] combine_rows_per_src=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=342576 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266222128 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532101680 weight_movement_bytes=69206016 total_nvlink_bytes=601307696 weight_to_token_ratio=0.130062 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64928 dup_rows=24108 dup_rows_remote=21084 dedup_ratio=0.245239 router_tokens_per_expert=[764, 769, 786, 761, 766, 777, 764, 744, 768, 771, 747, 829, 787, 788, 762, 779, 776, 758, 791, 753, 754, 756, 752, 781, 765, 762, 797, 778, 729, 778, 762, 761, 755, 734, 778, 784, 772, 762, 721, 766, 794, 805, 758, 747, 777, 718, 787, 760, 783, 766, 748, 778, 789, 797, 784, 819, 766, 760, 738, 747, 750, 765, 767, 762] payload_rows_per_dst=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] dup_rows_per_dst=[3014, 3060, 2990, 2924, 3032, 3024, 3122, 2942] combine_rows_per_src=[9248, 9402, 9252, 9340, 9112, 9268, 9406, 9168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65120 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[786, 711, 763, 773, 816, 749, 758, 728, 754, 741, 768, 716, 738, 760, 750, 799, 759, 822, 693, 746, 782, 806, 831, 760, 782, 788, 772, 809, 785, 777, 789, 802, 787, 797, 778, 776, 738, 786, 786, 770, 805, 711, 758, 793, 743, 758, 716, 794, 753, 776, 800, 760, 769, 732, 770, 787, 751, 718, 755, 760, 729, 788, 780, 815] payload_rows_per_dst=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] dup_rows_per_dst=[2794, 2902, 2804, 3164, 3020, 3144, 3164, 2954] combine_rows_per_src=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64850 dup_rows=23998 dup_rows_remote=20842 dedup_ratio=0.244120 router_tokens_per_expert=[766, 781, 813, 758, 792, 759, 744, 733, 770, 748, 738, 770, 818, 734, 740, 783, 733, 759, 873, 746, 788, 776, 751, 746, 739, 772, 765, 734, 755, 756, 748, 827, 783, 809, 799, 819, 780, 776, 790, 750, 731, 788, 765, 728, 766, 768, 775, 745, 730, 747, 719, 808, 766, 793, 724, 766, 790, 772, 780, 804, 760, 768, 744, 794] payload_rows_per_dst=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] dup_rows_per_dst=[2902, 2990, 2978, 2958, 3156, 2978, 2924, 3112] combine_rows_per_src=[9390, 9212, 9366, 9234, 9456, 9154, 9182, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343752 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231496 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119240 weight_movement_bytes=34603008 total_nvlink_bytes=566722248 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264568832 dispatch_meta_bytes=341080 combine_payload_bytes=264568832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264909912 ingress_bytes=299171840 link_bytes=299171840 token_bytes=529478744 weight_movement_bytes=34603008 total_nvlink_bytes=564081752 weight_to_token_ratio=0.065353 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125759549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65044 dup_rows=23964 dup_rows_remote=20982 dedup_ratio=0.243774 router_tokens_per_expert=[766, 828, 752, 814, 844, 757, 764, 737, 744, 743, 797, 805, 742, 743, 790, 756, 789, 781, 795, 787, 848, 754, 763, 731, 745, 752, 788, 756, 775, 749, 798, 726, 750, 766, 797, 779, 741, 717, 784, 764, 754, 738, 731, 784, 789, 753, 766, 701, 771, 751, 751, 778, 759, 762, 771, 796, 770, 751, 755, 767, 768, 801, 788, 780] payload_rows_per_dst=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] dup_rows_per_dst=[3042, 2960, 3124, 2992, 2968, 2876, 2982, 3020] combine_rows_per_src=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64818 dup_rows=24132 dup_rows_remote=21156 dedup_ratio=0.245483 router_tokens_per_expert=[712, 785, 757, 726, 798, 765, 794, 782, 743, 808, 810, 732, 799, 753, 746, 723, 766, 757, 796, 785, 784, 788, 773, 812, 745, 794, 786, 749, 781, 809, 747, 754, 741, 741, 760, 755, 761, 779, 737, 762, 809, 779, 775, 747, 765, 786, 774, 729, 763, 730, 778, 776, 764, 750, 758, 787, 741, 761, 769, 814, 781, 753, 808, 760] payload_rows_per_dst=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] dup_rows_per_dst=[3010, 3068, 3116, 2976, 2888, 3078, 2992, 3004] combine_rows_per_src=[9228, 9160, 9406, 9354, 9184, 9250, 9220, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74030 payload_rows_remote=64844 dup_rows=24274 dup_rows_remote=21350 dedup_ratio=0.246928 router_tokens_per_expert=[793, 782, 716, 806, 755, 775, 786, 793, 759, 769, 732, 767, 801, 756, 803, 723, 703, 747, 790, 713, 782, 760, 781, 779, 718, 738, 749, 767, 751, 760, 816, 764, 775, 770, 789, 814, 776, 802, 783, 761, 792, 750, 774, 786, 770, 739, 721, 821, 719, 775, 732, 792, 758, 778, 805, 738, 773, 738, 759, 815, 801, 704, 809, 799] payload_rows_per_dst=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] dup_rows_per_dst=[3146, 3004, 2924, 2964, 3154, 3066, 2918, 3098] combine_rows_per_src=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=342608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091088 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531839568 weight_movement_bytes=34603008 total_nvlink_bytes=566442576 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344048 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266289136 ingress_bytes=300548096 link_bytes=300548096 token_bytes=532234224 weight_movement_bytes=34603008 total_nvlink_bytes=566837232 weight_to_token_ratio=0.065015 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130938289e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344448 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075968 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807488 weight_movement_bytes=103809024 total_nvlink_bytes=637616512 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=342768 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265968368 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531593968 weight_movement_bytes=69206016 total_nvlink_bytes=600799984 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343896 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838424 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332952 weight_movement_bytes=69206016 total_nvlink_bytes=600538968 weight_to_token_ratio=0.130250 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344888 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109176 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873464 weight_movement_bytes=34603008 total_nvlink_bytes=568476472 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64850 dup_rows=24256 dup_rows_remote=21260 dedup_ratio=0.246745 router_tokens_per_expert=[726, 780, 789, 745, 752, 810, 763, 792, 756, 750, 741, 771, 744, 726, 795, 744, 778, 780, 759, 753, 753, 771, 849, 785, 777, 713, 712, 797, 809, 793, 778, 787, 728, 767, 766, 746, 798, 761, 781, 768, 717, 728, 732, 766, 812, 813, 730, 799, 739, 770, 752, 794, 769, 799, 777, 779, 724, 773, 770, 780, 801, 780, 785, 770] payload_rows_per_dst=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] dup_rows_per_dst=[3012, 2946, 3072, 3046, 3038, 2996, 3040, 3106] combine_rows_per_src=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344104 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764328 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184552 weight_movement_bytes=34603008 total_nvlink_bytes=567787560 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65042 dup_rows=23944 dup_rows_remote=21008 dedup_ratio=0.243571 router_tokens_per_expert=[781, 718, 777, 793, 745, 767, 779, 806, 776, 797, 714, 757, 735, 735, 785, 779, 741, 772, 768, 729, 738, 826, 779, 774, 782, 765, 796, 776, 750, 791, 739, 804, 776, 741, 753, 791, 766, 798, 724, 764, 774, 778, 763, 800, 727, 790, 761, 751, 789, 781, 798, 729, 742, 772, 725, 818, 777, 802, 816, 757, 756, 752, 748, 759] payload_rows_per_dst=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] dup_rows_per_dst=[3034, 2930, 2936, 3122, 3002, 2970, 2974, 2976] combine_rows_per_src=[9298, 9226, 9318, 9284, 9224, 9318, 9334, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65120 dup_rows=23946 dup_rows_remote=20992 dedup_ratio=0.243591 router_tokens_per_expert=[786, 711, 763, 773, 816, 749, 758, 728, 754, 741, 768, 716, 738, 760, 750, 799, 759, 822, 693, 746, 782, 806, 831, 760, 782, 788, 772, 809, 785, 777, 789, 802, 787, 797, 778, 776, 738, 786, 786, 770, 805, 711, 758, 793, 743, 758, 716, 794, 753, 776, 800, 760, 769, 732, 770, 787, 751, 718, 755, 760, 729, 788, 780, 815] payload_rows_per_dst=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] dup_rows_per_dst=[2794, 2902, 2804, 3164, 3020, 3144, 3164, 2954] combine_rows_per_src=[9010, 9150, 9090, 9444, 9416, 9516, 9494, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=344776 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265945800 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546824 weight_movement_bytes=103809024 total_nvlink_bytes=635355848 weight_to_token_ratio=0.195296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65122 dup_rows=23950 dup_rows_remote=21034 dedup_ratio=0.243632 router_tokens_per_expert=[728, 715, 775, 786, 804, 762, 759, 718, 800, 768, 779, 792, 751, 749, 778, 770, 789, 798, 735, 792, 762, 768, 788, 783, 764, 824, 813, 804, 747, 757, 770, 773, 729, 804, 747, 770, 755, 776, 779, 766, 767, 779, 782, 751, 783, 754, 776, 782, 790, 742, 752, 766, 757, 764, 752, 751, 710, 789, 710, 746, 803, 764, 752, 803] payload_rows_per_dst=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] dup_rows_per_dst=[2854, 3058, 3066, 3082, 2994, 3016, 2916, 2964] combine_rows_per_src=[9240, 9316, 9364, 9422, 9258, 9332, 9232, 9190] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64744 dup_rows=24206 dup_rows_remote=21150 dedup_ratio=0.246236 router_tokens_per_expert=[743, 769, 735, 767, 788, 747, 779, 792, 782, 780, 767, 797, 796, 738, 745, 800, 746, 783, 779, 783, 786, 780, 783, 813, 765, 772, 780, 762, 756, 738, 777, 749, 787, 767, 743, 769, 769, 814, 768, 804, 746, 730, 737, 761, 790, 782, 798, 788, 753, 702, 768, 807, 751, 758, 737, 741, 758, 768, 743, 755, 823, 723, 757, 778] payload_rows_per_dst=[9256, 9354, 9294, 9296, 9366, 9242, 9080, 9210] dup_rows_per_dst=[2984, 3056, 3212, 2902, 3076, 3022, 2954, 3000] combine_rows_per_src=[9256, 9354, 9294, 9296, 9366, 9242, 9080, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=344440 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265970040 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595640 weight_movement_bytes=34603008 total_nvlink_bytes=566198648 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65212 dup_rows=23880 dup_rows_remote=20966 dedup_ratio=0.242920 router_tokens_per_expert=[743, 774, 753, 750, 750, 737, 736, 725, 734, 783, 780, 781, 716, 778, 767, 755, 788, 763, 776, 804, 843, 782, 788, 774, 810, 777, 733, 786, 747, 760, 768, 764, 768, 770, 753, 748, 800, 725, 809, 755, 782, 791, 747, 756, 792, 784, 798, 773, 805, 793, 725, 776, 766, 770, 745, 773, 729, 768, 732, 798, 780, 780, 746, 790] payload_rows_per_dst=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] dup_rows_per_dst=[2914, 3186, 2890, 2896, 3014, 3102, 3034, 2844] combine_rows_per_src=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344200 combine_payload_bytes=266412032 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266756232 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533168264 weight_movement_bytes=69206016 total_nvlink_bytes=602374280 weight_to_token_ratio=0.129801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64756 dup_rows=24180 dup_rows_remote=21072 dedup_ratio=0.245972 router_tokens_per_expert=[766, 752, 797, 725, 764, 756, 790, 747, 784, 741, 772, 774, 785, 785, 817, 727, 856, 797, 756, 757, 791, 695, 745, 742, 752, 813, 775, 783, 747, 804, 772, 792, 749, 748, 770, 775, 787, 718, 801, 743, 781, 753, 766, 731, 744, 759, 733, 797, 739, 783, 772, 802, 764, 750, 781, 775, 789, 788, 805, 741, 771, 761, 763, 754] payload_rows_per_dst=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] dup_rows_per_dst=[3020, 3062, 3062, 3108, 2918, 2930, 3068, 3012] combine_rows_per_src=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344448 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267075968 ingress_bytes=370540544 link_bytes=370540544 token_bytes=533807488 weight_movement_bytes=103809024 total_nvlink_bytes=637616512 weight_to_token_ratio=0.194469 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394314236e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65182 dup_rows=23944 dup_rows_remote=21006 dedup_ratio=0.243571 router_tokens_per_expert=[767, 748, 801, 752, 759, 782, 751, 731, 763, 730, 750, 728, 736, 776, 775, 777, 765, 788, 735, 764, 757, 761, 776, 756, 753, 758, 787, 748, 801, 769, 787, 760, 768, 772, 733, 767, 728, 791, 743, 756, 802, 771, 852, 783, 779, 791, 785, 789, 692, 756, 789, 805, 769, 786, 727, 816, 797, 767, 765, 767, 838, 780, 766, 731] payload_rows_per_dst=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] dup_rows_per_dst=[2856, 2838, 2924, 3108, 2938, 3204, 3026, 3050] combine_rows_per_src=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64592 dup_rows=24028 dup_rows_remote=20678 dedup_ratio=0.244425 router_tokens_per_expert=[766, 787, 737, 765, 737, 792, 800, 782, 722, 768, 741, 732, 775, 757, 790, 778, 748, 764, 799, 770, 778, 765, 750, 714, 806, 742, 774, 798, 784, 769, 787, 767, 788, 793, 775, 779, 780, 786, 756, 800, 795, 850, 725, 773, 756, 757, 790, 705, 735, 776, 777, 796, 760, 776, 741, 759, 790, 762, 755, 729, 806, 742, 722, 774] payload_rows_per_dst=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] dup_rows_per_dst=[3350, 2972, 2908, 2784, 3094, 3002, 3064, 2854] combine_rows_per_src=[9684, 9184, 9238, 8968, 9420, 9300, 9324, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64850 dup_rows=24256 dup_rows_remote=21260 dedup_ratio=0.246745 router_tokens_per_expert=[726, 780, 789, 745, 752, 810, 763, 792, 756, 750, 741, 771, 744, 726, 795, 744, 778, 780, 759, 753, 753, 771, 849, 785, 777, 713, 712, 797, 809, 793, 778, 787, 728, 767, 766, 746, 798, 761, 781, 768, 717, 728, 732, 766, 812, 813, 730, 799, 739, 770, 752, 794, 769, 799, 777, 779, 724, 773, 770, 780, 801, 780, 785, 770] payload_rows_per_dst=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] dup_rows_per_dst=[3012, 2946, 3072, 3046, 3038, 2996, 3040, 3106] combine_rows_per_src=[9302, 9108, 9384, 9286, 9192, 9198, 9318, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344624 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267084336 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533824048 weight_movement_bytes=34603008 total_nvlink_bytes=568427056 weight_to_token_ratio=0.064821 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64756 dup_rows=24090 dup_rows_remote=21024 dedup_ratio=0.245056 router_tokens_per_expert=[760, 780, 763, 782, 755, 780, 773, 767, 740, 745, 737, 773, 818, 727, 747, 746, 761, 752, 768, 797, 731, 754, 764, 748, 792, 703, 766, 765, 834, 790, 738, 793, 772, 709, 753, 766, 767, 764, 764, 791, 750, 807, 798, 776, 797, 758, 817, 803, 722, 750, 781, 822, 747, 745, 741, 744, 797, 769, 789, 805, 763, 791, 760, 785] payload_rows_per_dst=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] dup_rows_per_dst=[3098, 2852, 2908, 3016, 3008, 3208, 2934, 3066] combine_rows_per_src=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65128 dup_rows=23930 dup_rows_remote=21094 dedup_ratio=0.243429 router_tokens_per_expert=[792, 740, 848, 760, 779, 846, 772, 786, 779, 730, 792, 785, 803, 765, 738, 788, 751, 796, 727, 798, 755, 760, 730, 781, 739, 791, 783, 737, 778, 772, 779, 814, 751, 754, 755, 766, 738, 762, 767, 748, 710, 738, 791, 740, 757, 759, 733, 809, 742, 770, 801, 730, 763, 813, 768, 745, 779, 764, 749, 807, 745, 778, 742, 784] payload_rows_per_dst=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] dup_rows_per_dst=[3260, 3058, 2928, 2934, 2836, 2940, 3042, 2932] combine_rows_per_src=[9386, 9302, 9268, 9452, 9246, 9134, 9222, 9364] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=343576 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265535000 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530726424 weight_movement_bytes=34603008 total_nvlink_bytes=565329432 weight_to_token_ratio=0.065199 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65222 dup_rows=23816 dup_rows_remote=20964 dedup_ratio=0.242269 router_tokens_per_expert=[786, 805, 754, 721, 752, 822, 815, 772, 787, 815, 746, 748, 770, 770, 803, 739, 771, 781, 762, 763, 800, 809, 776, 785, 707, 743, 781, 748, 787, 783, 761, 738, 737, 767, 758, 725, 815, 794, 794, 751, 764, 779, 734, 756, 734, 794, 789, 747, 776, 774, 727, 738, 763, 824, 732, 725, 798, 734, 773, 765, 770, 717, 810, 788] payload_rows_per_dst=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] dup_rows_per_dst=[3066, 2970, 3048, 2940, 3032, 2954, 2852, 2954] combine_rows_per_src=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64756 dup_rows=24180 dup_rows_remote=21072 dedup_ratio=0.245972 router_tokens_per_expert=[766, 752, 797, 725, 764, 756, 790, 747, 784, 741, 772, 774, 785, 785, 817, 727, 856, 797, 756, 757, 791, 695, 745, 742, 752, 813, 775, 783, 747, 804, 772, 792, 749, 748, 770, 775, 787, 718, 801, 743, 781, 753, 766, 731, 744, 759, 733, 797, 739, 783, 772, 802, 764, 750, 781, 775, 789, 788, 805, 741, 771, 761, 763, 754] payload_rows_per_dst=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] dup_rows_per_dst=[3020, 3062, 3062, 3108, 2918, 2930, 3068, 3012] combine_rows_per_src=[9174, 9308, 9216, 9368, 9264, 9198, 9264, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65070 dup_rows=24132 dup_rows_remote=21230 dedup_ratio=0.245483 router_tokens_per_expert=[763, 810, 797, 735, 800, 733, 741, 749, 762, 766, 759, 804, 764, 759, 725, 786, 790, 733, 706, 764, 759, 746, 767, 737, 778, 822, 773, 787, 759, 770, 778, 806, 784, 786, 782, 748, 765, 746, 761, 793, 749, 764, 735, 791, 751, 739, 761, 748, 757, 768, 783, 771, 775, 811, 790, 780, 779, 809, 757, 737, 746, 833, 742, 783] payload_rows_per_dst=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] dup_rows_per_dst=[2986, 2978, 2902, 3136, 3040, 2944, 3036, 3110] combine_rows_per_src=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344712 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267453064 ingress_bytes=336314368 link_bytes=336314368 token_bytes=534561416 weight_movement_bytes=69206016 total_nvlink_bytes=603767432 weight_to_token_ratio=0.129463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265523891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264568832 dispatch_meta_bytes=341080 combine_payload_bytes=264568832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264909912 ingress_bytes=299171840 link_bytes=299171840 token_bytes=529478744 weight_movement_bytes=34603008 total_nvlink_bytes=564081752 weight_to_token_ratio=0.065353 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125759549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=344440 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265970040 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531595640 weight_movement_bytes=34603008 total_nvlink_bytes=566198648 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343120 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265583696 ingress_bytes=369049600 link_bytes=369049600 token_bytes=530824272 weight_movement_bytes=103809024 total_nvlink_bytes=634633296 weight_to_token_ratio=0.195562 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388703934e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344888 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109176 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873464 weight_movement_bytes=34603008 total_nvlink_bytes=568476472 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343312 combine_payload_bytes=265240576 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265583888 ingress_bytes=403652608 link_bytes=403652608 token_bytes=530824464 weight_movement_bytes=138412032 total_nvlink_bytes=669236496 weight_to_token_ratio=0.260749 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518912267e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344752 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330224 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315696 weight_movement_bytes=34603008 total_nvlink_bytes=568918704 weight_to_token_ratio=0.064761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344744 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267494056 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643368 weight_movement_bytes=34603008 total_nvlink_bytes=569246376 weight_to_token_ratio=0.064722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=345200 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871920 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533398640 weight_movement_bytes=34603008 total_nvlink_bytes=568001648 weight_to_token_ratio=0.064873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65044 dup_rows=23964 dup_rows_remote=20982 dedup_ratio=0.243774 router_tokens_per_expert=[766, 828, 752, 814, 844, 757, 764, 737, 744, 743, 797, 805, 742, 743, 790, 756, 789, 781, 795, 787, 848, 754, 763, 731, 745, 752, 788, 756, 775, 749, 798, 726, 750, 766, 797, 779, 741, 717, 784, 764, 754, 738, 731, 784, 789, 753, 766, 701, 771, 751, 751, 778, 759, 762, 771, 796, 770, 751, 755, 767, 768, 801, 788, 780] payload_rows_per_dst=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] dup_rows_per_dst=[3042, 2960, 3124, 2992, 2968, 2876, 2982, 3020] combine_rows_per_src=[9482, 9280, 9372, 9186, 9228, 9156, 9296, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65352 dup_rows=23816 dup_rows_remote=21004 dedup_ratio=0.242269 router_tokens_per_expert=[736, 783, 815, 769, 760, 729, 768, 800, 710, 759, 757, 717, 757, 762, 787, 725, 794, 723, 771, 769, 800, 796, 712, 741, 784, 761, 776, 775, 764, 801, 753, 764, 790, 786, 779, 744, 783, 771, 756, 768, 774, 797, 769, 754, 804, 796, 766, 780, 771, 815, 727, 762, 801, 759, 767, 767, 761, 759, 745, 824, 762, 772, 774, 751] payload_rows_per_dst=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] dup_rows_per_dst=[2964, 2812, 3000, 2986, 2976, 3070, 3054, 2954] combine_rows_per_src=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344104 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764328 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184552 weight_movement_bytes=34603008 total_nvlink_bytes=567787560 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65212 dup_rows=23880 dup_rows_remote=20966 dedup_ratio=0.242920 router_tokens_per_expert=[743, 774, 753, 750, 750, 737, 736, 725, 734, 783, 780, 781, 716, 778, 767, 755, 788, 763, 776, 804, 843, 782, 788, 774, 810, 777, 733, 786, 747, 760, 768, 764, 768, 770, 753, 748, 800, 725, 809, 755, 782, 791, 747, 756, 792, 784, 798, 773, 805, 793, 725, 776, 766, 770, 745, 773, 729, 768, 732, 798, 780, 780, 746, 790] payload_rows_per_dst=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] dup_rows_per_dst=[2914, 3186, 2890, 2896, 3014, 3102, 3034, 2844] combine_rows_per_src=[9212, 9554, 9234, 9250, 9292, 9344, 9272, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64736 dup_rows=24064 dup_rows_remote=20898 dedup_ratio=0.244792 router_tokens_per_expert=[768, 752, 805, 761, 787, 729, 761, 742, 802, 796, 788, 751, 791, 744, 776, 794, 820, 746, 739, 781, 794, 748, 779, 738, 784, 762, 732, 788, 770, 794, 775, 759, 789, 735, 756, 763, 720, 766, 754, 781, 836, 727, 743, 798, 768, 718, 748, 816, 734, 782, 808, 783, 798, 754, 759, 761, 757, 756, 749, 773, 761, 731, 812, 760] payload_rows_per_dst=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] dup_rows_per_dst=[2630, 3074, 3372, 2800, 3000, 2974, 3048, 3166] combine_rows_per_src=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65072 dup_rows=23898 dup_rows_remote=20918 dedup_ratio=0.243103 router_tokens_per_expert=[746, 770, 762, 800, 752, 765, 806, 692, 809, 779, 763, 763, 724, 732, 781, 788, 801, 800, 813, 813, 771, 792, 767, 775, 777, 746, 769, 712, 763, 797, 804, 800, 803, 749, 746, 738, 776, 723, 785, 787, 738, 745, 800, 799, 793, 748, 774, 760, 753, 782, 793, 785, 801, 718, 775, 695, 759, 778, 762, 759, 737, 753, 738, 768] payload_rows_per_dst=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] dup_rows_per_dst=[3040, 2970, 3122, 2938, 2934, 2980, 2992, 2922] combine_rows_per_src=[9146, 9308, 9542, 9398, 9280, 9334, 9212, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74030 payload_rows_remote=64844 dup_rows=24274 dup_rows_remote=21350 dedup_ratio=0.246928 router_tokens_per_expert=[793, 782, 716, 806, 755, 775, 786, 793, 759, 769, 732, 767, 801, 756, 803, 723, 703, 747, 790, 713, 782, 760, 781, 779, 718, 738, 749, 767, 751, 760, 816, 764, 775, 770, 789, 814, 776, 802, 783, 761, 792, 750, 774, 786, 770, 739, 721, 821, 719, 775, 732, 792, 758, 778, 805, 738, 773, 738, 759, 815, 801, 704, 809, 799] payload_rows_per_dst=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] dup_rows_per_dst=[3146, 3004, 2924, 2964, 3154, 3066, 2918, 3098] combine_rows_per_src=[9266, 9216, 9186, 9162, 9386, 9240, 9276, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65182 dup_rows=23944 dup_rows_remote=21006 dedup_ratio=0.243571 router_tokens_per_expert=[767, 748, 801, 752, 759, 782, 751, 731, 763, 730, 750, 728, 736, 776, 775, 777, 765, 788, 735, 764, 757, 761, 776, 756, 753, 758, 787, 748, 801, 769, 787, 760, 768, 772, 733, 767, 728, 791, 743, 756, 802, 771, 852, 783, 779, 791, 785, 789, 692, 756, 789, 805, 769, 786, 727, 816, 797, 767, 765, 767, 838, 780, 766, 731] payload_rows_per_dst=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] dup_rows_per_dst=[2856, 2838, 2924, 3108, 2938, 3204, 3026, 3050] combine_rows_per_src=[9326, 9232, 9280, 9218, 9178, 9500, 9254, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64704 dup_rows=24250 dup_rows_remote=21248 dedup_ratio=0.246684 router_tokens_per_expert=[711, 771, 764, 766, 813, 767, 740, 789, 775, 776, 812, 791, 774, 767, 775, 770, 748, 757, 779, 773, 794, 745, 795, 744, 820, 761, 778, 770, 758, 736, 736, 817, 843, 762, 742, 777, 747, 731, 722, 773, 686, 785, 803, 780, 778, 749, 751, 799, 820, 782, 751, 790, 796, 790, 797, 760, 708, 755, 751, 729, 761, 730, 766, 766] payload_rows_per_dst=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] dup_rows_per_dst=[2960, 3156, 2964, 3002, 3088, 3082, 3138, 2860] combine_rows_per_src=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267681792 dispatch_meta_bytes=345424 combine_payload_bytes=267681792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268027216 ingress_bytes=302284800 link_bytes=302284800 token_bytes=535709008 weight_movement_bytes=34603008 total_nvlink_bytes=570312016 weight_to_token_ratio=0.064593 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137473366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65166 dup_rows=24010 dup_rows_remote=21090 dedup_ratio=0.244242 router_tokens_per_expert=[826, 772, 816, 784, 784, 782, 782, 777, 803, 783, 793, 768, 787, 725, 736, 742, 763, 723, 750, 785, 757, 754, 743, 749, 744, 738, 774, 767, 748, 738, 769, 741, 837, 773, 779, 765, 767, 776, 799, 754, 758, 783, 721, 761, 756, 782, 803, 740, 750, 788, 808, 748, 774, 723, 787, 766, 770, 789, 717, 844, 720, 742, 770, 799] payload_rows_per_dst=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] dup_rows_per_dst=[3128, 3038, 2920, 2824, 3120, 2958, 2996, 3026] combine_rows_per_src=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64756 dup_rows=24090 dup_rows_remote=21024 dedup_ratio=0.245056 router_tokens_per_expert=[760, 780, 763, 782, 755, 780, 773, 767, 740, 745, 737, 773, 818, 727, 747, 746, 761, 752, 768, 797, 731, 754, 764, 748, 792, 703, 766, 765, 834, 790, 738, 793, 772, 709, 753, 766, 767, 764, 764, 791, 750, 807, 798, 776, 797, 758, 817, 803, 722, 750, 781, 822, 747, 745, 741, 744, 797, 769, 789, 805, 763, 791, 760, 785] payload_rows_per_dst=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] dup_rows_per_dst=[3098, 2852, 2908, 3016, 3008, 3208, 2934, 3066] combine_rows_per_src=[9222, 9214, 9242, 9108, 9396, 9404, 9170, 9458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65352 dup_rows=23816 dup_rows_remote=21004 dedup_ratio=0.242269 router_tokens_per_expert=[736, 783, 815, 769, 760, 729, 768, 800, 710, 759, 757, 717, 757, 762, 787, 725, 794, 723, 771, 769, 800, 796, 712, 741, 784, 761, 776, 775, 764, 801, 753, 764, 790, 786, 779, 744, 783, 771, 756, 768, 774, 797, 769, 754, 804, 796, 766, 780, 771, 815, 727, 762, 801, 759, 767, 767, 761, 759, 745, 824, 762, 772, 774, 751] payload_rows_per_dst=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] dup_rows_per_dst=[2964, 2812, 3000, 2986, 2976, 3070, 3054, 2954] combine_rows_per_src=[9356, 9136, 9212, 9370, 9378, 9410, 9284, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344712 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267453064 ingress_bytes=336314368 link_bytes=336314368 token_bytes=534561416 weight_movement_bytes=69206016 total_nvlink_bytes=603767432 weight_to_token_ratio=0.129463 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265523891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=342536 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265501192 ingress_bytes=299761664 link_bytes=299761664 token_bytes=530659848 weight_movement_bytes=34603008 total_nvlink_bytes=565262856 weight_to_token_ratio=0.065208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127979009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344752 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330224 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315696 weight_movement_bytes=34603008 total_nvlink_bytes=568918704 weight_to_token_ratio=0.064761 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=343960 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266878872 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533413784 weight_movement_bytes=69206016 total_nvlink_bytes=602619800 weight_to_token_ratio=0.129742 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343808 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265371392 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398976 weight_movement_bytes=69206016 total_nvlink_bytes=599604992 weight_to_token_ratio=0.130479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=345024 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264960 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184896 weight_movement_bytes=34603008 total_nvlink_bytes=568787904 weight_to_token_ratio=0.064777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65222 dup_rows=23816 dup_rows_remote=20964 dedup_ratio=0.242269 router_tokens_per_expert=[786, 805, 754, 721, 752, 822, 815, 772, 787, 815, 746, 748, 770, 770, 803, 739, 771, 781, 762, 763, 800, 809, 776, 785, 707, 743, 781, 748, 787, 783, 761, 738, 737, 767, 758, 725, 815, 794, 794, 751, 764, 779, 734, 756, 734, 794, 789, 747, 776, 774, 727, 738, 763, 824, 732, 725, 798, 734, 773, 765, 770, 717, 810, 788] payload_rows_per_dst=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] dup_rows_per_dst=[3066, 2970, 3048, 2940, 3032, 2954, 2852, 2954] combine_rows_per_src=[9388, 9386, 9446, 9156, 9250, 9240, 9266, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64842 dup_rows=24176 dup_rows_remote=21092 dedup_ratio=0.245931 router_tokens_per_expert=[743, 782, 760, 777, 790, 763, 759, 752, 798, 766, 739, 807, 776, 782, 793, 724, 744, 755, 725, 716, 750, 767, 768, 719, 761, 773, 761, 813, 737, 798, 800, 761, 762, 801, 807, 766, 749, 827, 792, 793, 793, 769, 781, 720, 758, 797, 767, 758, 771, 802, 774, 751, 760, 736, 741, 817, 778, 772, 757, 764, 768, 740, 758, 764] payload_rows_per_dst=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] dup_rows_per_dst=[3054, 3084, 2770, 3072, 3208, 3006, 2986, 2996] combine_rows_per_src=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64862 dup_rows=24164 dup_rows_remote=21050 dedup_ratio=0.245809 router_tokens_per_expert=[715, 766, 766, 744, 773, 779, 771, 759, 797, 770, 754, 762, 812, 750, 778, 766, 762, 788, 724, 737, 736, 770, 778, 834, 766, 738, 716, 788, 800, 795, 775, 787, 797, 764, 769, 789, 796, 771, 734, 791, 800, 760, 797, 763, 776, 735, 812, 742, 759, 781, 796, 782, 746, 767, 779, 744, 773, 750, 754, 750, 766, 748, 755, 750] payload_rows_per_dst=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] dup_rows_per_dst=[3114, 3306, 3014, 2892, 3042, 2840, 3184, 2772] combine_rows_per_src=[9278, 9660, 9284, 9152, 9380, 8956, 9274, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=344776 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265945800 ingress_bytes=369410048 link_bytes=369410048 token_bytes=531546824 weight_movement_bytes=103809024 total_nvlink_bytes=635355848 weight_to_token_ratio=0.195296 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390060271e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64704 dup_rows=24250 dup_rows_remote=21248 dedup_ratio=0.246684 router_tokens_per_expert=[711, 771, 764, 766, 813, 767, 740, 789, 775, 776, 812, 791, 774, 767, 775, 770, 748, 757, 779, 773, 794, 745, 795, 744, 820, 761, 778, 770, 758, 736, 736, 817, 843, 762, 742, 777, 747, 731, 722, 773, 686, 785, 803, 780, 778, 749, 751, 799, 820, 782, 751, 790, 796, 790, 797, 760, 708, 755, 751, 729, 761, 730, 766, 766] payload_rows_per_dst=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] dup_rows_per_dst=[2960, 3156, 2964, 3002, 3088, 3082, 3138, 2860] combine_rows_per_src=[9282, 9324, 9306, 9350, 9106, 9180, 9434, 9072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65170 dup_rows=23806 dup_rows_remote=20820 dedup_ratio=0.242167 router_tokens_per_expert=[720, 773, 831, 779, 793, 720, 788, 776, 770, 750, 773, 756, 741, 761, 806, 748, 734, 767, 756, 756, 824, 748, 791, 783, 751, 750, 816, 811, 761, 769, 732, 785, 756, 783, 753, 767, 741, 744, 738, 735, 828, 795, 754, 774, 770, 827, 769, 803, 755, 732, 773, 787, 731, 779, 780, 755, 785, 727, 800, 770, 771, 783, 755, 713] payload_rows_per_dst=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] dup_rows_per_dst=[2982, 2966, 2958, 3020, 2882, 3006, 3006, 2986] combine_rows_per_src=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344744 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267494056 ingress_bytes=301752320 link_bytes=301752320 token_bytes=534643368 weight_movement_bytes=34603008 total_nvlink_bytes=569246376 weight_to_token_ratio=0.064722 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135469687e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65164 dup_rows=24006 dup_rows_remote=21008 dedup_ratio=0.244202 router_tokens_per_expert=[781, 800, 814, 754, 752, 771, 750, 766, 747, 776, 767, 762, 776, 804, 780, 732, 765, 761, 783, 815, 737, 753, 740, 761, 747, 779, 750, 719, 816, 795, 804, 796, 752, 752, 733, 760, 739, 763, 791, 776, 753, 780, 765, 799, 774, 735, 775, 736, 804, 750, 733, 740, 810, 792, 767, 766, 787, 781, 757, 732, 770, 799, 741, 787] payload_rows_per_dst=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] dup_rows_per_dst=[3050, 3040, 2910, 3036, 2998, 2930, 3036, 3006] combine_rows_per_src=[9326, 9248, 9320, 9376, 9134, 9304, 9288, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343120 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265583696 ingress_bytes=369049600 link_bytes=369049600 token_bytes=530824272 weight_movement_bytes=103809024 total_nvlink_bytes=634633296 weight_to_token_ratio=0.195562 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388703934e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64914 dup_rows=24174 dup_rows_remote=21198 dedup_ratio=0.245911 router_tokens_per_expert=[770, 796, 745, 768, 794, 743, 742, 761, 732, 778, 750, 754, 797, 766, 819, 802, 764, 794, 770, 718, 753, 768, 754, 758, 747, 750, 721, 727, 777, 789, 839, 746, 741, 725, 763, 787, 799, 758, 755, 769, 750, 797, 810, 819, 828, 744, 757, 736, 791, 790, 750, 717, 743, 771, 778, 780, 748, 774, 763, 765, 836, 786, 732, 798] payload_rows_per_dst=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] dup_rows_per_dst=[2968, 3070, 2974, 2976, 2966, 3192, 2938, 3090] combine_rows_per_src=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64944 dup_rows=24020 dup_rows_remote=20888 dedup_ratio=0.244344 router_tokens_per_expert=[765, 752, 806, 756, 771, 761, 783, 758, 761, 724, 735, 784, 753, 775, 760, 720, 747, 807, 742, 754, 793, 761, 768, 761, 725, 776, 819, 804, 759, 785, 740, 765, 798, 733, 800, 759, 724, 754, 785, 739, 778, 771, 761, 808, 768, 798, 794, 758, 743, 808, 761, 767, 797, 764, 762, 774, 715, 722, 805, 765, 810, 798, 782, 781] payload_rows_per_dst=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] dup_rows_per_dst=[3006, 2924, 2974, 2926, 3010, 3132, 3074, 2974] combine_rows_per_src=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65176 dup_rows=23878 dup_rows_remote=20922 dedup_ratio=0.242900 router_tokens_per_expert=[841, 724, 787, 750, 775, 768, 760, 739, 784, 751, 699, 783, 799, 782, 765, 749, 766, 746, 802, 787, 763, 773, 758, 757, 773, 763, 751, 784, 770, 744, 741, 784, 787, 782, 742, 795, 781, 822, 736, 770, 783, 768, 747, 788, 756, 790, 800, 768, 771, 743, 789, 771, 757, 733, 813, 726, 753, 734, 755, 769, 805, 733, 768, 799] payload_rows_per_dst=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] dup_rows_per_dst=[3014, 2992, 2982, 2914, 3020, 3056, 2956, 2944] combine_rows_per_src=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267681792 dispatch_meta_bytes=345424 combine_payload_bytes=267681792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268027216 ingress_bytes=302284800 link_bytes=302284800 token_bytes=535709008 weight_movement_bytes=34603008 total_nvlink_bytes=570312016 weight_to_token_ratio=0.064593 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137473366e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343736 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936568 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529400 weight_movement_bytes=34603008 total_nvlink_bytes=566132408 weight_to_token_ratio=0.065101 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65170 dup_rows=23890 dup_rows_remote=20994 dedup_ratio=0.243022 router_tokens_per_expert=[791, 798, 811, 780, 758, 837, 771, 787, 819, 787, 774, 749, 769, 782, 773, 767, 710, 743, 804, 771, 773, 734, 751, 784, 753, 778, 767, 753, 749, 767, 778, 770, 765, 795, 772, 773, 766, 766, 752, 769, 720, 798, 751, 753, 723, 752, 816, 775, 807, 757, 778, 749, 720, 746, 749, 767, 766, 702, 758, 775, 782, 772, 836, 704] payload_rows_per_dst=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] dup_rows_per_dst=[3114, 2970, 2896, 2998, 3030, 3024, 2918, 2940] combine_rows_per_src=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343648 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018400 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693152 weight_movement_bytes=34603008 total_nvlink_bytes=566296160 weight_to_token_ratio=0.065081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265027584 dispatch_meta_bytes=343808 combine_payload_bytes=265027584 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265371392 ingress_bytes=334233600 link_bytes=334233600 token_bytes=530398976 weight_movement_bytes=69206016 total_nvlink_bytes=599604992 weight_to_token_ratio=0.130479 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257694129e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344688 combine_payload_bytes=266911744 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267256432 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534168176 weight_movement_bytes=69206016 total_nvlink_bytes=603374192 weight_to_token_ratio=0.129558 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=344392 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267305288 ingress_bytes=370769920 link_bytes=370769920 token_bytes=534266184 weight_movement_bytes=103809024 total_nvlink_bytes=638075208 weight_to_token_ratio=0.194302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395177360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64944 dup_rows=24020 dup_rows_remote=20888 dedup_ratio=0.244344 router_tokens_per_expert=[765, 752, 806, 756, 771, 761, 783, 758, 761, 724, 735, 784, 753, 775, 760, 720, 747, 807, 742, 754, 793, 761, 768, 761, 725, 776, 819, 804, 759, 785, 740, 765, 798, 733, 800, 759, 724, 754, 785, 739, 778, 771, 761, 808, 768, 798, 794, 758, 743, 808, 761, 767, 797, 764, 762, 774, 715, 722, 805, 765, 810, 798, 782, 781] payload_rows_per_dst=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] dup_rows_per_dst=[3006, 2924, 2974, 2926, 3010, 3132, 3074, 2974] combine_rows_per_src=[9298, 9100, 9292, 9420, 9174, 9340, 9278, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65070 dup_rows=24132 dup_rows_remote=21230 dedup_ratio=0.245483 router_tokens_per_expert=[763, 810, 797, 735, 800, 733, 741, 749, 762, 766, 759, 804, 764, 759, 725, 786, 790, 733, 706, 764, 759, 746, 767, 737, 778, 822, 773, 787, 759, 770, 778, 806, 784, 786, 782, 748, 765, 746, 761, 793, 749, 764, 735, 791, 751, 739, 761, 748, 757, 768, 783, 771, 775, 811, 790, 780, 779, 809, 757, 737, 746, 833, 742, 783] payload_rows_per_dst=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] dup_rows_per_dst=[2986, 2978, 2902, 3136, 3040, 2944, 3036, 3110] combine_rows_per_src=[9270, 9272, 9102, 9410, 9290, 9132, 9434, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64736 dup_rows=24064 dup_rows_remote=20898 dedup_ratio=0.244792 router_tokens_per_expert=[768, 752, 805, 761, 787, 729, 761, 742, 802, 796, 788, 751, 791, 744, 776, 794, 820, 746, 739, 781, 794, 748, 779, 738, 784, 762, 732, 788, 770, 794, 775, 759, 789, 735, 756, 763, 720, 766, 754, 781, 836, 727, 743, 798, 768, 718, 748, 816, 734, 782, 808, 783, 798, 754, 759, 761, 757, 756, 749, 773, 761, 731, 812, 760] payload_rows_per_dst=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] dup_rows_per_dst=[2630, 3074, 3372, 2800, 3000, 2974, 3048, 3166] combine_rows_per_src=[8896, 9410, 9602, 9056, 9096, 9334, 9342, 9504] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343960 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267280280 ingress_bytes=336142336 link_bytes=336142336 token_bytes=534216600 weight_movement_bytes=69206016 total_nvlink_bytes=603422616 weight_to_token_ratio=0.129547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264876549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=344448 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266232192 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119936 weight_movement_bytes=34603008 total_nvlink_bytes=566722944 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343328 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353952 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364576 weight_movement_bytes=69206016 total_nvlink_bytes=601570592 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=345200 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871920 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533398640 weight_movement_bytes=34603008 total_nvlink_bytes=568001648 weight_to_token_ratio=0.064873 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64842 dup_rows=24176 dup_rows_remote=21092 dedup_ratio=0.245931 router_tokens_per_expert=[743, 782, 760, 777, 790, 763, 759, 752, 798, 766, 739, 807, 776, 782, 793, 724, 744, 755, 725, 716, 750, 767, 768, 719, 761, 773, 761, 813, 737, 798, 800, 761, 762, 801, 807, 766, 749, 827, 792, 793, 793, 769, 781, 720, 758, 797, 767, 758, 771, 802, 774, 751, 760, 736, 741, 817, 778, 772, 757, 764, 768, 740, 758, 764] payload_rows_per_dst=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] dup_rows_per_dst=[3054, 3084, 2770, 3072, 3208, 3006, 2986, 2996] combine_rows_per_src=[9198, 9286, 9118, 9336, 9386, 9280, 9318, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=344656 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267280976 ingress_bytes=301539328 link_bytes=301539328 token_bytes=534217296 weight_movement_bytes=34603008 total_nvlink_bytes=568820304 weight_to_token_ratio=0.064773 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134668215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65176 dup_rows=23878 dup_rows_remote=20922 dedup_ratio=0.242900 router_tokens_per_expert=[841, 724, 787, 750, 775, 768, 760, 739, 784, 751, 699, 783, 799, 782, 765, 749, 766, 746, 802, 787, 763, 773, 758, 757, 773, 763, 751, 784, 770, 744, 741, 784, 787, 782, 742, 795, 781, 822, 736, 770, 783, 768, 747, 788, 756, 790, 800, 768, 771, 743, 789, 771, 757, 733, 813, 726, 753, 734, 755, 769, 805, 733, 768, 799] payload_rows_per_dst=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] dup_rows_per_dst=[3014, 2992, 2982, 2914, 3020, 3056, 2956, 2944] combine_rows_per_src=[9274, 9232, 9322, 9306, 9410, 9344, 9250, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65000 dup_rows=24020 dup_rows_remote=21000 dedup_ratio=0.244344 router_tokens_per_expert=[790, 774, 758, 777, 787, 814, 747, 737, 748, 764, 736, 730, 784, 803, 805, 782, 765, 754, 753, 753, 746, 748, 775, 780, 790, 735, 768, 711, 779, 773, 792, 775, 793, 775, 758, 805, 778, 817, 765, 773, 752, 796, 744, 766, 725, 802, 780, 719, 771, 769, 763, 781, 789, 743, 804, 775, 766, 749, 760, 730, 737, 773, 789, 772] payload_rows_per_dst=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] dup_rows_per_dst=[3030, 3020, 2948, 3008, 3078, 2994, 3030, 2912] combine_rows_per_src=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=64920 dup_rows=23938 dup_rows_remote=20714 dedup_ratio=0.243510 router_tokens_per_expert=[809, 769, 778, 753, 778, 781, 749, 772, 757, 738, 778, 815, 737, 698, 748, 775, 758, 746, 754, 738, 772, 797, 747, 850, 775, 774, 788, 819, 787, 812, 764, 747, 796, 725, 770, 746, 755, 790, 771, 804, 786, 798, 763, 748, 790, 742, 767, 762, 703, 795, 778, 703, 764, 749, 764, 769, 773, 785, 797, 769, 785, 724, 774, 744] payload_rows_per_dst=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] dup_rows_per_dst=[3224, 2918, 2992, 3136, 2914, 2946, 2774, 3034] combine_rows_per_src=[9446, 9242, 9332, 9536, 9260, 9188, 9094, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64914 dup_rows=24174 dup_rows_remote=21198 dedup_ratio=0.245911 router_tokens_per_expert=[770, 796, 745, 768, 794, 743, 742, 761, 732, 778, 750, 754, 797, 766, 819, 802, 764, 794, 770, 718, 753, 768, 754, 758, 747, 750, 721, 727, 777, 789, 839, 746, 741, 725, 763, 787, 799, 758, 755, 769, 750, 797, 810, 819, 828, 744, 757, 736, 791, 790, 750, 717, 743, 771, 778, 780, 748, 774, 763, 765, 836, 786, 732, 798] payload_rows_per_dst=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] dup_rows_per_dst=[2968, 3070, 2974, 2976, 2966, 3192, 2938, 3090] combine_rows_per_src=[9270, 9326, 9184, 9216, 9228, 9290, 9302, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64996 dup_rows=23926 dup_rows_remote=20884 dedup_ratio=0.243388 router_tokens_per_expert=[744, 789, 762, 760, 820, 774, 767, 759, 788, 782, 707, 726, 819, 778, 705, 764, 768, 808, 844, 796, 783, 764, 747, 807, 736, 793, 790, 740, 790, 757, 717, 755, 754, 732, 784, 741, 750, 756, 721, 799, 776, 725, 797, 765, 781, 768, 741, 778, 785, 783, 794, 792, 824, 711, 753, 770, 753, 707, 723, 781, 789, 793, 757, 830] payload_rows_per_dst=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] dup_rows_per_dst=[2956, 2938, 3114, 2928, 2874, 3012, 3042, 3062] combine_rows_per_src=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=342536 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265501192 ingress_bytes=299761664 link_bytes=299761664 token_bytes=530659848 weight_movement_bytes=34603008 total_nvlink_bytes=565262856 weight_to_token_ratio=0.065208 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127979009e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=344392 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267305288 ingress_bytes=370769920 link_bytes=370769920 token_bytes=534266184 weight_movement_bytes=103809024 total_nvlink_bytes=638075208 weight_to_token_ratio=0.194302 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395177360e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64984 dup_rows=23974 dup_rows_remote=20944 dedup_ratio=0.243876 router_tokens_per_expert=[777, 756, 755, 769, 766, 754, 793, 761, 738, 761, 774, 736, 797, 802, 837, 750, 720, 746, 785, 726, 739, 791, 772, 757, 742, 730, 751, 761, 726, 719, 804, 778, 727, 729, 767, 776, 813, 776, 807, 793, 780, 756, 785, 769, 807, 792, 745, 742, 777, 770, 789, 755, 810, 766, 761, 768, 776, 781, 803, 738, 800, 762, 811, 748] payload_rows_per_dst=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] dup_rows_per_dst=[3068, 3094, 2886, 2846, 3030, 2962, 3044, 3044] combine_rows_per_src=[9194, 9296, 9186, 9176, 9346, 9390, 9348, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343736 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265936568 ingress_bytes=300195840 link_bytes=300195840 token_bytes=531529400 weight_movement_bytes=34603008 total_nvlink_bytes=566132408 weight_to_token_ratio=0.065101 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129612778e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342536 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266254856 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532167176 weight_movement_bytes=69206016 total_nvlink_bytes=601373192 weight_to_token_ratio=0.130046 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=344448 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266232192 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532119936 weight_movement_bytes=34603008 total_nvlink_bytes=566722944 weight_to_token_ratio=0.065029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343520 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266567136 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790752 weight_movement_bytes=69206016 total_nvlink_bytes=601996768 weight_to_token_ratio=0.129893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65140 dup_rows=23872 dup_rows_remote=20858 dedup_ratio=0.242839 router_tokens_per_expert=[746, 817, 776, 759, 764, 790, 804, 735, 764, 781, 734, 778, 766, 733, 754, 770, 788, 754, 757, 751, 783, 810, 780, 722, 761, 750, 785, 754, 793, 761, 791, 758, 785, 748, 757, 786, 815, 755, 770, 795, 753, 738, 791, 741, 713, 781, 751, 772, 772, 765, 737, 788, 733, 771, 736, 807, 763, 760, 786, 799, 779, 799, 748, 789] payload_rows_per_dst=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] dup_rows_per_dst=[3010, 3016, 2952, 3014, 3078, 2866, 2872, 3064] combine_rows_per_src=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65100 dup_rows=24062 dup_rows_remote=21088 dedup_ratio=0.244771 router_tokens_per_expert=[765, 763, 797, 815, 744, 768, 796, 770, 785, 783, 803, 836, 775, 795, 750, 763, 788, 746, 810, 728, 782, 825, 779, 790, 767, 745, 823, 793, 716, 757, 774, 704, 757, 745, 767, 783, 813, 754, 740, 726, 745, 743, 808, 764, 720, 757, 766, 765, 830, 776, 802, 759, 736, 776, 777, 770, 787, 745, 726, 755, 716, 687, 794, 728] payload_rows_per_dst=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] dup_rows_per_dst=[3082, 2940, 2972, 2862, 3032, 3186, 3014, 2974] combine_rows_per_src=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64980 dup_rows=24182 dup_rows_remote=21224 dedup_ratio=0.245992 router_tokens_per_expert=[733, 760, 771, 730, 755, 725, 775, 778, 761, 790, 800, 772, 791, 755, 731, 805, 765, 732, 767, 745, 760, 739, 806, 736, 785, 757, 756, 789, 753, 803, 788, 742, 781, 797, 730, 757, 807, 762, 742, 809, 762, 781, 773, 759, 780, 738, 788, 729, 767, 818, 760, 764, 732, 795, 820, 778, 780, 721, 746, 763, 791, 825, 779, 763] payload_rows_per_dst=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] dup_rows_per_dst=[2858, 3032, 2958, 3066, 3092, 3018, 3054, 3104] combine_rows_per_src=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74510 payload_rows_remote=65298 dup_rows=23794 dup_rows_remote=20908 dedup_ratio=0.242045 router_tokens_per_expert=[756, 771, 745, 760, 723, 796, 742, 793, 775, 740, 735, 723, 762, 764, 754, 758, 728, 784, 767, 724, 795, 755, 786, 767, 757, 760, 794, 753, 781, 787, 771, 778, 748, 811, 788, 756, 763, 711, 781, 812, 778, 751, 778, 792, 749, 744, 725, 732, 729, 780, 780, 747, 800, 790, 800, 813, 800, 785, 819, 810, 771, 753, 773, 799] payload_rows_per_dst=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] dup_rows_per_dst=[2894, 2876, 2988, 2936, 3008, 2886, 3070, 3136] combine_rows_per_src=[9278, 9146, 9224, 9426, 9332, 9212, 9408, 9484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343712 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518176 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692640 weight_movement_bytes=34603008 total_nvlink_bytes=567295648 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65166 dup_rows=24010 dup_rows_remote=21090 dedup_ratio=0.244242 router_tokens_per_expert=[826, 772, 816, 784, 784, 782, 782, 777, 803, 783, 793, 768, 787, 725, 736, 742, 763, 723, 750, 785, 757, 754, 743, 749, 744, 738, 774, 767, 748, 738, 769, 741, 837, 773, 779, 765, 767, 776, 799, 754, 758, 783, 721, 761, 756, 782, 803, 740, 750, 788, 808, 748, 774, 723, 787, 766, 770, 789, 717, 844, 720, 742, 770, 799] payload_rows_per_dst=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] dup_rows_per_dst=[3128, 3038, 2920, 2824, 3120, 2958, 2996, 3026] combine_rows_per_src=[9518, 9236, 9128, 9214, 9380, 9250, 9292, 9276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344000 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584000 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824000 weight_movement_bytes=34603008 total_nvlink_bytes=567427008 weight_to_token_ratio=0.064943 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=342536 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266254856 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532167176 weight_movement_bytes=69206016 total_nvlink_bytes=601373192 weight_to_token_ratio=0.130046 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=343712 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266518176 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532692640 weight_movement_bytes=34603008 total_nvlink_bytes=567295648 weight_to_token_ratio=0.064959 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344752 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266994352 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643952 weight_movement_bytes=34603008 total_nvlink_bytes=568246960 weight_to_token_ratio=0.064843 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343992 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157432 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970872 weight_movement_bytes=34603008 total_nvlink_bytes=568573880 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344816 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266502896 ingress_bytes=369967104 link_bytes=369967104 token_bytes=532660976 weight_movement_bytes=103809024 total_nvlink_bytes=636470000 weight_to_token_ratio=0.194888 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392156428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267460608 dispatch_meta_bytes=344824 combine_payload_bytes=267460608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267805432 ingress_bytes=371269632 link_bytes=371269632 token_bytes=535266040 weight_movement_bytes=103809024 total_nvlink_bytes=639075064 weight_to_token_ratio=0.193939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397057736e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65170 dup_rows=23806 dup_rows_remote=20820 dedup_ratio=0.242167 router_tokens_per_expert=[720, 773, 831, 779, 793, 720, 788, 776, 770, 750, 773, 756, 741, 761, 806, 748, 734, 767, 756, 756, 824, 748, 791, 783, 751, 750, 816, 811, 761, 769, 732, 785, 756, 783, 753, 767, 741, 744, 738, 735, 828, 795, 754, 774, 770, 827, 769, 803, 755, 732, 773, 787, 731, 779, 780, 755, 785, 727, 800, 770, 771, 783, 755, 713] payload_rows_per_dst=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] dup_rows_per_dst=[2982, 2966, 2958, 3020, 2882, 3006, 3006, 2986] combine_rows_per_src=[9174, 9244, 9360, 9330, 9240, 9464, 9358, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65000 dup_rows=24020 dup_rows_remote=21000 dedup_ratio=0.244344 router_tokens_per_expert=[790, 774, 758, 777, 787, 814, 747, 737, 748, 764, 736, 730, 784, 803, 805, 782, 765, 754, 753, 753, 746, 748, 775, 780, 790, 735, 768, 711, 779, 773, 792, 775, 793, 775, 758, 805, 778, 817, 765, 773, 752, 796, 744, 766, 725, 802, 780, 719, 771, 769, 763, 781, 789, 743, 804, 775, 766, 749, 760, 730, 737, 773, 789, 772] payload_rows_per_dst=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] dup_rows_per_dst=[3030, 3020, 2948, 3008, 3078, 2994, 3030, 2912] combine_rows_per_src=[9338, 9284, 9200, 9238, 9450, 9174, 9360, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64996 dup_rows=23926 dup_rows_remote=20884 dedup_ratio=0.243388 router_tokens_per_expert=[744, 789, 762, 760, 820, 774, 767, 759, 788, 782, 707, 726, 819, 778, 705, 764, 768, 808, 844, 796, 783, 764, 747, 807, 736, 793, 790, 740, 790, 757, 717, 755, 754, 732, 784, 741, 750, 756, 721, 799, 776, 725, 797, 765, 781, 768, 741, 778, 785, 783, 794, 792, 824, 711, 753, 770, 753, 707, 723, 781, 789, 793, 757, 830] payload_rows_per_dst=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] dup_rows_per_dst=[2956, 2938, 3114, 2928, 2874, 3012, 3042, 3062] combine_rows_per_src=[9394, 9200, 9520, 9228, 9200, 9250, 9382, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64826 dup_rows=24050 dup_rows_remote=20902 dedup_ratio=0.244649 router_tokens_per_expert=[760, 794, 738, 770, 760, 839, 696, 771, 768, 809, 810, 803, 801, 723, 797, 777, 758, 749, 744, 757, 828, 784, 774, 803, 794, 712, 810, 751, 781, 770, 746, 782, 744, 745, 765, 791, 781, 760, 804, 760, 779, 753, 766, 722, 772, 760, 765, 786, 756, 733, 786, 739, 784, 763, 768, 750, 747, 772, 786, 773, 776, 768, 737, 702] payload_rows_per_dst=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] dup_rows_per_dst=[2980, 3148, 3084, 2992, 2918, 2942, 2974, 3012] combine_rows_per_src=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343520 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266567136 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790752 weight_movement_bytes=69206016 total_nvlink_bytes=601996768 weight_to_token_ratio=0.129893 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65048 dup_rows=24090 dup_rows_remote=21164 dedup_ratio=0.245056 router_tokens_per_expert=[748, 714, 775, 760, 727, 773, 779, 770, 778, 737, 802, 747, 740, 779, 775, 803, 742, 758, 757, 784, 732, 758, 806, 738, 747, 797, 778, 808, 729, 769, 708, 786, 787, 760, 756, 750, 768, 757, 820, 779, 751, 717, 767, 790, 795, 794, 772, 803, 817, 787, 730, 795, 766, 775, 758, 743, 742, 773, 804, 797, 767, 762, 804, 762] payload_rows_per_dst=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] dup_rows_per_dst=[2926, 3062, 3138, 3216, 3080, 3024, 2802, 2842] combine_rows_per_src=[9166, 9342, 9484, 9530, 9274, 9272, 9068, 9078] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=345024 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264960 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184896 weight_movement_bytes=34603008 total_nvlink_bytes=568787904 weight_to_token_ratio=0.064777 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65140 dup_rows=23872 dup_rows_remote=20858 dedup_ratio=0.242839 router_tokens_per_expert=[746, 817, 776, 759, 764, 790, 804, 735, 764, 781, 734, 778, 766, 733, 754, 770, 788, 754, 757, 751, 783, 810, 780, 722, 761, 750, 785, 754, 793, 761, 791, 758, 785, 748, 757, 786, 815, 755, 770, 795, 753, 738, 791, 741, 713, 781, 751, 772, 772, 765, 737, 788, 733, 771, 736, 807, 763, 760, 786, 799, 779, 799, 748, 789] payload_rows_per_dst=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] dup_rows_per_dst=[3010, 3016, 2952, 3014, 3078, 2866, 2872, 3064] combine_rows_per_src=[9372, 9144, 9338, 9292, 9344, 9214, 9346, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64806 dup_rows=24130 dup_rows_remote=21032 dedup_ratio=0.245463 router_tokens_per_expert=[755, 776, 770, 774, 787, 791, 752, 730, 752, 717, 800, 752, 763, 712, 779, 831, 728, 800, 825, 742, 787, 758, 828, 737, 783, 773, 753, 786, 782, 775, 758, 772, 703, 710, 782, 774, 736, 795, 752, 774, 798, 755, 794, 740, 756, 768, 731, 764, 806, 794, 788, 802, 751, 796, 745, 751, 799, 735, 761, 774, 757, 739, 795, 799] payload_rows_per_dst=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] dup_rows_per_dst=[3000, 3016, 3116, 3022, 2928, 2932, 3098, 3018] combine_rows_per_src=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65084 dup_rows=23886 dup_rows_remote=20902 dedup_ratio=0.242981 router_tokens_per_expert=[789, 773, 759, 768, 738, 790, 761, 776, 772, 715, 778, 787, 756, 797, 736, 802, 761, 768, 763, 763, 760, 771, 777, 754, 742, 815, 737, 783, 757, 745, 799, 709, 793, 738, 725, 754, 764, 807, 760, 804, 790, 791, 788, 730, 760, 768, 756, 785, 769, 784, 794, 773, 764, 766, 784, 745, 767, 786, 826, 773, 753, 748, 761, 745] payload_rows_per_dst=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] dup_rows_per_dst=[3014, 2914, 2956, 2932, 2972, 2982, 3132, 2984] combine_rows_per_src=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64808 dup_rows=24162 dup_rows_remote=21158 dedup_ratio=0.245789 router_tokens_per_expert=[725, 734, 763, 769, 731, 763, 768, 773, 779, 780, 724, 735, 804, 774, 776, 827, 763, 789, 767, 785, 705, 804, 774, 782, 754, 734, 754, 787, 798, 783, 809, 775, 741, 773, 782, 733, 778, 745, 787, 732, 759, 780, 739, 821, 731, 787, 737, 823, 771, 814, 757, 732, 808, 786, 808, 748, 783, 742, 789, 763, 733, 748, 749, 785] payload_rows_per_dst=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] dup_rows_per_dst=[2956, 3030, 3004, 3000, 2946, 3070, 3168, 2988] combine_rows_per_src=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64900 dup_rows=24042 dup_rows_remote=20970 dedup_ratio=0.244568 router_tokens_per_expert=[737, 746, 776, 768, 754, 756, 766, 710, 746, 758, 783, 759, 781, 811, 779, 736, 744, 787, 754, 753, 732, 812, 803, 773, 776, 768, 760, 802, 774, 763, 783, 791, 745, 727, 786, 765, 807, 717, 728, 730, 774, 785, 794, 825, 759, 786, 753, 786, 812, 732, 789, 770, 756, 792, 785, 769, 743, 745, 793, 778, 751, 740, 777, 812] payload_rows_per_dst=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] dup_rows_per_dst=[2870, 2952, 3076, 3072, 2852, 3194, 3036, 2990] combine_rows_per_src=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74100 payload_rows_remote=64862 dup_rows=24204 dup_rows_remote=21140 dedup_ratio=0.246216 router_tokens_per_expert=[720, 772, 791, 771, 781, 759, 767, 783, 747, 755, 759, 723, 799, 738, 776, 780, 773, 773, 725, 801, 739, 791, 810, 728, 780, 775, 704, 746, 739, 791, 787, 764, 765, 765, 758, 780, 784, 758, 765, 776, 745, 827, 760, 826, 784, 761, 778, 774, 749, 717, 746, 775, 815, 767, 768, 763, 790, 800, 793, 773, 776, 731, 761, 775] payload_rows_per_dst=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] dup_rows_per_dst=[3030, 2972, 3054, 2972, 3064, 3082, 2956, 3074] combine_rows_per_src=[9258, 9182, 9226, 9200, 9238, 9428, 9244, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343960 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267280280 ingress_bytes=336142336 link_bytes=336142336 token_bytes=534216600 weight_movement_bytes=69206016 total_nvlink_bytes=603422616 weight_to_token_ratio=0.129547 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264876549e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344000 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584000 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824000 weight_movement_bytes=34603008 total_nvlink_bytes=567427008 weight_to_token_ratio=0.064943 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344848 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266781456 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533218064 weight_movement_bytes=103809024 total_nvlink_bytes=637027088 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343992 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157432 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970872 weight_movement_bytes=34603008 total_nvlink_bytes=568573880 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343352 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265788728 ingress_bytes=369254400 link_bytes=369254400 token_bytes=531234104 weight_movement_bytes=103809024 total_nvlink_bytes=635043128 weight_to_token_ratio=0.195411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389474580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65054 dup_rows=23956 dup_rows_remote=20908 dedup_ratio=0.243693 router_tokens_per_expert=[783, 766, 774, 771, 743, 752, 740, 798, 800, 762, 807, 765, 724, 777, 802, 777, 841, 783, 751, 759, 735, 725, 793, 720, 794, 757, 730, 740, 744, 757, 773, 762, 752, 775, 778, 756, 757, 773, 746, 764, 776, 820, 757, 766, 707, 746, 802, 797, 801, 771, 777, 809, 780, 796, 751, 783, 762, 785, 749, 736, 746, 771, 772, 786] payload_rows_per_dst=[9298, 9356, 9244, 9124, 9236, 9294, 9554, 9242] dup_rows_per_dst=[2956, 3072, 2970, 2990, 2966, 3048, 2982, 2972] combine_rows_per_src=[9298, 9356, 9244, 9124, 9236, 9294, 9554, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344008 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018760 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693512 weight_movement_bytes=34603008 total_nvlink_bytes=566296520 weight_to_token_ratio=0.065081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=342912 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870208 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397504 weight_movement_bytes=103809024 total_nvlink_bytes=635206528 weight_to_token_ratio=0.195351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65170 dup_rows=23890 dup_rows_remote=20994 dedup_ratio=0.243022 router_tokens_per_expert=[791, 798, 811, 780, 758, 837, 771, 787, 819, 787, 774, 749, 769, 782, 773, 767, 710, 743, 804, 771, 773, 734, 751, 784, 753, 778, 767, 753, 749, 767, 778, 770, 765, 795, 772, 773, 766, 766, 752, 769, 720, 798, 751, 753, 723, 752, 816, 775, 807, 757, 778, 749, 720, 746, 749, 767, 766, 702, 758, 775, 782, 772, 836, 704] payload_rows_per_dst=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] dup_rows_per_dst=[3114, 2970, 2896, 2998, 3030, 3024, 2918, 2940] combine_rows_per_src=[9552, 9470, 9244, 9232, 9286, 9152, 9228, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344848 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266781456 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533218064 weight_movement_bytes=103809024 total_nvlink_bytes=637027088 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=343944 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266928008 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512072 weight_movement_bytes=69206016 total_nvlink_bytes=602718088 weight_to_token_ratio=0.129718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=344008 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018760 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693512 weight_movement_bytes=34603008 total_nvlink_bytes=566296520 weight_to_token_ratio=0.065081 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343864 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797432 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531251000 weight_movement_bytes=69206016 total_nvlink_bytes=600457016 weight_to_token_ratio=0.130270 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343480 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173880 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004280 weight_movement_bytes=34603008 total_nvlink_bytes=566607288 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64806 dup_rows=24130 dup_rows_remote=21032 dedup_ratio=0.245463 router_tokens_per_expert=[755, 776, 770, 774, 787, 791, 752, 730, 752, 717, 800, 752, 763, 712, 779, 831, 728, 800, 825, 742, 787, 758, 828, 737, 783, 773, 753, 786, 782, 775, 758, 772, 703, 710, 782, 774, 736, 795, 752, 774, 798, 755, 794, 740, 756, 768, 731, 764, 806, 794, 788, 802, 751, 796, 745, 751, 799, 735, 761, 774, 757, 739, 795, 799] payload_rows_per_dst=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] dup_rows_per_dst=[3000, 3016, 3116, 3022, 2928, 2932, 3098, 3018] combine_rows_per_src=[9270, 9196, 9294, 9342, 9124, 9280, 9368, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343848 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266805032 ingress_bytes=301064192 link_bytes=301064192 token_bytes=533266216 weight_movement_bytes=34603008 total_nvlink_bytes=567869224 weight_to_token_ratio=0.064889 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132880317e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65100 dup_rows=24062 dup_rows_remote=21088 dedup_ratio=0.244771 router_tokens_per_expert=[765, 763, 797, 815, 744, 768, 796, 770, 785, 783, 803, 836, 775, 795, 750, 763, 788, 746, 810, 728, 782, 825, 779, 790, 767, 745, 823, 793, 716, 757, 774, 704, 757, 745, 767, 783, 813, 754, 740, 726, 745, 743, 808, 764, 720, 757, 766, 765, 830, 776, 802, 759, 736, 776, 777, 770, 787, 745, 726, 755, 716, 687, 794, 728] payload_rows_per_dst=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] dup_rows_per_dst=[3082, 2940, 2972, 2862, 3032, 3186, 3014, 2974] combine_rows_per_src=[9354, 9268, 9148, 9296, 9274, 9322, 9438, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343352 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265788728 ingress_bytes=369254400 link_bytes=369254400 token_bytes=531234104 weight_movement_bytes=103809024 total_nvlink_bytes=635043128 weight_to_token_ratio=0.195411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389474580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64990 dup_rows=24166 dup_rows_remote=21152 dedup_ratio=0.245829 router_tokens_per_expert=[786, 788, 698, 763, 734, 782, 752, 792, 788, 751, 756, 762, 794, 743, 740, 747, 750, 750, 751, 822, 819, 790, 749, 779, 757, 796, 695, 757, 764, 764, 746, 777, 799, 763, 740, 752, 780, 793, 746, 759, 784, 765, 794, 782, 804, 767, 785, 749, 762, 795, 773, 758, 725, 760, 760, 757, 744, 793, 826, 795, 822, 784, 782, 712] payload_rows_per_dst=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] dup_rows_per_dst=[2894, 3014, 3102, 2818, 3028, 3150, 2978, 3182] combine_rows_per_src=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65134 dup_rows=23828 dup_rows_remote=20806 dedup_ratio=0.242391 router_tokens_per_expert=[721, 776, 798, 739, 803, 749, 735, 786, 766, 792, 728, 749, 775, 768, 814, 741, 813, 735, 776, 810, 739, 783, 782, 792, 714, 807, 778, 733, 744, 764, 755, 739, 813, 790, 748, 750, 754, 775, 767, 778, 764, 788, 772, 739, 775, 755, 776, 744, 734, 814, 727, 752, 801, 804, 790, 761, 788, 759, 777, 762, 723, 815, 791, 762] payload_rows_per_dst=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] dup_rows_per_dst=[3022, 2892, 3098, 3184, 3006, 2932, 2730, 2964] combine_rows_per_src=[9342, 9264, 9398, 9368, 9344, 9294, 9076, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64826 dup_rows=24050 dup_rows_remote=20902 dedup_ratio=0.244649 router_tokens_per_expert=[760, 794, 738, 770, 760, 839, 696, 771, 768, 809, 810, 803, 801, 723, 797, 777, 758, 749, 744, 757, 828, 784, 774, 803, 794, 712, 810, 751, 781, 770, 746, 782, 744, 745, 765, 791, 781, 760, 804, 760, 779, 753, 766, 722, 772, 760, 765, 786, 756, 733, 786, 739, 784, 763, 768, 750, 747, 772, 786, 773, 776, 768, 737, 702] payload_rows_per_dst=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] dup_rows_per_dst=[2980, 3148, 3084, 2992, 2918, 2942, 2974, 3012] combine_rows_per_src=[9276, 9428, 9310, 9300, 9382, 9264, 9184, 9110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65094 dup_rows=23904 dup_rows_remote=20850 dedup_ratio=0.243164 router_tokens_per_expert=[777, 763, 789, 811, 750, 779, 793, 784, 767, 721, 767, 740, 774, 795, 796, 739, 752, 813, 771, 779, 767, 779, 779, 772, 788, 731, 718, 793, 771, 744, 775, 758, 800, 747, 777, 789, 785, 754, 775, 779, 752, 756, 779, 707, 757, 770, 787, 757, 721, 790, 765, 793, 790, 760, 755, 806, 775, 748, 724, 787, 749, 769, 773, 741] payload_rows_per_dst=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] dup_rows_per_dst=[3066, 2922, 3134, 2850, 3072, 2910, 3054, 2896] combine_rows_per_src=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=344656 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267280976 ingress_bytes=301539328 link_bytes=301539328 token_bytes=534217296 weight_movement_bytes=34603008 total_nvlink_bytes=568820304 weight_to_token_ratio=0.064773 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134668215e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64900 dup_rows=24042 dup_rows_remote=20970 dedup_ratio=0.244568 router_tokens_per_expert=[737, 746, 776, 768, 754, 756, 766, 710, 746, 758, 783, 759, 781, 811, 779, 736, 744, 787, 754, 753, 732, 812, 803, 773, 776, 768, 760, 802, 774, 763, 783, 791, 745, 727, 786, 765, 807, 717, 728, 730, 774, 785, 794, 825, 759, 786, 753, 786, 812, 732, 789, 770, 756, 792, 785, 769, 743, 745, 793, 778, 751, 740, 777, 812] payload_rows_per_dst=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] dup_rows_per_dst=[2870, 2952, 3076, 3072, 2852, 3194, 3036, 2990] combine_rows_per_src=[9156, 9354, 9240, 9362, 9158, 9330, 9374, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64710 dup_rows=24134 dup_rows_remote=20908 dedup_ratio=0.245504 router_tokens_per_expert=[752, 709, 715, 774, 763, 788, 743, 801, 796, 827, 745, 749, 818, 766, 774, 801, 739, 727, 782, 740, 757, 766, 765, 779, 777, 785, 815, 761, 767, 803, 758, 760, 743, 762, 770, 747, 777, 755, 775, 765, 774, 758, 759, 797, 752, 791, 739, 800, 747, 764, 732, 756, 766, 754, 768, 795, 766, 803, 783, 773, 795, 730, 770, 784] payload_rows_per_dst=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] dup_rows_per_dst=[2944, 2888, 2932, 2894, 3064, 3172, 3014, 3226] combine_rows_per_src=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65124 dup_rows=24032 dup_rows_remote=21214 dedup_ratio=0.244466 router_tokens_per_expert=[752, 773, 785, 763, 757, 787, 794, 759, 763, 737, 777, 791, 769, 775, 769, 795, 740, 745, 777, 754, 719, 720, 761, 767, 820, 784, 783, 778, 759, 773, 811, 771, 785, 744, 773, 751, 765, 745, 792, 781, 762, 755, 793, 710, 767, 751, 762, 777, 736, 787, 791, 855, 767, 748, 789, 791, 715, 790, 779, 808, 751, 726, 740, 758] payload_rows_per_dst=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] dup_rows_per_dst=[3060, 3076, 2818, 3158, 2998, 2892, 3062, 2968] combine_rows_per_src=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74004 payload_rows_remote=64786 dup_rows=24300 dup_rows_remote=21310 dedup_ratio=0.247192 router_tokens_per_expert=[772, 737, 779, 734, 733, 753, 765, 755, 769, 771, 773, 788, 768, 783, 703, 739, 771, 790, 762, 735, 808, 768, 772, 755, 801, 750, 727, 783, 773, 853, 792, 763, 769, 760, 734, 775, 797, 784, 764, 721, 753, 773, 750, 762, 733, 778, 788, 740, 780, 785, 811, 759, 777, 800, 762, 801, 812, 802, 772, 729, 783, 780, 721, 772] payload_rows_per_dst=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] dup_rows_per_dst=[2976, 2944, 2986, 3118, 2990, 3038, 3202, 3046] combine_rows_per_src=[9080, 9244, 9336, 9366, 9218, 9116, 9348, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344752 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266994352 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643952 weight_movement_bytes=34603008 total_nvlink_bytes=568246960 weight_to_token_ratio=0.064843 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343760 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132624 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921488 weight_movement_bytes=34603008 total_nvlink_bytes=568524496 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=342912 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265870208 ingress_bytes=369336320 link_bytes=369336320 token_bytes=531397504 weight_movement_bytes=103809024 total_nvlink_bytes=635206528 weight_to_token_ratio=0.195351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389782838e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343776 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968800 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593824 weight_movement_bytes=69206016 total_nvlink_bytes=602799840 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64770 dup_rows=24190 dup_rows_remote=21094 dedup_ratio=0.246073 router_tokens_per_expert=[820, 743, 754, 728, 764, 751, 781, 728, 780, 792, 787, 775, 773, 728, 767, 772, 720, 755, 776, 737, 772, 794, 817, 737, 755, 738, 763, 776, 813, 802, 806, 767, 788, 708, 765, 747, 713, 797, 752, 772, 786, 741, 791, 742, 771, 744, 714, 750, 789, 766, 845, 830, 754, 783, 752, 748, 807, 800, 785, 772, 770, 770, 714, 815] payload_rows_per_dst=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] dup_rows_per_dst=[2962, 2962, 3032, 3096, 2972, 2926, 3170, 3070] combine_rows_per_src=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64928 dup_rows=24010 dup_rows_remote=20902 dedup_ratio=0.244242 router_tokens_per_expert=[761, 769, 768, 808, 770, 723, 756, 787, 810, 752, 798, 779, 778, 762, 765, 780, 741, 770, 721, 727, 734, 783, 781, 751, 763, 761, 768, 760, 766, 754, 761, 781, 734, 756, 762, 812, 758, 771, 805, 780, 768, 773, 809, 809, 768, 753, 754, 803, 744, 750, 740, 759, 759, 756, 735, 765, 782, 748, 797, 767, 780, 802, 768, 797] payload_rows_per_dst=[9292, 9360, 9214, 9220, 9262, 9366, 9114, 9466] dup_rows_per_dst=[2992, 3088, 2802, 3008, 3094, 3108, 2902, 3016] combine_rows_per_src=[9292, 9360, 9214, 9220, 9262, 9366, 9114, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343480 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266173880 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004280 weight_movement_bytes=34603008 total_nvlink_bytes=566607288 weight_to_token_ratio=0.065043 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344384 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707840 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071296 weight_movement_bytes=34603008 total_nvlink_bytes=565674304 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344568 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266543608 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742648 weight_movement_bytes=103809024 total_nvlink_bytes=636551672 weight_to_token_ratio=0.194858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343760 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132624 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921488 weight_movement_bytes=34603008 total_nvlink_bytes=568524496 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64980 dup_rows=24182 dup_rows_remote=21224 dedup_ratio=0.245992 router_tokens_per_expert=[733, 760, 771, 730, 755, 725, 775, 778, 761, 790, 800, 772, 791, 755, 731, 805, 765, 732, 767, 745, 760, 739, 806, 736, 785, 757, 756, 789, 753, 803, 788, 742, 781, 797, 730, 757, 807, 762, 742, 809, 762, 781, 773, 759, 780, 738, 788, 729, 767, 818, 760, 764, 732, 795, 820, 778, 780, 721, 746, 763, 791, 825, 779, 763] payload_rows_per_dst=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] dup_rows_per_dst=[2858, 3032, 2958, 3066, 3092, 3018, 3054, 3104] combine_rows_per_src=[9196, 9378, 9142, 9280, 9278, 9202, 9414, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=342472 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265394632 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446792 weight_movement_bytes=34603008 total_nvlink_bytes=565049800 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=345352 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267093256 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533841160 weight_movement_bytes=34603008 total_nvlink_bytes=568444168 weight_to_token_ratio=0.064819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65094 dup_rows=23904 dup_rows_remote=20850 dedup_ratio=0.243164 router_tokens_per_expert=[777, 763, 789, 811, 750, 779, 793, 784, 767, 721, 767, 740, 774, 795, 796, 739, 752, 813, 771, 779, 767, 779, 779, 772, 788, 731, 718, 793, 771, 744, 775, 758, 800, 747, 777, 789, 785, 754, 775, 779, 752, 756, 779, 707, 757, 770, 787, 757, 721, 790, 765, 793, 790, 760, 755, 806, 775, 748, 724, 787, 749, 769, 773, 741] payload_rows_per_dst=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] dup_rows_per_dst=[3066, 2922, 3134, 2850, 3072, 2910, 3054, 2896] combine_rows_per_src=[9426, 9276, 9290, 9306, 9340, 9220, 9306, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344384 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707840 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071296 weight_movement_bytes=34603008 total_nvlink_bytes=565674304 weight_to_token_ratio=0.065157 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=343456 combine_payload_bytes=265297920 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265641376 ingress_bytes=369106944 link_bytes=369106944 token_bytes=530939296 weight_movement_bytes=103809024 total_nvlink_bytes=634748320 weight_to_token_ratio=0.195520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343320 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288408 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233496 weight_movement_bytes=69206016 total_nvlink_bytes=601439512 weight_to_token_ratio=0.130029 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65084 dup_rows=23886 dup_rows_remote=20902 dedup_ratio=0.242981 router_tokens_per_expert=[789, 773, 759, 768, 738, 790, 761, 776, 772, 715, 778, 787, 756, 797, 736, 802, 761, 768, 763, 763, 760, 771, 777, 754, 742, 815, 737, 783, 757, 745, 799, 709, 793, 738, 725, 754, 764, 807, 760, 804, 790, 791, 788, 730, 760, 768, 756, 785, 769, 784, 794, 773, 764, 766, 784, 745, 767, 786, 826, 773, 753, 748, 761, 745] payload_rows_per_dst=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] dup_rows_per_dst=[3014, 2914, 2956, 2932, 2972, 2982, 3132, 2984] combine_rows_per_src=[9294, 9212, 9120, 9336, 9318, 9354, 9450, 9334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=343776 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266968800 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533593824 weight_movement_bytes=69206016 total_nvlink_bytes=602799840 weight_to_token_ratio=0.129698 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65100 dup_rows=23904 dup_rows_remote=20922 dedup_ratio=0.243164 router_tokens_per_expert=[761, 701, 784, 806, 757, 762, 788, 790, 805, 737, 753, 768, 773, 782, 730, 793, 790, 742, 754, 759, 777, 790, 739, 751, 765, 811, 750, 751, 721, 760, 740, 759, 799, 734, 742, 782, 737, 770, 809, 761, 851, 764, 829, 755, 747, 785, 707, 756, 791, 786, 785, 769, 785, 797, 797, 742, 733, 748, 793, 797, 722, 753, 795, 782] payload_rows_per_dst=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] dup_rows_per_dst=[3020, 2982, 3000, 2902, 2938, 3062, 3086, 2914] combine_rows_per_src=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64732 dup_rows=24012 dup_rows_remote=20676 dedup_ratio=0.244263 router_tokens_per_expert=[751, 753, 710, 733, 802, 774, 779, 798, 748, 776, 786, 750, 759, 823, 794, 788, 760, 813, 746, 758, 760, 785, 779, 795, 774, 769, 786, 793, 738, 732, 770, 755, 787, 728, 788, 760, 715, 753, 736, 746, 759, 772, 753, 804, 783, 731, 792, 748, 714, 787, 741, 737, 784, 790, 840, 784, 740, 735, 806, 761, 784, 744, 816, 797] payload_rows_per_dst=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] dup_rows_per_dst=[3336, 3078, 3038, 2992, 3094, 2840, 2734, 2900] combine_rows_per_src=[9560, 9400, 9354, 9242, 9528, 9032, 9024, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344816 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266502896 ingress_bytes=369967104 link_bytes=369967104 token_bytes=532660976 weight_movement_bytes=103809024 total_nvlink_bytes=636470000 weight_to_token_ratio=0.194888 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392156428e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64890 dup_rows=24172 dup_rows_remote=21252 dedup_ratio=0.245890 router_tokens_per_expert=[774, 752, 775, 737, 796, 775, 794, 793, 784, 792, 727, 811, 708, 822, 728, 787, 777, 828, 745, 717, 809, 770, 737, 782, 767, 771, 787, 750, 774, 752, 775, 797, 758, 754, 746, 783, 745, 762, 817, 783, 797, 758, 807, 802, 762, 706, 683, 756, 767, 821, 746, 722, 769, 775, 720, 761, 786, 733, 734, 766, 761, 781, 761, 837] payload_rows_per_dst=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] dup_rows_per_dst=[3010, 3062, 2992, 3092, 3028, 2988, 2920, 3080] combine_rows_per_src=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65054 dup_rows=24010 dup_rows_remote=20948 dedup_ratio=0.244242 router_tokens_per_expert=[784, 758, 768, 756, 762, 749, 785, 772, 791, 813, 761, 759, 755, 809, 776, 779, 803, 806, 785, 757, 768, 720, 783, 805, 721, 724, 751, 733, 773, 769, 716, 680, 765, 746, 769, 767, 751, 761, 775, 747, 778, 819, 718, 804, 781, 759, 811, 748, 810, 768, 730, 759, 767, 812, 796, 789, 730, 752, 767, 801, 773, 785, 789, 754] payload_rows_per_dst=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] dup_rows_per_dst=[2998, 3030, 2836, 2758, 2892, 3386, 3048, 3062] combine_rows_per_src=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64822 dup_rows=24056 dup_rows_remote=20910 dedup_ratio=0.244710 router_tokens_per_expert=[725, 741, 748, 771, 728, 795, 697, 800, 727, 749, 754, 741, 782, 790, 734, 787, 823, 781, 731, 773, 828, 787, 762, 801, 768, 748, 775, 752, 774, 795, 742, 791, 760, 812, 725, 797, 760, 754, 778, 802, 780, 743, 778, 778, 785, 789, 764, 800, 763, 810, 773, 777, 728, 773, 779, 784, 749, 741, 739, 797, 784, 744, 749, 757] payload_rows_per_dst=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] dup_rows_per_dst=[2912, 2884, 3146, 3036, 3016, 3074, 3044, 2944] combine_rows_per_src=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64990 dup_rows=24166 dup_rows_remote=21152 dedup_ratio=0.245829 router_tokens_per_expert=[786, 788, 698, 763, 734, 782, 752, 792, 788, 751, 756, 762, 794, 743, 740, 747, 750, 750, 751, 822, 819, 790, 749, 779, 757, 796, 695, 757, 764, 764, 746, 777, 799, 763, 740, 752, 780, 793, 746, 759, 784, 765, 794, 782, 804, 767, 785, 749, 762, 795, 773, 758, 725, 760, 760, 757, 744, 793, 826, 795, 822, 784, 782, 712] payload_rows_per_dst=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] dup_rows_per_dst=[2894, 3014, 3102, 2818, 3028, 3150, 2978, 3182] combine_rows_per_src=[9296, 9148, 9318, 9294, 9236, 9310, 9202, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64770 dup_rows=24190 dup_rows_remote=21094 dedup_ratio=0.246073 router_tokens_per_expert=[820, 743, 754, 728, 764, 751, 781, 728, 780, 792, 787, 775, 773, 728, 767, 772, 720, 755, 776, 737, 772, 794, 817, 737, 755, 738, 763, 776, 813, 802, 806, 767, 788, 708, 765, 747, 713, 797, 752, 772, 786, 741, 791, 742, 771, 744, 714, 750, 789, 766, 845, 830, 754, 783, 752, 748, 807, 800, 785, 772, 770, 770, 714, 815] payload_rows_per_dst=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] dup_rows_per_dst=[2962, 2962, 3032, 3096, 2972, 2926, 3170, 3070] combine_rows_per_src=[9176, 9386, 9184, 9344, 9112, 9152, 9364, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64988 dup_rows=23966 dup_rows_remote=20808 dedup_ratio=0.243795 router_tokens_per_expert=[762, 769, 771, 753, 707, 845, 810, 777, 781, 773, 755, 766, 768, 758, 752, 759, 806, 752, 780, 721, 756, 778, 798, 711, 771, 734, 790, 749, 778, 749, 792, 754, 768, 754, 798, 800, 797, 750, 764, 823, 776, 757, 769, 762, 755, 821, 763, 798, 765, 758, 786, 731, 768, 759, 749, 760, 784, 737, 753, 789, 723, 780, 772, 758] payload_rows_per_dst=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] dup_rows_per_dst=[3004, 2984, 2960, 2958, 3158, 2970, 2942, 2990] combine_rows_per_src=[9384, 9240, 9244, 9276, 9350, 9432, 9210, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341632 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483904 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626176 weight_movement_bytes=34603008 total_nvlink_bytes=565229184 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=343944 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266928008 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512072 weight_movement_bytes=69206016 total_nvlink_bytes=602718088 weight_to_token_ratio=0.129718 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64872 dup_rows=24108 dup_rows_remote=21104 dedup_ratio=0.245239 router_tokens_per_expert=[728, 774, 736, 762, 759, 785, 760, 782, 793, 781, 781, 784, 754, 741, 751, 764, 742, 742, 768, 779, 728, 781, 768, 783, 740, 788, 747, 762, 735, 757, 805, 830, 750, 763, 754, 800, 754, 763, 741, 797, 772, 800, 791, 779, 761, 735, 754, 722, 800, 775, 771, 776, 751, 757, 772, 792, 765, 718, 811, 807, 779, 791, 745, 816] payload_rows_per_dst=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] dup_rows_per_dst=[2944, 3062, 2942, 3004, 3024, 2964, 3124, 3044] combine_rows_per_src=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65134 dup_rows=23928 dup_rows_remote=20876 dedup_ratio=0.243408 router_tokens_per_expert=[782, 726, 725, 743, 792, 793, 801, 756, 776, 795, 795, 752, 735, 807, 778, 716, 713, 794, 750, 809, 787, 794, 739, 789, 831, 763, 763, 781, 726, 738, 765, 766, 746, 755, 790, 737, 749, 785, 747, 775, 791, 763, 747, 754, 785, 774, 779, 754, 782, 783, 786, 793, 791, 781, 788, 788, 733, 810, 757, 787, 753, 739, 741, 729] payload_rows_per_dst=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] dup_rows_per_dst=[3030, 2936, 3032, 2960, 3008, 3052, 3072, 2838] combine_rows_per_src=[9206, 9372, 9318, 9306, 9160, 9242, 9512, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344088 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266993688 ingress_bytes=405061632 link_bytes=405061632 token_bytes=533643288 weight_movement_bytes=138412032 total_nvlink_bytes=672055320 weight_to_token_ratio=0.259372 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524214311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=341632 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265483904 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530626176 weight_movement_bytes=34603008 total_nvlink_bytes=565229184 weight_to_token_ratio=0.065212 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344568 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266134008 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923448 weight_movement_bytes=69206016 total_nvlink_bytes=601129464 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344568 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266543608 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742648 weight_movement_bytes=103809024 total_nvlink_bytes=636551672 weight_to_token_ratio=0.194858 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265297920 dispatch_meta_bytes=343456 combine_payload_bytes=265297920 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265641376 ingress_bytes=369106944 link_bytes=369106944 token_bytes=530939296 weight_movement_bytes=103809024 total_nvlink_bytes=634748320 weight_to_token_ratio=0.195520 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388919715e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344008 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805192 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266376 weight_movement_bytes=103809024 total_nvlink_bytes=637075400 weight_to_token_ratio=0.194666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64890 dup_rows=24172 dup_rows_remote=21252 dedup_ratio=0.245890 router_tokens_per_expert=[774, 752, 775, 737, 796, 775, 794, 793, 784, 792, 727, 811, 708, 822, 728, 787, 777, 828, 745, 717, 809, 770, 737, 782, 767, 771, 787, 750, 774, 752, 775, 797, 758, 754, 746, 783, 745, 762, 817, 783, 797, 758, 807, 802, 762, 706, 683, 756, 767, 821, 746, 722, 769, 775, 720, 761, 786, 733, 734, 766, 761, 781, 761, 837] payload_rows_per_dst=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] dup_rows_per_dst=[3010, 3062, 2992, 3092, 3028, 2988, 2920, 3080] combine_rows_per_src=[9382, 9256, 9338, 9254, 9268, 9154, 9242, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342928 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265853840 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531364752 weight_movement_bytes=69206016 total_nvlink_bytes=600570768 weight_to_token_ratio=0.130242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=343184 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266534032 ingress_bytes=300793856 link_bytes=300793856 token_bytes=532724880 weight_movement_bytes=34603008 total_nvlink_bytes=567327888 weight_to_token_ratio=0.064955 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131863064e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64808 dup_rows=24162 dup_rows_remote=21158 dedup_ratio=0.245789 router_tokens_per_expert=[725, 734, 763, 769, 731, 763, 768, 773, 779, 780, 724, 735, 804, 774, 776, 827, 763, 789, 767, 785, 705, 804, 774, 782, 754, 734, 754, 787, 798, 783, 809, 775, 741, 773, 782, 733, 778, 745, 787, 732, 759, 780, 739, 821, 731, 787, 737, 823, 771, 814, 757, 732, 808, 786, 808, 748, 783, 742, 789, 763, 733, 748, 749, 785] payload_rows_per_dst=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] dup_rows_per_dst=[2956, 3030, 3004, 3000, 2946, 3070, 3168, 2988] combine_rows_per_src=[9096, 9368, 9334, 9388, 9196, 9284, 9280, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343904 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059616 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775328 weight_movement_bytes=34603008 total_nvlink_bytes=566378336 weight_to_token_ratio=0.065071 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344040 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132904 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921768 weight_movement_bytes=34603008 total_nvlink_bytes=568524776 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=344568 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266134008 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531923448 weight_movement_bytes=69206016 total_nvlink_bytes=601129464 weight_to_token_ratio=0.130105 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64974 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[757, 823, 788, 790, 818, 723, 753, 728, 768, 722, 742, 810, 773, 752, 800, 789, 786, 699, 762, 789, 747, 789, 750, 803, 793, 726, 740, 770, 754, 765, 725, 747, 746, 766, 741, 780, 782, 784, 770, 767, 751, 790, 737, 752, 787, 799, 803, 772, 759, 739, 771, 791, 775, 786, 771, 751, 749, 784, 791, 809, 778, 784, 738, 768] payload_rows_per_dst=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] dup_rows_per_dst=[3064, 2994, 2976, 2916, 2988, 3032, 2972, 3070] combine_rows_per_src=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64858 dup_rows=24046 dup_rows_remote=21024 dedup_ratio=0.244609 router_tokens_per_expert=[758, 711, 790, 765, 804, 748, 762, 804, 779, 797, 782, 744, 811, 770, 767, 760, 762, 690, 805, 743, 797, 772, 792, 778, 770, 777, 770, 760, 790, 780, 745, 771, 764, 779, 761, 723, 813, 776, 727, 784, 803, 768, 766, 724, 785, 796, 753, 770, 747, 727, 775, 776, 789, 754, 743, 813, 749, 750, 707, 800, 740, 825, 775, 736] payload_rows_per_dst=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] dup_rows_per_dst=[3022, 3108, 3344, 2942, 3036, 2992, 2684, 2918] combine_rows_per_src=[9400, 9340, 9580, 9218, 9218, 9338, 8918, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64710 dup_rows=24134 dup_rows_remote=20908 dedup_ratio=0.245504 router_tokens_per_expert=[752, 709, 715, 774, 763, 788, 743, 801, 796, 827, 745, 749, 818, 766, 774, 801, 739, 727, 782, 740, 757, 766, 765, 779, 777, 785, 815, 761, 767, 803, 758, 760, 743, 762, 770, 747, 777, 755, 775, 765, 774, 758, 759, 797, 752, 791, 739, 800, 747, 764, 732, 756, 766, 754, 768, 795, 766, 803, 783, 773, 795, 730, 770, 784] payload_rows_per_dst=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] dup_rows_per_dst=[2944, 2888, 2932, 2894, 3064, 3172, 3014, 3226] combine_rows_per_src=[9146, 9136, 9178, 9292, 9390, 9344, 9224, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265453568 dispatch_meta_bytes=343864 combine_payload_bytes=265453568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265797432 ingress_bytes=334659584 link_bytes=334659584 token_bytes=531251000 weight_movement_bytes=69206016 total_nvlink_bytes=600457016 weight_to_token_ratio=0.130270 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259297072e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65186 dup_rows=23848 dup_rows_remote=20752 dedup_ratio=0.242594 router_tokens_per_expert=[762, 772, 761, 805, 789, 797, 752, 751, 797, 755, 761, 760, 744, 758, 760, 740, 766, 747, 808, 715, 751, 750, 773, 769, 798, 760, 783, 772, 781, 756, 769, 760, 744, 750, 723, 753, 771, 761, 753, 753, 803, 723, 796, 799, 804, 780, 757, 799, 740, 786, 806, 784, 795, 759, 794, 770, 753, 780, 759, 724, 754, 755, 786, 816] payload_rows_per_dst=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] dup_rows_per_dst=[3044, 2828, 2932, 2938, 2896, 3000, 3114, 3096] combine_rows_per_src=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64652 dup_rows=24202 dup_rows_remote=21020 dedup_ratio=0.246195 router_tokens_per_expert=[759, 739, 765, 754, 775, 751, 744, 782, 723, 729, 743, 745, 775, 835, 743, 757, 807, 789, 759, 815, 776, 801, 779, 790, 806, 766, 774, 782, 732, 784, 730, 791, 764, 764, 739, 789, 761, 764, 759, 776, 712, 776, 754, 767, 784, 786, 796, 746, 762, 783, 759, 791, 779, 736, 792, 752, 772, 747, 814, 783, 779, 743, 784, 739] payload_rows_per_dst=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] dup_rows_per_dst=[2926, 2906, 3182, 3054, 3002, 3056, 2964, 3112] combine_rows_per_src=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65274 dup_rows=23790 dup_rows_remote=20824 dedup_ratio=0.242004 router_tokens_per_expert=[776, 766, 776, 784, 702, 775, 796, 730, 770, 758, 767, 769, 775, 748, 790, 807, 756, 769, 751, 777, 764, 830, 766, 806, 712, 755, 771, 733, 766, 744, 743, 737, 741, 784, 727, 734, 815, 783, 803, 798, 738, 802, 774, 790, 793, 806, 741, 725, 801, 774, 779, 681, 736, 822, 755, 755, 769, 806, 806, 786, 774, 781, 757, 747] payload_rows_per_dst=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] dup_rows_per_dst=[2932, 2996, 3026, 2818, 3070, 2956, 2966, 3026] combine_rows_per_src=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65100 dup_rows=23904 dup_rows_remote=20922 dedup_ratio=0.243164 router_tokens_per_expert=[761, 701, 784, 806, 757, 762, 788, 790, 805, 737, 753, 768, 773, 782, 730, 793, 790, 742, 754, 759, 777, 790, 739, 751, 765, 811, 750, 751, 721, 760, 740, 759, 799, 734, 742, 782, 737, 770, 809, 761, 851, 764, 829, 755, 747, 785, 707, 756, 791, 786, 785, 769, 785, 797, 797, 742, 733, 748, 793, 797, 722, 753, 795, 782] payload_rows_per_dst=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] dup_rows_per_dst=[3020, 2982, 3000, 2902, 2938, 3062, 3086, 2914] combine_rows_per_src=[9278, 9300, 9204, 9212, 9330, 9326, 9418, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64872 dup_rows=24108 dup_rows_remote=21104 dedup_ratio=0.245239 router_tokens_per_expert=[728, 774, 736, 762, 759, 785, 760, 782, 793, 781, 781, 784, 754, 741, 751, 764, 742, 742, 768, 779, 728, 781, 768, 783, 740, 788, 747, 762, 735, 757, 805, 830, 750, 763, 754, 800, 754, 763, 741, 797, 772, 800, 791, 779, 761, 735, 754, 722, 800, 775, 771, 776, 751, 757, 772, 792, 765, 718, 811, 807, 779, 791, 745, 816] payload_rows_per_dst=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] dup_rows_per_dst=[2944, 3062, 2942, 3004, 3024, 2964, 3124, 3044] combine_rows_per_src=[9228, 9236, 9240, 9324, 9220, 9264, 9264, 9420] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343528 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266001896 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660264 weight_movement_bytes=34603008 total_nvlink_bytes=566263272 weight_to_token_ratio=0.065085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64738 dup_rows=24380 dup_rows_remote=21362 dedup_ratio=0.248006 router_tokens_per_expert=[721, 790, 775, 838, 759, 764, 789, 726, 724, 773, 780, 720, 758, 795, 744, 752, 810, 792, 771, 771, 754, 806, 768, 759, 755, 781, 751, 742, 790, 769, 753, 759, 776, 775, 778, 739, 751, 746, 724, 813, 712, 757, 811, 748, 773, 777, 795, 826, 783, 752, 777, 730, 736, 816, 763, 822, 791, 780, 735, 792, 721, 711, 803, 800] payload_rows_per_dst=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] dup_rows_per_dst=[3084, 2842, 3188, 3054, 3018, 3106, 3040, 3048] combine_rows_per_src=[9240, 9250, 9274, 9146, 9186, 9292, 9318, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20782 dedup_ratio=0.243022 router_tokens_per_expert=[830, 732, 798, 730, 797, 779, 791, 713, 784, 797, 769, 802, 783, 740, 759, 771, 751, 745, 731, 738, 746, 770, 773, 757, 765, 768, 808, 812, 797, 757, 713, 779, 798, 761, 774, 719, 780, 768, 754, 764, 755, 808, 799, 787, 774, 773, 785, 772, 792, 769, 746, 757, 740, 769, 770, 737, 742, 785, 805, 765, 784, 756, 734, 745] payload_rows_per_dst=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] dup_rows_per_dst=[3060, 3012, 2836, 3108, 2936, 3106, 2904, 2928] combine_rows_per_src=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65026 dup_rows=24022 dup_rows_remote=20938 dedup_ratio=0.244364 router_tokens_per_expert=[803, 745, 771, 776, 800, 755, 738, 752, 746, 757, 744, 721, 759, 737, 780, 794, 763, 739, 773, 770, 813, 800, 744, 759, 782, 761, 744, 757, 775, 786, 768, 748, 772, 788, 734, 760, 790, 797, 782, 798, 768, 744, 781, 786, 810, 756, 769, 756, 755, 757, 756, 775, 793, 798, 722, 784, 747, 731, 744, 788, 768, 803, 797, 783] payload_rows_per_dst=[9332, 9150, 9256, 9300, 9388, 9256, 9288, 9312] dup_rows_per_dst=[2948, 2926, 3066, 2942, 3054, 3084, 2992, 3010] combine_rows_per_src=[9332, 9150, 9256, 9300, 9388, 9256, 9288, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=342472 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265394632 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446792 weight_movement_bytes=34603008 total_nvlink_bytes=565049800 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344392 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706696 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069000 weight_movement_bytes=69206016 total_nvlink_bytes=604275016 weight_to_token_ratio=0.129340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343968 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477472 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532610976 weight_movement_bytes=34603008 total_nvlink_bytes=567213984 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343528 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266001896 ingress_bytes=300261376 link_bytes=300261376 token_bytes=531660264 weight_movement_bytes=34603008 total_nvlink_bytes=566263272 weight_to_token_ratio=0.065085 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129859385e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=344088 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266993688 ingress_bytes=405061632 link_bytes=405061632 token_bytes=533643288 weight_movement_bytes=138412032 total_nvlink_bytes=672055320 weight_to_token_ratio=0.259372 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524214311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343904 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059616 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775328 weight_movement_bytes=34603008 total_nvlink_bytes=566378336 weight_to_token_ratio=0.065071 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343752 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345608 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347464 weight_movement_bytes=34603008 total_nvlink_bytes=568950472 weight_to_token_ratio=0.064758 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65274 dup_rows=23790 dup_rows_remote=20824 dedup_ratio=0.242004 router_tokens_per_expert=[776, 766, 776, 784, 702, 775, 796, 730, 770, 758, 767, 769, 775, 748, 790, 807, 756, 769, 751, 777, 764, 830, 766, 806, 712, 755, 771, 733, 766, 744, 743, 737, 741, 784, 727, 734, 815, 783, 803, 798, 738, 802, 774, 790, 793, 806, 741, 725, 801, 774, 779, 681, 736, 822, 755, 755, 769, 806, 806, 786, 774, 781, 757, 747] payload_rows_per_dst=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] dup_rows_per_dst=[2932, 2996, 3026, 2818, 3070, 2956, 2966, 3026] combine_rows_per_src=[9278, 9372, 9412, 9104, 9300, 9382, 9240, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342688 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265157280 ingress_bytes=299417600 link_bytes=299417600 token_bytes=529971872 weight_movement_bytes=34603008 total_nvlink_bytes=564574880 weight_to_token_ratio=0.065292 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126684324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=344400 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265511248 ingress_bytes=334372864 link_bytes=334372864 token_bytes=530678096 weight_movement_bytes=69206016 total_nvlink_bytes=599884112 weight_to_token_ratio=0.130411 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258218168e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65124 dup_rows=24032 dup_rows_remote=21214 dedup_ratio=0.244466 router_tokens_per_expert=[752, 773, 785, 763, 757, 787, 794, 759, 763, 737, 777, 791, 769, 775, 769, 795, 740, 745, 777, 754, 719, 720, 761, 767, 820, 784, 783, 778, 759, 773, 811, 771, 785, 744, 773, 751, 765, 745, 792, 781, 762, 755, 793, 710, 767, 751, 762, 777, 736, 787, 791, 855, 767, 748, 789, 791, 715, 790, 779, 808, 751, 726, 740, 758] payload_rows_per_dst=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] dup_rows_per_dst=[3060, 3076, 2818, 3158, 2998, 2892, 3062, 2968] combine_rows_per_src=[9280, 9276, 9148, 9400, 9274, 9262, 9466, 9166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343624 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267091528 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839432 weight_movement_bytes=103809024 total_nvlink_bytes=637648456 weight_to_token_ratio=0.194457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343856 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266690352 ingress_bytes=335552512 link_bytes=335552512 token_bytes=533036848 weight_movement_bytes=69206016 total_nvlink_bytes=602242864 weight_to_token_ratio=0.129833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262657088e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344392 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706696 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069000 weight_movement_bytes=69206016 total_nvlink_bytes=604275016 weight_to_token_ratio=0.129340 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65118 dup_rows=23854 dup_rows_remote=20932 dedup_ratio=0.242655 router_tokens_per_expert=[796, 731, 774, 759, 754, 771, 770, 753, 765, 779, 785, 759, 756, 750, 772, 761, 788, 750, 780, 772, 759, 742, 731, 772, 798, 794, 750, 824, 705, 753, 756, 775, 768, 773, 772, 780, 758, 766, 757, 764, 742, 792, 810, 738, 756, 790, 786, 797, 775, 737, 766, 832, 784, 729, 778, 811, 746, 767, 773, 789, 731, 752, 787, 762] payload_rows_per_dst=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] dup_rows_per_dst=[2936, 2922, 2944, 2996, 3042, 3060, 3038, 2916] combine_rows_per_src=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=64910 dup_rows=23906 dup_rows_remote=20714 dedup_ratio=0.243184 router_tokens_per_expert=[733, 763, 770, 795, 769, 761, 800, 757, 770, 807, 786, 767, 776, 737, 792, 765, 723, 771, 719, 738, 786, 807, 760, 775, 753, 862, 777, 748, 788, 782, 773, 744, 799, 770, 739, 748, 777, 786, 782, 769, 751, 749, 769, 785, 764, 772, 783, 779, 789, 769, 757, 772, 762, 733, 748, 781, 751, 770, 755, 741, 762, 779, 733, 774] payload_rows_per_dst=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] dup_rows_per_dst=[3192, 3132, 2986, 3572, 2986, 2474, 2674, 2890] combine_rows_per_src=[9488, 9412, 9176, 9976, 9360, 8736, 9164, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=345352 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267093256 ingress_bytes=301350912 link_bytes=301350912 token_bytes=533841160 weight_movement_bytes=34603008 total_nvlink_bytes=568444168 weight_to_token_ratio=0.064819 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133959221e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65054 dup_rows=24010 dup_rows_remote=20948 dedup_ratio=0.244242 router_tokens_per_expert=[784, 758, 768, 756, 762, 749, 785, 772, 791, 813, 761, 759, 755, 809, 776, 779, 803, 806, 785, 757, 768, 720, 783, 805, 721, 724, 751, 733, 773, 769, 716, 680, 765, 746, 769, 767, 751, 761, 775, 747, 778, 819, 718, 804, 781, 759, 811, 748, 810, 768, 730, 759, 767, 812, 796, 789, 730, 752, 767, 801, 773, 785, 789, 754] payload_rows_per_dst=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] dup_rows_per_dst=[2998, 3030, 2836, 2758, 2892, 3386, 3048, 3062] combine_rows_per_src=[9270, 9456, 9080, 8976, 9158, 9588, 9526, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64432 dup_rows=24150 dup_rows_remote=20778 dedup_ratio=0.245667 router_tokens_per_expert=[745, 756, 698, 796, 747, 831, 737, 710, 805, 725, 733, 762, 782, 855, 802, 757, 786, 748, 801, 716, 776, 779, 764, 769, 779, 783, 799, 785, 797, 778, 755, 797, 729, 780, 791, 770, 754, 795, 760, 725, 761, 771, 786, 750, 777, 793, 750, 741, 805, 783, 754, 803, 743, 747, 729, 771, 762, 773, 774, 731, 807, 744, 779, 761] payload_rows_per_dst=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] dup_rows_per_dst=[2926, 3072, 2686, 3186, 3358, 2960, 2590, 3372] combine_rows_per_src=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65104 dup_rows=24098 dup_rows_remote=21214 dedup_ratio=0.245138 router_tokens_per_expert=[772, 805, 764, 750, 768, 764, 765, 792, 815, 788, 753, 780, 745, 744, 756, 761, 752, 717, 724, 773, 781, 771, 734, 741, 749, 816, 769, 768, 739, 729, 796, 809, 763, 762, 793, 776, 787, 755, 786, 765, 753, 745, 786, 806, 775, 751, 761, 735, 809, 753, 807, 780, 766, 784, 757, 832, 794, 760, 781, 705, 773, 730, 729, 803] payload_rows_per_dst=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] dup_rows_per_dst=[3000, 3090, 2884, 3092, 3082, 2942, 3114, 2894] combine_rows_per_src=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64872 dup_rows=24158 dup_rows_remote=21150 dedup_ratio=0.245748 router_tokens_per_expert=[768, 764, 749, 727, 774, 737, 752, 776, 810, 816, 760, 783, 741, 775, 725, 792, 725, 786, 766, 792, 738, 792, 738, 783, 795, 747, 781, 752, 783, 782, 796, 769, 780, 751, 776, 770, 767, 742, 775, 757, 781, 792, 775, 821, 673, 745, 770, 744, 797, 755, 742, 777, 770, 768, 778, 754, 816, 750, 726, 750, 788, 787, 790, 811] payload_rows_per_dst=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] dup_rows_per_dst=[2978, 3024, 2994, 3022, 3052, 2976, 3008, 3104] combine_rows_per_src=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64974 dup_rows=24012 dup_rows_remote=21018 dedup_ratio=0.244263 router_tokens_per_expert=[757, 823, 788, 790, 818, 723, 753, 728, 768, 722, 742, 810, 773, 752, 800, 789, 786, 699, 762, 789, 747, 789, 750, 803, 793, 726, 740, 770, 754, 765, 725, 747, 746, 766, 741, 780, 782, 784, 770, 767, 751, 790, 737, 752, 787, 799, 803, 772, 759, 739, 771, 791, 775, 786, 771, 751, 749, 784, 791, 809, 778, 784, 738, 768] payload_rows_per_dst=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] dup_rows_per_dst=[3064, 2994, 2976, 2916, 2988, 3032, 2972, 3070] combine_rows_per_src=[9296, 9318, 9274, 9124, 9284, 9350, 9314, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20782 dedup_ratio=0.243022 router_tokens_per_expert=[830, 732, 798, 730, 797, 779, 791, 713, 784, 797, 769, 802, 783, 740, 759, 771, 751, 745, 731, 738, 746, 770, 773, 757, 765, 768, 808, 812, 797, 757, 713, 779, 798, 761, 774, 719, 780, 768, 754, 764, 755, 808, 799, 787, 774, 773, 785, 772, 792, 769, 746, 757, 740, 769, 770, 737, 742, 785, 805, 765, 784, 756, 734, 745] payload_rows_per_dst=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] dup_rows_per_dst=[3060, 3012, 2836, 3108, 2936, 3106, 2904, 2928] combine_rows_per_src=[9280, 9398, 9186, 9290, 9300, 9400, 9256, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=342496 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266213856 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532085216 weight_movement_bytes=34603008 total_nvlink_bytes=566688224 weight_to_token_ratio=0.065033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65174 dup_rows=24028 dup_rows_remote=21148 dedup_ratio=0.244425 router_tokens_per_expert=[788, 753, 765, 796, 731, 805, 784, 743, 718, 818, 728, 750, 778, 742, 752, 745, 747, 786, 789, 745, 772, 779, 793, 753, 764, 795, 783, 758, 800, 789, 783, 786, 739, 751, 728, 772, 747, 719, 762, 773, 794, 729, 768, 802, 746, 785, 761, 798, 762, 748, 743, 782, 866, 801, 743, 753, 750, 794, 776, 816, 762, 736, 760, 768] payload_rows_per_dst=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] dup_rows_per_dst=[3020, 2924, 3070, 3100, 2880, 2990, 3050, 2994] combine_rows_per_src=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344008 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805192 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266376 weight_movement_bytes=103809024 total_nvlink_bytes=637075400 weight_to_token_ratio=0.194666 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65128 dup_rows=24002 dup_rows_remote=21126 dedup_ratio=0.244161 router_tokens_per_expert=[801, 773, 799, 729, 796, 792, 818, 759, 803, 774, 746, 750, 761, 791, 772, 816, 735, 792, 773, 761, 794, 795, 770, 780, 778, 771, 772, 723, 765, 720, 748, 748, 761, 748, 754, 754, 779, 769, 726, 784, 736, 812, 805, 825, 777, 776, 748, 761, 713, 749, 795, 742, 763, 752, 812, 754, 758, 723, 805, 751, 730, 758, 793, 734] payload_rows_per_dst=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] dup_rows_per_dst=[3132, 3136, 3020, 2876, 3012, 3086, 2906, 2834] combine_rows_per_src=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344200 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067528 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790856 weight_movement_bytes=34603008 total_nvlink_bytes=568393864 weight_to_token_ratio=0.064825 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65304 dup_rows=23700 dup_rows_remote=20700 dedup_ratio=0.241089 router_tokens_per_expert=[836, 756, 726, 756, 788, 758, 770, 712, 769, 791, 791, 749, 775, 739, 725, 768, 785, 805, 771, 777, 804, 797, 740, 757, 769, 775, 757, 689, 734, 740, 787, 813, 813, 763, 752, 796, 775, 755, 768, 716, 757, 755, 790, 772, 771, 771, 734, 800, 766, 732, 813, 807, 804, 721, 784, 795, 759, 753, 774, 788, 733, 780, 775, 771] payload_rows_per_dst=[9314, 9298, 9434, 9240, 9332, 9300, 9362, 9324] dup_rows_per_dst=[2890, 2916, 3038, 2888, 2944, 3000, 3082, 2942] combine_rows_per_src=[9314, 9298, 9434, 9240, 9332, 9300, 9362, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344088 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266059800 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775512 weight_movement_bytes=103809024 total_nvlink_bytes=635584536 weight_to_token_ratio=0.195212 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343968 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477472 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532610976 weight_movement_bytes=34603008 total_nvlink_bytes=567213984 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343624 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267091528 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839432 weight_movement_bytes=103809024 total_nvlink_bytes=637648456 weight_to_token_ratio=0.194457 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65174 dup_rows=24028 dup_rows_remote=21148 dedup_ratio=0.244425 router_tokens_per_expert=[788, 753, 765, 796, 731, 805, 784, 743, 718, 818, 728, 750, 778, 742, 752, 745, 747, 786, 789, 745, 772, 779, 793, 753, 764, 795, 783, 758, 800, 789, 783, 786, 739, 751, 728, 772, 747, 719, 762, 773, 794, 729, 768, 802, 746, 785, 761, 798, 762, 748, 743, 782, 866, 801, 743, 753, 750, 794, 776, 816, 762, 736, 760, 768] payload_rows_per_dst=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] dup_rows_per_dst=[3020, 2924, 3070, 3100, 2880, 2990, 3050, 2994] combine_rows_per_src=[9310, 9138, 9258, 9416, 9102, 9376, 9346, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=342496 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266213856 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532085216 weight_movement_bytes=34603008 total_nvlink_bytes=566688224 weight_to_token_ratio=0.065033 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263913472 dispatch_meta_bytes=340840 combine_payload_bytes=263913472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264254312 ingress_bytes=298516480 link_bytes=298516480 token_bytes=528167784 weight_movement_bytes=34603008 total_nvlink_bytes=562770792 weight_to_token_ratio=0.065515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123293482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=345288 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297992 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250696 weight_movement_bytes=34603008 total_nvlink_bytes=568853704 weight_to_token_ratio=0.064769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64822 dup_rows=24056 dup_rows_remote=20910 dedup_ratio=0.244710 router_tokens_per_expert=[725, 741, 748, 771, 728, 795, 697, 800, 727, 749, 754, 741, 782, 790, 734, 787, 823, 781, 731, 773, 828, 787, 762, 801, 768, 748, 775, 752, 774, 795, 742, 791, 760, 812, 725, 797, 760, 754, 778, 802, 780, 743, 778, 778, 785, 789, 764, 800, 763, 810, 773, 777, 728, 773, 779, 784, 749, 741, 739, 797, 784, 744, 749, 757] payload_rows_per_dst=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] dup_rows_per_dst=[2912, 2884, 3146, 3036, 3016, 3074, 3044, 2944] combine_rows_per_src=[9098, 9244, 9426, 9254, 9360, 9360, 9330, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64872 dup_rows=24158 dup_rows_remote=21150 dedup_ratio=0.245748 router_tokens_per_expert=[768, 764, 749, 727, 774, 737, 752, 776, 810, 816, 760, 783, 741, 775, 725, 792, 725, 786, 766, 792, 738, 792, 738, 783, 795, 747, 781, 752, 783, 782, 796, 769, 780, 751, 776, 770, 767, 742, 775, 757, 781, 792, 775, 821, 673, 745, 770, 744, 797, 755, 742, 777, 770, 768, 778, 754, 816, 750, 726, 750, 788, 787, 790, 811] payload_rows_per_dst=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] dup_rows_per_dst=[2978, 3024, 2994, 3022, 3052, 2976, 3008, 3104] combine_rows_per_src=[9116, 9380, 9246, 9388, 9184, 9226, 9274, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=345272 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267011256 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533677240 weight_movement_bytes=69206016 total_nvlink_bytes=602883256 weight_to_token_ratio=0.129678 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=345016 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267109304 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533873592 weight_movement_bytes=69206016 total_nvlink_bytes=603079608 weight_to_token_ratio=0.129630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344016 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267829200 ingress_bytes=371294208 link_bytes=371294208 token_bytes=535314384 weight_movement_bytes=103809024 total_nvlink_bytes=639123408 weight_to_token_ratio=0.193922 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397150213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=345288 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267297992 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534250696 weight_movement_bytes=34603008 total_nvlink_bytes=568853704 weight_to_token_ratio=0.064769 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=344088 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266059800 ingress_bytes=369524736 link_bytes=369524736 token_bytes=531775512 weight_movement_bytes=103809024 total_nvlink_bytes=635584536 weight_to_token_ratio=0.195212 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390491832e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64980 dup_rows=24080 dup_rows_remote=21100 dedup_ratio=0.244954 router_tokens_per_expert=[754, 727, 715, 791, 744, 779, 758, 743, 801, 757, 779, 767, 775, 745, 754, 734, 759, 796, 770, 783, 791, 795, 783, 794, 776, 787, 751, 770, 772, 791, 767, 778, 828, 742, 774, 774, 751, 791, 725, 740, 760, 757, 806, 730, 791, 740, 776, 752, 711, 786, 822, 750, 776, 767, 815, 790, 786, 778, 767, 774, 744, 772, 736, 755] payload_rows_per_dst=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] dup_rows_per_dst=[2928, 2980, 3012, 3064, 3042, 2946, 3106, 3002] combine_rows_per_src=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64658 dup_rows=24190 dup_rows_remote=21042 dedup_ratio=0.246073 router_tokens_per_expert=[811, 725, 754, 783, 764, 747, 762, 764, 752, 774, 782, 776, 738, 744, 759, 798, 762, 733, 730, 745, 780, 774, 756, 731, 759, 785, 792, 723, 792, 768, 781, 789, 828, 780, 739, 755, 766, 760, 786, 768, 764, 812, 783, 791, 768, 770, 792, 768, 767, 819, 772, 745, 782, 759, 795, 778, 735, 784, 767, 751, 698, 791, 775, 771] payload_rows_per_dst=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] dup_rows_per_dst=[3148, 3230, 3050, 2938, 3352, 2784, 2794, 2894] combine_rows_per_src=[9456, 9480, 9214, 9056, 9698, 9026, 8934, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342928 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265853840 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531364752 weight_movement_bytes=69206016 total_nvlink_bytes=600570768 weight_to_token_ratio=0.130242 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64708 dup_rows=24130 dup_rows_remote=20982 dedup_ratio=0.245463 router_tokens_per_expert=[753, 785, 754, 772, 739, 762, 746, 742, 702, 761, 788, 753, 781, 751, 766, 822, 782, 787, 788, 825, 790, 798, 745, 737, 741, 778, 744, 723, 780, 760, 762, 774, 760, 762, 785, 723, 743, 760, 801, 772, 787, 796, 777, 759, 756, 769, 818, 744, 811, 748, 741, 757, 775, 739, 841, 760, 742, 787, 766, 772, 760, 800, 792, 758] payload_rows_per_dst=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] dup_rows_per_dst=[2894, 3068, 2886, 3000, 3040, 2946, 3148, 3148] combine_rows_per_src=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64666 dup_rows=24154 dup_rows_remote=21130 dedup_ratio=0.245707 router_tokens_per_expert=[782, 808, 818, 755, 783, 744, 693, 751, 724, 812, 776, 740, 794, 859, 769, 796, 758, 764, 810, 785, 769, 736, 822, 810, 783, 791, 728, 780, 749, 778, 757, 784, 800, 751, 759, 777, 755, 744, 771, 737, 727, 756, 820, 767, 776, 746, 772, 749, 764, 799, 746, 751, 779, 750, 757, 736, 778, 803, 763, 745, 773, 757, 738, 698] payload_rows_per_dst=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] dup_rows_per_dst=[3068, 3092, 3024, 3066, 2984, 3008, 2978, 2934] combine_rows_per_src=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65172 dup_rows=23812 dup_rows_remote=20842 dedup_ratio=0.242228 router_tokens_per_expert=[779, 770, 782, 793, 790, 740, 768, 759, 718, 800, 761, 752, 784, 751, 703, 817, 755, 749, 773, 748, 768, 761, 765, 804, 760, 782, 769, 786, 810, 758, 734, 746, 761, 839, 738, 752, 742, 774, 763, 768, 743, 771, 770, 787, 781, 737, 780, 780, 806, 713, 738, 751, 814, 775, 784, 740, 746, 752, 761, 790, 768, 832, 798, 763] payload_rows_per_dst=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] dup_rows_per_dst=[3022, 2896, 3042, 2970, 3030, 2914, 2900, 3038] combine_rows_per_src=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65186 dup_rows=23848 dup_rows_remote=20752 dedup_ratio=0.242594 router_tokens_per_expert=[762, 772, 761, 805, 789, 797, 752, 751, 797, 755, 761, 760, 744, 758, 760, 740, 766, 747, 808, 715, 751, 750, 773, 769, 798, 760, 783, 772, 781, 756, 769, 760, 744, 750, 723, 753, 771, 761, 753, 753, 803, 723, 796, 799, 804, 780, 757, 799, 740, 786, 806, 784, 795, 759, 794, 770, 753, 780, 759, 724, 754, 755, 786, 816] payload_rows_per_dst=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] dup_rows_per_dst=[3044, 2828, 2932, 2938, 2896, 3000, 3114, 3096] combine_rows_per_src=[9334, 9322, 9226, 9280, 9120, 9456, 9448, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65132 dup_rows=23962 dup_rows_remote=20920 dedup_ratio=0.243754 router_tokens_per_expert=[764, 773, 783, 826, 724, 751, 791, 777, 750, 761, 775, 756, 784, 742, 772, 777, 739, 792, 776, 756, 756, 760, 790, 786, 772, 771, 756, 780, 812, 767, 751, 767, 816, 775, 776, 763, 736, 755, 748, 742, 789, 759, 740, 762, 828, 740, 783, 751, 784, 732, 770, 747, 786, 796, 739, 772, 758, 765, 758, 766, 774, 796, 744, 765] payload_rows_per_dst=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] dup_rows_per_dst=[3022, 3016, 2946, 3032, 2986, 2944, 3042, 2974] combine_rows_per_src=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65118 dup_rows=23854 dup_rows_remote=20932 dedup_ratio=0.242655 router_tokens_per_expert=[796, 731, 774, 759, 754, 771, 770, 753, 765, 779, 785, 759, 756, 750, 772, 761, 788, 750, 780, 772, 759, 742, 731, 772, 798, 794, 750, 824, 705, 753, 756, 775, 768, 773, 772, 780, 758, 766, 757, 764, 742, 792, 810, 738, 756, 790, 786, 797, 775, 737, 766, 832, 784, 729, 778, 811, 746, 767, 773, 789, 731, 752, 787, 762] payload_rows_per_dst=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] dup_rows_per_dst=[2936, 2922, 2944, 2996, 3042, 3060, 3038, 2916] combine_rows_per_src=[9280, 9332, 9244, 9314, 9234, 9362, 9386, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73818 payload_rows_remote=64530 dup_rows=24486 dup_rows_remote=21518 dedup_ratio=0.249084 router_tokens_per_expert=[741, 727, 792, 779, 754, 777, 777, 737, 728, 820, 786, 792, 723, 753, 782, 786, 764, 783, 754, 815, 771, 745, 777, 793, 709, 751, 775, 785, 741, 794, 754, 790, 794, 794, 751, 803, 740, 751, 753, 752, 783, 730, 783, 755, 756, 776, 799, 746, 779, 779, 781, 790, 771, 778, 785, 785, 729, 740, 740, 755, 826, 757, 755, 781] payload_rows_per_dst=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] dup_rows_per_dst=[3038, 3056, 3200, 3008, 3050, 2968, 3112, 3054] combine_rows_per_src=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344320 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266502400 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660480 weight_movement_bytes=69206016 total_nvlink_bytes=601866496 weight_to_token_ratio=0.129925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=342800 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265181968 ingress_bytes=299442176 link_bytes=299442176 token_bytes=530021136 weight_movement_bytes=34603008 total_nvlink_bytes=564624144 weight_to_token_ratio=0.065286 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126776801e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65128 dup_rows=24002 dup_rows_remote=21126 dedup_ratio=0.244161 router_tokens_per_expert=[801, 773, 799, 729, 796, 792, 818, 759, 803, 774, 746, 750, 761, 791, 772, 816, 735, 792, 773, 761, 794, 795, 770, 780, 778, 771, 772, 723, 765, 720, 748, 748, 761, 748, 754, 754, 779, 769, 726, 784, 736, 812, 805, 825, 777, 776, 748, 761, 713, 749, 795, 742, 763, 752, 812, 754, 758, 723, 805, 751, 730, 758, 793, 734] payload_rows_per_dst=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] dup_rows_per_dst=[3132, 3136, 3020, 2876, 3012, 3086, 2906, 2834] combine_rows_per_src=[9402, 9290, 9380, 9174, 9138, 9394, 9254, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65288 dup_rows=23886 dup_rows_remote=20984 dedup_ratio=0.242981 router_tokens_per_expert=[741, 764, 750, 772, 783, 761, 781, 772, 736, 764, 775, 765, 777, 772, 770, 768, 718, 753, 791, 757, 794, 764, 757, 804, 803, 801, 741, 783, 784, 766, 759, 786, 706, 772, 740, 738, 749, 783, 778, 750, 740, 778, 755, 719, 800, 825, 813, 759, 801, 794, 736, 789, 769, 771, 774, 794, 746, 756, 758, 742, 791, 757, 798, 759] payload_rows_per_dst=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] dup_rows_per_dst=[2926, 2960, 2936, 3060, 2902, 3020, 3066, 3016] combine_rows_per_src=[9322, 9294, 9340, 9386, 9130, 9358, 9390, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73818 payload_rows_remote=64530 dup_rows=24486 dup_rows_remote=21518 dedup_ratio=0.249084 router_tokens_per_expert=[741, 727, 792, 779, 754, 777, 777, 737, 728, 820, 786, 792, 723, 753, 782, 786, 764, 783, 754, 815, 771, 745, 777, 793, 709, 751, 775, 785, 741, 794, 754, 790, 794, 794, 751, 803, 740, 751, 753, 752, 783, 730, 783, 755, 756, 776, 799, 746, 779, 779, 781, 790, 771, 778, 785, 785, 729, 740, 740, 755, 826, 757, 755, 781] payload_rows_per_dst=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] dup_rows_per_dst=[3038, 3056, 3200, 3008, 3050, 2968, 3112, 3054] combine_rows_per_src=[9130, 9284, 9204, 9190, 9226, 9288, 9384, 9112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343752 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345608 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347464 weight_movement_bytes=34603008 total_nvlink_bytes=568950472 weight_to_token_ratio=0.064758 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344208 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267124880 ingress_bytes=370589696 link_bytes=370589696 token_bytes=533905552 weight_movement_bytes=103809024 total_nvlink_bytes=637714576 weight_to_token_ratio=0.194433 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394499191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344200 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067528 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790856 weight_movement_bytes=34603008 total_nvlink_bytes=568393864 weight_to_token_ratio=0.064825 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=342760 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265386728 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530430696 weight_movement_bytes=34603008 total_nvlink_bytes=565033704 weight_to_token_ratio=0.065236 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64652 dup_rows=24202 dup_rows_remote=21020 dedup_ratio=0.246195 router_tokens_per_expert=[759, 739, 765, 754, 775, 751, 744, 782, 723, 729, 743, 745, 775, 835, 743, 757, 807, 789, 759, 815, 776, 801, 779, 790, 806, 766, 774, 782, 732, 784, 730, 791, 764, 764, 739, 789, 761, 764, 759, 776, 712, 776, 754, 767, 784, 786, 796, 746, 762, 783, 759, 791, 779, 736, 792, 752, 772, 747, 814, 783, 779, 743, 784, 739] payload_rows_per_dst=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] dup_rows_per_dst=[2926, 2906, 3182, 3054, 3002, 3056, 2964, 3112] combine_rows_per_src=[9212, 9194, 9450, 9276, 9230, 9186, 9344, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=345016 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267109304 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533873592 weight_movement_bytes=69206016 total_nvlink_bytes=603079608 weight_to_token_ratio=0.129630 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=345088 combine_payload_bytes=267419648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267764736 ingress_bytes=302022656 link_bytes=302022656 token_bytes=535184384 weight_movement_bytes=34603008 total_nvlink_bytes=569787392 weight_to_token_ratio=0.064656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136486940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65132 dup_rows=23962 dup_rows_remote=20920 dedup_ratio=0.243754 router_tokens_per_expert=[764, 773, 783, 826, 724, 751, 791, 777, 750, 761, 775, 756, 784, 742, 772, 777, 739, 792, 776, 756, 756, 760, 790, 786, 772, 771, 756, 780, 812, 767, 751, 767, 816, 775, 776, 763, 736, 755, 748, 742, 789, 759, 740, 762, 828, 740, 783, 751, 784, 732, 770, 747, 786, 796, 739, 772, 758, 765, 758, 766, 774, 796, 744, 765] payload_rows_per_dst=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] dup_rows_per_dst=[3022, 3016, 2946, 3032, 2986, 2944, 3042, 2974] combine_rows_per_src=[9356, 9218, 9364, 9320, 9236, 9360, 9210, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=343184 combine_payload_bytes=264871936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265215120 ingress_bytes=403283968 link_bytes=403283968 token_bytes=530087056 weight_movement_bytes=138412032 total_nvlink_bytes=668499088 weight_to_token_ratio=0.261112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517525105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344056 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288568 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233080 weight_movement_bytes=34603008 total_nvlink_bytes=568836088 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264314880 dispatch_meta_bytes=344192 combine_payload_bytes=264314880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264659072 ingress_bytes=333520896 link_bytes=333520896 token_bytes=528973952 weight_movement_bytes=69206016 total_nvlink_bytes=598179968 weight_to_token_ratio=0.130831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.255012281e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267419648 dispatch_meta_bytes=345088 combine_payload_bytes=267419648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267764736 ingress_bytes=302022656 link_bytes=302022656 token_bytes=535184384 weight_movement_bytes=34603008 total_nvlink_bytes=569787392 weight_to_token_ratio=0.064656 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136486940e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233030848 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6120463552 ingress_bytes=7531290624 link_bytes=7531290624 total_nvlink_bytes=13651754176 weight_to_token_ratio=0.115975 prefetch_share_of_ingress=0.082702 dedup_ratio=0.244611 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232158274e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12244255600 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6126084976 ingress_bytes=7571496960 link_bytes=7571496960 total_nvlink_bytes=13697581936 weight_to_token_ratio=0.118695 prefetch_share_of_ingress=0.091403 dedup_ratio=0.244925 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027065 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246151200 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6127030304 ingress_bytes=7745462272 link_bytes=7745462272 total_nvlink_bytes=13872492576 weight_to_token_ratio=0.132804 prefetch_share_of_ingress=0.075948 dedup_ratio=0.244592 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267197868e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344208 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267124880 ingress_bytes=370589696 link_bytes=370589696 token_bytes=533905552 weight_movement_bytes=103809024 total_nvlink_bytes=637714576 weight_to_token_ratio=0.194433 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394499191e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65186 dup_rows=23816 dup_rows_remote=20840 dedup_ratio=0.242269 router_tokens_per_expert=[767, 768, 775, 725, 743, 794, 787, 783, 755, 745, 777, 786, 708, 807, 775, 786, 747, 793, 766, 732, 771, 731, 740, 825, 782, 753, 782, 763, 785, 790, 780, 757, 744, 750, 716, 829, 772, 741, 793, 795, 787, 792, 760, 787, 756, 737, 811, 777, 749, 787, 753, 780, 771, 778, 727, 758, 727, 783, 779, 753, 785, 817, 709, 771] payload_rows_per_dst=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] dup_rows_per_dst=[3014, 2976, 2968, 3010, 2920, 3046, 2936, 2946] combine_rows_per_src=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64830 dup_rows=24096 dup_rows_remote=20920 dedup_ratio=0.245117 router_tokens_per_expert=[785, 824, 713, 776, 796, 770, 821, 750, 750, 717, 799, 799, 743, 743, 762, 787, 778, 782, 832, 754, 741, 772, 790, 771, 781, 739, 755, 779, 792, 745, 735, 728, 715, 811, 754, 777, 754, 731, 754, 796, 776, 794, 736, 757, 784, 749, 793, 754, 789, 803, 764, 768, 782, 773, 817, 762, 775, 739, 741, 726, 754, 759, 772, 784] payload_rows_per_dst=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] dup_rows_per_dst=[3176, 2970, 3228, 2896, 2964, 2954, 3006, 2902] combine_rows_per_src=[9378, 9146, 9488, 9082, 9220, 9226, 9470, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342688 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265157280 ingress_bytes=299417600 link_bytes=299417600 token_bytes=529971872 weight_movement_bytes=34603008 total_nvlink_bytes=564574880 weight_to_token_ratio=0.065292 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126684324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64432 dup_rows=24150 dup_rows_remote=20778 dedup_ratio=0.245667 router_tokens_per_expert=[745, 756, 698, 796, 747, 831, 737, 710, 805, 725, 733, 762, 782, 855, 802, 757, 786, 748, 801, 716, 776, 779, 764, 769, 779, 783, 799, 785, 797, 778, 755, 797, 729, 780, 791, 770, 754, 795, 760, 725, 761, 771, 786, 750, 777, 793, 750, 741, 805, 783, 754, 803, 743, 747, 729, 771, 762, 773, 774, 731, 807, 744, 779, 761] payload_rows_per_dst=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] dup_rows_per_dst=[2926, 3072, 2686, 3186, 3358, 2960, 2590, 3372] combine_rows_per_src=[9114, 9354, 8984, 9360, 9458, 9298, 8864, 9722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344104 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345960 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347816 weight_movement_bytes=34603008 total_nvlink_bytes=568950824 weight_to_token_ratio=0.064757 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64886 dup_rows=24160 dup_rows_remote=21078 dedup_ratio=0.245768 router_tokens_per_expert=[777, 768, 809, 775, 738, 728, 810, 769, 728, 761, 755, 745, 764, 762, 751, 761, 742, 789, 730, 750, 790, 754, 796, 756, 798, 753, 784, 780, 839, 744, 731, 724, 747, 778, 729, 797, 773, 750, 758, 809, 772, 749, 770, 768, 798, 759, 774, 780, 809, 751, 780, 810, 748, 808, 808, 786, 761, 784, 785, 728, 772, 746, 737, 767] payload_rows_per_dst=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] dup_rows_per_dst=[3024, 2936, 2954, 3004, 3020, 3082, 3132, 3008] combine_rows_per_src=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343000 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886680 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430360 weight_movement_bytes=34603008 total_nvlink_bytes=566033368 weight_to_token_ratio=0.065113 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=20992 dedup_ratio=0.244832 router_tokens_per_expert=[763, 785, 776, 711, 744, 770, 782, 733, 759, 775, 757, 731, 783, 734, 798, 777, 762, 766, 682, 748, 805, 771, 765, 788, 797, 779, 731, 757, 776, 789, 793, 772, 811, 728, 792, 743, 784, 753, 770, 737, 815, 805, 810, 799, 788, 755, 780, 749, 740, 825, 762, 800, 775, 752, 775, 746, 746, 717, 753, 810, 778, 735, 746, 814] payload_rows_per_dst=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] dup_rows_per_dst=[2974, 3014, 2952, 3098, 2854, 3184, 3076, 2916] combine_rows_per_src=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64980 dup_rows=24080 dup_rows_remote=21100 dedup_ratio=0.244954 router_tokens_per_expert=[754, 727, 715, 791, 744, 779, 758, 743, 801, 757, 779, 767, 775, 745, 754, 734, 759, 796, 770, 783, 791, 795, 783, 794, 776, 787, 751, 770, 772, 791, 767, 778, 828, 742, 774, 774, 751, 791, 725, 740, 760, 757, 806, 730, 791, 740, 776, 752, 711, 786, 822, 750, 776, 767, 815, 790, 786, 778, 767, 774, 744, 772, 736, 755] payload_rows_per_dst=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] dup_rows_per_dst=[2928, 2980, 3012, 3064, 3042, 2946, 3106, 3002] combine_rows_per_src=[9094, 9244, 9530, 9320, 9208, 9278, 9328, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65052 dup_rows=23858 dup_rows_remote=20868 dedup_ratio=0.242696 router_tokens_per_expert=[761, 783, 735, 766, 761, 814, 843, 790, 737, 782, 768, 780, 791, 793, 814, 781, 780, 734, 744, 790, 775, 799, 800, 764, 739, 756, 750, 743, 711, 745, 809, 818, 790, 802, 749, 761, 760, 811, 776, 743, 775, 752, 797, 796, 743, 745, 719, 786, 723, 758, 761, 799, 781, 723, 746, 741, 733, 768, 782, 764, 797, 728, 755, 732] payload_rows_per_dst=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] dup_rows_per_dst=[3130, 3054, 3010, 2920, 2990, 2920, 2876, 2958] combine_rows_per_src=[9376, 9438, 9362, 9222, 9394, 9306, 9188, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65172 dup_rows=23812 dup_rows_remote=20842 dedup_ratio=0.242228 router_tokens_per_expert=[779, 770, 782, 793, 790, 740, 768, 759, 718, 800, 761, 752, 784, 751, 703, 817, 755, 749, 773, 748, 768, 761, 765, 804, 760, 782, 769, 786, 810, 758, 734, 746, 761, 839, 738, 752, 742, 774, 763, 768, 743, 771, 770, 787, 781, 737, 780, 780, 806, 713, 738, 751, 814, 775, 784, 740, 746, 752, 761, 790, 768, 832, 798, 763] payload_rows_per_dst=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] dup_rows_per_dst=[3022, 2896, 3042, 2970, 3030, 2914, 2900, 3038] combine_rows_per_src=[9340, 9276, 9204, 9320, 9244, 9384, 9342, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263913472 dispatch_meta_bytes=340840 combine_payload_bytes=263913472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264254312 ingress_bytes=298516480 link_bytes=298516480 token_bytes=528167784 weight_movement_bytes=34603008 total_nvlink_bytes=562770792 weight_to_token_ratio=0.065515 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123293482e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64886 dup_rows=24160 dup_rows_remote=21078 dedup_ratio=0.245768 router_tokens_per_expert=[777, 768, 809, 775, 738, 728, 810, 769, 728, 761, 755, 745, 764, 762, 751, 761, 742, 789, 730, 750, 790, 754, 796, 756, 798, 753, 784, 780, 839, 744, 731, 724, 747, 778, 729, 797, 773, 750, 758, 809, 772, 749, 770, 768, 798, 759, 774, 780, 809, 751, 780, 810, 748, 808, 808, 786, 761, 784, 785, 728, 772, 746, 737, 767] payload_rows_per_dst=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] dup_rows_per_dst=[3024, 2936, 2954, 3004, 3020, 3082, 3132, 3008] combine_rows_per_src=[9324, 9118, 9260, 9302, 9262, 9258, 9468, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343816 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428168 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512520 weight_movement_bytes=34603008 total_nvlink_bytes=567115528 weight_to_token_ratio=0.064981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344320 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266502400 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660480 weight_movement_bytes=69206016 total_nvlink_bytes=601866496 weight_to_token_ratio=0.129925 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344056 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267288568 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233080 weight_movement_bytes=34603008 total_nvlink_bytes=568836088 weight_to_token_ratio=0.064771 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343680 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266796672 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533249664 weight_movement_bytes=34603008 total_nvlink_bytes=567852672 weight_to_token_ratio=0.064891 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65104 dup_rows=24098 dup_rows_remote=21214 dedup_ratio=0.245138 router_tokens_per_expert=[772, 805, 764, 750, 768, 764, 765, 792, 815, 788, 753, 780, 745, 744, 756, 761, 752, 717, 724, 773, 781, 771, 734, 741, 749, 816, 769, 768, 739, 729, 796, 809, 763, 762, 793, 776, 787, 755, 786, 765, 753, 745, 786, 806, 775, 751, 761, 735, 809, 753, 807, 780, 766, 784, 757, 832, 794, 760, 781, 705, 773, 730, 729, 803] payload_rows_per_dst=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] dup_rows_per_dst=[3000, 3090, 2884, 3092, 3082, 2942, 3114, 2894] combine_rows_per_src=[9360, 9194, 9102, 9258, 9292, 9282, 9462, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=20992 dedup_ratio=0.244832 router_tokens_per_expert=[763, 785, 776, 711, 744, 770, 782, 733, 759, 775, 757, 731, 783, 734, 798, 777, 762, 766, 682, 748, 805, 771, 765, 788, 797, 779, 731, 757, 776, 789, 793, 772, 811, 728, 792, 743, 784, 753, 770, 737, 815, 805, 810, 799, 788, 755, 780, 749, 740, 825, 762, 800, 775, 752, 775, 746, 746, 717, 753, 810, 778, 735, 746, 814] payload_rows_per_dst=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] dup_rows_per_dst=[2974, 3014, 2952, 3098, 2854, 3184, 3076, 2916] combine_rows_per_src=[9154, 9214, 9222, 9290, 9382, 9418, 9274, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256593296 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6132254096 ingress_bytes=7266238464 link_bytes=7266238464 total_nvlink_bytes=13398492560 weight_to_token_ratio=0.093166 prefetch_share_of_ingress=0.104768 dedup_ratio=0.244306 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.188794363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343856 combine_payload_bytes=265773056 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266116912 ingress_bytes=404185088 link_bytes=404185088 token_bytes=531889968 weight_movement_bytes=138412032 total_nvlink_bytes=670302000 weight_to_token_ratio=0.260227 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520915947e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343680 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266796672 ingress_bytes=301056000 link_bytes=301056000 token_bytes=533249664 weight_movement_bytes=34603008 total_nvlink_bytes=567852672 weight_to_token_ratio=0.064891 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132849491e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343816 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428168 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512520 weight_movement_bytes=34603008 total_nvlink_bytes=567115528 weight_to_token_ratio=0.064981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259426776 prefetch_bytes=692060160 reduce_bytes=830472192 weight_movement_bytes=1522532352 egress_bytes=6133670360 ingress_bytes=7648288768 link_bytes=7648288768 total_nvlink_bytes=13781959128 weight_to_token_ratio=0.124193 prefetch_share_of_ingress=0.090486 dedup_ratio=0.244186 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=24 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.251299778e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64956 dup_rows=24072 dup_rows_remote=21046 dedup_ratio=0.244873 router_tokens_per_expert=[773, 768, 780, 733, 752, 775, 777, 767, 720, 751, 769, 822, 751, 790, 759, 738, 784, 767, 793, 717, 838, 770, 810, 791, 791, 786, 775, 738, 791, 744, 769, 704, 767, 742, 751, 807, 749, 721, 791, 765, 752, 778, 773, 782, 740, 740, 779, 838, 782, 781, 725, 778, 743, 737, 731, 789, 739, 779, 823, 757, 756, 805, 758, 801] payload_rows_per_dst=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] dup_rows_per_dst=[3026, 3026, 3104, 2992, 3138, 2836, 2894, 3056] combine_rows_per_src=[9276, 9266, 9406, 9204, 9310, 9152, 9238, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262441760 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6135178016 ingress_bytes=7580590080 link_bytes=7580590080 total_nvlink_bytes=13715768096 weight_to_token_ratio=0.118519 prefetch_share_of_ingress=0.091294 dedup_ratio=0.244013 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240223921e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12250644616 prefetch_bytes=761266176 reduce_bytes=346030080 weight_movement_bytes=1107296256 egress_bytes=6129279112 ingress_bytes=7228661760 link_bytes=7228661760 total_nvlink_bytes=13357940872 weight_to_token_ratio=0.090387 prefetch_share_of_ingress=0.105312 dedup_ratio=0.244650 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=10 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.182646619e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262441760 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6135178016 ingress_bytes=7580590080 link_bytes=7580590080 total_nvlink_bytes=13715768096 weight_to_token_ratio=0.118519 prefetch_share_of_ingress=0.091294 dedup_ratio=0.244013 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240223921e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246151200 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6127030304 ingress_bytes=7745462272 link_bytes=7745462272 total_nvlink_bytes=13872492576 weight_to_token_ratio=0.132804 prefetch_share_of_ingress=0.075948 dedup_ratio=0.244592 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267197868e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266665984 dispatch_meta_bytes=345272 combine_payload_bytes=266665984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267011256 ingress_bytes=335872000 link_bytes=335872000 token_bytes=533677240 weight_movement_bytes=69206016 total_nvlink_bytes=602883256 weight_to_token_ratio=0.129678 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263859296e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64708 dup_rows=24130 dup_rows_remote=20982 dedup_ratio=0.245463 router_tokens_per_expert=[753, 785, 754, 772, 739, 762, 746, 742, 702, 761, 788, 753, 781, 751, 766, 822, 782, 787, 788, 825, 790, 798, 745, 737, 741, 778, 744, 723, 780, 760, 762, 774, 760, 762, 785, 723, 743, 760, 801, 772, 787, 796, 777, 759, 756, 769, 818, 744, 811, 748, 741, 757, 775, 739, 841, 760, 742, 787, 766, 772, 760, 800, 792, 758] payload_rows_per_dst=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] dup_rows_per_dst=[2894, 3068, 2886, 3000, 3040, 2946, 3148, 3148] combine_rows_per_src=[9212, 9180, 9184, 9298, 9172, 9264, 9398, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259426776 prefetch_bytes=692060160 reduce_bytes=830472192 weight_movement_bytes=1522532352 egress_bytes=6133670360 ingress_bytes=7648288768 link_bytes=7648288768 total_nvlink_bytes=13781959128 weight_to_token_ratio=0.124193 prefetch_share_of_ingress=0.090486 dedup_ratio=0.244186 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=24 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.251299778e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344008 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403784 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463560 weight_movement_bytes=34603008 total_nvlink_bytes=567066568 weight_to_token_ratio=0.064987 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65186 dup_rows=23816 dup_rows_remote=20840 dedup_ratio=0.242269 router_tokens_per_expert=[767, 768, 775, 725, 743, 794, 787, 783, 755, 745, 777, 786, 708, 807, 775, 786, 747, 793, 766, 732, 771, 731, 740, 825, 782, 753, 782, 763, 785, 790, 780, 757, 744, 750, 716, 829, 772, 741, 793, 795, 787, 792, 760, 787, 756, 737, 811, 777, 749, 787, 753, 780, 771, 778, 727, 758, 727, 783, 779, 753, 785, 817, 709, 771] payload_rows_per_dst=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] dup_rows_per_dst=[3014, 2976, 2968, 3010, 2920, 3046, 2936, 2946] combine_rows_per_src=[9270, 9302, 9242, 9374, 9360, 9368, 9270, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=342760 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265386728 ingress_bytes=299646976 link_bytes=299646976 token_bytes=530430696 weight_movement_bytes=34603008 total_nvlink_bytes=565033704 weight_to_token_ratio=0.065236 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127547447e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233467688 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6120679208 ingress_bytes=7393099776 link_bytes=7393099776 total_nvlink_bytes=13513778984 weight_to_token_ratio=0.104656 prefetch_share_of_ingress=0.093609 dedup_ratio=0.244264 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.209549533e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344104 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345960 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347816 weight_movement_bytes=34603008 total_nvlink_bytes=568950824 weight_to_token_ratio=0.064757 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233030848 prefetch_bytes=622854144 reduce_bytes=795869184 weight_movement_bytes=1418723328 egress_bytes=6120463552 ingress_bytes=7531290624 link_bytes=7531290624 total_nvlink_bytes=13651754176 weight_to_token_ratio=0.115975 prefetch_share_of_ingress=0.082702 dedup_ratio=0.244611 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=23 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232158274e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64666 dup_rows=24154 dup_rows_remote=21130 dedup_ratio=0.245707 router_tokens_per_expert=[782, 808, 818, 755, 783, 744, 693, 751, 724, 812, 776, 740, 794, 859, 769, 796, 758, 764, 810, 785, 769, 736, 822, 810, 783, 791, 728, 780, 749, 778, 757, 784, 800, 751, 759, 777, 755, 744, 771, 737, 727, 756, 820, 767, 776, 746, 772, 749, 764, 799, 746, 751, 779, 750, 757, 736, 778, 803, 763, 745, 773, 757, 738, 698] payload_rows_per_dst=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] dup_rows_per_dst=[3068, 3092, 3024, 3066, 2984, 3008, 2978, 2934] combine_rows_per_src=[9200, 9448, 9484, 9234, 9204, 9218, 9186, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256593296 prefetch_bytes=761266176 reduce_bytes=380633088 weight_movement_bytes=1141899264 egress_bytes=6132254096 ingress_bytes=7266238464 link_bytes=7266238464 total_nvlink_bytes=13398492560 weight_to_token_ratio=0.093166 prefetch_share_of_ingress=0.104768 dedup_ratio=0.244306 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=11 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.188794363e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=264871936 dispatch_meta_bytes=343184 combine_payload_bytes=264871936 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265215120 ingress_bytes=403283968 link_bytes=403283968 token_bytes=530087056 weight_movement_bytes=138412032 total_nvlink_bytes=668499088 weight_to_token_ratio=0.261112 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.517525105e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12244255600 prefetch_bytes=692060160 reduce_bytes=761266176 weight_movement_bytes=1453326336 egress_bytes=6126084976 ingress_bytes=7571496960 link_bytes=7571496960 total_nvlink_bytes=13697581936 weight_to_token_ratio=0.118695 prefetch_share_of_ingress=0.091403 dedup_ratio=0.244925 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=22 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027065 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.238736239e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none - [2026-08-26 16:56:38] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 793.9 | throughput per GPU (TFLOP/s/GPU): 160.7 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.993162E+00 | loss scale: 1.0 | grad norm: 3.651 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64944 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[782, 754, 721, 754, 801, 810, 763, 797, 761, 754, 732, 773, 747, 747, 747, 815, 784, 793, 795, 764, 774, 801, 805, 766, 781, 776, 760, 770, 765, 752, 756, 743, 737, 768, 777, 782, 804, 751, 750, 780, 720, 772, 761, 776, 752, 771, 789, 730, 751, 779, 818, 778, 753, 800, 746, 767, 759, 740, 743, 755, 750, 829, 770, 751] payload_rows_per_dst=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] dup_rows_per_dst=[3056, 2994, 3134, 2982, 3034, 2990, 3006, 2940] combine_rows_per_src=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64762 dup_rows=24212 dup_rows_remote=21154 dedup_ratio=0.246297 router_tokens_per_expert=[790, 711, 776, 798, 781, 784, 776, 764, 753, 819, 787, 798, 778, 749, 763, 747, 764, 802, 731, 771, 808, 740, 773, 781, 783, 731, 727, 793, 733, 791, 734, 710, 761, 747, 811, 775, 795, 762, 727, 747, 752, 786, 743, 765, 774, 775, 723, 777, 739, 802, 792, 787, 764, 780, 764, 772, 783, 727, 794, 778, 827, 741, 777, 759] payload_rows_per_dst=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] dup_rows_per_dst=[3070, 3058, 3060, 2932, 2938, 2992, 3120, 3042] combine_rows_per_src=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] + [2026-08-26 18:08:44] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 789.4 | throughput per GPU (TFLOP/s/GPU): 161.6 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 8.993162E+00 | loss scale: 1.0 | grad norm: 3.651 | number of skipped iterations: 0 | number of nan iterations: 0 | WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65100 dup_rows=23906 dup_rows_remote=20858 dedup_ratio=0.243184 router_tokens_per_expert=[744, 803, 749, 781, 753, 754, 783, 764, 725, 752, 761, 732, 765, 817, 782, 776, 785, 752, 759, 785, 771, 736, 788, 752, 748, 796, 760, 760, 735, 724, 763, 811, 841, 700, 782, 792, 783, 731, 738, 776, 781, 803, 704, 769, 736, 832, 815, 733, 774, 771, 775, 754, 787, 812, 773, 788, 759, 741, 767, 750, 771, 790, 768, 790] payload_rows_per_dst=[9274, 9152, 9334, 9280, 9352, 9298, 9436, 9272] dup_rows_per_dst=[2988, 3068, 2922, 2914, 2934, 3048, 3032, 3000] combine_rows_per_src=[9274, 9152, 9334, 9280, 9352, 9298, 9436, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65066 dup_rows=23916 dup_rows_remote=20908 dedup_ratio=0.243286 router_tokens_per_expert=[759, 786, 762, 775, 787, 748, 755, 757, 790, 819, 819, 715, 765, 742, 774, 753, 751, 808, 836, 787, 763, 755, 827, 760, 790, 757, 717, 791, 773, 787, 767, 735, 799, 809, 784, 749, 751, 765, 737, 771, 738, 742, 740, 801, 771, 787, 770, 731, 731, 763, 768, 760, 755, 788, 758, 747, 783, 728, 780, 788, 763, 781, 768, 736] payload_rows_per_dst=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] dup_rows_per_dst=[2904, 2934, 3174, 3004, 3008, 2970, 2990, 2932] combine_rows_per_src=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21078 dedup_ratio=0.244568 router_tokens_per_expert=[719, 788, 784, 747, 786, 787, 784, 781, 802, 800, 791, 814, 734, 758, 717, 779, 785, 738, 719, 805, 805, 768, 778, 745, 819, 735, 766, 763, 753, 716, 736, 740, 748, 709, 727, 772, 763, 756, 755, 807, 823, 769, 763, 781, 777, 713, 815, 775, 751, 749, 744, 781, 772, 776, 804, 766, 770, 841, 745, 813, 730, 778, 740, 797] payload_rows_per_dst=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] dup_rows_per_dst=[3076, 3074, 2964, 2858, 2884, 3058, 3028, 3100] combine_rows_per_src=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74372 payload_rows_remote=65164 dup_rows=23932 dup_rows_remote=20974 dedup_ratio=0.243449 router_tokens_per_expert=[796, 763, 763, 752, 748, 756, 771, 776, 717, 810, 749, 803, 769, 768, 782, 743, 771, 757, 728, 779, 781, 751, 783, 796, 782, 757, 771, 789, 781, 793, 786, 798, 776, 756, 737, 807, 744, 753, 731, 752, 749, 804, 761, 791, 765, 757, 788, 741, 771, 790, 752, 767, 789, 808, 786, 786, 778, 714, 762, 795, 735, 748, 749, 741] payload_rows_per_dst=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] dup_rows_per_dst=[2724, 2916, 2788, 3368, 3118, 3040, 3020, 2958] combine_rows_per_src=[8918, 9244, 9096, 9754, 9402, 9272, 9478, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64944 dup_rows=24136 dup_rows_remote=21154 dedup_ratio=0.245524 router_tokens_per_expert=[782, 754, 721, 754, 801, 810, 763, 797, 761, 754, 732, 773, 747, 747, 747, 815, 784, 793, 795, 764, 774, 801, 805, 766, 781, 776, 760, 770, 765, 752, 756, 743, 737, 768, 777, 782, 804, 751, 750, 780, 720, 772, 761, 776, 752, 771, 789, 730, 751, 779, 818, 778, 753, 800, 746, 767, 759, 740, 743, 755, 750, 829, 770, 751] payload_rows_per_dst=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] dup_rows_per_dst=[3056, 2994, 3134, 2982, 3034, 2990, 3006, 2940] combine_rows_per_src=[9308, 9158, 9430, 9224, 9264, 9152, 9378, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21078 dedup_ratio=0.244568 router_tokens_per_expert=[719, 788, 784, 747, 786, 787, 784, 781, 802, 800, 791, 814, 734, 758, 717, 779, 785, 738, 719, 805, 805, 768, 778, 745, 819, 735, 766, 763, 753, 716, 736, 740, 748, 709, 727, 772, 763, 756, 755, 807, 823, 769, 763, 781, 777, 713, 815, 775, 751, 749, 744, 781, 772, 776, 804, 766, 770, 841, 745, 813, 730, 778, 740, 797] payload_rows_per_dst=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] dup_rows_per_dst=[3076, 3074, 2964, 2858, 2884, 3058, 3028, 3100] combine_rows_per_src=[9276, 9316, 9322, 9198, 9190, 9374, 9258, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65066 dup_rows=23916 dup_rows_remote=20908 dedup_ratio=0.243286 router_tokens_per_expert=[759, 786, 762, 775, 787, 748, 755, 757, 790, 819, 819, 715, 765, 742, 774, 753, 751, 808, 836, 787, 763, 755, 827, 760, 790, 757, 717, 791, 773, 787, 767, 735, 799, 809, 784, 749, 751, 765, 737, 771, 738, 742, 740, 801, 771, 787, 770, 731, 731, 763, 768, 760, 755, 788, 758, 747, 783, 728, 780, 788, 763, 781, 768, 736] payload_rows_per_dst=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] dup_rows_per_dst=[2904, 2934, 3174, 3004, 3008, 2970, 2990, 2932] combine_rows_per_src=[9354, 9420, 9400, 9230, 9322, 9190, 9150, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64762 dup_rows=24212 dup_rows_remote=21154 dedup_ratio=0.246297 router_tokens_per_expert=[790, 711, 776, 798, 781, 784, 776, 764, 753, 819, 787, 798, 778, 749, 763, 747, 764, 802, 731, 771, 808, 740, 773, 781, 783, 731, 727, 793, 733, 791, 734, 710, 761, 747, 811, 775, 795, 762, 727, 747, 752, 786, 743, 765, 774, 775, 723, 777, 739, 802, 792, 787, 764, 780, 764, 772, 783, 727, 794, 778, 827, 741, 777, 759] payload_rows_per_dst=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] dup_rows_per_dst=[3070, 3058, 3060, 2932, 2938, 2992, 3120, 3042] combine_rows_per_src=[9290, 9330, 9280, 9072, 9312, 9198, 9280, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344392 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355016 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365640 weight_movement_bytes=34603008 total_nvlink_bytes=566968648 weight_to_token_ratio=0.064999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65160 dup_rows=23934 dup_rows_remote=20920 dedup_ratio=0.243469 router_tokens_per_expert=[798, 779, 747, 803, 830, 805, 756, 795, 773, 758, 763, 784, 766, 722, 762, 780, 752, 753, 732, 712, 759, 765, 799, 755, 804, 705, 767, 780, 735, 726, 768, 822, 738, 768, 790, 711, 790, 772, 769, 768, 783, 775, 806, 762, 776, 736, 753, 777, 732, 746, 752, 797, 785, 734, 763, 803, 761, 813, 770, 778, 744, 786, 786, 773] payload_rows_per_dst=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] dup_rows_per_dst=[3158, 2970, 2880, 2926, 2914, 3032, 3014, 3040] combine_rows_per_src=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265265152 dispatch_meta_bytes=343664 combine_payload_bytes=265265152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265608816 ingress_bytes=334471168 link_bytes=334471168 token_bytes=530873968 weight_movement_bytes=69206016 total_nvlink_bytes=600079984 weight_to_token_ratio=0.130362 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258588078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266649600 dispatch_meta_bytes=343832 combine_payload_bytes=266649600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266993432 ingress_bytes=301252608 link_bytes=301252608 token_bytes=533643032 weight_movement_bytes=34603008 total_nvlink_bytes=568246040 weight_to_token_ratio=0.064843 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133589311e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343896 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854232 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364568 weight_movement_bytes=34603008 total_nvlink_bytes=567967576 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344552 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256296 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534168040 weight_movement_bytes=34603008 total_nvlink_bytes=568771048 weight_to_token_ratio=0.064779 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344392 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266355016 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365640 weight_movement_bytes=34603008 total_nvlink_bytes=566968648 weight_to_token_ratio=0.064999 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344072 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338312 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332552 weight_movement_bytes=69206016 total_nvlink_bytes=601538568 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343896 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854232 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364568 weight_movement_bytes=34603008 total_nvlink_bytes=567967576 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64696 dup_rows=24248 dup_rows_remote=21168 dedup_ratio=0.246663 router_tokens_per_expert=[771, 744, 811, 756, 798, 768, 758, 731, 787, 812, 741, 781, 737, 781, 771, 785, 826, 734, 761, 789, 730, 775, 770, 782, 782, 801, 725, 755, 721, 806, 803, 744, 767, 752, 791, 765, 778, 784, 746, 784, 801, 797, 752, 719, 776, 775, 775, 783, 760, 770, 755, 742, 769, 771, 758, 748, 746, 743, 751, 815, 782, 764, 768, 729] payload_rows_per_dst=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] dup_rows_per_dst=[3080, 3138, 3106, 2984, 2996, 3122, 2852, 2970] combine_rows_per_src=[9360, 9286, 9240, 9290, 9338, 9264, 9052, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344552 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267256296 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534168040 weight_movement_bytes=34603008 total_nvlink_bytes=568771048 weight_to_token_ratio=0.064779 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65160 dup_rows=23934 dup_rows_remote=20920 dedup_ratio=0.243469 router_tokens_per_expert=[798, 779, 747, 803, 830, 805, 756, 795, 773, 758, 763, 784, 766, 722, 762, 780, 752, 753, 732, 712, 759, 765, 799, 755, 804, 705, 767, 780, 735, 726, 768, 822, 738, 768, 790, 711, 790, 772, 769, 768, 783, 775, 806, 762, 776, 736, 753, 777, 732, 746, 752, 797, 785, 734, 763, 803, 761, 813, 770, 778, 744, 786, 786, 773] payload_rows_per_dst=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] dup_rows_per_dst=[3158, 2970, 2880, 2926, 2914, 3032, 3014, 3040] combine_rows_per_src=[9468, 9246, 9174, 9288, 9298, 9304, 9210, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265265152 dispatch_meta_bytes=343664 combine_payload_bytes=265265152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265608816 ingress_bytes=334471168 link_bytes=334471168 token_bytes=530873968 weight_movement_bytes=69206016 total_nvlink_bytes=600079984 weight_to_token_ratio=0.130362 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258588078e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344320 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267239680 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534135040 weight_movement_bytes=138412032 total_nvlink_bytes=672547072 weight_to_token_ratio=0.259133 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65038 dup_rows=24022 dup_rows_remote=21030 dedup_ratio=0.244364 router_tokens_per_expert=[746, 772, 749, 802, 784, 772, 762, 826, 754, 777, 764, 792, 789, 778, 751, 767, 765, 767, 748, 744, 758, 761, 768, 813, 715, 778, 782, 744, 751, 773, 776, 762, 752, 790, 792, 808, 749, 752, 797, 749, 775, 763, 769, 726, 751, 776, 760, 798, 741, 736, 796, 747, 722, 761, 813, 761, 756, 696, 781, 786, 771, 802, 791, 795] payload_rows_per_dst=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] dup_rows_per_dst=[3066, 2962, 3038, 2850, 3044, 2992, 2992, 3078] combine_rows_per_src=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74476 payload_rows_remote=65114 dup_rows=23828 dup_rows_remote=20812 dedup_ratio=0.242391 router_tokens_per_expert=[752, 749, 764, 724, 793, 748, 753, 759, 767, 794, 765, 724, 743, 767, 791, 801, 786, 735, 819, 786, 762, 741, 758, 827, 799, 776, 794, 792, 738, 762, 737, 791, 743, 785, 782, 753, 758, 783, 736, 797, 784, 728, 812, 732, 796, 806, 816, 758, 823, 783, 739, 770, 762, 763, 773, 719, 804, 749, 821, 746, 717, 735, 744, 738] payload_rows_per_dst=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] dup_rows_per_dst=[2852, 2988, 3096, 3016, 2970, 3022, 2966, 2918] combine_rows_per_src=[9232, 9316, 9332, 9362, 9304, 9442, 9298, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65030 dup_rows=23964 dup_rows_remote=20860 dedup_ratio=0.243774 router_tokens_per_expert=[752, 787, 756, 750, 787, 774, 756, 730, 796, 741, 816, 801, 740, 773, 748, 792, 776, 786, 793, 783, 802, 751, 737, 788, 762, 747, 773, 770, 778, 778, 735, 756, 740, 786, 754, 755, 763, 781, 747, 777, 774, 728, 796, 771, 757, 722, 811, 797, 725, 861, 787, 789, 749, 747, 742, 786, 765, 755, 792, 743, 758, 732, 764, 784] payload_rows_per_dst=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] dup_rows_per_dst=[2916, 3104, 3050, 2972, 2980, 2944, 3060, 2938] combine_rows_per_src=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64970 dup_rows=23942 dup_rows_remote=20832 dedup_ratio=0.243551 router_tokens_per_expert=[778, 750, 785, 759, 791, 772, 766, 721, 797, 808, 786, 761, 783, 761, 756, 806, 747, 785, 731, 781, 749, 732, 767, 801, 732, 795, 763, 745, 742, 759, 779, 799, 722, 742, 752, 701, 793, 785, 729, 845, 776, 735, 806, 828, 765, 761, 783, 763, 762, 763, 798, 779, 737, 768, 790, 750, 735, 766, 746, 784, 743, 798, 811, 749] payload_rows_per_dst=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] dup_rows_per_dst=[2972, 2968, 2960, 2912, 2940, 3016, 3064, 3110] combine_rows_per_src=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=65038 dup_rows=24022 dup_rows_remote=21030 dedup_ratio=0.244364 router_tokens_per_expert=[746, 772, 749, 802, 784, 772, 762, 826, 754, 777, 764, 792, 789, 778, 751, 767, 765, 767, 748, 744, 758, 761, 768, 813, 715, 778, 782, 744, 751, 773, 776, 762, 752, 790, 792, 808, 749, 752, 797, 749, 775, 763, 769, 726, 751, 776, 760, 798, 741, 736, 796, 747, 722, 761, 813, 761, 756, 696, 781, 786, 771, 802, 791, 795] payload_rows_per_dst=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] dup_rows_per_dst=[3066, 2962, 3038, 2850, 3044, 2992, 2992, 3078] combine_rows_per_src=[9360, 9382, 9210, 9312, 9334, 9244, 9162, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344320 combine_payload_bytes=266895360 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267239680 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534135040 weight_movement_bytes=138412032 total_nvlink_bytes=672547072 weight_to_token_ratio=0.259133 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64960 dup_rows=23998 dup_rows_remote=20934 dedup_ratio=0.244120 router_tokens_per_expert=[797, 777, 761, 703, 803, 713, 783, 777, 789, 775, 772, 765, 718, 752, 781, 791, 758, 763, 792, 791, 753, 773, 776, 732, 804, 801, 779, 767, 758, 739, 815, 742, 770, 823, 804, 746, 774, 740, 779, 769, 772, 756, 809, 788, 778, 774, 772, 747, 792, 737, 769, 749, 837, 766, 702, 752, 810, 754, 731, 774, 698, 761, 766, 753] payload_rows_per_dst=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] dup_rows_per_dst=[3018, 2978, 2992, 3036, 3064, 3034, 2934, 2942] combine_rows_per_src=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65118 dup_rows=23946 dup_rows_remote=20888 dedup_ratio=0.243591 router_tokens_per_expert=[791, 769, 767, 824, 791, 763, 788, 745, 739, 765, 749, 774, 805, 767, 773, 788, 770, 770, 750, 783, 772, 779, 787, 738, 762, 789, 795, 716, 718, 813, 784, 726, 752, 776, 759, 789, 788, 796, 764, 739, 758, 759, 725, 792, 762, 763, 745, 762, 721, 760, 813, 734, 749, 806, 719, 762, 753, 770, 802, 762, 792, 790, 760, 780] payload_rows_per_dst=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] dup_rows_per_dst=[3084, 2946, 3058, 2966, 3018, 2950, 2952, 2972] combine_rows_per_src=[9392, 9374, 9240, 9240, 9308, 9182, 9176, 9446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343704 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267050648 ingress_bytes=405118976 link_bytes=405118976 token_bytes=533757592 weight_movement_bytes=138412032 total_nvlink_bytes=672169624 weight_to_token_ratio=0.259316 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524430092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=64970 dup_rows=23942 dup_rows_remote=20832 dedup_ratio=0.243551 router_tokens_per_expert=[778, 750, 785, 759, 791, 772, 766, 721, 797, 808, 786, 761, 783, 761, 756, 806, 747, 785, 731, 781, 749, 732, 767, 801, 732, 795, 763, 745, 742, 759, 779, 799, 722, 742, 752, 701, 793, 785, 729, 845, 776, 735, 806, 828, 765, 761, 783, 763, 762, 763, 798, 779, 737, 768, 790, 750, 735, 766, 746, 784, 743, 798, 811, 749] payload_rows_per_dst=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] dup_rows_per_dst=[2972, 2968, 2960, 2912, 2940, 3016, 3064, 3110] combine_rows_per_src=[9272, 9262, 9274, 9288, 9226, 9296, 9352, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343560 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266706440 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533069320 weight_movement_bytes=103809024 total_nvlink_bytes=636878344 weight_to_token_ratio=0.194738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64960 dup_rows=23998 dup_rows_remote=20934 dedup_ratio=0.244120 router_tokens_per_expert=[797, 777, 761, 703, 803, 713, 783, 777, 789, 775, 772, 765, 718, 752, 781, 791, 758, 763, 792, 791, 753, 773, 776, 732, 804, 801, 779, 767, 758, 739, 815, 742, 770, 823, 804, 746, 774, 740, 779, 769, 772, 756, 809, 788, 778, 774, 772, 747, 792, 737, 769, 749, 837, 766, 702, 752, 810, 754, 731, 774, 698, 761, 766, 753] payload_rows_per_dst=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] dup_rows_per_dst=[3018, 2978, 2992, 3036, 3064, 3034, 2934, 2942] combine_rows_per_src=[9210, 9308, 9284, 9374, 9346, 9358, 9274, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65030 dup_rows=23964 dup_rows_remote=20860 dedup_ratio=0.243774 router_tokens_per_expert=[752, 787, 756, 750, 787, 774, 756, 730, 796, 741, 816, 801, 740, 773, 748, 792, 776, 786, 793, 783, 802, 751, 737, 788, 762, 747, 773, 770, 778, 778, 735, 756, 740, 786, 754, 755, 763, 781, 747, 777, 774, 728, 796, 771, 757, 722, 811, 797, 725, 861, 787, 789, 749, 747, 742, 786, 765, 755, 792, 743, 758, 732, 764, 784] payload_rows_per_dst=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] dup_rows_per_dst=[2916, 3104, 3050, 2972, 2980, 2944, 3060, 2938] combine_rows_per_src=[9268, 9310, 9382, 9226, 9226, 9368, 9312, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344272 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266739920 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135568 weight_movement_bytes=69206016 total_nvlink_bytes=602341584 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343208 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460328 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532577448 weight_movement_bytes=34603008 total_nvlink_bytes=567180456 weight_to_token_ratio=0.064973 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343576 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419736 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495896 weight_movement_bytes=34603008 total_nvlink_bytes=567098904 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266706944 dispatch_meta_bytes=343704 combine_payload_bytes=266706944 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267050648 ingress_bytes=405118976 link_bytes=405118976 token_bytes=533757592 weight_movement_bytes=138412032 total_nvlink_bytes=672169624 weight_to_token_ratio=0.259316 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524430092e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344024 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067352 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790680 weight_movement_bytes=34603008 total_nvlink_bytes=568393688 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343208 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460328 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532577448 weight_movement_bytes=34603008 total_nvlink_bytes=567180456 weight_to_token_ratio=0.064973 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64792 dup_rows=24068 dup_rows_remote=21060 dedup_ratio=0.244832 router_tokens_per_expert=[786, 767, 789, 737, 732, 729, 739, 797, 776, 813, 728, 719, 768, 747, 814, 781, 756, 769, 748, 738, 755, 754, 765, 806, 811, 765, 759, 792, 806, 761, 775, 767, 769, 732, 752, 763, 787, 801, 753, 806, 725, 741, 774, 728, 802, 796, 786, 756, 815, 758, 724, 763, 819, 783, 752, 786, 804, 704, 771, 765, 782, 784, 767, 755] payload_rows_per_dst=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] dup_rows_per_dst=[3008, 3060, 2986, 2990, 3040, 2952, 3010, 3022] combine_rows_per_src=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=343576 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266419736 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532495896 weight_movement_bytes=34603008 total_nvlink_bytes=567098904 weight_to_token_ratio=0.064983 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65206 dup_rows=23730 dup_rows_remote=20736 dedup_ratio=0.241394 router_tokens_per_expert=[739, 776, 746, 778, 723, 752, 790, 801, 765, 727, 782, 728, 748, 772, 778, 754, 823, 764, 790, 718, 749, 806, 740, 713, 860, 756, 747, 779, 770, 778, 796, 805, 739, 762, 797, 772, 767, 745, 784, 743, 751, 723, 810, 814, 734, 733, 780, 776, 794, 755, 788, 763, 763, 769, 745, 804, 730, 775, 727, 797, 814, 773, 754, 818] payload_rows_per_dst=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] dup_rows_per_dst=[2902, 2866, 2996, 3130, 2982, 2888, 2994, 2972] combine_rows_per_src=[9308, 9242, 9210, 9452, 9236, 9354, 9368, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65038 dup_rows=24052 dup_rows_remote=21064 dedup_ratio=0.244670 router_tokens_per_expert=[856, 814, 775, 795, 780, 754, 759, 785, 729, 762, 783, 790, 719, 796, 733, 780, 738, 792, 755, 744, 746, 777, 777, 751, 806, 773, 783, 728, 782, 730, 777, 722, 772, 751, 774, 733, 737, 796, 768, 760, 736, 769, 756, 815, 794, 756, 754, 786, 736, 764, 812, 732, 782, 781, 715, 764, 844, 736, 754, 793, 772, 764, 768, 787] payload_rows_per_dst=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] dup_rows_per_dst=[3142, 2958, 2868, 2988, 3016, 3026, 2948, 3106] combine_rows_per_src=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343560 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266706440 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533069320 weight_movement_bytes=103809024 total_nvlink_bytes=636878344 weight_to_token_ratio=0.194738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65214 dup_rows=23954 dup_rows_remote=21062 dedup_ratio=0.243673 router_tokens_per_expert=[786, 747, 752, 782, 784, 767, 828, 770, 725, 736, 814, 737, 751, 751, 754, 746, 794, 779, 796, 797, 771, 756, 750, 754, 786, 776, 794, 803, 765, 738, 737, 723, 723, 784, 783, 792, 706, 772, 812, 764, 770, 745, 762, 752, 782, 783, 769, 792, 767, 808, 763, 795, 774, 774, 834, 767, 752, 731, 750, 774, 751, 741, 774, 757] payload_rows_per_dst=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] dup_rows_per_dst=[2952, 2892, 3080, 3020, 2942, 3086, 3102, 2880] combine_rows_per_src=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64792 dup_rows=24068 dup_rows_remote=21060 dedup_ratio=0.244832 router_tokens_per_expert=[786, 767, 789, 737, 732, 729, 739, 797, 776, 813, 728, 719, 768, 747, 814, 781, 756, 769, 748, 738, 755, 754, 765, 806, 811, 765, 759, 792, 806, 761, 775, 767, 769, 732, 752, 763, 787, 801, 753, 806, 725, 741, 774, 728, 802, 796, 786, 756, 815, 758, 724, 763, 819, 783, 752, 786, 804, 704, 771, 765, 782, 784, 767, 755] payload_rows_per_dst=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] dup_rows_per_dst=[3008, 3060, 2986, 2990, 3040, 2952, 3010, 3022] combine_rows_per_src=[9444, 9238, 9196, 9178, 9286, 9262, 9390, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343768 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427544 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511320 weight_movement_bytes=34603008 total_nvlink_bytes=569114328 weight_to_token_ratio=0.064738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64846 dup_rows=24074 dup_rows_remote=21014 dedup_ratio=0.244893 router_tokens_per_expert=[765, 775, 745, 761, 760, 726, 756, 769, 747, 753, 807, 762, 775, 797, 760, 760, 766, 768, 780, 780, 749, 788, 735, 786, 771, 782, 762, 759, 822, 749, 761, 778, 812, 732, 729, 735, 751, 751, 810, 744, 744, 785, 779, 759, 822, 768, 781, 784, 806, 757, 784, 739, 782, 747, 789, 754, 753, 778, 782, 801, 755, 754, 766, 765] payload_rows_per_dst=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] dup_rows_per_dst=[3012, 2994, 3040, 3116, 2804, 3060, 3042, 3006] combine_rows_per_src=[9102, 9328, 9264, 9252, 9324, 9384, 9274, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64552 dup_rows=24086 dup_rows_remote=20658 dedup_ratio=0.245015 router_tokens_per_expert=[773, 775, 803, 761, 715, 757, 775, 779, 738, 777, 746, 779, 761, 773, 787, 763, 736, 801, 753, 742, 829, 783, 791, 784, 786, 812, 776, 751, 743, 735, 765, 799, 791, 773, 772, 761, 774, 768, 771, 742, 716, 747, 732, 849, 758, 754, 776, 735, 764, 828, 758, 822, 737, 699, 752, 743, 763, 752, 789, 814, 760, 766, 748, 790] payload_rows_per_dst=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] dup_rows_per_dst=[2676, 2936, 3066, 2898, 3034, 3026, 3022, 3428] combine_rows_per_src=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=65038 dup_rows=24052 dup_rows_remote=21064 dedup_ratio=0.244670 router_tokens_per_expert=[856, 814, 775, 795, 780, 754, 759, 785, 729, 762, 783, 790, 719, 796, 733, 780, 738, 792, 755, 744, 746, 777, 777, 751, 806, 773, 783, 728, 782, 730, 777, 722, 772, 751, 774, 733, 737, 796, 768, 760, 736, 769, 756, 815, 794, 756, 754, 786, 736, 764, 812, 732, 782, 781, 715, 764, 844, 736, 754, 793, 772, 764, 768, 787] payload_rows_per_dst=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] dup_rows_per_dst=[3142, 2958, 2868, 2988, 3016, 3026, 2948, 3106] combine_rows_per_src=[9494, 9226, 9292, 9214, 9166, 9306, 9224, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=343408 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265731440 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531119472 weight_movement_bytes=34603008 total_nvlink_bytes=565722480 weight_to_token_ratio=0.065151 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343768 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427544 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511320 weight_movement_bytes=34603008 total_nvlink_bytes=569114328 weight_to_token_ratio=0.064738 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64896 dup_rows=24172 dup_rows_remote=21168 dedup_ratio=0.245890 router_tokens_per_expert=[752, 758, 751, 782, 769, 793, 755, 755, 765, 758, 799, 802, 754, 772, 713, 785, 755, 772, 751, 778, 797, 763, 765, 741, 746, 742, 758, 851, 789, 792, 777, 762, 729, 703, 796, 739, 802, 790, 738, 823, 739, 775, 772, 779, 800, 753, 763, 781, 804, 755, 827, 769, 752, 759, 835, 766, 749, 730, 754, 735, 765, 724, 755, 789] payload_rows_per_dst=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] dup_rows_per_dst=[3048, 2982, 2972, 3130, 3004, 3028, 3088, 2920] combine_rows_per_src=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344408 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266740056 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135704 weight_movement_bytes=69206016 total_nvlink_bytes=602341720 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64938 dup_rows=23946 dup_rows_remote=20842 dedup_ratio=0.243591 router_tokens_per_expert=[774, 707, 772, 773, 752, 764, 786, 788, 765, 730, 801, 749, 745, 778, 815, 801, 790, 801, 796, 744, 803, 771, 788, 769, 829, 754, 768, 743, 797, 793, 741, 761, 781, 757, 769, 794, 751, 750, 799, 781, 766, 762, 736, 737, 775, 786, 744, 775, 756, 788, 741, 752, 792, 775, 747, 806, 768, 733, 767, 745, 758, 770, 721, 722] payload_rows_per_dst=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] dup_rows_per_dst=[3002, 3022, 3104, 3100, 2998, 2918, 2968, 2834] combine_rows_per_src=[9230, 9346, 9420, 9272, 9366, 9244, 9346, 9134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=345104 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267461648 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534578192 weight_movement_bytes=69206016 total_nvlink_bytes=603784208 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=64552 dup_rows=24086 dup_rows_remote=20658 dedup_ratio=0.245015 router_tokens_per_expert=[773, 775, 803, 761, 715, 757, 775, 779, 738, 777, 746, 779, 761, 773, 787, 763, 736, 801, 753, 742, 829, 783, 791, 784, 786, 812, 776, 751, 743, 735, 765, 799, 791, 773, 772, 761, 774, 768, 771, 742, 716, 747, 732, 849, 758, 754, 776, 735, 764, 828, 758, 822, 737, 699, 752, 743, 763, 752, 789, 814, 760, 766, 748, 790] payload_rows_per_dst=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] dup_rows_per_dst=[2676, 2936, 3066, 2898, 3034, 3026, 3022, 3428] combine_rows_per_src=[8870, 9282, 9332, 9314, 9270, 9230, 9254, 9666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64896 dup_rows=24172 dup_rows_remote=21168 dedup_ratio=0.245890 router_tokens_per_expert=[752, 758, 751, 782, 769, 793, 755, 755, 765, 758, 799, 802, 754, 772, 713, 785, 755, 772, 751, 778, 797, 763, 765, 741, 746, 742, 758, 851, 789, 792, 777, 762, 729, 703, 796, 739, 802, 790, 738, 823, 739, 775, 772, 779, 800, 753, 763, 781, 804, 755, 827, 769, 752, 759, 835, 766, 749, 730, 754, 735, 765, 724, 755, 789] payload_rows_per_dst=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] dup_rows_per_dst=[3048, 2982, 2972, 3130, 3004, 3028, 3088, 2920] combine_rows_per_src=[9182, 9314, 9272, 9304, 9236, 9296, 9446, 9082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65214 dup_rows=23954 dup_rows_remote=21062 dedup_ratio=0.243673 router_tokens_per_expert=[786, 747, 752, 782, 784, 767, 828, 770, 725, 736, 814, 737, 751, 751, 754, 746, 794, 779, 796, 797, 771, 756, 750, 754, 786, 776, 794, 803, 765, 738, 737, 723, 723, 784, 783, 792, 706, 772, 812, 764, 770, 745, 762, 752, 782, 783, 769, 792, 767, 808, 763, 795, 774, 774, 834, 767, 752, 731, 750, 774, 751, 741, 774, 757] payload_rows_per_dst=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] dup_rows_per_dst=[2952, 2892, 3080, 3020, 2942, 3086, 3102, 2880] combine_rows_per_src=[9480, 9136, 9314, 9224, 9330, 9224, 9462, 9180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343440 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265952656 ingress_bytes=300212224 link_bytes=300212224 token_bytes=531561872 weight_movement_bytes=34603008 total_nvlink_bytes=566164880 weight_to_token_ratio=0.065097 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129674430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264404992 dispatch_meta_bytes=340840 combine_payload_bytes=264404992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264745832 ingress_bytes=299008000 link_bytes=299008000 token_bytes=529150824 weight_movement_bytes=34603008 total_nvlink_bytes=563753832 weight_to_token_ratio=0.065393 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125143032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344256 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266158272 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531972288 weight_movement_bytes=34603008 total_nvlink_bytes=566575296 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344408 combine_payload_bytes=266395648 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266740056 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135704 weight_movement_bytes=69206016 total_nvlink_bytes=602341720 weight_to_token_ratio=0.129809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343120 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266329168 ingress_bytes=369795072 link_bytes=369795072 token_bytes=532315216 weight_movement_bytes=103809024 total_nvlink_bytes=636124240 weight_to_token_ratio=0.195014 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391509085e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264404992 dispatch_meta_bytes=340840 combine_payload_bytes=264404992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264745832 ingress_bytes=299008000 link_bytes=299008000 token_bytes=529150824 weight_movement_bytes=34603008 total_nvlink_bytes=563753832 weight_to_token_ratio=0.065393 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125143032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65042 dup_rows=23832 dup_rows_remote=20630 dedup_ratio=0.242432 router_tokens_per_expert=[753, 743, 770, 800, 791, 745, 789, 761, 820, 794, 762, 760, 787, 802, 751, 736, 800, 781, 779, 799, 732, 763, 760, 793, 730, 779, 758, 762, 799, 782, 722, 792, 755, 757, 769, 775, 752, 750, 780, 794, 815, 763, 736, 723, 748, 750, 746, 790, 764, 787, 764, 728, 765, 759, 774, 797, 808, 785, 729, 768, 735, 729, 767, 795] payload_rows_per_dst=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] dup_rows_per_dst=[3202, 3098, 2860, 2898, 3200, 2928, 2708, 2938] combine_rows_per_src=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64980 dup_rows=24058 dup_rows_remote=21058 dedup_ratio=0.244731 router_tokens_per_expert=[766, 810, 804, 728, 728, 751, 760, 803, 758, 768, 734, 752, 774, 795, 772, 808, 776, 747, 778, 737, 704, 782, 764, 808, 810, 785, 744, 759, 743, 774, 740, 778, 767, 735, 769, 753, 768, 785, 762, 749, 783, 798, 775, 801, 741, 791, 749, 781, 768, 758, 737, 797, 769, 761, 794, 825, 770, 767, 752, 768, 783, 776, 707, 773] payload_rows_per_dst=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] dup_rows_per_dst=[3016, 3070, 2990, 3000, 2902, 3096, 3092, 2892] combine_rows_per_src=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=344256 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266158272 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531972288 weight_movement_bytes=34603008 total_nvlink_bytes=566575296 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65030 dup_rows=23928 dup_rows_remote=20972 dedup_ratio=0.243408 router_tokens_per_expert=[819, 733, 730, 782, 847, 798, 819, 713, 790, 799, 765, 734, 757, 765, 804, 813, 736, 795, 769, 728, 736, 778, 728, 827, 778, 765, 745, 797, 752, 765, 759, 756, 723, 800, 757, 767, 757, 791, 745, 710, 801, 767, 757, 756, 762, 741, 812, 771, 765, 772, 777, 765, 771, 743, 777, 781, 744, 724, 746, 776, 723, 820, 793, 776] payload_rows_per_dst=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] dup_rows_per_dst=[3126, 3102, 2896, 2922, 2928, 3018, 2956, 2980] combine_rows_per_src=[9356, 9352, 9298, 9312, 9172, 9316, 9346, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65006 dup_rows=23960 dup_rows_remote=20994 dedup_ratio=0.243734 router_tokens_per_expert=[783, 756, 819, 785, 774, 763, 760, 768, 816, 759, 781, 721, 775, 786, 765, 749, 750, 792, 766, 772, 742, 708, 760, 785, 768, 743, 793, 779, 744, 749, 772, 776, 766, 739, 803, 766, 807, 768, 769, 756, 769, 758, 735, 769, 782, 728, 781, 783, 748, 769, 814, 750, 730, 792, 831, 741, 739, 800, 758, 777, 777, 769, 766, 753] payload_rows_per_dst=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] dup_rows_per_dst=[3042, 2966, 2936, 2942, 3008, 2974, 3084, 3008] combine_rows_per_src=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267116544 dispatch_meta_bytes=345104 combine_payload_bytes=267116544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267461648 ingress_bytes=336322560 link_bytes=336322560 token_bytes=534578192 weight_movement_bytes=69206016 total_nvlink_bytes=603784208 weight_to_token_ratio=0.129459 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265554717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342688 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266754720 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533166752 weight_movement_bytes=69206016 total_nvlink_bytes=602372768 weight_to_token_ratio=0.129802 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21104 dedup_ratio=0.244568 router_tokens_per_expert=[797, 738, 741, 796, 796, 738, 777, 767, 787, 732, 723, 789, 758, 737, 788, 775, 764, 795, 766, 791, 794, 773, 771, 764, 770, 771, 722, 758, 715, 822, 757, 737, 793, 744, 783, 759, 781, 776, 792, 753, 748, 744, 766, 782, 790, 812, 754, 734, 749, 807, 774, 769, 803, 783, 780, 803, 729, 761, 820, 759, 771, 745, 756, 723] payload_rows_per_dst=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] dup_rows_per_dst=[3076, 2874, 3080, 2922, 3082, 2938, 3146, 2924] combine_rows_per_src=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344152 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502232 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660312 weight_movement_bytes=69206016 total_nvlink_bytes=601866328 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65042 dup_rows=23832 dup_rows_remote=20630 dedup_ratio=0.242432 router_tokens_per_expert=[753, 743, 770, 800, 791, 745, 789, 761, 820, 794, 762, 760, 787, 802, 751, 736, 800, 781, 779, 799, 732, 763, 760, 793, 730, 779, 758, 762, 799, 782, 722, 792, 755, 757, 769, 775, 752, 750, 780, 794, 815, 763, 736, 723, 748, 750, 746, 790, 764, 787, 764, 728, 765, 759, 774, 797, 808, 785, 729, 768, 735, 729, 767, 795] payload_rows_per_dst=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] dup_rows_per_dst=[3202, 3098, 2860, 2898, 3200, 2928, 2708, 2938] combine_rows_per_src=[9430, 9336, 9226, 9340, 9514, 9214, 9118, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344008 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266706888 ingress_bytes=335568896 link_bytes=335568896 token_bytes=533069768 weight_movement_bytes=69206016 total_nvlink_bytes=602275784 weight_to_token_ratio=0.129825 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262718740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64828 dup_rows=24052 dup_rows_remote=20946 dedup_ratio=0.244670 router_tokens_per_expert=[756, 754, 757, 755, 766, 737, 767, 743, 752, 751, 729, 788, 800, 774, 751, 737, 800, 735, 685, 775, 793, 802, 755, 763, 741, 801, 744, 792, 778, 781, 758, 802, 831, 748, 788, 799, 773, 778, 761, 787, 795, 751, 769, 772, 763, 763, 804, 803, 712, 801, 774, 800, 776, 778, 754, 725, 775, 743, 752, 753, 738, 778, 818, 768] payload_rows_per_dst=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] dup_rows_per_dst=[2906, 2952, 2998, 3036, 3106, 3034, 3018, 3002] combine_rows_per_src=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64974 dup_rows=24134 dup_rows_remote=21224 dedup_ratio=0.245504 router_tokens_per_expert=[750, 737, 735, 752, 744, 741, 755, 784, 761, 722, 811, 772, 802, 766, 772, 778, 768, 740, 707, 765, 753, 758, 774, 788, 768, 770, 790, 731, 778, 758, 776, 803, 782, 740, 802, 750, 756, 760, 768, 779, 753, 753, 799, 773, 737, 799, 737, 751, 776, 773, 837, 818, 802, 770, 815, 751, 817, 758, 802, 804, 763, 743, 722, 753] payload_rows_per_dst=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] dup_rows_per_dst=[2870, 3070, 2910, 3048, 2990, 3024, 3178, 3044] combine_rows_per_src=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64940 dup_rows=24042 dup_rows_remote=21104 dedup_ratio=0.244568 router_tokens_per_expert=[797, 738, 741, 796, 796, 738, 777, 767, 787, 732, 723, 789, 758, 737, 788, 775, 764, 795, 766, 791, 794, 773, 771, 764, 770, 771, 722, 758, 715, 822, 757, 737, 793, 744, 783, 759, 781, 776, 792, 753, 748, 744, 766, 782, 790, 812, 754, 734, 749, 807, 774, 769, 803, 783, 780, 803, 729, 761, 820, 759, 771, 745, 756, 723] payload_rows_per_dst=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] dup_rows_per_dst=[3076, 2874, 3080, 2922, 3082, 2938, 3146, 2924] combine_rows_per_src=[9224, 9304, 9356, 9182, 9280, 9322, 9390, 9204] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74144 payload_rows_remote=64498 dup_rows=24160 dup_rows_remote=20790 dedup_ratio=0.245768 router_tokens_per_expert=[776, 769, 769, 790, 788, 775, 760, 782, 758, 809, 764, 786, 784, 811, 767, 756, 795, 763, 773, 760, 771, 764, 767, 744, 759, 703, 769, 779, 765, 724, 784, 772, 771, 741, 765, 801, 750, 749, 784, 824, 785, 730, 781, 782, 750, 740, 806, 717, 801, 782, 778, 761, 755, 797, 742, 780, 762, 734, 730, 745, 804, 755, 765, 749] payload_rows_per_dst=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] dup_rows_per_dst=[3032, 3068, 3014, 2788, 3106, 3138, 2644, 3370] combine_rows_per_src=[9386, 9402, 9260, 9050, 9250, 9316, 8834, 9646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64980 dup_rows=24058 dup_rows_remote=21058 dedup_ratio=0.244731 router_tokens_per_expert=[766, 810, 804, 728, 728, 751, 760, 803, 758, 768, 734, 752, 774, 795, 772, 808, 776, 747, 778, 737, 704, 782, 764, 808, 810, 785, 744, 759, 743, 774, 740, 778, 767, 735, 769, 753, 768, 785, 762, 749, 783, 798, 775, 801, 741, 791, 749, 781, 768, 758, 737, 797, 769, 761, 794, 825, 770, 767, 752, 768, 783, 776, 707, 773] payload_rows_per_dst=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] dup_rows_per_dst=[3016, 3070, 2990, 3000, 2902, 3096, 3092, 2892] combine_rows_per_src=[9284, 9252, 9202, 9266, 9274, 9342, 9326, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344000 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608576 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873152 weight_movement_bytes=34603008 total_nvlink_bytes=567476160 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64974 dup_rows=24134 dup_rows_remote=21224 dedup_ratio=0.245504 router_tokens_per_expert=[750, 737, 735, 752, 744, 741, 755, 784, 761, 722, 811, 772, 802, 766, 772, 778, 768, 740, 707, 765, 753, 758, 774, 788, 768, 770, 790, 731, 778, 758, 776, 803, 782, 740, 802, 750, 756, 760, 768, 779, 753, 753, 799, 773, 737, 799, 737, 751, 776, 773, 837, 818, 802, 770, 815, 751, 817, 758, 802, 804, 763, 743, 722, 753] payload_rows_per_dst=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] dup_rows_per_dst=[2870, 3070, 2910, 3048, 2990, 3024, 3178, 3044] combine_rows_per_src=[9126, 9298, 9196, 9300, 9284, 9180, 9506, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64828 dup_rows=24052 dup_rows_remote=20946 dedup_ratio=0.244670 router_tokens_per_expert=[756, 754, 757, 755, 766, 737, 767, 743, 752, 751, 729, 788, 800, 774, 751, 737, 800, 735, 685, 775, 793, 802, 755, 763, 741, 801, 744, 792, 778, 781, 758, 802, 831, 748, 788, 799, 773, 778, 761, 787, 795, 751, 769, 772, 763, 763, 804, 803, 712, 801, 774, 800, 776, 778, 754, 725, 775, 743, 752, 753, 738, 778, 818, 768] payload_rows_per_dst=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] dup_rows_per_dst=[2906, 2952, 2998, 3036, 3106, 3034, 3018, 3002] combine_rows_per_src=[9164, 9212, 9218, 9358, 9424, 9406, 9222, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=342688 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266754720 ingress_bytes=335618048 link_bytes=335618048 token_bytes=533166752 weight_movement_bytes=69206016 total_nvlink_bytes=602372768 weight_to_token_ratio=0.129802 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262903695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65006 dup_rows=23960 dup_rows_remote=20994 dedup_ratio=0.243734 router_tokens_per_expert=[783, 756, 819, 785, 774, 763, 760, 768, 816, 759, 781, 721, 775, 786, 765, 749, 750, 792, 766, 772, 742, 708, 760, 785, 768, 743, 793, 779, 744, 749, 772, 776, 766, 739, 803, 766, 807, 768, 769, 756, 769, 758, 735, 769, 782, 728, 781, 783, 748, 769, 814, 750, 730, 792, 831, 741, 739, 800, 758, 777, 777, 769, 766, 753] payload_rows_per_dst=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] dup_rows_per_dst=[3042, 2966, 2936, 2942, 3008, 2974, 3084, 3008] combine_rows_per_src=[9374, 9338, 9214, 9306, 9340, 9236, 9266, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344176 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338416 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332656 weight_movement_bytes=34603008 total_nvlink_bytes=566935664 weight_to_token_ratio=0.065003 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343096 combine_payload_bytes=265535488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265878584 ingress_bytes=403947520 link_bytes=403947520 token_bytes=531414072 weight_movement_bytes=138412032 total_nvlink_bytes=669826104 weight_to_token_ratio=0.260460 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520021997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344792 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478296 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611800 weight_movement_bytes=34603008 total_nvlink_bytes=567214808 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64804 dup_rows=24170 dup_rows_remote=21138 dedup_ratio=0.245870 router_tokens_per_expert=[792, 823, 765, 769, 749, 755, 744, 783, 713, 807, 805, 764, 762, 745, 765, 784, 790, 802, 791, 725, 773, 720, 730, 818, 784, 814, 757, 778, 824, 759, 780, 781, 791, 757, 763, 783, 814, 755, 758, 758, 763, 765, 772, 741, 797, 727, 782, 794, 787, 763, 748, 773, 743, 765, 752, 771, 779, 764, 768, 752, 729, 729, 724, 734] payload_rows_per_dst=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] dup_rows_per_dst=[3032, 3044, 3228, 3274, 2832, 3038, 2896, 2826] combine_rows_per_src=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264183808 dispatch_meta_bytes=341152 combine_payload_bytes=264183808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264524960 ingress_bytes=298786816 link_bytes=298786816 token_bytes=528708768 weight_movement_bytes=34603008 total_nvlink_bytes=563311776 weight_to_token_ratio=0.065448 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124310734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74688 payload_rows_remote=65360 dup_rows=23616 dup_rows_remote=20714 dedup_ratio=0.240234 router_tokens_per_expert=[798, 781, 759, 791, 782, 778, 783, 783, 738, 791, 758, 758, 778, 739, 725, 776, 798, 775, 801, 792, 762, 795, 758, 821, 778, 814, 754, 753, 755, 752, 737, 772, 797, 768, 742, 714, 780, 772, 700, 727, 739, 805, 854, 801, 746, 819, 769, 713, 789, 758, 742, 751, 742, 760, 771, 781, 743, 751, 742, 761, 766, 783, 775, 756] payload_rows_per_dst=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] dup_rows_per_dst=[3088, 2880, 3040, 2902, 2842, 3046, 2894, 2924] combine_rows_per_src=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344152 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502232 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660312 weight_movement_bytes=69206016 total_nvlink_bytes=601866328 weight_to_token_ratio=0.129925 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64736 dup_rows=24178 dup_rows_remote=21102 dedup_ratio=0.245951 router_tokens_per_expert=[800, 783, 801, 746, 743, 776, 801, 771, 769, 783, 777, 766, 693, 757, 763, 790, 725, 781, 717, 754, 765, 754, 748, 805, 756, 824, 782, 752, 747, 803, 813, 763, 768, 797, 772, 783, 747, 757, 775, 825, 780, 735, 729, 764, 723, 752, 762, 763, 782, 782, 794, 739, 780, 814, 791, 751, 761, 785, 796, 765, 750, 799, 690, 733] payload_rows_per_dst=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] dup_rows_per_dst=[3074, 2982, 2942, 3112, 3136, 2896, 3076, 2960] combine_rows_per_src=[9368, 9214, 9156, 9368, 9312, 9120, 9390, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343768 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780952 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531218136 weight_movement_bytes=34603008 total_nvlink_bytes=565821144 weight_to_token_ratio=0.065139 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343096 combine_payload_bytes=265535488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265878584 ingress_bytes=403947520 link_bytes=403947520 token_bytes=531414072 weight_movement_bytes=138412032 total_nvlink_bytes=669826104 weight_to_token_ratio=0.260460 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520021997e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344792 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478296 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611800 weight_movement_bytes=34603008 total_nvlink_bytes=567214808 weight_to_token_ratio=0.064969 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344000 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608576 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873152 weight_movement_bytes=34603008 total_nvlink_bytes=567476160 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267714560 dispatch_meta_bytes=344296 combine_payload_bytes=267714560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268058856 ingress_bytes=336920576 link_bytes=336920576 token_bytes=535773416 weight_movement_bytes=69206016 total_nvlink_bytes=604979432 weight_to_token_ratio=0.129170 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267805003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64902 dup_rows=23984 dup_rows_remote=21158 dedup_ratio=0.243978 router_tokens_per_expert=[778, 758, 749, 710, 770, 756, 754, 803, 758, 757, 728, 779, 797, 758, 804, 741, 745, 735, 790, 798, 736, 780, 814, 809, 786, 771, 743, 779, 750, 772, 806, 732, 746, 763, 785, 771, 757, 754, 787, 727, 773, 775, 792, 782, 801, 771, 751, 805, 812, 772, 785, 760, 769, 811, 813, 821, 735, 713, 777, 752, 722, 730, 717, 777] payload_rows_per_dst=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] dup_rows_per_dst=[2908, 2826, 3096, 3062, 2948, 3162, 3210, 2772] combine_rows_per_src=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64804 dup_rows=24170 dup_rows_remote=21138 dedup_ratio=0.245870 router_tokens_per_expert=[792, 823, 765, 769, 749, 755, 744, 783, 713, 807, 805, 764, 762, 745, 765, 784, 790, 802, 791, 725, 773, 720, 730, 818, 784, 814, 757, 778, 824, 759, 780, 781, 791, 757, 763, 783, 814, 755, 758, 758, 763, 765, 772, 741, 797, 727, 782, 794, 787, 763, 748, 773, 743, 765, 752, 771, 779, 764, 768, 752, 729, 729, 724, 734] payload_rows_per_dst=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] dup_rows_per_dst=[3032, 3044, 3228, 3274, 2832, 3038, 2896, 2826] combine_rows_per_src=[9330, 9258, 9604, 9352, 8978, 9244, 9236, 9132] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343352 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265502008 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660664 weight_movement_bytes=103809024 total_nvlink_bytes=634469688 weight_to_token_ratio=0.195622 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65004 dup_rows=24016 dup_rows_remote=20986 dedup_ratio=0.244303 router_tokens_per_expert=[725, 727, 809, 809, 735, 763, 743, 762, 709, 734, 740, 754, 743, 813, 758, 751, 731, 773, 786, 791, 790, 819, 826, 756, 765, 793, 729, 791, 741, 797, 816, 755, 766, 739, 785, 804, 778, 752, 790, 743, 758, 772, 757, 788, 753, 765, 781, 731, 797, 761, 735, 814, 774, 745, 769, 787, 786, 777, 808, 753, 763, 744, 768, 775] payload_rows_per_dst=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] dup_rows_per_dst=[2900, 2832, 3160, 3034, 3030, 2906, 3040, 3114] combine_rows_per_src=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=64958 dup_rows=23878 dup_rows_remote=20776 dedup_ratio=0.242900 router_tokens_per_expert=[767, 766, 753, 763, 788, 785, 735, 815, 762, 762, 784, 803, 772, 731, 745, 788, 747, 743, 768, 761, 788, 764, 760, 747, 794, 761, 769, 770, 786, 778, 826, 768, 769, 726, 782, 808, 778, 778, 795, 762, 780, 755, 798, 815, 760, 759, 759, 736, 777, 732, 749, 762, 732, 747, 711, 768, 730, 790, 751, 755, 781, 798, 782, 778] payload_rows_per_dst=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] dup_rows_per_dst=[3034, 3020, 2960, 3054, 2988, 2884, 2836, 3102] combine_rows_per_src=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65068 dup_rows=23942 dup_rows_remote=20906 dedup_ratio=0.243551 router_tokens_per_expert=[747, 742, 755, 779, 775, 792, 782, 794, 746, 793, 789, 746, 781, 722, 746, 770, 767, 806, 778, 777, 726, 768, 748, 749, 786, 798, 744, 794, 720, 780, 798, 807, 773, 782, 777, 757, 755, 744, 755, 790, 793, 829, 722, 755, 781, 795, 738, 752, 751, 743, 772, 788, 769, 765, 739, 780, 778, 756, 773, 737, 795, 733, 825, 745] payload_rows_per_dst=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] dup_rows_per_dst=[3026, 2902, 2954, 3068, 2980, 3036, 2944, 3032] combine_rows_per_src=[9306, 9284, 9284, 9386, 9286, 9294, 9270, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64966 dup_rows=23994 dup_rows_remote=20992 dedup_ratio=0.244080 router_tokens_per_expert=[759, 784, 773, 764, 779, 732, 733, 777, 723, 802, 755, 780, 769, 813, 771, 735, 781, 795, 809, 764, 723, 748, 770, 783, 766, 747, 779, 775, 739, 750, 782, 774, 794, 775, 797, 785, 732, 774, 736, 742, 737, 754, 754, 795, 755, 764, 769, 785, 768, 731, 785, 785, 757, 776, 787, 740, 778, 792, 752, 788, 821, 769, 776, 765] payload_rows_per_dst=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] dup_rows_per_dst=[3004, 2956, 3002, 3020, 2954, 2950, 2954, 3154] combine_rows_per_src=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=64958 dup_rows=23878 dup_rows_remote=20776 dedup_ratio=0.242900 router_tokens_per_expert=[767, 766, 753, 763, 788, 785, 735, 815, 762, 762, 784, 803, 772, 731, 745, 788, 747, 743, 768, 761, 788, 764, 760, 747, 794, 761, 769, 770, 786, 778, 826, 768, 769, 726, 782, 808, 778, 778, 795, 762, 780, 755, 798, 815, 760, 759, 759, 736, 777, 732, 749, 762, 732, 747, 711, 768, 730, 790, 751, 755, 781, 798, 782, 778] payload_rows_per_dst=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] dup_rows_per_dst=[3034, 3020, 2960, 3054, 2988, 2884, 2836, 3102] combine_rows_per_src=[9310, 9274, 9196, 9448, 9408, 9200, 9122, 9468] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74688 payload_rows_remote=65360 dup_rows=23616 dup_rows_remote=20714 dedup_ratio=0.240234 router_tokens_per_expert=[798, 781, 759, 791, 782, 778, 783, 783, 738, 791, 758, 758, 778, 739, 725, 776, 798, 775, 801, 792, 762, 795, 758, 821, 778, 814, 754, 753, 755, 752, 737, 772, 797, 768, 742, 714, 780, 772, 700, 727, 739, 805, 854, 801, 746, 819, 769, 713, 789, 758, 742, 751, 742, 760, 771, 781, 743, 751, 742, 761, 766, 783, 775, 756] payload_rows_per_dst=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] dup_rows_per_dst=[3088, 2880, 3040, 2902, 2842, 3046, 2894, 2924] combine_rows_per_src=[9422, 9246, 9564, 9328, 9158, 9446, 9294, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64966 dup_rows=23994 dup_rows_remote=20992 dedup_ratio=0.244080 router_tokens_per_expert=[759, 784, 773, 764, 779, 732, 733, 777, 723, 802, 755, 780, 769, 813, 771, 735, 781, 795, 809, 764, 723, 748, 770, 783, 766, 747, 779, 775, 739, 750, 782, 774, 794, 775, 797, 785, 732, 774, 736, 742, 737, 754, 754, 795, 755, 764, 769, 785, 768, 731, 785, 785, 757, 776, 787, 740, 778, 792, 752, 788, 821, 769, 776, 765] payload_rows_per_dst=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] dup_rows_per_dst=[3004, 2956, 3002, 3020, 2954, 2950, 2954, 3154] combine_rows_per_src=[9198, 9340, 9344, 9204, 9316, 9276, 9304, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65004 dup_rows=24016 dup_rows_remote=20986 dedup_ratio=0.244303 router_tokens_per_expert=[725, 727, 809, 809, 735, 763, 743, 762, 709, 734, 740, 754, 743, 813, 758, 751, 731, 773, 786, 791, 790, 819, 826, 756, 765, 793, 729, 791, 741, 797, 816, 755, 766, 739, 785, 804, 778, 752, 790, 743, 758, 772, 757, 788, 753, 765, 781, 731, 797, 761, 735, 814, 774, 745, 769, 787, 786, 777, 808, 753, 763, 744, 768, 775] payload_rows_per_dst=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] dup_rows_per_dst=[2900, 2832, 3160, 3034, 3030, 2906, 3040, 3114] combine_rows_per_src=[9246, 9172, 9384, 9340, 9284, 9304, 9324, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343768 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780952 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531218136 weight_movement_bytes=34603008 total_nvlink_bytes=565821144 weight_to_token_ratio=0.065139 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64902 dup_rows=23984 dup_rows_remote=21158 dedup_ratio=0.243978 router_tokens_per_expert=[778, 758, 749, 710, 770, 756, 754, 803, 758, 757, 728, 779, 797, 758, 804, 741, 745, 735, 790, 798, 736, 780, 814, 809, 786, 771, 743, 779, 750, 772, 806, 732, 746, 763, 785, 771, 757, 754, 787, 727, 773, 775, 792, 782, 801, 771, 751, 805, 812, 772, 785, 760, 769, 811, 813, 821, 735, 713, 777, 752, 722, 730, 717, 777] payload_rows_per_dst=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] dup_rows_per_dst=[2908, 2826, 3096, 3062, 2948, 3162, 3210, 2772] combine_rows_per_src=[9248, 9418, 9318, 9216, 9232, 9338, 9476, 9074] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344240 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182832 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532021424 weight_movement_bytes=34603008 total_nvlink_bytes=566624432 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342936 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266410904 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532478872 weight_movement_bytes=34603008 total_nvlink_bytes=567081880 weight_to_token_ratio=0.064985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343896 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266862424 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533380952 weight_movement_bytes=69206016 total_nvlink_bytes=602586968 weight_to_token_ratio=0.129750 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343960 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266600344 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532856728 weight_movement_bytes=34603008 total_nvlink_bytes=567459736 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343832 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444568 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545304 weight_movement_bytes=69206016 total_nvlink_bytes=601751320 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65186 dup_rows=23818 dup_rows_remote=20750 dedup_ratio=0.242289 router_tokens_per_expert=[781, 755, 747, 797, 791, 774, 743, 734, 733, 810, 770, 712, 818, 762, 735, 749, 717, 748, 748, 774, 801, 781, 759, 776, 785, 771, 762, 790, 748, 775, 786, 756, 743, 817, 752, 735, 789, 773, 794, 760, 790, 780, 797, 767, 782, 755, 761, 764, 768, 780, 746, 759, 781, 822, 755, 744, 775, 778, 822, 747, 732, 729, 792, 775] payload_rows_per_dst=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] dup_rows_per_dst=[3068, 3356, 2672, 2840, 2952, 3058, 2922, 2950] combine_rows_per_src=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64988 dup_rows=24154 dup_rows_remote=21188 dedup_ratio=0.245707 router_tokens_per_expert=[766, 801, 788, 734, 781, 801, 796, 756, 762, 736, 753, 749, 732, 797, 741, 794, 751, 771, 767, 787, 790, 774, 757, 796, 757, 731, 763, 732, 775, 762, 752, 792, 795, 730, 757, 777, 811, 745, 752, 793, 786, 785, 736, 785, 728, 809, 788, 815, 716, 809, 747, 761, 772, 733, 751, 790, 730, 734, 757, 782, 795, 805, 763, 771] payload_rows_per_dst=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] dup_rows_per_dst=[3124, 2950, 3096, 2966, 3062, 3092, 2908, 2956] combine_rows_per_src=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=342936 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266410904 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532478872 weight_movement_bytes=34603008 total_nvlink_bytes=567081880 weight_to_token_ratio=0.064985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64914 dup_rows=24004 dup_rows_remote=21012 dedup_ratio=0.244181 router_tokens_per_expert=[763, 741, 714, 772, 784, 797, 760, 792, 768, 772, 726, 744, 747, 745, 787, 756, 760, 745, 742, 758, 791, 807, 782, 777, 717, 752, 798, 775, 761, 790, 757, 776, 817, 795, 753, 823, 793, 755, 787, 815, 776, 716, 780, 749, 721, 793, 728, 746, 773, 779, 762, 785, 741, 785, 769, 795, 787, 764, 819, 783, 776, 734, 745, 752] payload_rows_per_dst=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] dup_rows_per_dst=[2906, 2900, 3038, 3004, 3248, 2954, 2992, 2962] combine_rows_per_src=[9340, 9190, 9286, 9248, 9428, 9064, 9386, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267714560 dispatch_meta_bytes=344296 combine_payload_bytes=267714560 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268058856 ingress_bytes=336920576 link_bytes=336920576 token_bytes=535773416 weight_movement_bytes=69206016 total_nvlink_bytes=604979432 weight_to_token_ratio=0.129170 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267805003e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343960 combine_payload_bytes=266256384 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266600344 ingress_bytes=300859392 link_bytes=300859392 token_bytes=532856728 weight_movement_bytes=34603008 total_nvlink_bytes=567459736 weight_to_token_ratio=0.064939 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132109671e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344240 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266182832 ingress_bytes=300441600 link_bytes=300441600 token_bytes=532021424 weight_movement_bytes=34603008 total_nvlink_bytes=566624432 weight_to_token_ratio=0.065041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130537553e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343744 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345600 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347456 weight_movement_bytes=34603008 total_nvlink_bytes=568950464 weight_to_token_ratio=0.064758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344704 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266535552 ingress_bytes=335396864 link_bytes=335396864 token_bytes=532726400 weight_movement_bytes=69206016 total_nvlink_bytes=601932416 weight_to_token_ratio=0.129909 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262071398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343704 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266231448 ingress_bytes=335093760 link_bytes=335093760 token_bytes=532119192 weight_movement_bytes=69206016 total_nvlink_bytes=601325208 weight_to_token_ratio=0.130057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260930842e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64934 dup_rows=23974 dup_rows_remote=20856 dedup_ratio=0.243876 router_tokens_per_expert=[761, 751, 729, 745, 780, 741, 751, 815, 825, 758, 778, 802, 765, 787, 756, 786, 781, 775, 738, 781, 769, 781, 787, 725, 797, 762, 790, 773, 751, 685, 745, 774, 789, 746, 736, 725, 755, 759, 719, 805, 774, 777, 772, 768, 756, 722, 794, 761, 803, 800, 773, 775, 761, 765, 779, 811, 773, 796, 745, 817, 753, 767, 712, 820] payload_rows_per_dst=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] dup_rows_per_dst=[2964, 3118, 3004, 2966, 2854, 2942, 3054, 3072] combine_rows_per_src=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65186 dup_rows=23818 dup_rows_remote=20750 dedup_ratio=0.242289 router_tokens_per_expert=[781, 755, 747, 797, 791, 774, 743, 734, 733, 810, 770, 712, 818, 762, 735, 749, 717, 748, 748, 774, 801, 781, 759, 776, 785, 771, 762, 790, 748, 775, 786, 756, 743, 817, 752, 735, 789, 773, 794, 760, 790, 780, 797, 767, 782, 755, 761, 764, 768, 780, 746, 759, 781, 822, 755, 744, 775, 778, 822, 747, 732, 729, 792, 775] payload_rows_per_dst=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] dup_rows_per_dst=[3068, 3356, 2672, 2840, 2952, 3058, 2922, 2950] combine_rows_per_src=[9300, 9708, 9006, 9150, 9250, 9386, 9336, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64578 dup_rows=24058 dup_rows_remote=20864 dedup_ratio=0.244731 router_tokens_per_expert=[761, 768, 786, 794, 792, 782, 778, 805, 762, 785, 716, 774, 764, 748, 761, 739, 756, 751, 798, 818, 818, 752, 767, 759, 766, 768, 790, 732, 792, 733, 818, 784, 816, 723, 717, 775, 772, 759, 786, 778, 769, 730, 780, 750, 723, 786, 779, 765, 740, 783, 756, 751, 770, 741, 769, 801, 802, 758, 788, 775, 762, 738, 754, 739] payload_rows_per_dst=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] dup_rows_per_dst=[2800, 2972, 2966, 3058, 3066, 3058, 2944, 3194] combine_rows_per_src=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65024 dup_rows=24056 dup_rows_remote=21194 dedup_ratio=0.244710 router_tokens_per_expert=[789, 734, 761, 757, 698, 748, 790, 784, 702, 798, 771, 755, 739, 784, 758, 749, 820, 784, 778, 790, 778, 779, 781, 798, 754, 794, 761, 790, 768, 769, 758, 795, 791, 778, 765, 704, 779, 777, 747, 753, 755, 780, 757, 774, 713, 747, 780, 737, 792, 789, 766, 734, 755, 797, 775, 795, 817, 743, 804, 755, 762, 779, 761, 777] payload_rows_per_dst=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] dup_rows_per_dst=[2984, 2820, 3236, 3050, 2984, 2862, 3044, 3076] combine_rows_per_src=[9138, 9292, 9380, 9328, 9204, 9224, 9362, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21076 dedup_ratio=0.244548 router_tokens_per_expert=[788, 737, 760, 773, 784, 784, 715, 782, 745, 816, 734, 743, 757, 772, 762, 740, 793, 764, 722, 731, 751, 781, 775, 776, 791, 785, 772, 808, 786, 771, 789, 835, 784, 765, 757, 774, 778, 764, 761, 799, 770, 780, 770, 749, 803, 723, 762, 781, 761, 775, 730, 776, 776, 777, 749, 774, 779, 738, 765, 803, 754, 788, 755, 710] payload_rows_per_dst=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] dup_rows_per_dst=[3050, 2976, 3038, 3100, 2964, 2970, 2988, 2954] combine_rows_per_src=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65000 dup_rows=24024 dup_rows_remote=20932 dedup_ratio=0.244385 router_tokens_per_expert=[744, 791, 762, 797, 773, 816, 835, 752, 806, 745, 793, 792, 781, 754, 759, 753, 781, 780, 718, 777, 807, 795, 741, 787, 774, 727, 769, 800, 762, 750, 790, 740, 748, 778, 803, 800, 717, 775, 795, 772, 728, 775, 761, 779, 768, 773, 775, 778, 776, 795, 751, 732, 787, 807, 763, 708, 727, 740, 763, 759, 786, 740, 733, 709] payload_rows_per_dst=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] dup_rows_per_dst=[3132, 2970, 3092, 3010, 3004, 3066, 2930, 2820] combine_rows_per_src=[9408, 9396, 9280, 9214, 9372, 9208, 9308, 9094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64988 dup_rows=24154 dup_rows_remote=21188 dedup_ratio=0.245707 router_tokens_per_expert=[766, 801, 788, 734, 781, 801, 796, 756, 762, 736, 753, 749, 732, 797, 741, 794, 751, 771, 767, 787, 790, 774, 757, 796, 757, 731, 763, 732, 775, 762, 752, 792, 795, 730, 757, 777, 811, 745, 752, 793, 786, 785, 736, 785, 728, 809, 788, 815, 716, 809, 747, 761, 772, 733, 751, 790, 730, 734, 757, 782, 795, 805, 763, 771] payload_rows_per_dst=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] dup_rows_per_dst=[3124, 2950, 3096, 2966, 3062, 3092, 2908, 2956] combine_rows_per_src=[9322, 9178, 9290, 9162, 9258, 9372, 9250, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64578 dup_rows=24058 dup_rows_remote=20864 dedup_ratio=0.244731 router_tokens_per_expert=[761, 768, 786, 794, 792, 782, 778, 805, 762, 785, 716, 774, 764, 748, 761, 739, 756, 751, 798, 818, 818, 752, 767, 759, 766, 768, 790, 732, 792, 733, 818, 784, 816, 723, 717, 775, 772, 759, 786, 778, 769, 730, 780, 750, 723, 786, 779, 765, 740, 783, 756, 751, 770, 741, 769, 801, 802, 758, 788, 775, 762, 738, 754, 739] payload_rows_per_dst=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] dup_rows_per_dst=[2800, 2972, 2966, 3058, 3066, 3058, 2944, 3194] combine_rows_per_src=[9102, 9126, 9250, 9308, 9408, 9106, 9278, 9668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21076 dedup_ratio=0.244548 router_tokens_per_expert=[788, 737, 760, 773, 784, 784, 715, 782, 745, 816, 734, 743, 757, 772, 762, 740, 793, 764, 722, 731, 751, 781, 775, 776, 791, 785, 772, 808, 786, 771, 789, 835, 784, 765, 757, 774, 778, 764, 761, 799, 770, 780, 770, 749, 803, 723, 762, 781, 761, 775, 730, 776, 776, 777, 749, 774, 779, 738, 765, 803, 754, 788, 755, 710] payload_rows_per_dst=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] dup_rows_per_dst=[3050, 2976, 3038, 3100, 2964, 2970, 2988, 2954] combine_rows_per_src=[9196, 9162, 9148, 9574, 9400, 9306, 9248, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=343744 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267345600 ingress_bytes=301604864 link_bytes=301604864 token_bytes=534347456 weight_movement_bytes=34603008 total_nvlink_bytes=568950464 weight_to_token_ratio=0.064758 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134914822e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343160 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312824 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282488 weight_movement_bytes=34603008 total_nvlink_bytes=566885496 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64824 dup_rows=24212 dup_rows_remote=21116 dedup_ratio=0.246297 router_tokens_per_expert=[764, 746, 783, 762, 776, 771, 770, 765, 784, 741, 757, 748, 772, 748, 795, 777, 763, 812, 791, 771, 728, 728, 741, 810, 772, 765, 744, 799, 744, 748, 816, 794, 792, 781, 769, 819, 751, 759, 763, 749, 767, 807, 754, 755, 779, 768, 825, 753, 745, 752, 764, 790, 822, 751, 752, 764, 713, 766, 753, 764, 718, 760, 806, 756] payload_rows_per_dst=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] dup_rows_per_dst=[3018, 2992, 3100, 3096, 2956, 3172, 3012, 2866] combine_rows_per_src=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264511488 dispatch_meta_bytes=341768 combine_payload_bytes=264511488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264853256 ingress_bytes=299114496 link_bytes=299114496 token_bytes=529364744 weight_movement_bytes=34603008 total_nvlink_bytes=563967752 weight_to_token_ratio=0.065367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125543768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344872 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266683176 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533021480 weight_movement_bytes=69206016 total_nvlink_bytes=602227496 weight_to_token_ratio=0.129837 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64996 dup_rows=23894 dup_rows_remote=20786 dedup_ratio=0.243062 router_tokens_per_expert=[759, 783, 742, 731, 769, 763, 780, 796, 814, 776, 786, 795, 758, 792, 736, 763, 788, 740, 762, 752, 773, 770, 764, 711, 781, 773, 736, 772, 777, 765, 760, 803, 757, 755, 806, 753, 769, 792, 776, 740, 769, 789, 781, 742, 772, 743, 787, 768, 743, 775, 751, 807, 780, 823, 788, 770, 768, 797, 756, 708, 751, 711, 772, 783] payload_rows_per_dst=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] dup_rows_per_dst=[3108, 3134, 2868, 2996, 2962, 2960, 2982, 2884] combine_rows_per_src=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64934 dup_rows=23974 dup_rows_remote=20856 dedup_ratio=0.243876 router_tokens_per_expert=[761, 751, 729, 745, 780, 741, 751, 815, 825, 758, 778, 802, 765, 787, 756, 786, 781, 775, 738, 781, 769, 781, 787, 725, 797, 762, 790, 773, 751, 685, 745, 774, 789, 746, 736, 725, 755, 759, 719, 805, 774, 777, 772, 768, 756, 722, 794, 761, 803, 800, 773, 775, 761, 765, 779, 811, 773, 796, 745, 817, 753, 767, 712, 820] payload_rows_per_dst=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] dup_rows_per_dst=[2964, 3118, 3004, 2966, 2854, 2942, 3054, 3072] combine_rows_per_src=[9182, 9396, 9270, 9188, 9214, 9306, 9480, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343728 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583728 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823728 weight_movement_bytes=69206016 total_nvlink_bytes=602029744 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266190848 dispatch_meta_bytes=344704 combine_payload_bytes=266190848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266535552 ingress_bytes=335396864 link_bytes=335396864 token_bytes=532726400 weight_movement_bytes=69206016 total_nvlink_bytes=601932416 weight_to_token_ratio=0.129909 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262071398e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266026704 ingress_bytes=300285952 link_bytes=300285952 token_bytes=531709648 weight_movement_bytes=34603008 total_nvlink_bytes=566312656 weight_to_token_ratio=0.065079 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129951862e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65030 dup_rows=24130 dup_rows_remote=21126 dedup_ratio=0.245463 router_tokens_per_expert=[778, 771, 778, 790, 751, 798, 768, 756, 800, 732, 747, 760, 736, 817, 791, 796, 788, 776, 739, 760, 735, 759, 765, 707, 763, 758, 787, 756, 782, 742, 754, 743, 760, 767, 766, 743, 817, 755, 731, 770, 798, 792, 793, 801, 731, 849, 815, 799, 765, 790, 771, 738, 769, 752, 713, 776, 751, 773, 766, 746, 772, 782, 746, 772] payload_rows_per_dst=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] dup_rows_per_dst=[3000, 3086, 2976, 2962, 2892, 3244, 3004, 2966] combine_rows_per_src=[9380, 9272, 9082, 9208, 9326, 9512, 9144, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264511488 dispatch_meta_bytes=341768 combine_payload_bytes=264511488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264853256 ingress_bytes=299114496 link_bytes=299114496 token_bytes=529364744 weight_movement_bytes=34603008 total_nvlink_bytes=563967752 weight_to_token_ratio=0.065367 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125543768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343760 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265862864 ingress_bytes=334725120 link_bytes=334725120 token_bytes=531381968 weight_movement_bytes=69206016 total_nvlink_bytes=600587984 weight_to_token_ratio=0.130238 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259543679e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343128 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266566744 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790360 weight_movement_bytes=69206016 total_nvlink_bytes=601996376 weight_to_token_ratio=0.129894 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343160 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312824 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282488 weight_movement_bytes=34603008 total_nvlink_bytes=566885496 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344624 combine_payload_bytes=266362880 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266707504 ingress_bytes=370171904 link_bytes=370171904 token_bytes=533070384 weight_movement_bytes=103809024 total_nvlink_bytes=636879408 weight_to_token_ratio=0.194738 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392927073e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65172 dup_rows=23990 dup_rows_remote=21026 dedup_ratio=0.244039 router_tokens_per_expert=[795, 739, 793, 752, 784, 791, 781, 768, 730, 731, 774, 752, 769, 777, 732, 788, 786, 763, 773, 778, 782, 763, 772, 725, 824, 779, 745, 800, 759, 743, 755, 812, 786, 752, 751, 769, 768, 761, 735, 761, 818, 795, 756, 730, 744, 726, 729, 822, 824, 800, 772, 795, 758, 761, 752, 750, 782, 757, 809, 761, 747, 749, 752, 765] payload_rows_per_dst=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] dup_rows_per_dst=[2972, 2964, 2994, 3168, 2940, 2952, 2950, 3050] combine_rows_per_src=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64996 dup_rows=23894 dup_rows_remote=20786 dedup_ratio=0.243062 router_tokens_per_expert=[759, 783, 742, 731, 769, 763, 780, 796, 814, 776, 786, 795, 758, 792, 736, 763, 788, 740, 762, 752, 773, 770, 764, 711, 781, 773, 736, 772, 777, 765, 760, 803, 757, 755, 806, 753, 769, 792, 776, 740, 769, 789, 781, 742, 772, 743, 787, 768, 743, 775, 751, 807, 780, 823, 788, 770, 768, 797, 756, 708, 751, 711, 772, 783] payload_rows_per_dst=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] dup_rows_per_dst=[3108, 3134, 2868, 2996, 2962, 2960, 2982, 2884] combine_rows_per_src=[9414, 9456, 9328, 9214, 9334, 9076, 9380, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64756 dup_rows=24134 dup_rows_remote=21026 dedup_ratio=0.245504 router_tokens_per_expert=[772, 730, 772, 792, 761, 773, 777, 760, 755, 750, 759, 748, 803, 782, 733, 787, 765, 806, 754, 786, 827, 717, 774, 758, 773, 729, 759, 767, 743, 747, 770, 827, 725, 771, 775, 796, 762, 798, 819, 806, 765, 731, 749, 798, 800, 760, 721, 756, 740, 746, 770, 746, 767, 746, 779, 765, 770, 808, 773, 797, 755, 777, 749, 776] payload_rows_per_dst=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] dup_rows_per_dst=[3014, 2982, 2926, 3024, 2800, 3220, 3060, 3108] combine_rows_per_src=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65032 dup_rows=24030 dup_rows_remote=21084 dedup_ratio=0.244446 router_tokens_per_expert=[776, 763, 772, 794, 754, 719, 786, 760, 799, 771, 751, 783, 765, 737, 762, 742, 768, 733, 731, 780, 754, 814, 785, 789, 773, 719, 731, 792, 758, 761, 739, 774, 808, 769, 779, 750, 817, 815, 783, 750, 763, 783, 787, 764, 732, 746, 746, 773, 772, 738, 813, 754, 764, 797, 781, 766, 776, 750, 794, 734, 797, 799, 756, 761] payload_rows_per_dst=[9216, 9282, 9262, 9188, 9392, 9242, 9386, 9306] dup_rows_per_dst=[3032, 2938, 3046, 2906, 3150, 2946, 2984, 3028] combine_rows_per_src=[9216, 9282, 9262, 9188, 9392, 9242, 9386, 9306] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64984 dup_rows=24154 dup_rows_remote=21056 dedup_ratio=0.245707 router_tokens_per_expert=[718, 751, 808, 787, 795, 747, 738, 709, 750, 757, 793, 774, 728, 807, 746, 757, 753, 782, 756, 762, 776, 733, 798, 772, 755, 816, 778, 815, 765, 764, 778, 713, 774, 760, 822, 779, 800, 802, 787, 774, 757, 745, 802, 756, 733, 750, 734, 782, 764, 760, 777, 789, 797, 756, 744, 740, 765, 740, 762, 781, 783, 798, 795, 763] payload_rows_per_dst=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] dup_rows_per_dst=[2932, 2992, 3098, 3062, 3132, 2994, 2902, 3042] combine_rows_per_src=[9174, 9232, 9166, 9306, 9464, 9124, 9352, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64824 dup_rows=24212 dup_rows_remote=21116 dedup_ratio=0.246297 router_tokens_per_expert=[764, 746, 783, 762, 776, 771, 770, 765, 784, 741, 757, 748, 772, 748, 795, 777, 763, 812, 791, 771, 728, 728, 741, 810, 772, 765, 744, 799, 744, 748, 816, 794, 792, 781, 769, 819, 751, 759, 763, 749, 767, 807, 754, 755, 779, 768, 825, 753, 745, 752, 764, 790, 822, 751, 752, 764, 713, 766, 753, 764, 718, 760, 806, 756] payload_rows_per_dst=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] dup_rows_per_dst=[3018, 2992, 3100, 3096, 2956, 3172, 3012, 2866] combine_rows_per_src=[9256, 9252, 9188, 9268, 9410, 9244, 9268, 9206] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64764 dup_rows=24064 dup_rows_remote=20906 dedup_ratio=0.244792 router_tokens_per_expert=[750, 740, 730, 775, 747, 746, 775, 765, 749, 777, 760, 787, 788, 796, 777, 737, 764, 756, 743, 750, 766, 770, 794, 768, 735, 791, 795, 779, 808, 742, 775, 760, 757, 802, 784, 748, 808, 804, 807, 807, 785, 731, 733, 761, 755, 805, 748, 790, 788, 770, 761, 759, 726, 749, 747, 760, 773, 753, 780, 733, 778, 772, 778, 805] payload_rows_per_dst=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] dup_rows_per_dst=[2880, 2966, 3040, 3034, 3158, 2970, 2936, 3080] combine_rows_per_src=[9176, 9376, 9182, 9336, 9476, 9246, 9184, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64756 dup_rows=24134 dup_rows_remote=21026 dedup_ratio=0.245504 router_tokens_per_expert=[772, 730, 772, 792, 761, 773, 777, 760, 755, 750, 759, 748, 803, 782, 733, 787, 765, 806, 754, 786, 827, 717, 774, 758, 773, 729, 759, 767, 743, 747, 770, 827, 725, 771, 775, 796, 762, 798, 819, 806, 765, 731, 749, 798, 800, 760, 721, 756, 740, 746, 770, 746, 767, 746, 779, 765, 770, 808, 773, 797, 755, 777, 749, 776] payload_rows_per_dst=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] dup_rows_per_dst=[3014, 2982, 2926, 3024, 2800, 3220, 3060, 3108] combine_rows_per_src=[9260, 9252, 9050, 9206, 9274, 9370, 9344, 9414] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64886 dup_rows=24164 dup_rows_remote=21120 dedup_ratio=0.245809 router_tokens_per_expert=[723, 761, 759, 746, 770, 788, 727, 804, 764, 754, 789, 795, 717, 796, 766, 781, 780, 803, 779, 747, 767, 750, 756, 728, 788, 760, 756, 772, 741, 770, 762, 800, 737, 771, 780, 776, 780, 821, 803, 761, 827, 727, 786, 794, 773, 759, 795, 707, 809, 780, 769, 748, 781, 782, 769, 739, 750, 773, 772, 788, 701, 734, 796, 765] payload_rows_per_dst=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] dup_rows_per_dst=[2990, 3048, 3028, 3044, 3012, 3038, 3020, 2984] combine_rows_per_src=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344792 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289304 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233816 weight_movement_bytes=34603008 total_nvlink_bytes=568836824 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64678 dup_rows=24152 dup_rows_remote=21124 dedup_ratio=0.245687 router_tokens_per_expert=[808, 761, 778, 794, 724, 791, 751, 762, 752, 802, 715, 772, 828, 788, 789, 748, 786, 727, 781, 789, 791, 780, 809, 752, 715, 751, 777, 803, 778, 757, 758, 768, 777, 756, 787, 757, 780, 828, 767, 758, 756, 785, 719, 734, 756, 773, 787, 767, 789, 790, 755, 779, 752, 787, 727, 752, 719, 731, 795, 785, 792, 733, 745, 749] payload_rows_per_dst=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] dup_rows_per_dst=[3028, 3320, 3098, 3006, 2816, 2974, 2994, 2916] combine_rows_per_src=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343128 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266566744 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532790360 weight_movement_bytes=69206016 total_nvlink_bytes=601996376 weight_to_token_ratio=0.129894 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343128 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265583704 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824280 weight_movement_bytes=34603008 total_nvlink_bytes=565427288 weight_to_token_ratio=0.065187 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344464 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715536 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086608 weight_movement_bytes=34603008 total_nvlink_bytes=567689616 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344160 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266518624 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532693088 weight_movement_bytes=103809024 total_nvlink_bytes=636502112 weight_to_token_ratio=0.194876 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343760 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265862864 ingress_bytes=334725120 link_bytes=334725120 token_bytes=531381968 weight_movement_bytes=69206016 total_nvlink_bytes=600587984 weight_to_token_ratio=0.130238 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259543679e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65172 dup_rows=23990 dup_rows_remote=21026 dedup_ratio=0.244039 router_tokens_per_expert=[795, 739, 793, 752, 784, 791, 781, 768, 730, 731, 774, 752, 769, 777, 732, 788, 786, 763, 773, 778, 782, 763, 772, 725, 824, 779, 745, 800, 759, 743, 755, 812, 786, 752, 751, 769, 768, 761, 735, 761, 818, 795, 756, 730, 744, 726, 729, 822, 824, 800, 772, 795, 758, 761, 752, 750, 782, 757, 809, 761, 747, 749, 752, 765] payload_rows_per_dst=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] dup_rows_per_dst=[2972, 2964, 2994, 3168, 2940, 2952, 2950, 3050] combine_rows_per_src=[9434, 9142, 9290, 9266, 9226, 9288, 9474, 9194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65088 dup_rows=23960 dup_rows_remote=21026 dedup_ratio=0.243734 router_tokens_per_expert=[772, 766, 806, 778, 774, 775, 753, 795, 736, 734, 736, 791, 770, 779, 757, 768, 784, 790, 772, 801, 783, 786, 741, 779, 794, 771, 792, 727, 723, 765, 790, 778, 791, 737, 776, 731, 790, 737, 761, 719, 816, 759, 790, 767, 734, 790, 762, 773, 745, 762, 774, 750, 752, 753, 784, 775, 747, 814, 758, 751, 740, 796, 770, 782] payload_rows_per_dst=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] dup_rows_per_dst=[3026, 2948, 3040, 3052, 2894, 3036, 2934, 3030] combine_rows_per_src=[9412, 9194, 9432, 9228, 9190, 9346, 9256, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=342680 combine_payload_bytes=265273344 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265616024 ingress_bytes=403685376 link_bytes=403685376 token_bytes=530889368 weight_movement_bytes=138412032 total_nvlink_bytes=669301400 weight_to_token_ratio=0.260717 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519035571e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343128 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265583704 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824280 weight_movement_bytes=34603008 total_nvlink_bytes=565427288 weight_to_token_ratio=0.065187 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344024 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117080 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890136 weight_movement_bytes=34603008 total_nvlink_bytes=566493144 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343208 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264296 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185384 weight_movement_bytes=34603008 total_nvlink_bytes=564788392 weight_to_token_ratio=0.065266 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344792 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289304 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233816 weight_movement_bytes=34603008 total_nvlink_bytes=568836824 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344456 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266944904 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545352 weight_movement_bytes=34603008 total_nvlink_bytes=568148360 weight_to_token_ratio=0.064855 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65048 dup_rows=23878 dup_rows_remote=20798 dedup_ratio=0.242900 router_tokens_per_expert=[780, 805, 806, 733, 754, 746, 705, 745, 808, 755, 729, 797, 799, 775, 783, 783, 772, 745, 811, 787, 731, 728, 752, 772, 813, 778, 751, 819, 759, 718, 784, 727, 768, 801, 762, 788, 790, 773, 798, 727, 747, 779, 792, 763, 756, 756, 745, 717, 763, 759, 772, 753, 795, 810, 745, 866, 781, 721, 758, 748, 772, 762, 758, 777] payload_rows_per_dst=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] dup_rows_per_dst=[3004, 3080, 2912, 3038, 3022, 2860, 3086, 2876] combine_rows_per_src=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64580 dup_rows=24298 dup_rows_remote=21154 dedup_ratio=0.247172 router_tokens_per_expert=[768, 774, 753, 780, 785, 760, 771, 793, 754, 746, 776, 753, 791, 721, 792, 740, 794, 744, 737, 791, 750, 753, 731, 841, 800, 740, 788, 746, 759, 746, 778, 752, 747, 774, 717, 724, 744, 820, 769, 775, 784, 787, 789, 728, 745, 794, 795, 806, 778, 713, 758, 744, 744, 783, 759, 787, 776, 765, 763, 777, 801, 764, 818, 817] payload_rows_per_dst=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] dup_rows_per_dst=[3074, 2952, 3094, 2990, 3004, 3064, 2976, 3144] combine_rows_per_src=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64678 dup_rows=24152 dup_rows_remote=21124 dedup_ratio=0.245687 router_tokens_per_expert=[808, 761, 778, 794, 724, 791, 751, 762, 752, 802, 715, 772, 828, 788, 789, 748, 786, 727, 781, 789, 791, 780, 809, 752, 715, 751, 777, 803, 778, 757, 758, 768, 777, 756, 787, 757, 780, 828, 767, 758, 756, 785, 719, 734, 756, 773, 787, 767, 789, 790, 755, 779, 752, 787, 727, 752, 719, 731, 795, 785, 792, 733, 745, 749] payload_rows_per_dst=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] dup_rows_per_dst=[3028, 3320, 3098, 3006, 2816, 2974, 2994, 2916] combine_rows_per_src=[9474, 9472, 9392, 9158, 9026, 9180, 9268, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64986 dup_rows=23922 dup_rows_remote=21006 dedup_ratio=0.243347 router_tokens_per_expert=[742, 764, 782, 761, 719, 730, 750, 743, 763, 766, 752, 770, 762, 776, 764, 752, 750, 853, 741, 783, 801, 769, 775, 785, 775, 781, 723, 755, 751, 800, 780, 782, 765, 728, 778, 745, 755, 780, 769, 789, 780, 745, 744, 777, 774, 767, 778, 791, 780, 792, 762, 779, 735, 797, 736, 789, 804, 824, 769, 763, 747, 726, 763, 821] payload_rows_per_dst=[9142, 9266, 9366, 9334, 9218, 9396, 9310, 9350] dup_rows_per_dst=[2840, 2944, 3148, 2960, 3000, 2916, 3030, 3084] combine_rows_per_src=[9142, 9266, 9366, 9334, 9218, 9396, 9310, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65042 dup_rows=24030 dup_rows_remote=21172 dedup_ratio=0.244446 router_tokens_per_expert=[775, 797, 783, 733, 779, 768, 759, 795, 800, 750, 790, 792, 752, 750, 782, 747, 809, 762, 769, 734, 769, 762, 721, 719, 763, 796, 752, 727, 752, 773, 750, 794, 765, 717, 785, 789, 806, 814, 764, 775, 696, 821, 778, 768, 788, 748, 777, 785, 754, 768, 767, 755, 764, 772, 801, 792, 766, 758, 764, 769, 745, 737, 760, 800] payload_rows_per_dst=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] dup_rows_per_dst=[3000, 3100, 2858, 2986, 3068, 3024, 3022, 2972] combine_rows_per_src=[9378, 9226, 9232, 9228, 9362, 9298, 9324, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65088 dup_rows=24048 dup_rows_remote=21198 dedup_ratio=0.244629 router_tokens_per_expert=[727, 807, 780, 727, 787, 779, 761, 809, 785, 762, 820, 785, 759, 754, 755, 805, 844, 754, 782, 801, 738, 714, 784, 745, 751, 755, 766, 767, 769, 775, 771, 764, 755, 767, 739, 775, 768, 714, 722, 769, 814, 738, 811, 762, 734, 773, 760, 767, 766, 754, 767, 774, 782, 793, 766, 755, 728, 768, 779, 758, 754, 775, 835, 748] payload_rows_per_dst=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] dup_rows_per_dst=[3028, 3196, 2954, 3060, 2850, 2976, 2936, 3048] combine_rows_per_src=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64886 dup_rows=24164 dup_rows_remote=21120 dedup_ratio=0.245809 router_tokens_per_expert=[723, 761, 759, 746, 770, 788, 727, 804, 764, 754, 789, 795, 717, 796, 766, 781, 780, 803, 779, 747, 767, 750, 756, 728, 788, 760, 756, 772, 741, 770, 762, 800, 737, 771, 780, 776, 780, 821, 803, 761, 827, 727, 786, 794, 773, 759, 795, 707, 809, 780, 769, 748, 781, 782, 769, 739, 750, 773, 772, 788, 701, 734, 796, 765] payload_rows_per_dst=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] dup_rows_per_dst=[2990, 3048, 3028, 3044, 3012, 3038, 3020, 2984] combine_rows_per_src=[9166, 9276, 9192, 9254, 9446, 9298, 9334, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64580 dup_rows=24298 dup_rows_remote=21154 dedup_ratio=0.247172 router_tokens_per_expert=[768, 774, 753, 780, 785, 760, 771, 793, 754, 746, 776, 753, 791, 721, 792, 740, 794, 744, 737, 791, 750, 753, 731, 841, 800, 740, 788, 746, 759, 746, 778, 752, 747, 774, 717, 724, 744, 820, 769, 775, 784, 787, 789, 728, 745, 794, 795, 806, 778, 713, 758, 744, 744, 783, 759, 787, 776, 765, 763, 777, 801, 764, 818, 817] payload_rows_per_dst=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] dup_rows_per_dst=[3074, 2952, 3094, 2990, 3004, 3064, 2976, 3144] combine_rows_per_src=[9294, 9098, 9270, 9234, 9136, 9392, 9156, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=342936 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264862616 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529382296 weight_movement_bytes=34603008 total_nvlink_bytes=563985304 weight_to_token_ratio=0.065365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64956 dup_rows=24064 dup_rows_remote=20986 dedup_ratio=0.244792 router_tokens_per_expert=[763, 758, 763, 794, 732, 804, 791, 715, 742, 750, 765, 781, 747, 756, 758, 792, 761, 785, 782, 736, 776, 774, 787, 758, 761, 778, 715, 777, 750, 800, 781, 819, 774, 776, 794, 776, 754, 800, 758, 779, 768, 775, 755, 820, 769, 767, 766, 794, 789, 740, 726, 771, 747, 761, 711, 737, 786, 757, 778, 770, 793, 769, 778, 763] payload_rows_per_dst=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] dup_rows_per_dst=[3000, 2878, 3034, 3078, 3074, 3138, 2810, 3052] combine_rows_per_src=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64692 dup_rows=24216 dup_rows_remote=21106 dedup_ratio=0.246338 router_tokens_per_expert=[782, 744, 787, 771, 733, 760, 752, 773, 769, 741, 765, 749, 782, 728, 797, 802, 790, 790, 761, 792, 731, 779, 769, 737, 737, 761, 768, 781, 789, 783, 762, 753, 729, 748, 818, 783, 748, 793, 765, 782, 751, 808, 759, 791, 747, 748, 764, 780, 810, 760, 746, 726, 747, 739, 780, 795, 769, 737, 816, 779, 777, 798, 791, 750] payload_rows_per_dst=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] dup_rows_per_dst=[3110, 3010, 3070, 3020, 3064, 2932, 2880, 3130] combine_rows_per_src=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343384 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266779992 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533216600 weight_movement_bytes=138412032 total_nvlink_bytes=671628632 weight_to_token_ratio=0.259579 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65088 dup_rows=24048 dup_rows_remote=21198 dedup_ratio=0.244629 router_tokens_per_expert=[727, 807, 780, 727, 787, 779, 761, 809, 785, 762, 820, 785, 759, 754, 755, 805, 844, 754, 782, 801, 738, 714, 784, 745, 751, 755, 766, 767, 769, 775, 771, 764, 755, 767, 739, 775, 768, 714, 722, 769, 814, 738, 811, 762, 734, 773, 760, 767, 766, 754, 767, 774, 782, 793, 766, 755, 728, 768, 779, 758, 754, 775, 835, 748] payload_rows_per_dst=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] dup_rows_per_dst=[3028, 3196, 2954, 3060, 2850, 2976, 2936, 3048] combine_rows_per_src=[9326, 9254, 9370, 9176, 9168, 9342, 9378, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=264921088 dispatch_meta_bytes=343208 combine_payload_bytes=264921088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265264296 ingress_bytes=299524096 link_bytes=299524096 token_bytes=530185384 weight_movement_bytes=34603008 total_nvlink_bytes=564788392 weight_to_token_ratio=0.065266 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127085060e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343968 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526624 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532709280 weight_movement_bytes=69206016 total_nvlink_bytes=601915296 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344856 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756888 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168920 weight_movement_bytes=34603008 total_nvlink_bytes=567771928 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344024 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117080 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890136 weight_movement_bytes=34603008 total_nvlink_bytes=566493144 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65048 dup_rows=23878 dup_rows_remote=20798 dedup_ratio=0.242900 router_tokens_per_expert=[780, 805, 806, 733, 754, 746, 705, 745, 808, 755, 729, 797, 799, 775, 783, 783, 772, 745, 811, 787, 731, 728, 752, 772, 813, 778, 751, 819, 759, 718, 784, 727, 768, 801, 762, 788, 790, 773, 798, 727, 747, 779, 792, 763, 756, 756, 745, 717, 763, 759, 772, 753, 795, 810, 745, 866, 781, 721, 758, 748, 772, 762, 758, 777] payload_rows_per_dst=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] dup_rows_per_dst=[3004, 3080, 2912, 3038, 3022, 2860, 3086, 2876] combine_rows_per_src=[9144, 9378, 9284, 9260, 9392, 9250, 9440, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64974 dup_rows=24084 dup_rows_remote=21120 dedup_ratio=0.244995 router_tokens_per_expert=[779, 794, 767, 803, 753, 767, 751, 792, 791, 765, 724, 726, 758, 814, 743, 788, 764, 738, 787, 789, 766, 760, 763, 782, 760, 817, 787, 743, 777, 806, 773, 773, 748, 727, 776, 789, 760, 798, 756, 778, 765, 792, 838, 722, 794, 761, 764, 746, 718, 761, 763, 771, 767, 821, 778, 726, 742, 778, 756, 761, 769, 724, 784, 719] payload_rows_per_dst=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] dup_rows_per_dst=[3066, 2910, 3062, 3102, 3004, 3078, 2964, 2898] combine_rows_per_src=[9346, 9308, 9236, 9370, 9260, 9286, 9246, 9168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=345144 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945592 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533546040 weight_movement_bytes=34603008 total_nvlink_bytes=568149048 weight_to_token_ratio=0.064855 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264519680 dispatch_meta_bytes=342936 combine_payload_bytes=264519680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264862616 ingress_bytes=299122688 link_bytes=299122688 token_bytes=529382296 weight_movement_bytes=34603008 total_nvlink_bytes=563985304 weight_to_token_ratio=0.065365 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125574594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=343192 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265321624 ingress_bytes=299581440 link_bytes=299581440 token_bytes=530300056 weight_movement_bytes=34603008 total_nvlink_bytes=564903064 weight_to_token_ratio=0.065252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127300840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343768 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266403544 ingress_bytes=404471808 link_bytes=404471808 token_bytes=532463320 weight_movement_bytes=138412032 total_nvlink_bytes=670875352 weight_to_token_ratio=0.259947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521994851e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=343384 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266779992 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533216600 weight_movement_bytes=138412032 total_nvlink_bytes=671628632 weight_to_token_ratio=0.259579 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344376 combine_payload_bytes=266133504 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266477880 ingress_bytes=369942528 link_bytes=369942528 token_bytes=532611384 weight_movement_bytes=103809024 total_nvlink_bytes=636420408 weight_to_token_ratio=0.194906 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392063950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65016 dup_rows=23988 dup_rows_remote=21014 dedup_ratio=0.244019 router_tokens_per_expert=[787, 799, 760, 767, 794, 781, 789, 746, 753, 769, 770, 807, 727, 786, 737, 788, 768, 760, 776, 781, 767, 769, 798, 751, 755, 765, 790, 791, 775, 779, 746, 754, 746, 796, 762, 771, 748, 714, 808, 755, 771, 768, 763, 759, 730, 767, 752, 733, 753, 771, 801, 807, 760, 777, 763, 791, 766, 756, 745, 749, 754, 782, 776, 773] payload_rows_per_dst=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] dup_rows_per_dst=[3048, 2974, 2990, 3018, 3000, 2912, 3130, 2916] combine_rows_per_src=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64884 dup_rows=23944 dup_rows_remote=20578 dedup_ratio=0.243571 router_tokens_per_expert=[746, 767, 791, 741, 744, 771, 767, 777, 712, 775, 764, 734, 803, 762, 798, 752, 781, 782, 741, 743, 802, 759, 761, 777, 777, 814, 728, 751, 798, 729, 784, 761, 777, 762, 759, 755, 787, 748, 773, 787, 792, 748, 775, 768, 787, 817, 750, 786, 764, 764, 758, 792, 749, 748, 710, 755, 811, 747, 810, 752, 744, 787, 807, 791] payload_rows_per_dst=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] dup_rows_per_dst=[3032, 2868, 2776, 2954, 3054, 2832, 3062, 3366] combine_rows_per_src=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64692 dup_rows=24216 dup_rows_remote=21106 dedup_ratio=0.246338 router_tokens_per_expert=[782, 744, 787, 771, 733, 760, 752, 773, 769, 741, 765, 749, 782, 728, 797, 802, 790, 790, 761, 792, 731, 779, 769, 737, 737, 761, 768, 781, 789, 783, 762, 753, 729, 748, 818, 783, 748, 793, 765, 782, 751, 808, 759, 791, 747, 748, 764, 780, 810, 760, 746, 726, 747, 739, 780, 795, 769, 737, 816, 779, 777, 798, 791, 750] payload_rows_per_dst=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] dup_rows_per_dst=[3110, 3010, 3070, 3020, 3064, 2932, 2880, 3130] combine_rows_per_src=[9396, 9258, 9228, 9248, 9298, 9154, 9202, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65274 dup_rows=23884 dup_rows_remote=20876 dedup_ratio=0.242961 router_tokens_per_expert=[774, 795, 734, 775, 733, 717, 760, 798, 758, 801, 810, 749, 797, 722, 792, 763, 754, 756, 780, 775, 778, 760, 752, 760, 742, 792, 776, 749, 792, 767, 765, 814, 750, 732, 764, 797, 784, 784, 823, 759, 748, 791, 778, 752, 768, 810, 674, 756, 792, 732, 808, 786, 776, 813, 754, 754, 767, 745, 753, 788, 760, 717, 737, 810] payload_rows_per_dst=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] dup_rows_per_dst=[2932, 3002, 3000, 3106, 3008, 3008, 2944, 2884] combine_rows_per_src=[9240, 9382, 9230, 9288, 9378, 9146, 9486, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64976 dup_rows=24032 dup_rows_remote=21100 dedup_ratio=0.244466 router_tokens_per_expert=[768, 753, 758, 790, 755, 792, 783, 833, 741, 821, 744, 737, 769, 719, 747, 787, 762, 776, 765, 765, 741, 745, 788, 772, 813, 738, 770, 733, 770, 773, 779, 821, 769, 759, 780, 761, 724, 763, 746, 750, 698, 788, 746, 813, 779, 794, 801, 783, 761, 735, 754, 775, 760, 705, 793, 780, 765, 758, 819, 760, 783, 773, 759, 810] payload_rows_per_dst=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] dup_rows_per_dst=[3018, 2974, 2932, 3044, 2962, 3082, 2896, 3124] combine_rows_per_src=[9446, 9156, 9296, 9350, 9142, 9322, 9230, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64802 dup_rows=24196 dup_rows_remote=21214 dedup_ratio=0.246134 router_tokens_per_expert=[782, 774, 724, 801, 758, 744, 827, 775, 779, 744, 771, 725, 754, 744, 780, 740, 790, 762, 745, 794, 734, 721, 772, 781, 808, 824, 762, 809, 722, 746, 761, 763, 790, 754, 751, 763, 731, 793, 823, 739, 785, 763, 797, 788, 761, 795, 759, 747, 751, 773, 765, 744, 748, 746, 797, 750, 793, 777, 754, 826, 769, 770, 788, 746] payload_rows_per_dst=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] dup_rows_per_dst=[2972, 2968, 3016, 3150, 2982, 3042, 2948, 3118] combine_rows_per_src=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64956 dup_rows=24064 dup_rows_remote=20986 dedup_ratio=0.244792 router_tokens_per_expert=[763, 758, 763, 794, 732, 804, 791, 715, 742, 750, 765, 781, 747, 756, 758, 792, 761, 785, 782, 736, 776, 774, 787, 758, 761, 778, 715, 777, 750, 800, 781, 819, 774, 776, 794, 776, 754, 800, 758, 779, 768, 775, 755, 820, 769, 767, 766, 794, 789, 740, 726, 771, 747, 761, 711, 737, 786, 757, 778, 770, 793, 769, 778, 763] payload_rows_per_dst=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] dup_rows_per_dst=[3000, 2878, 3034, 3078, 3074, 3138, 2810, 3052] combine_rows_per_src=[9240, 9304, 9284, 9284, 9348, 9290, 9154, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64884 dup_rows=23944 dup_rows_remote=20578 dedup_ratio=0.243571 router_tokens_per_expert=[746, 767, 791, 741, 744, 771, 767, 777, 712, 775, 764, 734, 803, 762, 798, 752, 781, 782, 741, 743, 802, 759, 761, 777, 777, 814, 728, 751, 798, 729, 784, 761, 777, 762, 759, 755, 787, 748, 773, 787, 792, 748, 775, 768, 787, 817, 750, 786, 764, 764, 758, 792, 749, 748, 710, 755, 811, 747, 810, 752, 744, 787, 807, 791] payload_rows_per_dst=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] dup_rows_per_dst=[3032, 2868, 2776, 2954, 3054, 2832, 3062, 3366] combine_rows_per_src=[9176, 9332, 9172, 9150, 9296, 9272, 9486, 9476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64786 dup_rows=24246 dup_rows_remote=21208 dedup_ratio=0.246643 router_tokens_per_expert=[785, 739, 789, 788, 796, 718, 757, 747, 766, 774, 733, 762, 756, 781, 788, 762, 727, 811, 776, 793, 787, 778, 768, 803, 782, 791, 767, 729, 750, 748, 779, 796, 801, 763, 732, 824, 744, 776, 767, 764, 771, 726, 785, 767, 819, 747, 770, 725, 822, 735, 787, 783, 771, 751, 752, 777, 764, 791, 772, 708, 744, 737, 803, 748] payload_rows_per_dst=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] dup_rows_per_dst=[3038, 3378, 3202, 2988, 3008, 3006, 3052, 2574] combine_rows_per_src=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65046 dup_rows=23964 dup_rows_remote=21092 dedup_ratio=0.243774 router_tokens_per_expert=[802, 826, 789, 771, 775, 771, 740, 783, 777, 757, 734, 786, 740, 799, 775, 716, 768, 813, 818, 751, 754, 774, 785, 776, 783, 760, 733, 823, 750, 761, 782, 691, 755, 757, 766, 785, 750, 765, 800, 773, 772, 764, 755, 767, 753, 755, 751, 781, 771, 786, 723, 772, 729, 762, 762, 782, 770, 756, 760, 797, 764, 765, 819, 722] payload_rows_per_dst=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] dup_rows_per_dst=[2992, 3006, 3096, 2872, 3008, 2986, 2964, 3040] combine_rows_per_src=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=341848 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266106712 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531871576 weight_movement_bytes=69206016 total_nvlink_bytes=601077592 weight_to_token_ratio=0.130118 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64802 dup_rows=24196 dup_rows_remote=21214 dedup_ratio=0.246134 router_tokens_per_expert=[782, 774, 724, 801, 758, 744, 827, 775, 779, 744, 771, 725, 754, 744, 780, 740, 790, 762, 745, 794, 734, 721, 772, 781, 808, 824, 762, 809, 722, 746, 761, 763, 790, 754, 751, 763, 731, 793, 823, 739, 785, 763, 797, 788, 761, 795, 759, 747, 751, 773, 765, 744, 748, 746, 797, 750, 793, 777, 754, 826, 769, 770, 788, 746] payload_rows_per_dst=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] dup_rows_per_dst=[2972, 2968, 3016, 3150, 2982, 3042, 2948, 3118] combine_rows_per_src=[9398, 9106, 9182, 9240, 9306, 9348, 9200, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=264978432 dispatch_meta_bytes=343192 combine_payload_bytes=264978432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265321624 ingress_bytes=299581440 link_bytes=299581440 token_bytes=530300056 weight_movement_bytes=34603008 total_nvlink_bytes=564903064 weight_to_token_ratio=0.065252 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127300840e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344600 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706904 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535069208 weight_movement_bytes=69206016 total_nvlink_bytes=604275224 weight_to_token_ratio=0.129340 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344120 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649656 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955192 weight_movement_bytes=34603008 total_nvlink_bytes=567558200 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344304 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266486000 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627696 weight_movement_bytes=69206016 total_nvlink_bytes=601833712 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343768 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266403544 ingress_bytes=404471808 link_bytes=404471808 token_bytes=532463320 weight_movement_bytes=138412032 total_nvlink_bytes=670875352 weight_to_token_ratio=0.259947 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521994851e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=65016 dup_rows=23988 dup_rows_remote=21014 dedup_ratio=0.244019 router_tokens_per_expert=[787, 799, 760, 767, 794, 781, 789, 746, 753, 769, 770, 807, 727, 786, 737, 788, 768, 760, 776, 781, 767, 769, 798, 751, 755, 765, 790, 791, 775, 779, 746, 754, 746, 796, 762, 771, 748, 714, 808, 755, 771, 768, 763, 759, 730, 767, 752, 733, 753, 771, 801, 807, 760, 777, 763, 791, 766, 756, 745, 749, 754, 782, 776, 773] payload_rows_per_dst=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] dup_rows_per_dst=[3048, 2974, 2990, 3018, 3000, 2912, 3130, 2916] combine_rows_per_src=[9398, 9300, 9350, 9292, 9200, 9174, 9316, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74210 payload_rows_remote=64936 dup_rows=24094 dup_rows_remote=21082 dedup_ratio=0.245097 router_tokens_per_expert=[739, 734, 769, 786, 748, 765, 761, 745, 746, 818, 707, 767, 734, 762, 723, 770, 786, 804, 794, 796, 758, 762, 747, 814, 712, 812, 741, 805, 756, 762, 731, 767, 752, 811, 782, 780, 763, 734, 724, 760, 774, 776, 785, 834, 770, 772, 792, 756, 778, 794, 722, 738, 778, 767, 770, 796, 814, 775, 775, 787, 771, 744, 761, 796] payload_rows_per_dst=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] dup_rows_per_dst=[2922, 2956, 3098, 3016, 2986, 3064, 3012, 3040] combine_rows_per_src=[9172, 9098, 9424, 9156, 9226, 9454, 9274, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344064 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265773056 ingress_bytes=300032000 link_bytes=300032000 token_bytes=531202048 weight_movement_bytes=34603008 total_nvlink_bytes=565805056 weight_to_token_ratio=0.065141 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128996261e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=341848 combine_payload_bytes=265764864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266106712 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531871576 weight_movement_bytes=69206016 total_nvlink_bytes=601077592 weight_to_token_ratio=0.130118 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343976 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707432 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070888 weight_movement_bytes=34603008 total_nvlink_bytes=565673896 weight_to_token_ratio=0.065157 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344552 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772968 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201384 weight_movement_bytes=34603008 total_nvlink_bytes=567804392 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344120 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266649656 ingress_bytes=300908544 link_bytes=300908544 token_bytes=532955192 weight_movement_bytes=34603008 total_nvlink_bytes=567558200 weight_to_token_ratio=0.064927 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132294626e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=344072 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321928 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299784 weight_movement_bytes=34603008 total_nvlink_bytes=566902792 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65140 dup_rows=23846 dup_rows_remote=20806 dedup_ratio=0.242574 router_tokens_per_expert=[778, 737, 759, 786, 794, 736, 764, 751, 783, 766, 816, 734, 776, 804, 730, 770, 785, 765, 737, 782, 785, 783, 769, 787, 745, 775, 759, 754, 706, 822, 798, 771, 755, 777, 788, 820, 766, 722, 751, 787, 819, 747, 824, 743, 806, 746, 769, 761, 797, 750, 760, 731, 781, 754, 738, 757, 743, 763, 813, 740, 785, 747, 687, 818] payload_rows_per_dst=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] dup_rows_per_dst=[2934, 3040, 3004, 3010, 3004, 2998, 2948, 2908] combine_rows_per_src=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65096 dup_rows=23724 dup_rows_remote=20720 dedup_ratio=0.241333 router_tokens_per_expert=[747, 787, 740, 754, 785, 744, 724, 755, 776, 755, 750, 757, 749, 773, 745, 783, 752, 764, 761, 741, 775, 816, 738, 783, 781, 788, 752, 776, 773, 743, 701, 816, 756, 806, 780, 749, 772, 751, 758, 790, 801, 810, 785, 744, 761, 744, 751, 807, 788, 758, 759, 747, 742, 790, 786, 797, 784, 787, 762, 771, 850, 747, 795, 740] payload_rows_per_dst=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] dup_rows_per_dst=[2892, 2928, 2874, 3022, 3010, 2768, 3226, 3004] combine_rows_per_src=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74058 payload_rows_remote=64786 dup_rows=24246 dup_rows_remote=21208 dedup_ratio=0.246643 router_tokens_per_expert=[785, 739, 789, 788, 796, 718, 757, 747, 766, 774, 733, 762, 756, 781, 788, 762, 727, 811, 776, 793, 787, 778, 768, 803, 782, 791, 767, 729, 750, 748, 779, 796, 801, 763, 732, 824, 744, 776, 767, 764, 771, 726, 785, 767, 819, 747, 770, 725, 822, 735, 787, 783, 771, 751, 752, 777, 764, 791, 772, 708, 744, 737, 803, 748] payload_rows_per_dst=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] dup_rows_per_dst=[3038, 3378, 3202, 2988, 3008, 3006, 3052, 2574] combine_rows_per_src=[9272, 9628, 9318, 9190, 9334, 9214, 9304, 8798] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64962 dup_rows=24020 dup_rows_remote=20938 dedup_ratio=0.244344 router_tokens_per_expert=[772, 736, 770, 781, 739, 738, 789, 815, 784, 754, 741, 758, 766, 795, 754, 804, 791, 755, 760, 832, 763, 709, 769, 743, 746, 763, 739, 742, 754, 790, 812, 754, 758, 792, 740, 828, 710, 751, 820, 761, 771, 755, 800, 809, 789, 759, 744, 775, 790, 747, 748, 799, 717, 790, 768, 780, 778, 750, 762, 754, 747, 789, 785, 768] payload_rows_per_dst=[9336, 9236, 9240, 9242, 9304, 9322, 9332, 9272] dup_rows_per_dst=[2944, 3076, 3004, 2958, 3016, 3082, 2946, 2994] combine_rows_per_src=[9336, 9236, 9240, 9242, 9304, 9322, 9332, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64994 dup_rows=23980 dup_rows_remote=20954 dedup_ratio=0.243937 router_tokens_per_expert=[810, 772, 737, 799, 780, 796, 770, 702, 739, 745, 776, 723, 799, 786, 784, 778, 803, 793, 765, 759, 773, 734, 764, 787, 778, 812, 794, 751, 745, 779, 801, 784, 739, 707, 728, 777, 795, 768, 764, 767, 821, 822, 763, 733, 775, 761, 791, 766, 771, 726, 769, 730, 761, 724, 791, 776, 765, 766, 786, 781, 803, 759, 741, 708] payload_rows_per_dst=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] dup_rows_per_dst=[3032, 2896, 3026, 3028, 2858, 3108, 2968, 3064] combine_rows_per_src=[9300, 9364, 9330, 9460, 9232, 9356, 9128, 9154] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65234 dup_rows=23756 dup_rows_remote=20782 dedup_ratio=0.241659 router_tokens_per_expert=[804, 785, 755, 766, 720, 755, 727, 754, 776, 787, 769, 777, 812, 751, 800, 721, 735, 827, 742, 775, 786, 796, 760, 766, 719, 762, 753, 770, 766, 763, 745, 781, 767, 793, 748, 788, 788, 749, 758, 753, 805, 774, 783, 775, 714, 791, 755, 820, 765, 782, 800, 765, 766, 727, 730, 799, 778, 784, 779, 759, 757, 765, 784, 746] payload_rows_per_dst=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] dup_rows_per_dst=[2932, 3060, 3016, 2944, 2974, 2954, 2904, 2972] combine_rows_per_src=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65046 dup_rows=23964 dup_rows_remote=21092 dedup_ratio=0.243774 router_tokens_per_expert=[802, 826, 789, 771, 775, 771, 740, 783, 777, 757, 734, 786, 740, 799, 775, 716, 768, 813, 818, 751, 754, 774, 785, 776, 783, 760, 733, 823, 750, 761, 782, 691, 755, 757, 766, 785, 750, 765, 800, 773, 772, 764, 755, 767, 753, 755, 751, 781, 771, 786, 723, 772, 729, 762, 762, 782, 770, 756, 760, 797, 764, 765, 819, 722] payload_rows_per_dst=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] dup_rows_per_dst=[2992, 3006, 3096, 2872, 3008, 2986, 2964, 3040] combine_rows_per_src=[9522, 9162, 9382, 9294, 9294, 9210, 9210, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73942 payload_rows_remote=64714 dup_rows=24362 dup_rows_remote=21292 dedup_ratio=0.247823 router_tokens_per_expert=[764, 794, 758, 746, 757, 732, 745, 813, 774, 768, 803, 762, 805, 791, 763, 763, 731, 809, 772, 770, 756, 762, 766, 747, 764, 769, 785, 786, 777, 746, 744, 778, 737, 796, 800, 763, 757, 764, 735, 773, 795, 839, 757, 741, 746, 756, 732, 731, 797, 789, 760, 785, 808, 756, 750, 775, 740, 741, 790, 796, 760, 768, 776, 739] payload_rows_per_dst=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] dup_rows_per_dst=[3002, 3104, 2984, 3070, 3000, 3028, 3150, 3024] combine_rows_per_src=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64872 dup_rows=24078 dup_rows_remote=20938 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 780, 763, 796, 764, 744, 759, 770, 772, 792, 778, 738, 748, 738, 803, 723, 743, 761, 751, 795, 753, 786, 752, 802, 761, 790, 812, 804, 807, 722, 765, 773, 780, 750, 747, 763, 748, 775, 788, 751, 715, 774, 816, 758, 779, 729, 758, 826, 772, 757, 748, 759, 734, 770, 790, 814, 772, 752, 756, 775, 734, 785, 770] payload_rows_per_dst=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] dup_rows_per_dst=[3140, 3106, 2934, 3086, 2996, 2818, 3078, 2920] combine_rows_per_src=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65096 dup_rows=23724 dup_rows_remote=20720 dedup_ratio=0.241333 router_tokens_per_expert=[747, 787, 740, 754, 785, 744, 724, 755, 776, 755, 750, 757, 749, 773, 745, 783, 752, 764, 761, 741, 775, 816, 738, 783, 781, 788, 752, 776, 773, 743, 701, 816, 756, 806, 780, 749, 772, 751, 758, 790, 801, 810, 785, 744, 761, 744, 751, 807, 788, 758, 759, 747, 742, 790, 786, 797, 784, 787, 762, 771, 850, 747, 795, 740] payload_rows_per_dst=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] dup_rows_per_dst=[2892, 2928, 2874, 3022, 3010, 2768, 3226, 3004] combine_rows_per_src=[9180, 9248, 9152, 9472, 9314, 9174, 9556, 9484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343784 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157224 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970664 weight_movement_bytes=34603008 total_nvlink_bytes=568573672 weight_to_token_ratio=0.064803 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343264 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266976480 ingress_bytes=370442240 link_bytes=370442240 token_bytes=533609696 weight_movement_bytes=103809024 total_nvlink_bytes=637418720 weight_to_token_ratio=0.194541 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393944326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65234 dup_rows=23756 dup_rows_remote=20782 dedup_ratio=0.241659 router_tokens_per_expert=[804, 785, 755, 766, 720, 755, 727, 754, 776, 787, 769, 777, 812, 751, 800, 721, 735, 827, 742, 775, 786, 796, 760, 766, 719, 762, 753, 770, 766, 763, 745, 781, 767, 793, 748, 788, 788, 749, 758, 753, 805, 774, 783, 775, 714, 791, 755, 820, 765, 782, 800, 765, 766, 727, 730, 799, 778, 784, 779, 759, 757, 765, 784, 746] payload_rows_per_dst=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] dup_rows_per_dst=[2932, 3060, 3016, 2944, 2974, 2954, 2904, 2972] combine_rows_per_src=[9200, 9326, 9358, 9174, 9314, 9480, 9364, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343976 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707432 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070888 weight_movement_bytes=34603008 total_nvlink_bytes=565673896 weight_to_token_ratio=0.065157 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343600 combine_payload_bytes=266084352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266427952 ingress_bytes=404496384 link_bytes=404496384 token_bytes=532512304 weight_movement_bytes=138412032 total_nvlink_bytes=670924336 weight_to_token_ratio=0.259923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522087328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343792 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266559216 ingress_bytes=335421440 link_bytes=335421440 token_bytes=532774640 weight_movement_bytes=69206016 total_nvlink_bytes=601980656 weight_to_token_ratio=0.129897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262163875e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344552 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266772968 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201384 weight_movement_bytes=34603008 total_nvlink_bytes=567804392 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65140 dup_rows=23846 dup_rows_remote=20806 dedup_ratio=0.242574 router_tokens_per_expert=[778, 737, 759, 786, 794, 736, 764, 751, 783, 766, 816, 734, 776, 804, 730, 770, 785, 765, 737, 782, 785, 783, 769, 787, 745, 775, 759, 754, 706, 822, 798, 771, 755, 777, 788, 820, 766, 722, 751, 787, 819, 747, 824, 743, 806, 746, 769, 761, 797, 750, 760, 731, 781, 754, 738, 757, 743, 763, 813, 740, 785, 747, 687, 818] payload_rows_per_dst=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] dup_rows_per_dst=[2934, 3040, 3004, 3010, 3004, 2998, 2948, 2908] combine_rows_per_src=[9276, 9318, 9382, 9250, 9328, 9432, 9188, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65162 dup_rows=23872 dup_rows_remote=20884 dedup_ratio=0.242839 router_tokens_per_expert=[765, 787, 746, 791, 760, 731, 750, 805, 770, 826, 764, 757, 747, 750, 773, 759, 765, 748, 770, 775, 798, 776, 752, 761, 743, 795, 699, 790, 752, 743, 748, 756, 769, 727, 800, 778, 763, 738, 792, 781, 794, 801, 769, 752, 766, 809, 784, 752, 752, 775, 733, 807, 772, 770, 778, 742, 766, 758, 801, 761, 807, 736, 802, 765] payload_rows_per_dst=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] dup_rows_per_dst=[3048, 2946, 2960, 2904, 2984, 3016, 2988, 3026] combine_rows_per_src=[9222, 9346, 9330, 9148, 9312, 9438, 9270, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343600 combine_payload_bytes=266084352 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266427952 ingress_bytes=404496384 link_bytes=404496384 token_bytes=532512304 weight_movement_bytes=138412032 total_nvlink_bytes=670924336 weight_to_token_ratio=0.259923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522087328e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344064 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267542528 ingress_bytes=301801472 link_bytes=301801472 token_bytes=534740992 weight_movement_bytes=34603008 total_nvlink_bytes=569344000 weight_to_token_ratio=0.064710 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135654642e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344024 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265412568 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530481112 weight_movement_bytes=34603008 total_nvlink_bytes=565084120 weight_to_token_ratio=0.065229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343240 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266058952 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531774664 weight_movement_bytes=34603008 total_nvlink_bytes=566377672 weight_to_token_ratio=0.065071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343264 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266976480 ingress_bytes=370442240 link_bytes=370442240 token_bytes=533609696 weight_movement_bytes=103809024 total_nvlink_bytes=637418720 weight_to_token_ratio=0.194541 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393944326e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=343784 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157224 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533970664 weight_movement_bytes=34603008 total_nvlink_bytes=568573672 weight_to_token_ratio=0.064803 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344184 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247736 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151288 weight_movement_bytes=34603008 total_nvlink_bytes=568754296 weight_to_token_ratio=0.064781 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[781, 739, 803, 799, 779, 819, 714, 746, 794, 835, 756, 781, 774, 762, 803, 713, 767, 763, 805, 777, 758, 746, 733, 858, 749, 745, 756, 787, 742, 737, 748, 756, 746, 746, 740, 775, 744, 752, 788, 761, 769, 754, 791, 831, 754, 775, 759, 751, 774, 785, 741, 766, 724, 783, 817, 776, 770, 747, 774, 776, 730, 774, 772, 782] payload_rows_per_dst=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] dup_rows_per_dst=[3022, 3036, 3088, 2926, 2916, 3010, 3066, 2976] combine_rows_per_src=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65204 dup_rows=23628 dup_rows_remote=20530 dedup_ratio=0.240356 router_tokens_per_expert=[736, 726, 751, 748, 749, 763, 734, 739, 732, 763, 785, 745, 764, 782, 787, 802, 756, 776, 729, 792, 873, 790, 771, 798, 809, 785, 781, 703, 764, 782, 755, 832, 745, 798, 814, 759, 757, 771, 781, 756, 797, 731, 770, 778, 814, 760, 717, 802, 712, 721, 757, 777, 806, 791, 771, 707, 786, 742, 719, 806, 779, 737, 789, 800] payload_rows_per_dst=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] dup_rows_per_dst=[2812, 2944, 3098, 3058, 2926, 2970, 2866, 2954] combine_rows_per_src=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20928 dedup_ratio=0.244364 router_tokens_per_expert=[733, 794, 755, 773, 716, 757, 793, 810, 766, 826, 757, 715, 809, 743, 786, 787, 780, 767, 770, 827, 780, 799, 823, 730, 767, 755, 749, 741, 743, 806, 780, 758, 789, 746, 785, 789, 773, 779, 821, 749, 798, 802, 742, 796, 739, 725, 767, 762, 771, 743, 752, 735, 750, 816, 764, 745, 751, 736, 775, 719, 734, 751, 774, 779] payload_rows_per_dst=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] dup_rows_per_dst=[3026, 3140, 3128, 2942, 3094, 2940, 2840, 2912] combine_rows_per_src=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64644 dup_rows=24256 dup_rows_remote=21114 dedup_ratio=0.246745 router_tokens_per_expert=[779, 749, 756, 766, 745, 790, 825, 784, 787, 771, 773, 790, 727, 725, 786, 712, 795, 776, 765, 778, 820, 770, 757, 786, 787, 765, 780, 760, 828, 753, 781, 731, 732, 772, 792, 765, 775, 711, 756, 744, 759, 791, 779, 767, 737, 755, 751, 801, 721, 740, 759, 743, 760, 795, 813, 737, 785, 778, 794, 757, 761, 800, 791, 734] payload_rows_per_dst=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] dup_rows_per_dst=[3094, 2542, 3020, 3282, 3182, 3092, 2902, 3142] combine_rows_per_src=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64872 dup_rows=24078 dup_rows_remote=20938 dedup_ratio=0.244934 router_tokens_per_expert=[780, 782, 780, 763, 796, 764, 744, 759, 770, 772, 792, 778, 738, 748, 738, 803, 723, 743, 761, 751, 795, 753, 786, 752, 802, 761, 790, 812, 804, 807, 722, 765, 773, 780, 750, 747, 763, 748, 775, 788, 751, 715, 774, 816, 758, 779, 729, 758, 826, 772, 757, 748, 759, 734, 770, 790, 814, 772, 752, 756, 775, 734, 785, 770] payload_rows_per_dst=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] dup_rows_per_dst=[3140, 3106, 2934, 3086, 2996, 2818, 3078, 2920] combine_rows_per_src=[9354, 9246, 9216, 9440, 9356, 9162, 9234, 9218] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64858 dup_rows=24048 dup_rows_remote=20996 dedup_ratio=0.244629 router_tokens_per_expert=[720, 740, 752, 796, 756, 725, 783, 777, 770, 772, 817, 769, 798, 787, 765, 764, 707, 747, 773, 770, 804, 778, 785, 752, 764, 759, 758, 760, 781, 759, 744, 702, 731, 781, 757, 749, 749, 783, 785, 748, 796, 761, 769, 777, 788, 764, 736, 834, 809, 822, 718, 814, 767, 801, 809, 727, 762, 735, 746, 758, 769, 781, 770, 822] payload_rows_per_dst=[9094, 9344, 9226, 9160, 9230, 9398, 9454, 9350] dup_rows_per_dst=[3004, 3140, 3006, 2894, 2936, 3052, 3080, 2936] combine_rows_per_src=[9094, 9344, 9226, 9160, 9230, 9398, 9454, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74676 payload_rows_remote=65204 dup_rows=23628 dup_rows_remote=20530 dedup_ratio=0.240356 router_tokens_per_expert=[736, 726, 751, 748, 749, 763, 734, 739, 732, 763, 785, 745, 764, 782, 787, 802, 756, 776, 729, 792, 873, 790, 771, 798, 809, 785, 781, 703, 764, 782, 755, 832, 745, 798, 814, 759, 757, 771, 781, 756, 797, 731, 770, 778, 814, 760, 717, 802, 712, 721, 757, 777, 806, 791, 771, 707, 786, 742, 719, 806, 779, 737, 789, 800] payload_rows_per_dst=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] dup_rows_per_dst=[2812, 2944, 3098, 3058, 2926, 2970, 2866, 2954] combine_rows_per_src=[9080, 9376, 9472, 9364, 9436, 9368, 9218, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73942 payload_rows_remote=64714 dup_rows=24362 dup_rows_remote=21292 dedup_ratio=0.247823 router_tokens_per_expert=[764, 794, 758, 746, 757, 732, 745, 813, 774, 768, 803, 762, 805, 791, 763, 763, 731, 809, 772, 770, 756, 762, 766, 747, 764, 769, 785, 786, 777, 746, 744, 778, 737, 796, 800, 763, 757, 764, 735, 773, 795, 839, 757, 741, 746, 756, 732, 731, 797, 789, 760, 785, 808, 756, 750, 775, 740, 741, 790, 796, 760, 768, 776, 739] payload_rows_per_dst=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] dup_rows_per_dst=[3002, 3104, 2984, 3070, 3000, 3028, 3150, 3024] combine_rows_per_src=[9216, 9354, 9242, 9228, 9250, 9166, 9290, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65270 dup_rows=23762 dup_rows_remote=20956 dedup_ratio=0.241720 router_tokens_per_expert=[824, 786, 777, 737, 798, 798, 779, 787, 753, 764, 759, 729, 737, 770, 758, 780, 774, 803, 786, 777, 760, 789, 748, 783, 747, 706, 726, 773, 765, 764, 802, 756, 761, 739, 759, 821, 719, 754, 705, 799, 806, 741, 816, 777, 757, 748, 763, 766, 748, 750, 775, 780, 793, 730, 755, 793, 774, 777, 816, 753, 793, 750, 784, 755] payload_rows_per_dst=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] dup_rows_per_dst=[3120, 3008, 3028, 2806, 2810, 3028, 2958, 3004] combine_rows_per_src=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64726 dup_rows=24140 dup_rows_remote=20828 dedup_ratio=0.245565 router_tokens_per_expert=[788, 807, 782, 768, 783, 752, 749, 761, 776, 787, 812, 746, 791, 768, 761, 774, 761, 753, 755, 752, 744, 761, 702, 787, 712, 781, 747, 819, 807, 754, 783, 729, 723, 758, 787, 763, 798, 748, 795, 757, 751, 746, 798, 802, 764, 806, 793, 719, 754, 750, 764, 786, 778, 811, 748, 769, 737, 753, 796, 769, 783, 754, 756, 784] payload_rows_per_dst=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] dup_rows_per_dst=[3312, 3080, 2902, 2930, 3052, 2842, 3012, 3010] combine_rows_per_src=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74048 payload_rows_remote=64644 dup_rows=24256 dup_rows_remote=21114 dedup_ratio=0.246745 router_tokens_per_expert=[779, 749, 756, 766, 745, 790, 825, 784, 787, 771, 773, 790, 727, 725, 786, 712, 795, 776, 765, 778, 820, 770, 757, 786, 787, 765, 780, 760, 828, 753, 781, 731, 732, 772, 792, 765, 775, 711, 756, 744, 759, 791, 779, 767, 737, 755, 751, 801, 721, 740, 759, 743, 760, 795, 813, 737, 785, 778, 794, 757, 761, 800, 791, 734] payload_rows_per_dst=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] dup_rows_per_dst=[3094, 2542, 3020, 3282, 3182, 3092, 2902, 3142] combine_rows_per_src=[9294, 8820, 9328, 9442, 9338, 9188, 9234, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343472 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266026416 ingress_bytes=369491968 link_bytes=369491968 token_bytes=531709360 weight_movement_bytes=103809024 total_nvlink_bytes=635518384 weight_to_token_ratio=0.195236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390368529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64914 dup_rows=24022 dup_rows_remote=20928 dedup_ratio=0.244364 router_tokens_per_expert=[733, 794, 755, 773, 716, 757, 793, 810, 766, 826, 757, 715, 809, 743, 786, 787, 780, 767, 770, 827, 780, 799, 823, 730, 767, 755, 749, 741, 743, 806, 780, 758, 789, 746, 785, 789, 773, 779, 821, 749, 798, 802, 742, 796, 739, 725, 767, 762, 771, 743, 752, 735, 750, 816, 764, 745, 751, 736, 775, 719, 734, 751, 774, 779] payload_rows_per_dst=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] dup_rows_per_dst=[3026, 3140, 3128, 2942, 3094, 2940, 2840, 2912] combine_rows_per_src=[9236, 9238, 9424, 9256, 9368, 9322, 9312, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=343032 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265124856 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529906680 weight_movement_bytes=34603008 total_nvlink_bytes=564509688 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343240 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266058952 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531774664 weight_movement_bytes=34603008 total_nvlink_bytes=566377672 weight_to_token_ratio=0.065071 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343416 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266001784 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531660152 weight_movement_bytes=69206016 total_nvlink_bytes=600866168 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=342936 combine_payload_bytes=267075584 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267418520 ingress_bytes=370884608 link_bytes=370884608 token_bytes=534494104 weight_movement_bytes=103809024 total_nvlink_bytes=638303128 weight_to_token_ratio=0.194219 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395608921e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265068544 dispatch_meta_bytes=344024 combine_payload_bytes=265068544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265412568 ingress_bytes=299671552 link_bytes=299671552 token_bytes=530481112 weight_movement_bytes=34603008 total_nvlink_bytes=565084120 weight_to_token_ratio=0.065229 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127639925e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265887744 dispatch_meta_bytes=343368 combine_payload_bytes=265887744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266231112 ingress_bytes=300490752 link_bytes=300490752 token_bytes=532118856 weight_movement_bytes=34603008 total_nvlink_bytes=566721864 weight_to_token_ratio=0.065029 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130722508e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64864 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[781, 739, 803, 799, 779, 819, 714, 746, 794, 835, 756, 781, 774, 762, 803, 713, 767, 763, 805, 777, 758, 746, 733, 858, 749, 745, 756, 787, 742, 737, 748, 756, 746, 746, 740, 775, 744, 752, 788, 761, 769, 754, 791, 831, 754, 775, 759, 751, 774, 785, 741, 766, 724, 783, 817, 776, 770, 747, 774, 776, 730, 774, 772, 782] payload_rows_per_dst=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] dup_rows_per_dst=[3022, 3036, 3088, 2926, 2916, 3010, 3066, 2976] combine_rows_per_src=[9338, 9400, 9326, 9114, 9188, 9358, 9266, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65066 dup_rows=23898 dup_rows_remote=20842 dedup_ratio=0.243103 router_tokens_per_expert=[787, 760, 785, 770, 745, 794, 753, 774, 783, 829, 772, 769, 738, 751, 757, 782, 766, 748, 735, 718, 737, 774, 725, 798, 767, 757, 756, 741, 748, 771, 815, 797, 738, 819, 758, 780, 756, 759, 761, 720, 773, 755, 793, 769, 720, 777, 771, 811, 785, 762, 793, 772, 749, 775, 779, 783, 786, 767, 785, 779, 750, 777, 774, 774] payload_rows_per_dst=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] dup_rows_per_dst=[2988, 3040, 2842, 3026, 2914, 2998, 3056, 3034] combine_rows_per_src=[9348, 9322, 9160, 9278, 9268, 9340, 9340, 9350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=343416 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266001784 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531660152 weight_movement_bytes=69206016 total_nvlink_bytes=600866168 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342216 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265459912 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577608 weight_movement_bytes=34603008 total_nvlink_bytes=565180616 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344904 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690824 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535036744 weight_movement_bytes=69206016 total_nvlink_bytes=604242760 weight_to_token_ratio=0.129348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264781824 dispatch_meta_bytes=343032 combine_payload_bytes=264781824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265124856 ingress_bytes=299384832 link_bytes=299384832 token_bytes=529906680 weight_movement_bytes=34603008 total_nvlink_bytes=564509688 weight_to_token_ratio=0.065300 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126561020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343472 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266026416 ingress_bytes=369491968 link_bytes=369491968 token_bytes=531709360 weight_movement_bytes=103809024 total_nvlink_bytes=635518384 weight_to_token_ratio=0.195236 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390368529e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343632 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266853968 ingress_bytes=335716352 link_bytes=335716352 token_bytes=533364304 weight_movement_bytes=69206016 total_nvlink_bytes=602570320 weight_to_token_ratio=0.129754 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263273605e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65142 dup_rows=23876 dup_rows_remote=20872 dedup_ratio=0.242879 router_tokens_per_expert=[804, 758, 777, 748, 752, 778, 744, 759, 770, 759, 784, 778, 746, 781, 780, 747, 772, 758, 770, 739, 744, 792, 816, 777, 770, 755, 789, 756, 758, 778, 812, 775, 750, 819, 793, 751, 773, 802, 778, 796, 763, 768, 798, 790, 783, 766, 733, 756, 755, 742, 716, 771, 730, 687, 748, 769, 788, 761, 765, 761, 781, 780, 786, 767] payload_rows_per_dst=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] dup_rows_per_dst=[2990, 3004, 3054, 3058, 3040, 2942, 2768, 3020] combine_rows_per_src=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65006 dup_rows=24048 dup_rows_remote=21038 dedup_ratio=0.244629 router_tokens_per_expert=[783, 805, 741, 788, 736, 769, 757, 798, 769, 773, 754, 747, 720, 745, 744, 777, 780, 738, 780, 771, 759, 768, 788, 766, 754, 786, 759, 799, 761, 764, 802, 777, 783, 775, 721, 801, 748, 792, 797, 780, 781, 762, 719, 789, 815, 779, 776, 734, 791, 789, 828, 795, 794, 763, 735, 758, 767, 672, 747, 751, 758, 789, 763, 742] payload_rows_per_dst=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] dup_rows_per_dst=[2954, 2912, 2796, 3140, 2988, 2964, 3284, 3010] combine_rows_per_src=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64726 dup_rows=24140 dup_rows_remote=20828 dedup_ratio=0.245565 router_tokens_per_expert=[788, 807, 782, 768, 783, 752, 749, 761, 776, 787, 812, 746, 791, 768, 761, 774, 761, 753, 755, 752, 744, 761, 702, 787, 712, 781, 747, 819, 807, 754, 783, 729, 723, 758, 787, 763, 798, 748, 795, 757, 751, 746, 798, 802, 764, 806, 793, 719, 754, 750, 764, 786, 778, 811, 748, 769, 737, 753, 796, 769, 783, 754, 756, 784] payload_rows_per_dst=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] dup_rows_per_dst=[3312, 3080, 2902, 2930, 3052, 2842, 3012, 3010] combine_rows_per_src=[9438, 9424, 9058, 9334, 9206, 9232, 9218, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64990 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[767, 776, 746, 764, 731, 773, 781, 791, 752, 745, 775, 786, 796, 740, 776, 781, 778, 741, 788, 760, 750, 780, 761, 791, 806, 795, 796, 691, 804, 703, 757, 729, 789, 764, 734, 757, 791, 772, 753, 826, 772, 787, 775, 784, 750, 760, 782, 783, 820, 782, 751, 756, 767, 752, 812, 783, 759, 704, 788, 772, 746, 754, 756, 761] payload_rows_per_dst=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] dup_rows_per_dst=[3022, 3060, 2916, 2978, 3046, 2996, 3024, 2882] combine_rows_per_src=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65078 dup_rows=23908 dup_rows_remote=20896 dedup_ratio=0.243205 router_tokens_per_expert=[732, 784, 723, 715, 768, 724, 738, 736, 766, 773, 744, 772, 789, 768, 761, 735, 772, 783, 779, 764, 748, 760, 772, 787, 743, 758, 841, 796, 757, 747, 818, 797, 738, 778, 757, 733, 788, 803, 743, 808, 791, 746, 798, 771, 809, 755, 809, 787, 790, 749, 776, 744, 713, 747, 796, 803, 777, 787, 775, 758, 788, 759, 788, 738] payload_rows_per_dst=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] dup_rows_per_dst=[2738, 2988, 3012, 3080, 2966, 3106, 3036, 2982] combine_rows_per_src=[9102, 9228, 9318, 9434, 9330, 9426, 9200, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65270 dup_rows=23762 dup_rows_remote=20956 dedup_ratio=0.241720 router_tokens_per_expert=[824, 786, 777, 737, 798, 798, 779, 787, 753, 764, 759, 729, 737, 770, 758, 780, 774, 803, 786, 777, 760, 789, 748, 783, 747, 706, 726, 773, 765, 764, 802, 756, 761, 739, 759, 821, 719, 754, 705, 799, 806, 741, 816, 777, 757, 748, 763, 766, 748, 750, 775, 780, 793, 730, 755, 793, 774, 777, 816, 753, 793, 750, 784, 755] payload_rows_per_dst=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] dup_rows_per_dst=[3120, 3008, 3028, 2806, 2810, 3028, 2958, 3004] combine_rows_per_src=[9452, 9092, 9412, 9272, 9304, 9320, 9290, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65092 dup_rows=24024 dup_rows_remote=21016 dedup_ratio=0.244385 router_tokens_per_expert=[734, 749, 774, 796, 786, 794, 757, 771, 731, 746, 747, 755, 802, 769, 747, 794, 786, 816, 781, 772, 760, 751, 750, 793, 817, 789, 854, 769, 799, 818, 827, 774, 734, 773, 763, 754, 777, 773, 780, 744, 767, 765, 733, 747, 762, 746, 794, 747, 753, 768, 744, 783, 743, 749, 728, 782, 756, 738, 782, 712, 736, 738, 809, 764] payload_rows_per_dst=[9322, 9184, 9388, 9604, 9188, 9184, 9208, 9202] dup_rows_per_dst=[3000, 2998, 3030, 3290, 3008, 2938, 2892, 2868] combine_rows_per_src=[9322, 9184, 9388, 9604, 9188, 9184, 9208, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64990 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[767, 776, 746, 764, 731, 773, 781, 791, 752, 745, 775, 786, 796, 740, 776, 781, 778, 741, 788, 760, 750, 780, 761, 791, 806, 795, 796, 691, 804, 703, 757, 729, 789, 764, 734, 757, 791, 772, 753, 826, 772, 787, 775, 784, 750, 760, 782, 783, 820, 782, 751, 756, 767, 752, 812, 783, 759, 704, 788, 772, 746, 754, 756, 761] payload_rows_per_dst=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] dup_rows_per_dst=[3022, 3060, 2916, 2978, 3046, 2996, 3024, 2882] combine_rows_per_src=[9236, 9242, 9382, 9184, 9326, 9390, 9422, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65222 dup_rows=23778 dup_rows_remote=20852 dedup_ratio=0.241882 router_tokens_per_expert=[808, 743, 749, 736, 786, 779, 736, 778, 764, 763, 815, 721, 795, 759, 759, 766, 801, 768, 755, 789, 769, 714, 768, 752, 746, 758, 769, 743, 751, 746, 763, 806, 789, 783, 817, 760, 787, 721, 767, 761, 765, 803, 761, 792, 771, 770, 707, 767, 756, 805, 764, 792, 743, 733, 772, 793, 770, 773, 773, 811, 802, 802, 744, 743] payload_rows_per_dst=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] dup_rows_per_dst=[2926, 3306, 2874, 2638, 3004, 3002, 2994, 3034] combine_rows_per_src=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344176 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608752 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873328 weight_movement_bytes=34603008 total_nvlink_bytes=567476336 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=342216 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265459912 ingress_bytes=299720704 link_bytes=299720704 token_bytes=530577608 weight_movement_bytes=34603008 total_nvlink_bytes=565180616 weight_to_token_ratio=0.065218 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127824880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343672 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542712 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741752 weight_movement_bytes=69206016 total_nvlink_bytes=601947768 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65202 dup_rows=23844 dup_rows_remote=20976 dedup_ratio=0.242554 router_tokens_per_expert=[814, 767, 746, 758, 775, 807, 775, 755, 786, 763, 778, 763, 769, 805, 748, 820, 767, 785, 756, 784, 743, 766, 770, 774, 748, 758, 780, 756, 737, 772, 765, 747, 728, 752, 777, 810, 789, 733, 803, 824, 770, 802, 750, 770, 760, 715, 766, 769, 773, 757, 789, 816, 766, 748, 733, 752, 751, 779, 756, 730, 790, 737, 727, 793] payload_rows_per_dst=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] dup_rows_per_dst=[3022, 3036, 3014, 2868, 3114, 2888, 2970, 2932] combine_rows_per_src=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65006 dup_rows=24048 dup_rows_remote=21038 dedup_ratio=0.244629 router_tokens_per_expert=[783, 805, 741, 788, 736, 769, 757, 798, 769, 773, 754, 747, 720, 745, 744, 777, 780, 738, 780, 771, 759, 768, 788, 766, 754, 786, 759, 799, 761, 764, 802, 777, 783, 775, 721, 801, 748, 792, 797, 780, 781, 762, 719, 789, 815, 779, 776, 734, 791, 789, 828, 795, 794, 763, 735, 758, 767, 672, 747, 751, 758, 789, 763, 742] payload_rows_per_dst=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] dup_rows_per_dst=[2954, 2912, 2796, 3140, 2988, 2964, 3284, 3010] combine_rows_per_src=[9400, 9146, 8982, 9396, 9322, 9172, 9588, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344056 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165688 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987320 weight_movement_bytes=34603008 total_nvlink_bytes=568590328 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73964 payload_rows_remote=64752 dup_rows=24340 dup_rows_remote=21302 dedup_ratio=0.247599 router_tokens_per_expert=[757, 769, 822, 769, 764, 771, 770, 822, 713, 758, 769, 737, 756, 766, 796, 781, 764, 813, 768, 791, 782, 775, 789, 754, 781, 795, 752, 787, 783, 750, 782, 715, 759, 758, 784, 757, 776, 783, 718, 726, 741, 789, 789, 764, 770, 748, 717, 779, 758, 783, 758, 734, 746, 791, 758, 797, 752, 790, 739, 814, 731, 791, 797, 754] payload_rows_per_dst=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] dup_rows_per_dst=[3132, 3012, 3124, 3048, 3008, 2926, 3038, 3052] combine_rows_per_src=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343896 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266903384 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462872 weight_movement_bytes=103809024 total_nvlink_bytes=637271896 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267345920 dispatch_meta_bytes=344904 combine_payload_bytes=267345920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267690824 ingress_bytes=336551936 link_bytes=336551936 token_bytes=535036744 weight_movement_bytes=69206016 total_nvlink_bytes=604242760 weight_to_token_ratio=0.129348 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266417841e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344432 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266961264 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533578096 weight_movement_bytes=69206016 total_nvlink_bytes=602784112 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=343672 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266542712 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532741752 weight_movement_bytes=69206016 total_nvlink_bytes=601947768 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73964 payload_rows_remote=64752 dup_rows=24340 dup_rows_remote=21302 dedup_ratio=0.247599 router_tokens_per_expert=[757, 769, 822, 769, 764, 771, 770, 822, 713, 758, 769, 737, 756, 766, 796, 781, 764, 813, 768, 791, 782, 775, 789, 754, 781, 795, 752, 787, 783, 750, 782, 715, 759, 758, 784, 757, 776, 783, 718, 726, 741, 789, 789, 764, 770, 748, 717, 779, 758, 783, 758, 734, 746, 791, 758, 797, 752, 790, 739, 814, 731, 791, 797, 754] payload_rows_per_dst=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] dup_rows_per_dst=[3132, 3012, 3124, 3048, 3008, 2926, 3038, 3052] combine_rows_per_src=[9356, 9140, 9348, 9242, 9114, 9268, 9212, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65142 dup_rows=23876 dup_rows_remote=20872 dedup_ratio=0.242879 router_tokens_per_expert=[804, 758, 777, 748, 752, 778, 744, 759, 770, 759, 784, 778, 746, 781, 780, 747, 772, 758, 770, 739, 744, 792, 816, 777, 770, 755, 789, 756, 758, 778, 812, 775, 750, 819, 793, 751, 773, 802, 778, 796, 763, 768, 798, 790, 783, 766, 733, 756, 755, 742, 716, 771, 730, 687, 748, 769, 788, 761, 765, 761, 781, 780, 786, 767] payload_rows_per_dst=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] dup_rows_per_dst=[2990, 3004, 3054, 3058, 3040, 2942, 2768, 3020] combine_rows_per_src=[9250, 9286, 9282, 9328, 9484, 9372, 9068, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344712 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267412104 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534479496 weight_movement_bytes=103809024 total_nvlink_bytes=638288520 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344176 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608752 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532873328 weight_movement_bytes=34603008 total_nvlink_bytes=567476336 weight_to_token_ratio=0.064937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344296 combine_payload_bytes=267149312 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267493608 ingress_bytes=267149312 link_bytes=267493608 token_bytes=534642920 weight_movement_bytes=0 total_nvlink_bytes=534642920 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.006556912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265224192 dispatch_meta_bytes=344216 combine_payload_bytes=265224192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265568408 ingress_bytes=299827200 link_bytes=299827200 token_bytes=530792600 weight_movement_bytes=34603008 total_nvlink_bytes=565395608 weight_to_token_ratio=0.065191 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128225616e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65092 dup_rows=23980 dup_rows_remote=21074 dedup_ratio=0.243937 router_tokens_per_expert=[716, 761, 734, 765, 802, 772, 778, 816, 759, 771, 746, 746, 756, 763, 798, 730, 776, 753, 768, 746, 784, 770, 772, 751, 790, 749, 718, 803, 784, 775, 775, 810, 780, 788, 721, 825, 795, 756, 747, 761, 816, 759, 791, 760, 750, 765, 767, 816, 783, 759, 810, 795, 784, 771, 720, 751, 729, 776, 739, 756, 762, 760, 757, 766] payload_rows_per_dst=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] dup_rows_per_dst=[3038, 2906, 2952, 3016, 3068, 3128, 3066, 2806] combine_rows_per_src=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64850 dup_rows=24048 dup_rows_remote=21030 dedup_ratio=0.244629 router_tokens_per_expert=[774, 777, 781, 801, 795, 816, 729, 718, 751, 778, 792, 731, 756, 712, 695, 762, 746, 790, 753, 741, 754, 757, 729, 890, 742, 811, 759, 721, 795, 809, 753, 796, 723, 834, 747, 710, 740, 771, 773, 769, 755, 808, 786, 738, 788, 807, 780, 790, 780, 703, 740, 797, 762, 792, 800, 747, 803, 748, 762, 774, 735, 787, 832, 757] payload_rows_per_dst=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] dup_rows_per_dst=[3086, 2844, 3016, 3106, 2972, 3050, 2956, 3018] combine_rows_per_src=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344056 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165688 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987320 weight_movement_bytes=34603008 total_nvlink_bytes=568590328 weight_to_token_ratio=0.064801 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65222 dup_rows=23778 dup_rows_remote=20852 dedup_ratio=0.241882 router_tokens_per_expert=[808, 743, 749, 736, 786, 779, 736, 778, 764, 763, 815, 721, 795, 759, 759, 766, 801, 768, 755, 789, 769, 714, 768, 752, 746, 758, 769, 743, 751, 746, 763, 806, 789, 783, 817, 760, 787, 721, 767, 761, 765, 803, 761, 792, 771, 770, 707, 767, 756, 805, 764, 792, 743, 733, 772, 793, 770, 773, 773, 811, 802, 802, 744, 743] payload_rows_per_dst=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] dup_rows_per_dst=[2926, 3306, 2874, 2638, 3004, 3002, 2994, 3034] combine_rows_per_src=[9304, 9698, 9358, 8806, 9366, 9270, 9322, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64824 dup_rows=24110 dup_rows_remote=21062 dedup_ratio=0.245260 router_tokens_per_expert=[727, 793, 733, 739, 811, 759, 769, 740, 759, 786, 796, 770, 765, 741, 764, 691, 771, 762, 734, 738, 809, 736, 718, 805, 788, 802, 773, 748, 782, 786, 762, 752, 750, 795, 782, 807, 776, 801, 776, 774, 769, 732, 767, 796, 784, 796, 761, 804, 697, 760, 787, 769, 773, 812, 755, 750, 774, 775, 729, 762, 801, 744, 787, 798] payload_rows_per_dst=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] dup_rows_per_dst=[2868, 3006, 2978, 3026, 3120, 3048, 2982, 3082] combine_rows_per_src=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65316 dup_rows=23782 dup_rows_remote=20704 dedup_ratio=0.241923 router_tokens_per_expert=[733, 731, 782, 807, 756, 752, 752, 768, 748, 764, 772, 782, 762, 782, 779, 737, 762, 764, 779, 757, 776, 783, 793, 728, 794, 788, 728, 734, 748, 792, 756, 769, 774, 790, 807, 747, 805, 749, 764, 739, 782, 741, 786, 758, 796, 791, 755, 771, 804, 744, 741, 774, 763, 723, 788, 777, 781, 785, 770, 749, 760, 803, 764, 813] payload_rows_per_dst=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] dup_rows_per_dst=[2936, 2950, 3078, 2848, 2932, 3052, 2986, 3000] combine_rows_per_src=[9226, 9302, 9206, 9370, 9418, 9308, 9242, 9450] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65202 dup_rows=23844 dup_rows_remote=20976 dedup_ratio=0.242554 router_tokens_per_expert=[814, 767, 746, 758, 775, 807, 775, 755, 786, 763, 778, 763, 769, 805, 748, 820, 767, 785, 756, 784, 743, 766, 770, 774, 748, 758, 780, 756, 737, 772, 765, 747, 728, 752, 777, 810, 789, 733, 803, 824, 770, 802, 750, 770, 760, 715, 766, 769, 773, 757, 789, 816, 766, 748, 733, 752, 751, 779, 756, 730, 790, 737, 727, 793] payload_rows_per_dst=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] dup_rows_per_dst=[3022, 3036, 3014, 2868, 3114, 2888, 2970, 2932] combine_rows_per_src=[9372, 9428, 9276, 9258, 9318, 9316, 9298, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65066 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[752, 746, 767, 728, 762, 713, 765, 744, 785, 736, 752, 767, 788, 783, 797, 814, 752, 788, 743, 751, 789, 703, 755, 739, 788, 802, 774, 810, 795, 780, 809, 732, 801, 766, 732, 743, 745, 747, 785, 764, 766, 807, 770, 817, 784, 711, 783, 747, 772, 757, 785, 773, 791, 765, 733, 756, 785, 752, 778, 749, 799, 823, 778, 779] payload_rows_per_dst=[9126, 9414, 9078, 9354, 9302, 9398, 9238, 9458] dup_rows_per_dst=[2828, 3030, 2962, 3226, 2864, 2972, 3026, 3028] combine_rows_per_src=[9126, 9414, 9078, 9354, 9302, 9398, 9238, 9458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74194 payload_rows_remote=64824 dup_rows=24110 dup_rows_remote=21062 dedup_ratio=0.245260 router_tokens_per_expert=[727, 793, 733, 739, 811, 759, 769, 740, 759, 786, 796, 770, 765, 741, 764, 691, 771, 762, 734, 738, 809, 736, 718, 805, 788, 802, 773, 748, 782, 786, 762, 752, 750, 795, 782, 807, 776, 801, 776, 774, 769, 732, 767, 796, 784, 796, 761, 804, 697, 760, 787, 769, 773, 812, 755, 750, 774, 775, 729, 762, 801, 744, 787, 798] payload_rows_per_dst=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] dup_rows_per_dst=[2868, 3006, 2978, 3026, 3120, 3048, 2982, 3082] combine_rows_per_src=[9274, 9138, 9168, 9360, 9402, 9370, 9224, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64874 dup_rows=24118 dup_rows_remote=21050 dedup_ratio=0.245341 router_tokens_per_expert=[774, 820, 747, 769, 737, 721, 852, 720, 736, 774, 729, 746, 759, 744, 741, 793, 801, 789, 785, 799, 755, 732, 770, 778, 758, 774, 770, 796, 788, 783, 769, 752, 784, 742, 756, 768, 766, 768, 787, 711, 751, 774, 827, 751, 722, 789, 780, 720, 777, 733, 816, 770, 807, 779, 780, 816, 780, 741, 772, 721, 800, 755, 782, 766] payload_rows_per_dst=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] dup_rows_per_dst=[3030, 2880, 3088, 3068, 2966, 2938, 3156, 2992] combine_rows_per_src=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343520 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265969120 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594720 weight_movement_bytes=69206016 total_nvlink_bytes=600800736 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344664 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266961496 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533578328 weight_movement_bytes=34603008 total_nvlink_bytes=568181336 weight_to_token_ratio=0.064851 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64850 dup_rows=24048 dup_rows_remote=21030 dedup_ratio=0.244629 router_tokens_per_expert=[774, 777, 781, 801, 795, 816, 729, 718, 751, 778, 792, 731, 756, 712, 695, 762, 746, 790, 753, 741, 754, 757, 729, 890, 742, 811, 759, 721, 795, 809, 753, 796, 723, 834, 747, 710, 740, 771, 773, 769, 755, 808, 786, 738, 788, 807, 780, 790, 780, 703, 740, 797, 762, 792, 800, 747, 803, 748, 762, 774, 735, 787, 832, 757] payload_rows_per_dst=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] dup_rows_per_dst=[3086, 2844, 3016, 3106, 2972, 3050, 2956, 3018] combine_rows_per_src=[9296, 9110, 9304, 9238, 9162, 9454, 9286, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343544 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265862648 ingress_bytes=369328128 link_bytes=369328128 token_bytes=531381752 weight_movement_bytes=103809024 total_nvlink_bytes=635190776 weight_to_token_ratio=0.195357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389752012e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344296 combine_payload_bytes=267149312 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267493608 ingress_bytes=267149312 link_bytes=267493608 token_bytes=534642920 weight_movement_bytes=0 total_nvlink_bytes=534642920 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.006556912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267534336 dispatch_meta_bytes=344080 combine_payload_bytes=267534336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267878416 ingress_bytes=302137344 link_bytes=302137344 token_bytes=535412752 weight_movement_bytes=34603008 total_nvlink_bytes=570015760 weight_to_token_ratio=0.064629 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136918501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65018 dup_rows=23842 dup_rows_remote=20810 dedup_ratio=0.242533 router_tokens_per_expert=[718, 776, 769, 744, 760, 770, 785, 753, 759, 785, 834, 762, 734, 765, 778, 770, 726, 777, 765, 778, 791, 743, 761, 744, 784, 751, 801, 751, 777, 751, 748, 796, 753, 750, 727, 758, 753, 765, 767, 742, 824, 787, 776, 768, 763, 767, 781, 773, 765, 803, 758, 794, 779, 798, 787, 754, 737, 771, 753, 751, 822, 752, 764, 804] payload_rows_per_dst=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] dup_rows_per_dst=[2894, 3118, 2978, 2970, 2868, 3056, 3032, 2926] combine_rows_per_src=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=64690 dup_rows=24030 dup_rows_remote=20782 dedup_ratio=0.244446 router_tokens_per_expert=[814, 757, 743, 688, 784, 746, 810, 770, 755, 766, 771, 778, 822, 726, 765, 750, 822, 777, 766, 797, 787, 731, 725, 782, 772, 778, 800, 761, 774, 733, 801, 761, 725, 766, 792, 813, 793, 741, 791, 733, 742, 758, 792, 756, 770, 752, 778, 727, 751, 755, 826, 746, 768, 846, 763, 771, 741, 770, 739, 754, 757, 755, 799, 770] payload_rows_per_dst=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] dup_rows_per_dst=[3248, 3172, 3230, 2842, 2996, 2674, 2972, 2896] combine_rows_per_src=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=344712 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267412104 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534479496 weight_movement_bytes=103809024 total_nvlink_bytes=638288520 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=344552 combine_payload_bytes=266510336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266854888 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533365224 weight_movement_bytes=34603008 total_nvlink_bytes=567968232 weight_to_token_ratio=0.064877 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343544 combine_payload_bytes=265519104 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265862648 ingress_bytes=369328128 link_bytes=369328128 token_bytes=531381752 weight_movement_bytes=103809024 total_nvlink_bytes=635190776 weight_to_token_ratio=0.195357 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389752012e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343696 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067600 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531791504 weight_movement_bytes=69206016 total_nvlink_bytes=600997520 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65018 dup_rows=23842 dup_rows_remote=20810 dedup_ratio=0.242533 router_tokens_per_expert=[718, 776, 769, 744, 760, 770, 785, 753, 759, 785, 834, 762, 734, 765, 778, 770, 726, 777, 765, 778, 791, 743, 761, 744, 784, 751, 801, 751, 777, 751, 748, 796, 753, 750, 727, 758, 753, 765, 767, 742, 824, 787, 776, 768, 763, 767, 781, 773, 765, 803, 758, 794, 779, 798, 787, 754, 737, 771, 753, 751, 822, 752, 764, 804] payload_rows_per_dst=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] dup_rows_per_dst=[2894, 3118, 2978, 2970, 2868, 3056, 3032, 2926] combine_rows_per_src=[9256, 9256, 9192, 9348, 9162, 9422, 9444, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65092 dup_rows=23980 dup_rows_remote=21074 dedup_ratio=0.243937 router_tokens_per_expert=[716, 761, 734, 765, 802, 772, 778, 816, 759, 771, 746, 746, 756, 763, 798, 730, 776, 753, 768, 746, 784, 770, 772, 751, 790, 749, 718, 803, 784, 775, 775, 810, 780, 788, 721, 825, 795, 756, 747, 761, 816, 759, 791, 760, 750, 765, 767, 816, 783, 759, 810, 795, 784, 771, 720, 751, 729, 776, 739, 756, 762, 760, 757, 766] payload_rows_per_dst=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] dup_rows_per_dst=[3038, 2906, 2952, 3016, 3068, 3128, 3066, 2806] combine_rows_per_src=[9250, 9232, 9288, 9392, 9278, 9320, 9280, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343520 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265969120 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594720 weight_movement_bytes=69206016 total_nvlink_bytes=600800736 weight_to_token_ratio=0.130186 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=343312 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657040 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532970768 weight_movement_bytes=69206016 total_nvlink_bytes=602176784 weight_to_token_ratio=0.129850 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=341888 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312128 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530282368 weight_movement_bytes=34603008 total_nvlink_bytes=564885376 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65146 dup_rows=23866 dup_rows_remote=20900 dedup_ratio=0.242778 router_tokens_per_expert=[729, 791, 774, 765, 768, 748, 782, 704, 741, 709, 826, 757, 783, 768, 781, 764, 785, 793, 777, 734, 704, 812, 734, 758, 809, 772, 776, 752, 789, 776, 808, 739, 743, 760, 769, 776, 759, 748, 780, 807, 805, 790, 782, 776, 818, 734, 730, 766, 769, 745, 745, 744, 731, 786, 819, 798, 761, 764, 757, 766, 757, 801, 801, 757] payload_rows_per_dst=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] dup_rows_per_dst=[2918, 2966, 2960, 3036, 2956, 3020, 3028, 2982] combine_rows_per_src=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64742 dup_rows=23984 dup_rows_remote=20656 dedup_ratio=0.243978 router_tokens_per_expert=[780, 800, 796, 756, 779, 704, 729, 758, 770, 733, 787, 729, 738, 752, 817, 765, 773, 750, 741, 792, 778, 781, 773, 786, 763, 758, 781, 782, 720, 740, 801, 745, 768, 768, 793, 780, 745, 733, 770, 803, 762, 771, 728, 819, 766, 794, 811, 721, 810, 790, 746, 814, 794, 781, 758, 767, 816, 767, 710, 769, 756, 797, 748, 740] payload_rows_per_dst=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] dup_rows_per_dst=[2868, 2900, 3012, 3002, 3004, 3034, 2836, 3328] combine_rows_per_src=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=344664 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266961496 ingress_bytes=301219840 link_bytes=301219840 token_bytes=533578328 weight_movement_bytes=34603008 total_nvlink_bytes=568181336 weight_to_token_ratio=0.064851 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133466008e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65098 dup_rows=23948 dup_rows_remote=21144 dedup_ratio=0.243612 router_tokens_per_expert=[762, 805, 779, 767, 748, 748, 774, 781, 754, 822, 770, 802, 741, 795, 736, 755, 777, 790, 745, 777, 766, 781, 781, 788, 776, 756, 771, 769, 765, 819, 761, 755, 782, 830, 811, 777, 744, 746, 799, 763, 767, 741, 790, 750, 755, 753, 743, 732, 804, 734, 740, 797, 761, 725, 791, 776, 747, 761, 733, 782, 745, 771, 747, 739] payload_rows_per_dst=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] dup_rows_per_dst=[3056, 3030, 3010, 3116, 3062, 2804, 2950, 2920] combine_rows_per_src=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=64690 dup_rows=24030 dup_rows_remote=20782 dedup_ratio=0.244446 router_tokens_per_expert=[814, 757, 743, 688, 784, 746, 810, 770, 755, 766, 771, 778, 822, 726, 765, 750, 822, 777, 766, 797, 787, 731, 725, 782, 772, 778, 800, 761, 774, 733, 801, 761, 725, 766, 792, 813, 793, 741, 791, 733, 742, 758, 792, 756, 770, 752, 778, 727, 751, 755, 826, 746, 768, 846, 763, 771, 741, 770, 739, 754, 757, 755, 799, 770] payload_rows_per_dst=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] dup_rows_per_dst=[3248, 3172, 3230, 2842, 2996, 2674, 2972, 2896] combine_rows_per_src=[9584, 9362, 9584, 9082, 9328, 8848, 9302, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64978 dup_rows=24114 dup_rows_remote=21156 dedup_ratio=0.245300 router_tokens_per_expert=[807, 752, 806, 803, 737, 771, 763, 803, 751, 783, 731, 743, 782, 769, 779, 755, 800, 782, 751, 742, 772, 763, 730, 745, 786, 766, 834, 787, 744, 752, 761, 734, 761, 735, 777, 717, 797, 744, 767, 794, 739, 774, 807, 742, 754, 783, 720, 767, 793, 783, 743, 795, 787, 761, 800, 762, 760, 783, 795, 740, 785, 738, 799, 766] payload_rows_per_dst=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] dup_rows_per_dst=[3122, 3082, 2958, 2986, 2936, 2984, 3112, 2934] combine_rows_per_src=[9362, 9104, 9212, 9342, 9248, 9188, 9336, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64874 dup_rows=24118 dup_rows_remote=21050 dedup_ratio=0.245341 router_tokens_per_expert=[774, 820, 747, 769, 737, 721, 852, 720, 736, 774, 729, 746, 759, 744, 741, 793, 801, 789, 785, 799, 755, 732, 770, 778, 758, 774, 770, 796, 788, 783, 769, 752, 784, 742, 756, 768, 766, 768, 787, 711, 751, 774, 827, 751, 722, 789, 780, 720, 777, 733, 816, 770, 807, 779, 780, 816, 780, 741, 772, 721, 800, 755, 782, 766] payload_rows_per_dst=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] dup_rows_per_dst=[3030, 2880, 3088, 3068, 2966, 2938, 3156, 2992] combine_rows_per_src=[9250, 9164, 9330, 9312, 9198, 9290, 9400, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65046 dup_rows=23926 dup_rows_remote=20890 dedup_ratio=0.243388 router_tokens_per_expert=[844, 770, 753, 757, 814, 715, 754, 782, 795, 776, 755, 721, 774, 749, 783, 813, 799, 767, 791, 753, 769, 729, 768, 779, 767, 774, 726, 747, 757, 782, 771, 761, 773, 756, 754, 767, 801, 756, 773, 804, 812, 744, 768, 774, 788, 796, 759, 745, 825, 741, 750, 764, 762, 768, 789, 782, 763, 755, 764, 752, 756, 750, 755, 711] payload_rows_per_dst=[9368, 9300, 9300, 9264, 9332, 9378, 9262, 9174] dup_rows_per_dst=[3010, 3032, 3010, 2906, 3036, 2994, 3100, 2838] combine_rows_per_src=[9368, 9300, 9300, 9264, 9332, 9378, 9262, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64854 dup_rows=24054 dup_rows_remote=21150 dedup_ratio=0.244690 router_tokens_per_expert=[773, 756, 772, 745, 715, 760, 761, 785, 741, 755, 768, 796, 762, 786, 816, 738, 798, 750, 743, 775, 821, 724, 800, 770, 781, 798, 795, 772, 728, 764, 776, 736, 729, 818, 761, 826, 772, 703, 768, 754, 753, 724, 776, 746, 745, 783, 737, 708, 788, 773, 819, 767, 781, 733, 843, 769, 743, 785, 802, 812, 729, 783, 819, 743] payload_rows_per_dst=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] dup_rows_per_dst=[3000, 3002, 3090, 2904, 2964, 2894, 3114, 3086] combine_rows_per_src=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65098 dup_rows=23948 dup_rows_remote=21144 dedup_ratio=0.243612 router_tokens_per_expert=[762, 805, 779, 767, 748, 748, 774, 781, 754, 822, 770, 802, 741, 795, 736, 755, 777, 790, 745, 777, 766, 781, 781, 788, 776, 756, 771, 769, 765, 819, 761, 755, 782, 830, 811, 777, 744, 746, 799, 763, 767, 741, 790, 750, 755, 753, 743, 732, 804, 734, 740, 797, 761, 725, 791, 776, 747, 761, 733, 782, 745, 771, 747, 739] payload_rows_per_dst=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] dup_rows_per_dst=[3056, 3030, 3010, 3116, 3062, 2804, 2950, 2920] combine_rows_per_src=[9272, 9320, 9400, 9228, 9442, 9258, 9306, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344184 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182200 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020216 weight_movement_bytes=34603008 total_nvlink_bytes=568623224 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341592 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524824 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530708056 weight_movement_bytes=34603008 total_nvlink_bytes=565311064 weight_to_token_ratio=0.065202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266986376 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533627784 weight_movement_bytes=69206016 total_nvlink_bytes=602833800 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=64742 dup_rows=23984 dup_rows_remote=20656 dedup_ratio=0.243978 router_tokens_per_expert=[780, 800, 796, 756, 779, 704, 729, 758, 770, 733, 787, 729, 738, 752, 817, 765, 773, 750, 741, 792, 778, 781, 773, 786, 763, 758, 781, 782, 720, 740, 801, 745, 768, 768, 793, 780, 745, 733, 770, 803, 762, 771, 728, 819, 766, 794, 811, 721, 810, 790, 746, 814, 794, 781, 758, 767, 816, 767, 710, 769, 756, 797, 748, 740] payload_rows_per_dst=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] dup_rows_per_dst=[2868, 2900, 3012, 3002, 3004, 3034, 2836, 3328] combine_rows_per_src=[9186, 9282, 9336, 9208, 9222, 9404, 9104, 9578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=341888 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265312128 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530282368 weight_movement_bytes=34603008 total_nvlink_bytes=564885376 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344536 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494424 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644312 weight_movement_bytes=34603008 total_nvlink_bytes=567247320 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65092 dup_rows=23910 dup_rows_remote=20870 dedup_ratio=0.243225 router_tokens_per_expert=[749, 759, 807, 781, 760, 746, 766, 736, 748, 800, 779, 826, 803, 724, 725, 796, 749, 782, 819, 780, 748, 764, 794, 717, 749, 758, 777, 778, 816, 754, 747, 802, 720, 767, 750, 739, 734, 735, 771, 811, 833, 788, 790, 789, 773, 777, 712, 746, 759, 766, 810, 744, 791, 794, 771, 736, 789, 783, 770, 725, 793, 743, 764, 740] payload_rows_per_dst=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] dup_rows_per_dst=[2930, 3034, 2968, 3056, 2876, 3026, 3040, 2980] combine_rows_per_src=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64868 dup_rows=24090 dup_rows_remote=20986 dedup_ratio=0.245056 router_tokens_per_expert=[754, 788, 779, 755, 808, 786, 760, 794, 785, 752, 794, 764, 743, 786, 732, 784, 765, 704, 752, 753, 795, 760, 750, 759, 732, 761, 755, 756, 778, 736, 783, 769, 775, 766, 783, 786, 808, 804, 769, 770, 783, 763, 732, 780, 790, 782, 806, 806, 724, 759, 752, 745, 778, 783, 741, 781, 760, 787, 748, 765, 784, 736, 765, 769] payload_rows_per_dst=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] dup_rows_per_dst=[3104, 3170, 2884, 2836, 3138, 3030, 2914, 3014] combine_rows_per_src=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=343744 combine_payload_bytes=266428416 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266772160 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533200576 weight_movement_bytes=69206016 total_nvlink_bytes=602406592 weight_to_token_ratio=0.129794 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344016 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986000 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531627984 weight_movement_bytes=34603008 total_nvlink_bytes=566230992 weight_to_token_ratio=0.065089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343696 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266067600 ingress_bytes=334929920 link_bytes=334929920 token_bytes=531791504 weight_movement_bytes=69206016 total_nvlink_bytes=600997520 weight_to_token_ratio=0.130137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260314325e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65092 dup_rows=23910 dup_rows_remote=20870 dedup_ratio=0.243225 router_tokens_per_expert=[749, 759, 807, 781, 760, 746, 766, 736, 748, 800, 779, 826, 803, 724, 725, 796, 749, 782, 819, 780, 748, 764, 794, 717, 749, 758, 777, 778, 816, 754, 747, 802, 720, 767, 750, 739, 734, 735, 771, 811, 833, 788, 790, 789, 773, 777, 712, 746, 759, 766, 810, 744, 791, 794, 771, 736, 789, 783, 770, 725, 793, 743, 764, 740] payload_rows_per_dst=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] dup_rows_per_dst=[2930, 3034, 2968, 3056, 2876, 3026, 3040, 2980] combine_rows_per_src=[9278, 9368, 9338, 9306, 9178, 9390, 9302, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65146 dup_rows=23866 dup_rows_remote=20900 dedup_ratio=0.242778 router_tokens_per_expert=[729, 791, 774, 765, 768, 748, 782, 704, 741, 709, 826, 757, 783, 768, 781, 764, 785, 793, 777, 734, 704, 812, 734, 758, 809, 772, 776, 752, 789, 776, 808, 739, 743, 760, 769, 776, 759, 748, 780, 807, 805, 790, 782, 776, 818, 734, 730, 766, 769, 745, 745, 744, 731, 786, 819, 798, 761, 764, 757, 766, 757, 801, 801, 757] payload_rows_per_dst=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] dup_rows_per_dst=[2918, 2966, 2960, 3036, 2956, 3020, 3028, 2982] combine_rows_per_src=[9204, 9292, 9234, 9406, 9328, 9382, 9246, 9346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344968 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266986376 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533627784 weight_movement_bytes=69206016 total_nvlink_bytes=602833800 weight_to_token_ratio=0.129690 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=341592 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265524824 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530708056 weight_movement_bytes=34603008 total_nvlink_bytes=565311064 weight_to_token_ratio=0.065202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266616832 dispatch_meta_bytes=343848 combine_payload_bytes=266616832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266960680 ingress_bytes=335822848 link_bytes=335822848 token_bytes=533577512 weight_movement_bytes=69206016 total_nvlink_bytes=602783528 weight_to_token_ratio=0.129702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263674341e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343416 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042744 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742072 weight_movement_bytes=103809024 total_nvlink_bytes=635551096 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64982 dup_rows=24066 dup_rows_remote=21096 dedup_ratio=0.244812 router_tokens_per_expert=[790, 755, 787, 749, 740, 737, 788, 765, 742, 774, 737, 773, 766, 780, 750, 791, 738, 787, 800, 741, 763, 789, 774, 791, 757, 726, 745, 802, 751, 786, 766, 772, 727, 785, 726, 759, 758, 818, 760, 734, 817, 772, 785, 761, 772, 766, 731, 777, 824, 764, 737, 762, 774, 734, 784, 784, 739, 802, 787, 769, 766, 773, 815, 778] payload_rows_per_dst=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] dup_rows_per_dst=[3014, 2970, 3038, 2988, 2902, 3032, 3036, 3086] combine_rows_per_src=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64952 dup_rows=23988 dup_rows_remote=20918 dedup_ratio=0.244019 router_tokens_per_expert=[720, 735, 786, 766, 755, 748, 781, 767, 797, 795, 747, 746, 760, 752, 806, 737, 799, 738, 778, 818, 768, 760, 761, 764, 772, 798, 748, 754, 779, 743, 806, 753, 734, 832, 761, 765, 768, 767, 749, 813, 737, 805, 792, 767, 762, 790, 749, 754, 775, 728, 813, 741, 717, 799, 789, 786, 737, 746, 750, 825, 793, 770, 735, 766] payload_rows_per_dst=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] dup_rows_per_dst=[2898, 3030, 2956, 2968, 3126, 3012, 2928, 3070] combine_rows_per_src=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64820 dup_rows=24150 dup_rows_remote=21104 dedup_ratio=0.245667 router_tokens_per_expert=[782, 770, 778, 803, 773, 799, 740, 727, 691, 804, 812, 763, 806, 778, 770, 707, 762, 740, 777, 747, 722, 749, 760, 738, 745, 785, 783, 817, 803, 720, 740, 786, 746, 767, 749, 794, 734, 761, 778, 808, 818, 760, 757, 821, 756, 742, 753, 783, 776, 768, 747, 783, 811, 756, 834, 783, 726, 770, 792, 739, 734, 745, 798, 786] payload_rows_per_dst=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] dup_rows_per_dst=[3094, 3058, 2926, 3010, 3004, 3046, 3046, 2966] combine_rows_per_src=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344184 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182200 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020216 weight_movement_bytes=34603008 total_nvlink_bytes=568623224 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64868 dup_rows=24090 dup_rows_remote=20986 dedup_ratio=0.245056 router_tokens_per_expert=[754, 788, 779, 755, 808, 786, 760, 794, 785, 752, 794, 764, 743, 786, 732, 784, 765, 704, 752, 753, 795, 760, 750, 759, 732, 761, 755, 756, 778, 736, 783, 769, 775, 766, 783, 786, 808, 804, 769, 770, 783, 763, 732, 780, 790, 782, 806, 806, 724, 759, 752, 745, 778, 783, 741, 781, 760, 787, 748, 765, 784, 736, 765, 769] payload_rows_per_dst=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] dup_rows_per_dst=[3104, 3170, 2884, 2836, 3138, 3030, 2914, 3014] combine_rows_per_src=[9346, 9380, 9264, 9232, 9382, 9184, 9212, 9214] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=64806 dup_rows=24064 dup_rows_remote=21086 dedup_ratio=0.244792 router_tokens_per_expert=[721, 769, 789, 753, 734, 773, 763, 781, 816, 740, 802, 818, 776, 782, 721, 724, 803, 763, 761, 810, 715, 797, 780, 777, 739, 795, 815, 789, 741, 745, 820, 742, 738, 774, 746, 739, 775, 803, 769, 785, 779, 744, 779, 726, 733, 773, 774, 763, 740, 800, 751, 764, 783, 762, 715, 755, 772, 766, 767, 798, 769, 772, 806, 778] payload_rows_per_dst=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] dup_rows_per_dst=[2926, 2956, 2978, 3072, 2984, 3006, 2996, 3146] combine_rows_per_src=[9240, 9402, 9434, 9300, 9274, 9136, 9144, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64882 dup_rows=24078 dup_rows_remote=20878 dedup_ratio=0.244934 router_tokens_per_expert=[795, 731, 752, 782, 783, 768, 786, 802, 800, 769, 772, 756, 779, 773, 775, 770, 787, 756, 724, 720, 766, 790, 777, 766, 741, 740, 802, 743, 761, 801, 731, 697, 835, 813, 747, 792, 768, 761, 734, 822, 772, 773, 759, 774, 760, 747, 808, 764, 750, 724, 777, 760, 789, 726, 755, 733, 787, 765, 797, 789, 734, 811, 774, 757] payload_rows_per_dst=[9382, 9340, 9250, 9170, 9344, 9200, 9100, 9440] dup_rows_per_dst=[3016, 3048, 2922, 2862, 3200, 3114, 2928, 2988] combine_rows_per_src=[9382, 9340, 9250, 9170, 9344, 9200, 9100, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64876 dup_rows=24052 dup_rows_remote=21126 dedup_ratio=0.244670 router_tokens_per_expert=[745, 757, 795, 824, 766, 762, 783, 805, 750, 769, 793, 753, 693, 774, 813, 748, 783, 780, 773, 740, 811, 768, 770, 798, 739, 753, 783, 766, 730, 762, 809, 809, 754, 732, 800, 834, 753, 772, 750, 756, 795, 748, 751, 768, 779, 835, 766, 757, 759, 743, 756, 762, 742, 758, 782, 753, 771, 729, 722, 788, 795, 712, 771, 755] payload_rows_per_dst=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] dup_rows_per_dst=[3008, 3020, 3120, 2926, 3050, 3066, 2920, 2942] combine_rows_per_src=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64820 dup_rows=24150 dup_rows_remote=21104 dedup_ratio=0.245667 router_tokens_per_expert=[782, 770, 778, 803, 773, 799, 740, 727, 691, 804, 812, 763, 806, 778, 770, 707, 762, 740, 777, 747, 722, 749, 760, 738, 745, 785, 783, 817, 803, 720, 740, 786, 746, 767, 749, 794, 734, 761, 778, 808, 818, 760, 757, 821, 756, 742, 753, 783, 776, 768, 747, 783, 811, 756, 834, 783, 726, 770, 792, 739, 734, 745, 798, 786] payload_rows_per_dst=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] dup_rows_per_dst=[3094, 3058, 2926, 3010, 3004, 3046, 3046, 2966] combine_rows_per_src=[9250, 9204, 9064, 9348, 9270, 9334, 9470, 9214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344312 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510584 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676856 weight_movement_bytes=34603008 total_nvlink_bytes=567279864 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64854 dup_rows=24054 dup_rows_remote=21150 dedup_ratio=0.244690 router_tokens_per_expert=[773, 756, 772, 745, 715, 760, 761, 785, 741, 755, 768, 796, 762, 786, 816, 738, 798, 750, 743, 775, 821, 724, 800, 770, 781, 798, 795, 772, 728, 764, 776, 736, 729, 818, 761, 826, 772, 703, 768, 754, 753, 724, 776, 746, 745, 783, 737, 708, 788, 773, 819, 767, 781, 733, 843, 769, 743, 785, 802, 812, 729, 783, 819, 743] payload_rows_per_dst=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] dup_rows_per_dst=[3000, 3002, 3090, 2904, 2964, 2894, 3114, 3086] combine_rows_per_src=[9134, 9322, 9272, 9396, 9298, 9050, 9432, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343480 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266386872 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430264 weight_movement_bytes=69206016 total_nvlink_bytes=601636280 weight_to_token_ratio=0.129981 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64952 dup_rows=23988 dup_rows_remote=20918 dedup_ratio=0.244019 router_tokens_per_expert=[720, 735, 786, 766, 755, 748, 781, 767, 797, 795, 747, 746, 760, 752, 806, 737, 799, 738, 778, 818, 768, 760, 761, 764, 772, 798, 748, 754, 779, 743, 806, 753, 734, 832, 761, 765, 768, 767, 749, 813, 737, 805, 792, 767, 762, 790, 749, 754, 775, 728, 813, 741, 717, 799, 789, 786, 737, 746, 750, 825, 793, 770, 735, 766] payload_rows_per_dst=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] dup_rows_per_dst=[2898, 3030, 2956, 2968, 3126, 3012, 2928, 3070] combine_rows_per_src=[9218, 9250, 9368, 9338, 9298, 9288, 9192, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64496 dup_rows=24296 dup_rows_remote=21130 dedup_ratio=0.247152 router_tokens_per_expert=[771, 736, 729, 820, 752, 734, 740, 813, 767, 754, 771, 764, 754, 800, 739, 771, 777, 741, 779, 735, 804, 783, 773, 782, 748, 777, 802, 749, 714, 782, 747, 751, 768, 779, 788, 803, 753, 806, 788, 738, 774, 783, 780, 764, 781, 762, 801, 772, 797, 766, 783, 838, 758, 749, 736, 793, 745, 730, 747, 770, 741, 744, 765, 791] payload_rows_per_dst=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] dup_rows_per_dst=[3166, 3034, 3002, 3152, 3020, 3136, 3080, 2706] combine_rows_per_src=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343416 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042744 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742072 weight_movement_bytes=103809024 total_nvlink_bytes=635551096 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343568 combine_payload_bytes=265445376 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265788944 ingress_bytes=334651392 link_bytes=334651392 token_bytes=531234320 weight_movement_bytes=69206016 total_nvlink_bytes=600440336 weight_to_token_ratio=0.130274 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259266246e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343040 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266099712 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531856384 weight_movement_bytes=69206016 total_nvlink_bytes=601062400 weight_to_token_ratio=0.130122 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344016 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986000 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531627984 weight_movement_bytes=34603008 total_nvlink_bytes=566230992 weight_to_token_ratio=0.065089 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64894 dup_rows=24056 dup_rows_remote=21188 dedup_ratio=0.244710 router_tokens_per_expert=[777, 777, 750, 792, 760, 757, 778, 732, 750, 723, 754, 772, 766, 806, 747, 778, 800, 746, 756, 722, 775, 756, 816, 794, 805, 776, 781, 746, 727, 775, 767, 745, 754, 787, 765, 776, 764, 765, 771, 770, 771, 766, 797, 824, 785, 742, 786, 743, 740, 758, 753, 760, 739, 813, 779, 769, 792, 785, 790, 716, 752, 765, 804, 765] payload_rows_per_dst=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] dup_rows_per_dst=[2998, 2972, 3082, 3000, 3056, 3022, 2868, 3058] combine_rows_per_src=[9248, 9220, 9248, 9244, 9248, 9406, 9354, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344008 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266076104 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531808200 weight_movement_bytes=69206016 total_nvlink_bytes=601014216 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64982 dup_rows=24066 dup_rows_remote=21096 dedup_ratio=0.244812 router_tokens_per_expert=[790, 755, 787, 749, 740, 737, 788, 765, 742, 774, 737, 773, 766, 780, 750, 791, 738, 787, 800, 741, 763, 789, 774, 791, 757, 726, 745, 802, 751, 786, 766, 772, 727, 785, 726, 759, 758, 818, 760, 734, 817, 772, 785, 761, 772, 766, 731, 777, 824, 764, 737, 762, 774, 734, 784, 784, 739, 802, 787, 769, 766, 773, 815, 778] payload_rows_per_dst=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] dup_rows_per_dst=[3014, 2970, 3038, 2988, 2902, 3032, 3036, 3086] combine_rows_per_src=[9208, 9256, 9328, 9222, 9232, 9330, 9290, 9372] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343480 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266386872 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430264 weight_movement_bytes=69206016 total_nvlink_bytes=601636280 weight_to_token_ratio=0.129981 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264175616 dispatch_meta_bytes=342504 combine_payload_bytes=264175616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264518120 ingress_bytes=298778624 link_bytes=298778624 token_bytes=528693736 weight_movement_bytes=34603008 total_nvlink_bytes=563296744 weight_to_token_ratio=0.065450 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124279908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64492 dup_rows=24134 dup_rows_remote=20808 dedup_ratio=0.245504 router_tokens_per_expert=[765, 720, 761, 801, 764, 769, 751, 794, 773, 784, 717, 729, 760, 774, 771, 726, 766, 766, 786, 776, 763, 742, 773, 832, 777, 761, 781, 750, 771, 747, 797, 737, 743, 803, 756, 749, 811, 766, 780, 778, 776, 789, 732, 735, 757, 750, 804, 772, 743, 767, 759, 835, 764, 769, 782, 771, 770, 749, 779, 801, 794, 718, 798, 768] payload_rows_per_dst=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] dup_rows_per_dst=[2994, 2922, 2928, 2820, 2942, 3122, 3080, 3326] combine_rows_per_src=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344328 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266150152 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955976 weight_movement_bytes=103809024 total_nvlink_bytes=635765000 weight_to_token_ratio=0.195146 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65090 dup_rows=23862 dup_rows_remote=20876 dedup_ratio=0.242737 router_tokens_per_expert=[778, 799, 793, 771, 742, 746, 771, 744, 764, 790, 752, 760, 789, 781, 766, 767, 797, 769, 771, 759, 774, 762, 778, 735, 754, 724, 768, 761, 801, 792, 764, 764, 730, 793, 780, 765, 823, 775, 772, 762, 741, 753, 790, 763, 779, 729, 767, 744, 808, 762, 760, 798, 767, 736, 798, 793, 804, 742, 751, 736, 750, 776, 774, 745] payload_rows_per_dst=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] dup_rows_per_dst=[2968, 2986, 2978, 2936, 2972, 2916, 3136, 2970] combine_rows_per_src=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65228 dup_rows=23798 dup_rows_remote=20808 dedup_ratio=0.242086 router_tokens_per_expert=[776, 765, 722, 784, 768, 801, 772, 783, 736, 729, 758, 773, 712, 796, 818, 777, 749, 763, 732, 745, 740, 752, 773, 815, 789, 788, 785, 801, 767, 751, 756, 765, 770, 812, 738, 797, 755, 767, 746, 749, 764, 696, 780, 828, 789, 766, 790, 746, 785, 759, 785, 772, 793, 752, 753, 729, 773, 755, 760, 806, 809, 735, 760, 792] payload_rows_per_dst=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] dup_rows_per_dst=[2996, 3004, 2922, 2982, 2990, 2916, 2964, 3024] combine_rows_per_src=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64894 dup_rows=24112 dup_rows_remote=21108 dedup_ratio=0.245280 router_tokens_per_expert=[828, 730, 769, 802, 749, 752, 774, 745, 764, 730, 781, 786, 745, 785, 749, 735, 751, 760, 780, 796, 779, 790, 777, 718, 774, 788, 774, 768, 786, 781, 726, 814, 731, 722, 777, 752, 775, 778, 792, 741, 747, 802, 788, 759, 764, 780, 787, 811, 752, 776, 796, 797, 765, 754, 761, 805, 741, 731, 781, 763, 738, 815, 736, 749] payload_rows_per_dst=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] dup_rows_per_dst=[3052, 2904, 3004, 3100, 2876, 3182, 3042, 2952] combine_rows_per_src=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65258 dup_rows=23782 dup_rows_remote=20840 dedup_ratio=0.241923 router_tokens_per_expert=[786, 759, 796, 806, 772, 739, 795, 786, 739, 761, 810, 729, 735, 726, 733, 750, 743, 757, 790, 769, 782, 748, 796, 800, 753, 722, 765, 767, 746, 766, 795, 789, 752, 757, 790, 788, 729, 767, 745, 752, 704, 779, 820, 803, 791, 784, 755, 797, 762, 773, 744, 797, 786, 740, 779, 796, 787, 809, 754, 772, 757, 751, 799, 723] payload_rows_per_dst=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] dup_rows_per_dst=[2978, 2896, 3012, 2942, 2856, 3120, 2982, 2996] combine_rows_per_src=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952504 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561144 weight_movement_bytes=34603008 total_nvlink_bytes=568164152 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65058 dup_rows=23998 dup_rows_remote=21138 dedup_ratio=0.244120 router_tokens_per_expert=[765, 778, 788, 777, 778, 752, 748, 762, 771, 778, 780, 773, 801, 766, 747, 788, 781, 803, 779, 761, 786, 735, 756, 748, 809, 811, 775, 732, 749, 757, 825, 760, 742, 783, 731, 772, 828, 748, 730, 792, 768, 767, 743, 707, 742, 794, 779, 754, 771, 795, 746, 751, 792, 771, 764, 762, 736, 769, 751, 793, 800, 735, 747, 770] payload_rows_per_dst=[9326, 9348, 9290, 9308, 9260, 9248, 9258, 9268] dup_rows_per_dst=[2970, 3060, 3008, 3128, 2992, 2860, 3046, 2934] combine_rows_per_src=[9326, 9348, 9290, 9308, 9260, 9248, 9258, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64492 dup_rows=24134 dup_rows_remote=20808 dedup_ratio=0.245504 router_tokens_per_expert=[765, 720, 761, 801, 764, 769, 751, 794, 773, 784, 717, 729, 760, 774, 771, 726, 766, 766, 786, 776, 763, 742, 773, 832, 777, 761, 781, 750, 771, 747, 797, 737, 743, 803, 756, 749, 811, 766, 780, 778, 776, 789, 732, 735, 757, 750, 804, 772, 743, 767, 759, 835, 764, 769, 782, 771, 770, 749, 779, 801, 794, 718, 798, 768] payload_rows_per_dst=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] dup_rows_per_dst=[2994, 2922, 2928, 2820, 2942, 3122, 3080, 3326] combine_rows_per_src=[9212, 9146, 9152, 8914, 9372, 9316, 9380, 9678] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21024 dedup_ratio=0.244670 router_tokens_per_expert=[812, 804, 752, 767, 719, 770, 739, 800, 745, 770, 717, 762, 798, 800, 758, 762, 790, 790, 788, 758, 746, 758, 752, 826, 810, 823, 804, 751, 775, 700, 789, 798, 746, 763, 798, 763, 795, 763, 727, 780, 732, 778, 732, 779, 771, 773, 756, 771, 801, 796, 766, 769, 783, 767, 780, 712, 783, 762, 715, 735, 751, 779, 734, 759] payload_rows_per_dst=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] dup_rows_per_dst=[3028, 3304, 3048, 3098, 2954, 2986, 2726, 2908] combine_rows_per_src=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344144 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267518032 ingress_bytes=336379904 link_bytes=336379904 token_bytes=534691920 weight_movement_bytes=69206016 total_nvlink_bytes=603897936 weight_to_token_ratio=0.129432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265770498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344312 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510584 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676856 weight_movement_bytes=34603008 total_nvlink_bytes=567279864 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264159232 dispatch_meta_bytes=341200 combine_payload_bytes=264159232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264500432 ingress_bytes=298762240 link_bytes=298762240 token_bytes=528659664 weight_movement_bytes=34603008 total_nvlink_bytes=563262672 weight_to_token_ratio=0.065454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124218257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64894 dup_rows=24112 dup_rows_remote=21108 dedup_ratio=0.245280 router_tokens_per_expert=[828, 730, 769, 802, 749, 752, 774, 745, 764, 730, 781, 786, 745, 785, 749, 735, 751, 760, 780, 796, 779, 790, 777, 718, 774, 788, 774, 768, 786, 781, 726, 814, 731, 722, 777, 752, 775, 778, 792, 741, 747, 802, 788, 759, 764, 780, 787, 811, 752, 776, 796, 797, 765, 754, 761, 805, 741, 731, 781, 763, 738, 815, 736, 749] payload_rows_per_dst=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] dup_rows_per_dst=[3052, 2904, 3004, 3100, 2876, 3182, 3042, 2952] combine_rows_per_src=[9246, 9246, 9298, 9322, 9260, 9294, 9370, 9156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65228 dup_rows=23798 dup_rows_remote=20808 dedup_ratio=0.242086 router_tokens_per_expert=[776, 765, 722, 784, 768, 801, 772, 783, 736, 729, 758, 773, 712, 796, 818, 777, 749, 763, 732, 745, 740, 752, 773, 815, 789, 788, 785, 801, 767, 751, 756, 765, 770, 812, 738, 797, 755, 767, 746, 749, 764, 696, 780, 828, 789, 766, 790, 746, 785, 759, 785, 772, 793, 752, 753, 729, 773, 755, 760, 806, 809, 735, 760, 792] payload_rows_per_dst=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] dup_rows_per_dst=[2996, 3004, 2922, 2982, 2990, 2916, 2964, 3024] combine_rows_per_src=[9346, 9194, 9216, 9422, 9278, 9402, 9292, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64496 dup_rows=24296 dup_rows_remote=21130 dedup_ratio=0.247152 router_tokens_per_expert=[771, 736, 729, 820, 752, 734, 740, 813, 767, 754, 771, 764, 754, 800, 739, 771, 777, 741, 779, 735, 804, 783, 773, 782, 748, 777, 802, 749, 714, 782, 747, 751, 768, 779, 788, 803, 753, 806, 788, 738, 774, 783, 780, 764, 781, 762, 801, 772, 797, 766, 783, 838, 758, 749, 736, 793, 745, 730, 747, 770, 741, 744, 765, 791] payload_rows_per_dst=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] dup_rows_per_dst=[3166, 3034, 3002, 3152, 3020, 3136, 3080, 2706] combine_rows_per_src=[9512, 9240, 9346, 9330, 9084, 9298, 9326, 8872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64876 dup_rows=24052 dup_rows_remote=21126 dedup_ratio=0.244670 router_tokens_per_expert=[745, 757, 795, 824, 766, 762, 783, 805, 750, 769, 793, 753, 693, 774, 813, 748, 783, 780, 773, 740, 811, 768, 770, 798, 739, 753, 783, 766, 730, 762, 809, 809, 754, 732, 800, 834, 753, 772, 750, 756, 795, 748, 751, 768, 779, 835, 766, 757, 759, 743, 756, 762, 742, 758, 782, 753, 771, 729, 722, 788, 795, 712, 771, 755] payload_rows_per_dst=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] dup_rows_per_dst=[3008, 3020, 3120, 2926, 3050, 3066, 2920, 2942] combine_rows_per_src=[9466, 9166, 9326, 9376, 9252, 9332, 9190, 9144] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344784 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822352 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299920 weight_movement_bytes=34603008 total_nvlink_bytes=567902928 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=344008 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149832 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955656 weight_movement_bytes=103809024 total_nvlink_bytes=635764680 weight_to_token_ratio=0.195146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344392 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267641160 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937928 weight_movement_bytes=69206016 total_nvlink_bytes=604143944 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344144 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267518032 ingress_bytes=336379904 link_bytes=336379904 token_bytes=534691920 weight_movement_bytes=69206016 total_nvlink_bytes=603897936 weight_to_token_ratio=0.129432 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265770498e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=64956 dup_rows=23898 dup_rows_remote=20802 dedup_ratio=0.243103 router_tokens_per_expert=[766, 748, 763, 787, 733, 753, 742, 745, 718, 741, 744, 756, 789, 762, 798, 770, 758, 773, 788, 760, 784, 774, 750, 780, 749, 768, 790, 789, 742, 765, 779, 743, 758, 789, 736, 775, 779, 810, 765, 769, 791, 803, 770, 797, 755, 761, 723, 725, 735, 808, 791, 809, 790, 762, 771, 807, 731, 764, 773, 751, 792, 794, 797, 764] payload_rows_per_dst=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] dup_rows_per_dst=[2766, 2938, 3000, 3006, 3030, 3034, 3096, 3028] combine_rows_per_src=[9308, 9218, 9334, 9244, 9332, 9216, 9450, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344784 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822352 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299920 weight_movement_bytes=34603008 total_nvlink_bytes=567902928 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264175616 dispatch_meta_bytes=342504 combine_payload_bytes=264175616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264518120 ingress_bytes=298778624 link_bytes=298778624 token_bytes=528693736 weight_movement_bytes=34603008 total_nvlink_bytes=563296744 weight_to_token_ratio=0.065450 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124279908e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=344008 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266076104 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531808200 weight_movement_bytes=69206016 total_nvlink_bytes=601014216 weight_to_token_ratio=0.130133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264159232 dispatch_meta_bytes=341200 combine_payload_bytes=264159232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264500432 ingress_bytes=298762240 link_bytes=298762240 token_bytes=528659664 weight_movement_bytes=34603008 total_nvlink_bytes=563262672 weight_to_token_ratio=0.065454 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.124218257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343912 combine_payload_bytes=266051584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266395496 ingress_bytes=335257600 link_bytes=335257600 token_bytes=532447080 weight_movement_bytes=69206016 total_nvlink_bytes=601653096 weight_to_token_ratio=0.129977 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65090 dup_rows=23862 dup_rows_remote=20876 dedup_ratio=0.242737 router_tokens_per_expert=[778, 799, 793, 771, 742, 746, 771, 744, 764, 790, 752, 760, 789, 781, 766, 767, 797, 769, 771, 759, 774, 762, 778, 735, 754, 724, 768, 761, 801, 792, 764, 764, 730, 793, 780, 765, 823, 775, 772, 762, 741, 753, 790, 763, 779, 729, 767, 744, 808, 762, 760, 798, 767, 736, 798, 793, 804, 742, 751, 736, 750, 776, 774, 745] payload_rows_per_dst=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] dup_rows_per_dst=[2968, 2986, 2978, 2936, 2972, 2916, 3136, 2970] combine_rows_per_src=[9320, 9352, 9312, 9320, 9428, 9216, 9308, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64672 dup_rows=24280 dup_rows_remote=21132 dedup_ratio=0.246989 router_tokens_per_expert=[762, 781, 745, 788, 792, 765, 731, 767, 720, 751, 747, 776, 769, 745, 746, 786, 747, 856, 792, 758, 748, 798, 783, 785, 727, 805, 755, 756, 742, 771, 775, 785, 740, 815, 741, 787, 766, 682, 777, 806, 797, 755, 759, 751, 735, 767, 716, 789, 803, 769, 745, 761, 776, 803, 783, 733, 808, 825, 754, 725, 815, 819, 744, 752] payload_rows_per_dst=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] dup_rows_per_dst=[2880, 2914, 3126, 2966, 3076, 3214, 2956, 3148] combine_rows_per_src=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343032 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266402808 ingress_bytes=335265792 link_bytes=335265792 token_bytes=532462584 weight_movement_bytes=69206016 total_nvlink_bytes=601668600 weight_to_token_ratio=0.129973 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261578184e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65022 dup_rows=24054 dup_rows_remote=21070 dedup_ratio=0.244690 router_tokens_per_expert=[775, 745, 781, 815, 776, 805, 767, 759, 761, 820, 746, 746, 701, 765, 804, 763, 803, 780, 748, 763, 754, 742, 770, 802, 822, 765, 785, 782, 790, 753, 775, 756, 844, 777, 772, 739, 754, 808, 766, 777, 791, 727, 745, 743, 774, 792, 756, 733, 738, 778, 787, 741, 698, 804, 728, 769, 723, 756, 796, 781, 768, 759, 748, 761] payload_rows_per_dst=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] dup_rows_per_dst=[3070, 2984, 2956, 3114, 3074, 2938, 2976, 2942] combine_rows_per_src=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64898 dup_rows=24118 dup_rows_remote=21030 dedup_ratio=0.245341 router_tokens_per_expert=[749, 762, 791, 777, 798, 784, 753, 819, 780, 783, 788, 758, 728, 822, 753, 811, 791, 795, 769, 750, 791, 710, 758, 757, 757, 780, 781, 723, 740, 772, 790, 726, 737, 818, 829, 760, 732, 758, 775, 779, 740, 761, 747, 786, 788, 737, 753, 774, 792, 785, 777, 775, 734, 749, 796, 759, 714, 761, 800, 737, 833, 769, 724, 727] payload_rows_per_dst=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] dup_rows_per_dst=[3072, 3192, 2918, 2940, 3088, 2960, 3020, 2928] combine_rows_per_src=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64910 dup_rows=24068 dup_rows_remote=21166 dedup_ratio=0.244832 router_tokens_per_expert=[767, 781, 786, 803, 772, 752, 731, 752, 777, 768, 799, 762, 734, 787, 712, 778, 808, 823, 740, 791, 781, 789, 756, 750, 747, 729, 757, 786, 779, 796, 738, 782, 803, 746, 745, 729, 734, 783, 769, 750, 729, 787, 808, 738, 770, 792, 791, 791, 738, 770, 761, 770, 743, 760, 818, 759, 768, 783, 790, 757, 792, 747, 726, 792] payload_rows_per_dst=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] dup_rows_per_dst=[2904, 2936, 3194, 2902, 3018, 3056, 2994, 3064] combine_rows_per_src=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65258 dup_rows=23672 dup_rows_remote=20632 dedup_ratio=0.240804 router_tokens_per_expert=[768, 717, 740, 772, 762, 757, 780, 798, 782, 762, 788, 776, 795, 792, 776, 811, 783, 783, 776, 753, 748, 803, 782, 779, 779, 760, 795, 764, 771, 722, 741, 782, 746, 806, 739, 748, 769, 765, 769, 780, 814, 741, 792, 743, 734, 735, 776, 780, 761, 763, 763, 790, 706, 766, 725, 753, 734, 785, 779, 777, 781, 754, 810, 771] payload_rows_per_dst=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] dup_rows_per_dst=[2878, 3128, 3040, 2934, 2952, 2956, 2794, 2990] combine_rows_per_src=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344368 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674480 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004592 weight_movement_bytes=34603008 total_nvlink_bytes=567607600 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=65008 dup_rows=24082 dup_rows_remote=21184 dedup_ratio=0.244975 router_tokens_per_expert=[797, 767, 760, 788, 809, 783, 746, 763, 791, 741, 771, 742, 786, 777, 770, 719, 786, 834, 760, 795, 755, 762, 759, 771, 776, 750, 806, 752, 752, 783, 786, 729, 738, 758, 731, 734, 750, 736, 765, 729, 771, 724, 748, 778, 758, 759, 733, 785, 824, 731, 780, 741, 799, 804, 800, 782, 776, 826, 800, 748, 790, 759, 825, 704] payload_rows_per_dst=[9346, 9196, 9412, 9196, 9084, 9214, 9376, 9398] dup_rows_per_dst=[3080, 2998, 3032, 3072, 2798, 2898, 3146, 3058] combine_rows_per_src=[9346, 9196, 9412, 9196, 9084, 9214, 9376, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64672 dup_rows=24280 dup_rows_remote=21132 dedup_ratio=0.246989 router_tokens_per_expert=[762, 781, 745, 788, 792, 765, 731, 767, 720, 751, 747, 776, 769, 745, 746, 786, 747, 856, 792, 758, 748, 798, 783, 785, 727, 805, 755, 756, 742, 771, 775, 785, 740, 815, 741, 787, 766, 682, 777, 806, 797, 755, 759, 751, 735, 767, 716, 789, 803, 769, 745, 761, 776, 803, 783, 733, 808, 825, 754, 725, 815, 819, 744, 752] payload_rows_per_dst=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] dup_rows_per_dst=[2880, 2914, 3126, 2966, 3076, 3214, 2956, 3148] combine_rows_per_src=[8768, 9166, 9408, 9250, 9398, 9292, 9390, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64766 dup_rows=24118 dup_rows_remote=21080 dedup_ratio=0.245341 router_tokens_per_expert=[771, 765, 770, 779, 815, 749, 756, 772, 772, 774, 739, 720, 750, 710, 750, 704, 788, 741, 795, 774, 825, 722, 742, 791, 789, 796, 763, 776, 786, 794, 780, 784, 766, 800, 789, 793, 794, 798, 748, 739, 762, 758, 778, 745, 795, 770, 760, 765, 758, 746, 778, 776, 758, 779, 758, 773, 774, 747, 767, 750, 758, 767, 761, 800] payload_rows_per_dst=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] dup_rows_per_dst=[3038, 3152, 2850, 2964, 3122, 3088, 3086, 2818] combine_rows_per_src=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344304 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266215664 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532087024 weight_movement_bytes=34603008 total_nvlink_bytes=566690032 weight_to_token_ratio=0.065033 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=343864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952504 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561144 weight_movement_bytes=34603008 total_nvlink_bytes=568164152 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343216 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265239728 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530136240 weight_movement_bytes=103809024 total_nvlink_bytes=633945264 weight_to_token_ratio=0.195816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64974 dup_rows=24126 dup_rows_remote=21242 dedup_ratio=0.245422 router_tokens_per_expert=[785, 754, 766, 780, 729, 773, 751, 759, 776, 777, 746, 811, 737, 760, 766, 787, 733, 794, 770, 764, 755, 789, 738, 751, 762, 780, 788, 729, 752, 748, 814, 813, 746, 797, 786, 746, 730, 756, 785, 774, 773, 780, 777, 748, 805, 777, 753, 802, 778, 761, 740, 723, 760, 743, 772, 767, 800, 768, 808, 771, 790, 760, 770, 769] payload_rows_per_dst=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] dup_rows_per_dst=[2978, 3064, 2964, 2990, 2988, 3172, 2884, 3086] combine_rows_per_src=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=343560 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267640328 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937096 weight_movement_bytes=69206016 total_nvlink_bytes=604143112 weight_to_token_ratio=0.129372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65258 dup_rows=23672 dup_rows_remote=20632 dedup_ratio=0.240804 router_tokens_per_expert=[768, 717, 740, 772, 762, 757, 780, 798, 782, 762, 788, 776, 795, 792, 776, 811, 783, 783, 776, 753, 748, 803, 782, 779, 779, 760, 795, 764, 771, 722, 741, 782, 746, 806, 739, 748, 769, 765, 769, 780, 814, 741, 792, 743, 734, 735, 776, 780, 761, 763, 763, 790, 706, 766, 725, 753, 734, 785, 779, 777, 781, 754, 810, 771] payload_rows_per_dst=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] dup_rows_per_dst=[2878, 3128, 3040, 2934, 2952, 2956, 2794, 2990] combine_rows_per_src=[9310, 9436, 9374, 9294, 9292, 9274, 9260, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64898 dup_rows=24118 dup_rows_remote=21030 dedup_ratio=0.245341 router_tokens_per_expert=[749, 762, 791, 777, 798, 784, 753, 819, 780, 783, 788, 758, 728, 822, 753, 811, 791, 795, 769, 750, 791, 710, 758, 757, 757, 780, 781, 723, 740, 772, 790, 726, 737, 818, 829, 760, 732, 758, 775, 779, 740, 761, 747, 786, 788, 737, 753, 774, 792, 785, 777, 775, 734, 749, 796, 759, 714, 761, 800, 737, 833, 769, 724, 727] payload_rows_per_dst=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] dup_rows_per_dst=[3072, 3192, 2918, 2940, 3088, 2960, 3020, 2928] combine_rows_per_src=[9394, 9254, 9324, 9198, 9288, 9212, 9314, 9202] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344768 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617536 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532890304 weight_movement_bytes=34603008 total_nvlink_bytes=567493312 weight_to_token_ratio=0.064935 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21024 dedup_ratio=0.244670 router_tokens_per_expert=[812, 804, 752, 767, 719, 770, 739, 800, 745, 770, 717, 762, 798, 800, 758, 762, 790, 790, 788, 758, 746, 758, 752, 826, 810, 823, 804, 751, 775, 700, 789, 798, 746, 763, 798, 763, 795, 763, 727, 780, 732, 778, 732, 779, 771, 773, 756, 771, 801, 796, 766, 769, 783, 767, 780, 712, 783, 762, 715, 735, 751, 779, 734, 759] payload_rows_per_dst=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] dup_rows_per_dst=[3028, 3304, 3048, 3098, 2954, 2986, 2726, 2908] combine_rows_per_src=[9298, 9496, 9368, 9402, 9316, 9198, 9046, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65258 dup_rows=23782 dup_rows_remote=20840 dedup_ratio=0.241923 router_tokens_per_expert=[786, 759, 796, 806, 772, 739, 795, 786, 739, 761, 810, 729, 735, 726, 733, 750, 743, 757, 790, 769, 782, 748, 796, 800, 753, 722, 765, 767, 746, 766, 795, 789, 752, 757, 790, 788, 729, 767, 745, 752, 704, 779, 820, 803, 791, 784, 755, 797, 762, 773, 744, 797, 786, 740, 779, 796, 787, 809, 754, 772, 757, 751, 799, 723] payload_rows_per_dst=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] dup_rows_per_dst=[2978, 2896, 3012, 2942, 2856, 3120, 2982, 2996] combine_rows_per_src=[9500, 9070, 9358, 9264, 9304, 9346, 9372, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=343560 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267640328 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937096 weight_movement_bytes=69206016 total_nvlink_bytes=604143112 weight_to_token_ratio=0.129372 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64974 dup_rows=24126 dup_rows_remote=21242 dedup_ratio=0.245422 router_tokens_per_expert=[785, 754, 766, 780, 729, 773, 751, 759, 776, 777, 746, 811, 737, 760, 766, 787, 733, 794, 770, 764, 755, 789, 738, 751, 762, 780, 788, 729, 752, 748, 814, 813, 746, 797, 786, 746, 730, 756, 785, 774, 773, 780, 777, 748, 805, 777, 753, 802, 778, 761, 740, 723, 760, 743, 772, 767, 800, 768, 808, 771, 790, 760, 770, 769] payload_rows_per_dst=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] dup_rows_per_dst=[2978, 3064, 2964, 2990, 2988, 3172, 2884, 3086] combine_rows_per_src=[9216, 9256, 9224, 9382, 9252, 9258, 9204, 9386] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343712 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266165920 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988128 weight_movement_bytes=34603008 total_nvlink_bytes=566591136 weight_to_token_ratio=0.065045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343912 combine_payload_bytes=266051584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266395496 ingress_bytes=335257600 link_bytes=335257600 token_bytes=532447080 weight_movement_bytes=69206016 total_nvlink_bytes=601653096 weight_to_token_ratio=0.129977 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261547358e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65022 dup_rows=24054 dup_rows_remote=21070 dedup_ratio=0.244690 router_tokens_per_expert=[775, 745, 781, 815, 776, 805, 767, 759, 761, 820, 746, 746, 701, 765, 804, 763, 803, 780, 748, 763, 754, 742, 770, 802, 822, 765, 785, 782, 790, 753, 775, 756, 844, 777, 772, 739, 754, 808, 766, 777, 791, 727, 745, 743, 774, 792, 756, 733, 738, 778, 787, 741, 698, 804, 728, 769, 723, 756, 796, 781, 768, 759, 748, 761] payload_rows_per_dst=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] dup_rows_per_dst=[3070, 2984, 2956, 3114, 3074, 2938, 2976, 2942] combine_rows_per_src=[9376, 9228, 9368, 9342, 9400, 9184, 9110, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344392 combine_payload_bytes=267296768 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267641160 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534937928 weight_movement_bytes=69206016 total_nvlink_bytes=604143944 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=264896512 dispatch_meta_bytes=343216 combine_payload_bytes=264896512 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265239728 ingress_bytes=368705536 link_bytes=368705536 token_bytes=530136240 weight_movement_bytes=103809024 total_nvlink_bytes=633945264 weight_to_token_ratio=0.195816 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387409249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343384 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265624920 ingress_bytes=369090560 link_bytes=369090560 token_bytes=530906456 weight_movement_bytes=103809024 total_nvlink_bytes=634715480 weight_to_token_ratio=0.195532 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388858063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64802 dup_rows=23944 dup_rows_remote=20638 dedup_ratio=0.243571 router_tokens_per_expert=[742, 753, 758, 761, 739, 758, 784, 762, 737, 821, 800, 739, 803, 777, 747, 791, 770, 785, 753, 763, 719, 740, 771, 725, 763, 748, 732, 795, 766, 765, 720, 772, 725, 750, 792, 772, 748, 763, 775, 716, 724, 760, 771, 760, 785, 800, 770, 781, 763, 798, 785, 769, 798, 784, 799, 787, 774, 841, 801, 792, 777, 782, 795, 756] payload_rows_per_dst=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] dup_rows_per_dst=[2848, 2850, 2882, 2904, 3082, 2896, 3176, 3306] combine_rows_per_src=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65010 dup_rows=24032 dup_rows_remote=21054 dedup_ratio=0.244466 router_tokens_per_expert=[796, 769, 792, 731, 798, 747, 780, 806, 809, 791, 755, 731, 746, 735, 798, 743, 821, 782, 718, 781, 750, 814, 740, 778, 772, 792, 740, 756, 759, 762, 766, 746, 767, 833, 784, 783, 742, 760, 748, 779, 812, 747, 789, 703, 741, 761, 835, 732, 722, 713, 756, 788, 837, 784, 746, 811, 829, 743, 742, 746, 760, 766, 748, 741] payload_rows_per_dst=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] dup_rows_per_dst=[3040, 2978, 3034, 2936, 3090, 2978, 3008, 2968] combine_rows_per_src=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64924 dup_rows=23924 dup_rows_remote=20896 dedup_ratio=0.243368 router_tokens_per_expert=[771, 727, 780, 766, 752, 756, 762, 749, 759, 783, 739, 806, 763, 793, 828, 771, 791, 797, 754, 759, 786, 760, 751, 752, 764, 791, 757, 744, 781, 764, 804, 719, 742, 739, 767, 815, 751, 750, 775, 777, 791, 785, 752, 772, 728, 773, 767, 780, 820, 722, 756, 761, 774, 783, 816, 798, 779, 758, 774, 749, 794, 747, 757, 721] payload_rows_per_dst=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] dup_rows_per_dst=[2986, 3028, 2926, 3024, 2986, 2978, 3086, 2910] combine_rows_per_src=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21050 dedup_ratio=0.244629 router_tokens_per_expert=[751, 737, 718, 816, 789, 727, 785, 748, 750, 788, 788, 730, 778, 779, 827, 789, 776, 748, 732, 767, 790, 788, 773, 753, 738, 727, 772, 790, 770, 750, 788, 733, 741, 782, 813, 801, 785, 738, 776, 737, 798, 758, 768, 756, 791, 749, 751, 806, 778, 789, 788, 757, 762, 768, 802, 740, 768, 760, 743, 765, 792, 754, 735, 806] payload_rows_per_dst=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] dup_rows_per_dst=[2940, 3076, 3040, 2950, 2998, 3050, 2970, 3024] combine_rows_per_src=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65002 dup_rows=24062 dup_rows_remote=21152 dedup_ratio=0.244771 router_tokens_per_expert=[752, 759, 753, 805, 785, 783, 787, 748, 769, 779, 797, 762, 808, 819, 754, 765, 778, 734, 774, 776, 807, 805, 794, 816, 746, 738, 712, 753, 782, 760, 765, 819, 752, 746, 763, 801, 753, 787, 769, 752, 809, 714, 764, 742, 780, 739, 763, 772, 791, 765, 743, 759, 787, 778, 742, 793, 777, 770, 762, 752, 767, 727, 759, 690] payload_rows_per_dst=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] dup_rows_per_dst=[3088, 3088, 3122, 2910, 3018, 2902, 3068, 2866] combine_rows_per_src=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343280 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266271984 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532200688 weight_movement_bytes=69206016 total_nvlink_bytes=601406704 weight_to_token_ratio=0.130037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344368 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674480 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004592 weight_movement_bytes=34603008 total_nvlink_bytes=567607600 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=341760 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265770752 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531199744 weight_movement_bytes=69206016 total_nvlink_bytes=600405760 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65040 dup_rows=23930 dup_rows_remote=20960 dedup_ratio=0.243429 router_tokens_per_expert=[782, 827, 764, 839, 775, 765, 769, 783, 784, 727, 751, 791, 767, 770, 719, 761, 773, 711, 727, 796, 781, 765, 812, 787, 724, 750, 802, 733, 743, 797, 798, 792, 749, 723, 752, 825, 738, 732, 775, 728, 786, 763, 738, 813, 758, 752, 768, 791, 749, 754, 777, 792, 762, 742, 794, 766, 772, 783, 776, 794, 759, 761, 732, 783] payload_rows_per_dst=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] dup_rows_per_dst=[3104, 2882, 2970, 3064, 2912, 3034, 2870, 3094] combine_rows_per_src=[9504, 9258, 9334, 9214, 9132, 9304, 9402, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65010 dup_rows=24032 dup_rows_remote=21054 dedup_ratio=0.244466 router_tokens_per_expert=[796, 769, 792, 731, 798, 747, 780, 806, 809, 791, 755, 731, 746, 735, 798, 743, 821, 782, 718, 781, 750, 814, 740, 778, 772, 792, 740, 756, 759, 762, 766, 746, 767, 833, 784, 783, 742, 760, 748, 779, 812, 747, 789, 703, 741, 761, 835, 732, 722, 713, 756, 788, 837, 784, 746, 811, 829, 743, 742, 746, 760, 766, 748, 741] payload_rows_per_dst=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] dup_rows_per_dst=[3040, 2978, 3034, 2936, 3090, 2978, 3008, 2968] combine_rows_per_src=[9398, 9238, 9334, 9250, 9302, 9262, 9306, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64802 dup_rows=23944 dup_rows_remote=20638 dedup_ratio=0.243571 router_tokens_per_expert=[742, 753, 758, 761, 739, 758, 784, 762, 737, 821, 800, 739, 803, 777, 747, 791, 770, 785, 753, 763, 719, 740, 771, 725, 763, 748, 732, 795, 766, 765, 720, 772, 725, 750, 792, 772, 748, 763, 775, 716, 724, 760, 771, 760, 785, 800, 770, 781, 763, 798, 785, 769, 798, 784, 799, 787, 774, 841, 801, 792, 777, 782, 795, 756] payload_rows_per_dst=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] dup_rows_per_dst=[2848, 2850, 2882, 2904, 3082, 2896, 3176, 3306] combine_rows_per_src=[9180, 9218, 9170, 9218, 9362, 9186, 9468, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64908 dup_rows=24048 dup_rows_remote=21050 dedup_ratio=0.244629 router_tokens_per_expert=[751, 737, 718, 816, 789, 727, 785, 748, 750, 788, 788, 730, 778, 779, 827, 789, 776, 748, 732, 767, 790, 788, 773, 753, 738, 727, 772, 790, 770, 750, 788, 733, 741, 782, 813, 801, 785, 738, 776, 737, 798, 758, 768, 756, 791, 749, 751, 806, 778, 789, 788, 757, 762, 768, 802, 740, 768, 760, 743, 765, 792, 754, 735, 806] payload_rows_per_dst=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] dup_rows_per_dst=[2940, 3076, 3040, 2950, 2998, 3050, 2970, 3024] combine_rows_per_src=[9202, 9382, 9214, 9186, 9348, 9304, 9398, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344256 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266625216 ingress_bytes=370089984 link_bytes=370089984 token_bytes=532906176 weight_movement_bytes=103809024 total_nvlink_bytes=636715200 weight_to_token_ratio=0.194798 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392618815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344616 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592808 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841000 weight_movement_bytes=34603008 total_nvlink_bytes=567444008 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64766 dup_rows=24118 dup_rows_remote=21080 dedup_ratio=0.245341 router_tokens_per_expert=[771, 765, 770, 779, 815, 749, 756, 772, 772, 774, 739, 720, 750, 710, 750, 704, 788, 741, 795, 774, 825, 722, 742, 791, 789, 796, 763, 776, 786, 794, 780, 784, 766, 800, 789, 793, 794, 798, 748, 739, 762, 758, 778, 745, 795, 770, 760, 765, 758, 746, 778, 776, 758, 779, 758, 773, 774, 747, 767, 750, 758, 767, 761, 800] payload_rows_per_dst=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] dup_rows_per_dst=[3038, 3152, 2850, 2964, 3122, 3088, 3086, 2818] combine_rows_per_src=[9420, 9370, 8938, 9456, 9332, 9178, 9434, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64910 dup_rows=24068 dup_rows_remote=21166 dedup_ratio=0.244832 router_tokens_per_expert=[767, 781, 786, 803, 772, 752, 731, 752, 777, 768, 799, 762, 734, 787, 712, 778, 808, 823, 740, 791, 781, 789, 756, 750, 747, 729, 757, 786, 779, 796, 738, 782, 803, 746, 745, 729, 734, 783, 769, 750, 729, 787, 808, 738, 770, 792, 791, 791, 738, 770, 761, 770, 743, 760, 818, 759, 768, 783, 790, 757, 792, 747, 726, 792] payload_rows_per_dst=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] dup_rows_per_dst=[2904, 2936, 3194, 2902, 3018, 3056, 2994, 3064] combine_rows_per_src=[9384, 9298, 9282, 9326, 9100, 9356, 9244, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344000 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747840 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151680 weight_movement_bytes=34603008 total_nvlink_bytes=567754688 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343832 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266207000 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532070168 weight_movement_bytes=34603008 total_nvlink_bytes=566673176 weight_to_token_ratio=0.065035 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64790 dup_rows=24276 dup_rows_remote=21202 dedup_ratio=0.246948 router_tokens_per_expert=[780, 753, 723, 785, 808, 771, 799, 737, 794, 739, 793, 760, 764, 769, 795, 771, 748, 762, 742, 747, 780, 749, 684, 750, 738, 764, 781, 743, 724, 747, 759, 774, 755, 762, 774, 803, 768, 784, 787, 812, 786, 714, 725, 806, 749, 723, 759, 802, 716, 779, 787, 783, 780, 767, 797, 823, 775, 812, 823, 768, 773, 801, 779, 747] payload_rows_per_dst=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] dup_rows_per_dst=[3074, 3090, 3130, 3244, 3086, 2964, 2892, 2796] combine_rows_per_src=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=65070 dup_rows=24036 dup_rows_remote=21140 dedup_ratio=0.244507 router_tokens_per_expert=[799, 748, 777, 776, 780, 788, 747, 766, 717, 753, 788, 766, 822, 751, 745, 792, 735, 758, 756, 753, 770, 706, 799, 761, 791, 723, 787, 738, 745, 810, 793, 757, 804, 746, 711, 756, 777, 785, 785, 755, 821, 788, 751, 769, 753, 787, 808, 783, 774, 790, 757, 748, 743, 749, 765, 721, 768, 820, 770, 755, 791, 811, 792, 722] payload_rows_per_dst=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] dup_rows_per_dst=[2950, 3026, 2964, 3036, 2964, 3120, 2896, 3080] combine_rows_per_src=[9412, 9242, 9112, 9252, 9274, 9400, 9198, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344000 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747840 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151680 weight_movement_bytes=34603008 total_nvlink_bytes=567754688 weight_to_token_ratio=0.064903 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344256 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266625216 ingress_bytes=370089984 link_bytes=370089984 token_bytes=532906176 weight_movement_bytes=103809024 total_nvlink_bytes=636715200 weight_to_token_ratio=0.194798 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392618815e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265281536 dispatch_meta_bytes=343384 combine_payload_bytes=265281536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265624920 ingress_bytes=369090560 link_bytes=369090560 token_bytes=530906456 weight_movement_bytes=103809024 total_nvlink_bytes=634715480 weight_to_token_ratio=0.195532 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388858063e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64924 dup_rows=23924 dup_rows_remote=20896 dedup_ratio=0.243368 router_tokens_per_expert=[771, 727, 780, 766, 752, 756, 762, 749, 759, 783, 739, 806, 763, 793, 828, 771, 791, 797, 754, 759, 786, 760, 751, 752, 764, 791, 757, 744, 781, 764, 804, 719, 742, 739, 767, 815, 751, 750, 775, 777, 791, 785, 752, 772, 728, 773, 767, 780, 820, 722, 756, 761, 774, 783, 816, 798, 779, 758, 774, 749, 794, 747, 757, 721] payload_rows_per_dst=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] dup_rows_per_dst=[2986, 3028, 2926, 3024, 2986, 2978, 3086, 2910] combine_rows_per_src=[9140, 9456, 9374, 9224, 9246, 9318, 9374, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344304 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266215664 ingress_bytes=300474368 link_bytes=300474368 token_bytes=532087024 weight_movement_bytes=34603008 total_nvlink_bytes=566690032 weight_to_token_ratio=0.065033 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130660857e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=64964 dup_rows=23928 dup_rows_remote=20820 dedup_ratio=0.243408 router_tokens_per_expert=[748, 748, 789, 750, 757, 786, 768, 752, 789, 782, 813, 784, 766, 771, 748, 792, 801, 767, 781, 717, 817, 747, 711, 752, 771, 778, 813, 774, 729, 690, 752, 751, 757, 774, 795, 737, 735, 805, 789, 751, 777, 773, 736, 764, 735, 786, 745, 757, 771, 719, 783, 808, 792, 769, 800, 773, 800, 737, 801, 745, 815, 807, 777, 745] payload_rows_per_dst=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] dup_rows_per_dst=[2912, 2952, 2988, 2910, 2964, 3060, 3034, 3108] combine_rows_per_src=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=341760 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265770752 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531199744 weight_movement_bytes=69206016 total_nvlink_bytes=600405760 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343968 combine_payload_bytes=265379840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265723808 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103648 weight_movement_bytes=69206016 total_nvlink_bytes=600309664 weight_to_token_ratio=0.130306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65012 dup_rows=24026 dup_rows_remote=21082 dedup_ratio=0.244405 router_tokens_per_expert=[749, 708, 734, 812, 796, 756, 781, 757, 841, 803, 781, 719, 792, 787, 757, 750, 772, 786, 773, 742, 763, 756, 758, 752, 786, 749, 728, 767, 769, 778, 748, 754, 747, 835, 810, 745, 785, 787, 786, 742, 809, 713, 764, 724, 770, 764, 815, 746, 759, 821, 752, 782, 793, 808, 769, 770, 741, 774, 763, 797, 758, 754, 727, 738] payload_rows_per_dst=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] dup_rows_per_dst=[2944, 3064, 2984, 2996, 3106, 2944, 3090, 2898] combine_rows_per_src=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344840 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266871560 ingress_bytes=335732736 link_bytes=335732736 token_bytes=533398280 weight_movement_bytes=69206016 total_nvlink_bytes=602604296 weight_to_token_ratio=0.129745 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263335257e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64892 dup_rows=24248 dup_rows_remote=21226 dedup_ratio=0.246663 router_tokens_per_expert=[797, 757, 830, 776, 751, 738, 729, 786, 745, 765, 771, 731, 777, 790, 762, 752, 785, 726, 750, 735, 758, 783, 772, 792, 749, 743, 766, 740, 777, 758, 751, 766, 756, 765, 783, 743, 739, 775, 760, 802, 803, 778, 730, 782, 807, 782, 760, 805, 770, 756, 790, 757, 778, 757, 832, 747, 769, 773, 743, 813, 793, 772, 740, 784] payload_rows_per_dst=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] dup_rows_per_dst=[2998, 3022, 3032, 2966, 3034, 3140, 3014, 3042] combine_rows_per_src=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64940 dup_rows=24174 dup_rows_remote=21324 dedup_ratio=0.245911 router_tokens_per_expert=[771, 789, 776, 776, 795, 754, 734, 781, 768, 738, 787, 748, 768, 766, 773, 837, 750, 767, 732, 774, 786, 744, 833, 825, 786, 742, 796, 761, 788, 763, 720, 664, 758, 801, 773, 807, 775, 774, 741, 761, 766, 783, 758, 739, 751, 733, 739, 772, 771, 757, 760, 782, 778, 758, 775, 833, 794, 785, 750, 727, 735, 779, 772, 773] payload_rows_per_dst=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] dup_rows_per_dst=[3102, 3018, 3112, 2850, 3070, 2888, 3066, 3068] combine_rows_per_src=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343280 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266271984 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532200688 weight_movement_bytes=69206016 total_nvlink_bytes=601406704 weight_to_token_ratio=0.130037 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343136 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266435680 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528224 weight_movement_bytes=103809024 total_nvlink_bytes=636337248 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64994 dup_rows=24190 dup_rows_remote=21128 dedup_ratio=0.246073 router_tokens_per_expert=[722, 777, 745, 792, 779, 782, 792, 752, 749, 774, 769, 791, 806, 760, 745, 767, 753, 748, 719, 815, 777, 772, 742, 765, 761, 759, 807, 764, 806, 743, 724, 763, 758, 757, 757, 779, 732, 760, 735, 782, 794, 744, 734, 754, 775, 745, 767, 741, 783, 806, 778, 784, 785, 738, 794, 781, 804, 742, 789, 792, 802, 772, 797, 771] payload_rows_per_dst=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] dup_rows_per_dst=[3034, 3052, 3062, 2954, 2968, 2858, 3140, 3122] combine_rows_per_src=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21112 dedup_ratio=0.245911 router_tokens_per_expert=[780, 761, 780, 761, 786, 801, 764, 789, 710, 766, 771, 774, 766, 762, 754, 824, 774, 768, 780, 703, 770, 804, 759, 797, 775, 788, 745, 756, 709, 777, 816, 676, 780, 763, 728, 776, 801, 757, 797, 788, 793, 775, 749, 771, 768, 729, 736, 805, 759, 763, 798, 751, 759, 746, 769, 779, 788, 757, 768, 765, 721, 766, 786, 815] payload_rows_per_dst=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] dup_rows_per_dst=[3096, 2984, 3026, 2946, 3062, 3008, 3068, 2984] combine_rows_per_src=[9348, 9270, 9284, 9138, 9318, 9244, 9180, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266142104 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939736 weight_movement_bytes=103809024 total_nvlink_bytes=635748760 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344376 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633528 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922680 weight_movement_bytes=34603008 total_nvlink_bytes=567525688 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64994 dup_rows=24190 dup_rows_remote=21128 dedup_ratio=0.246073 router_tokens_per_expert=[722, 777, 745, 792, 779, 782, 792, 752, 749, 774, 769, 791, 806, 760, 745, 767, 753, 748, 719, 815, 777, 772, 742, 765, 761, 759, 807, 764, 806, 743, 724, 763, 758, 757, 757, 779, 732, 760, 735, 782, 794, 744, 734, 754, 775, 745, 767, 741, 783, 806, 778, 784, 785, 738, 794, 781, 804, 742, 789, 792, 802, 772, 797, 771] payload_rows_per_dst=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] dup_rows_per_dst=[3034, 3052, 3062, 2954, 2968, 2858, 3140, 3122] combine_rows_per_src=[9248, 9270, 9120, 9300, 9152, 9250, 9358, 9416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65012 dup_rows=24026 dup_rows_remote=21082 dedup_ratio=0.244405 router_tokens_per_expert=[749, 708, 734, 812, 796, 756, 781, 757, 841, 803, 781, 719, 792, 787, 757, 750, 772, 786, 773, 742, 763, 756, 758, 752, 786, 749, 728, 767, 769, 778, 748, 754, 747, 835, 810, 745, 785, 787, 786, 742, 809, 713, 764, 724, 770, 764, 815, 746, 759, 821, 752, 782, 793, 808, 769, 770, 741, 774, 763, 797, 758, 754, 727, 738] payload_rows_per_dst=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] dup_rows_per_dst=[2944, 3064, 2984, 2996, 3106, 2944, 3090, 2898] combine_rows_per_src=[9242, 9396, 9220, 9162, 9368, 9266, 9418, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=64964 dup_rows=23928 dup_rows_remote=20820 dedup_ratio=0.243408 router_tokens_per_expert=[748, 748, 789, 750, 757, 786, 768, 752, 789, 782, 813, 784, 766, 771, 748, 792, 801, 767, 781, 717, 817, 747, 711, 752, 771, 778, 813, 774, 729, 690, 752, 751, 757, 774, 795, 737, 735, 805, 789, 751, 777, 773, 736, 764, 735, 786, 745, 757, 771, 719, 783, 808, 792, 769, 800, 773, 800, 737, 801, 745, 815, 807, 777, 745] payload_rows_per_dst=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] dup_rows_per_dst=[2912, 2952, 2988, 2910, 2964, 3060, 3034, 3108] combine_rows_per_src=[9218, 9262, 9228, 9206, 9322, 9332, 9396, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=345056 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266339296 ingress_bytes=404406272 link_bytes=404406272 token_bytes=532333536 weight_movement_bytes=138412032 total_nvlink_bytes=670745568 weight_to_token_ratio=0.260010 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74028 payload_rows_remote=64790 dup_rows=24276 dup_rows_remote=21202 dedup_ratio=0.246948 router_tokens_per_expert=[780, 753, 723, 785, 808, 771, 799, 737, 794, 739, 793, 760, 764, 769, 795, 771, 748, 762, 742, 747, 780, 749, 684, 750, 738, 764, 781, 743, 724, 747, 759, 774, 755, 762, 774, 803, 768, 784, 787, 812, 786, 714, 725, 806, 749, 723, 759, 802, 716, 779, 787, 783, 780, 767, 797, 823, 775, 812, 823, 768, 773, 801, 779, 747] payload_rows_per_dst=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] dup_rows_per_dst=[3074, 3090, 3130, 3244, 3086, 2964, 2892, 2796] combine_rows_per_src=[9238, 9346, 9344, 9574, 9404, 9098, 9022, 9002] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343696 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265813648 ingress_bytes=300072960 link_bytes=300072960 token_bytes=531283600 weight_movement_bytes=34603008 total_nvlink_bytes=565886608 weight_to_token_ratio=0.065131 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129150391e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64968 dup_rows=23940 dup_rows_remote=20964 dedup_ratio=0.243530 router_tokens_per_expert=[764, 758, 741, 810, 790, 797, 756, 768, 768, 731, 743, 826, 758, 753, 750, 763, 747, 796, 755, 742, 812, 766, 780, 756, 722, 758, 764, 754, 754, 742, 807, 803, 716, 747, 715, 755, 724, 779, 736, 736, 766, 738, 751, 806, 753, 832, 775, 739, 758, 790, 841, 807, 746, 765, 745, 764, 769, 764, 802, 776, 795, 763, 839, 826] payload_rows_per_dst=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] dup_rows_per_dst=[2976, 2952, 2936, 3354, 3068, 3096, 2772, 2786] combine_rows_per_src=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64968 dup_rows=24098 dup_rows_remote=21068 dedup_ratio=0.245138 router_tokens_per_expert=[761, 769, 753, 742, 806, 773, 731, 774, 774, 762, 758, 829, 744, 722, 743, 759, 775, 744, 786, 760, 782, 797, 739, 774, 785, 747, 749, 784, 761, 752, 783, 793, 753, 813, 750, 774, 735, 762, 750, 755, 744, 777, 796, 758, 799, 764, 745, 798, 767, 762, 808, 756, 768, 757, 790, 726, 786, 761, 811, 758, 779, 779, 785, 775] payload_rows_per_dst=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] dup_rows_per_dst=[2918, 3114, 3000, 2944, 2972, 3080, 3030, 3040] combine_rows_per_src=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65002 dup_rows=24062 dup_rows_remote=21152 dedup_ratio=0.244771 router_tokens_per_expert=[752, 759, 753, 805, 785, 783, 787, 748, 769, 779, 797, 762, 808, 819, 754, 765, 778, 734, 774, 776, 807, 805, 794, 816, 746, 738, 712, 753, 782, 760, 765, 819, 752, 746, 763, 801, 753, 787, 769, 752, 809, 714, 764, 742, 780, 739, 763, 772, 791, 765, 743, 759, 787, 778, 742, 793, 777, 770, 762, 752, 767, 727, 759, 690] payload_rows_per_dst=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] dup_rows_per_dst=[3088, 3088, 3122, 2910, 3018, 2902, 3068, 2866] combine_rows_per_src=[9256, 9418, 9446, 9240, 9228, 9264, 9248, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=344488 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559912 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775336 weight_movement_bytes=34603008 total_nvlink_bytes=567378344 weight_to_token_ratio=0.064949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344376 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633528 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922680 weight_movement_bytes=34603008 total_nvlink_bytes=567525688 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64968 dup_rows=24098 dup_rows_remote=21068 dedup_ratio=0.245138 router_tokens_per_expert=[761, 769, 753, 742, 806, 773, 731, 774, 774, 762, 758, 829, 744, 722, 743, 759, 775, 744, 786, 760, 782, 797, 739, 774, 785, 747, 749, 784, 761, 752, 783, 793, 753, 813, 750, 774, 735, 762, 750, 755, 744, 777, 796, 758, 799, 764, 745, 798, 767, 762, 808, 756, 768, 757, 790, 726, 786, 761, 811, 758, 779, 779, 785, 775] payload_rows_per_dst=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] dup_rows_per_dst=[2918, 3114, 3000, 2944, 2972, 3080, 3030, 3040] combine_rows_per_src=[9300, 9068, 9314, 9364, 9212, 9282, 9238, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74056 payload_rows_remote=64892 dup_rows=24248 dup_rows_remote=21226 dedup_ratio=0.246663 router_tokens_per_expert=[797, 757, 830, 776, 751, 738, 729, 786, 745, 765, 771, 731, 777, 790, 762, 752, 785, 726, 750, 735, 758, 783, 772, 792, 749, 743, 766, 740, 777, 758, 751, 766, 756, 765, 783, 743, 739, 775, 760, 802, 803, 778, 730, 782, 807, 782, 760, 805, 770, 756, 790, 757, 778, 757, 832, 747, 769, 773, 743, 813, 793, 772, 740, 784] payload_rows_per_dst=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] dup_rows_per_dst=[2998, 3022, 3032, 2966, 3034, 3140, 3014, 3042] combine_rows_per_src=[9330, 9164, 9170, 9134, 9212, 9354, 9360, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344616 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592808 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532841000 weight_movement_bytes=34603008 total_nvlink_bytes=567444008 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343968 combine_payload_bytes=265379840 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265723808 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103648 weight_movement_bytes=69206016 total_nvlink_bytes=600309664 weight_to_token_ratio=0.130306 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65122 dup_rows=23974 dup_rows_remote=20814 dedup_ratio=0.243876 router_tokens_per_expert=[787, 778, 745, 807, 765, 756, 746, 753, 799, 721, 744, 752, 743, 709, 779, 712, 756, 800, 776, 791, 798, 851, 761, 808, 748, 781, 771, 813, 806, 721, 779, 771, 797, 803, 773, 765, 745, 785, 746, 768, 747, 769, 763, 773, 744, 755, 773, 773, 783, 790, 770, 744, 781, 728, 756, 756, 794, 749, 813, 750, 764, 758, 747, 763] payload_rows_per_dst=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] dup_rows_per_dst=[2976, 2740, 2970, 2980, 3044, 3008, 3096, 3160] combine_rows_per_src=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343136 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266435680 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532528224 weight_movement_bytes=103809024 total_nvlink_bytes=636337248 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343728 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452656 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561584 weight_movement_bytes=69206016 total_nvlink_bytes=601767600 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64786 dup_rows=24132 dup_rows_remote=21106 dedup_ratio=0.245483 router_tokens_per_expert=[731, 763, 787, 731, 814, 756, 758, 790, 760, 824, 751, 751, 785, 803, 721, 811, 795, 771, 749, 830, 741, 790, 743, 769, 821, 788, 779, 778, 815, 822, 722, 811, 850, 741, 769, 724, 742, 757, 787, 711, 750, 783, 727, 845, 761, 756, 757, 765, 745, 778, 723, 763, 729, 738, 718, 723, 772, 751, 763, 792, 773, 744, 730, 825] payload_rows_per_dst=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] dup_rows_per_dst=[2962, 3026, 3184, 3246, 2998, 2970, 2772, 2974] combine_rows_per_src=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64794 dup_rows=24200 dup_rows_remote=21210 dedup_ratio=0.246175 router_tokens_per_expert=[739, 779, 776, 791, 781, 732, 794, 796, 774, 765, 740, 803, 772, 757, 754, 763, 724, 781, 769, 789, 763, 768, 791, 754, 757, 730, 793, 752, 747, 784, 753, 790, 791, 752, 775, 775, 750, 729, 774, 767, 759, 751, 789, 758, 738, 814, 777, 764, 778, 754, 756, 802, 772, 790, 784, 739, 712, 793, 737, 779, 794, 806, 746, 786] payload_rows_per_dst=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] dup_rows_per_dst=[3060, 2994, 3010, 2954, 2976, 2990, 3056, 3160] combine_rows_per_src=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344144 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266453072 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532562000 weight_movement_bytes=69206016 total_nvlink_bytes=601768016 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64910 dup_rows=24222 dup_rows_remote=21022 dedup_ratio=0.246399 router_tokens_per_expert=[728, 780, 776, 821, 774, 764, 786, 767, 742, 770, 736, 751, 795, 782, 824, 760, 740, 806, 742, 747, 747, 790, 746, 759, 800, 744, 772, 780, 775, 757, 800, 758, 802, 773, 793, 746, 728, 745, 737, 774, 758, 777, 747, 761, 785, 741, 773, 756, 757, 782, 797, 753, 772, 763, 761, 812, 791, 800, 734, 807, 739, 718, 765, 786] payload_rows_per_dst=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] dup_rows_per_dst=[3118, 3000, 2912, 3200, 2940, 3050, 2976, 3026] combine_rows_per_src=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64996 dup_rows=24072 dup_rows_remote=21100 dedup_ratio=0.244873 router_tokens_per_expert=[751, 746, 750, 774, 806, 752, 748, 736, 738, 754, 762, 791, 731, 769, 758, 778, 761, 773, 774, 809, 782, 746, 768, 734, 756, 798, 772, 783, 755, 832, 746, 797, 757, 767, 750, 766, 776, 760, 777, 751, 778, 788, 732, 753, 752, 772, 798, 805, 771, 780, 791, 765, 744, 809, 798, 780, 792, 729, 739, 740, 784, 736, 819, 763] payload_rows_per_dst=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] dup_rows_per_dst=[2926, 3024, 3016, 3098, 2972, 3034, 3042, 2960] combine_rows_per_src=[9200, 9138, 9278, 9380, 9236, 9322, 9434, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343568 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707024 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070480 weight_movement_bytes=34603008 total_nvlink_bytes=565673488 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64794 dup_rows=24200 dup_rows_remote=21210 dedup_ratio=0.246175 router_tokens_per_expert=[739, 779, 776, 791, 781, 732, 794, 796, 774, 765, 740, 803, 772, 757, 754, 763, 724, 781, 769, 789, 763, 768, 791, 754, 757, 730, 793, 752, 747, 784, 753, 790, 791, 752, 775, 775, 750, 729, 774, 767, 759, 751, 789, 758, 738, 814, 777, 764, 778, 754, 756, 802, 772, 790, 784, 739, 712, 793, 737, 779, 794, 806, 746, 786] payload_rows_per_dst=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] dup_rows_per_dst=[3060, 2994, 3010, 2954, 2976, 2990, 3056, 3160] combine_rows_per_src=[9316, 9262, 9268, 9258, 9250, 9310, 9294, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266142104 ingress_bytes=369606656 link_bytes=369606656 token_bytes=531939736 weight_movement_bytes=103809024 total_nvlink_bytes=635748760 weight_to_token_ratio=0.195152 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390800091e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74024 payload_rows_remote=64884 dup_rows=24280 dup_rows_remote=21304 dedup_ratio=0.246989 router_tokens_per_expert=[744, 759, 766, 804, 753, 775, 799, 709, 787, 847, 753, 739, 816, 780, 763, 769, 727, 733, 734, 773, 783, 793, 782, 733, 765, 757, 796, 809, 755, 762, 772, 798, 757, 748, 778, 760, 801, 800, 752, 752, 745, 750, 766, 772, 808, 784, 775, 808, 785, 703, 750, 755, 766, 780, 788, 727, 776, 746, 724, 789, 752, 790, 803, 727] payload_rows_per_dst=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] dup_rows_per_dst=[2978, 3106, 2976, 3164, 3032, 3100, 2908, 3016] combine_rows_per_src=[9240, 9402, 9140, 9264, 9264, 9316, 9200, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343744 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083456 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823168 weight_movement_bytes=34603008 total_nvlink_bytes=568426176 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=344016 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265740240 ingress_bytes=299999232 link_bytes=299999232 token_bytes=531136464 weight_movement_bytes=34603008 total_nvlink_bytes=565739472 weight_to_token_ratio=0.065149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128872958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=343728 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266215088 ingress_bytes=369680384 link_bytes=369680384 token_bytes=532086448 weight_movement_bytes=103809024 total_nvlink_bytes=635895472 weight_to_token_ratio=0.195098 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391077523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65122 dup_rows=23974 dup_rows_remote=20814 dedup_ratio=0.243876 router_tokens_per_expert=[787, 778, 745, 807, 765, 756, 746, 753, 799, 721, 744, 752, 743, 709, 779, 712, 756, 800, 776, 791, 798, 851, 761, 808, 748, 781, 771, 813, 806, 721, 779, 771, 797, 803, 773, 765, 745, 785, 746, 768, 747, 769, 763, 773, 744, 755, 773, 773, 783, 790, 770, 744, 781, 728, 756, 756, 794, 749, 813, 750, 764, 758, 747, 763] payload_rows_per_dst=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] dup_rows_per_dst=[2976, 2740, 2970, 2980, 3044, 3008, 3096, 3160] combine_rows_per_src=[9298, 9178, 9440, 9222, 9336, 9272, 9376, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64940 dup_rows=24174 dup_rows_remote=21324 dedup_ratio=0.245911 router_tokens_per_expert=[771, 789, 776, 776, 795, 754, 734, 781, 768, 738, 787, 748, 768, 766, 773, 837, 750, 767, 732, 774, 786, 744, 833, 825, 786, 742, 796, 761, 788, 763, 720, 664, 758, 801, 773, 807, 775, 774, 741, 761, 766, 783, 758, 739, 751, 733, 739, 772, 771, 757, 760, 782, 778, 758, 775, 833, 794, 785, 750, 727, 735, 779, 772, 773] payload_rows_per_dst=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] dup_rows_per_dst=[3102, 3018, 3112, 2850, 3070, 2888, 3066, 3068] combine_rows_per_src=[9250, 9352, 9310, 9190, 9310, 9194, 9362, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64968 dup_rows=23940 dup_rows_remote=20964 dedup_ratio=0.243530 router_tokens_per_expert=[764, 758, 741, 810, 790, 797, 756, 768, 768, 731, 743, 826, 758, 753, 750, 763, 747, 796, 755, 742, 812, 766, 780, 756, 722, 758, 764, 754, 754, 742, 807, 803, 716, 747, 715, 755, 724, 779, 736, 736, 766, 738, 751, 806, 753, 832, 775, 739, 758, 790, 841, 807, 746, 765, 745, 764, 769, 764, 802, 776, 795, 763, 839, 826] payload_rows_per_dst=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] dup_rows_per_dst=[2976, 2952, 2936, 3354, 3068, 3096, 2772, 2786] combine_rows_per_src=[9396, 9228, 9372, 9780, 9300, 9224, 8890, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344384 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266568000 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791616 weight_movement_bytes=103809024 total_nvlink_bytes=636600640 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64986 dup_rows=23964 dup_rows_remote=20954 dedup_ratio=0.243774 router_tokens_per_expert=[738, 745, 728, 738, 777, 730, 766, 798, 732, 749, 745, 786, 745, 792, 766, 785, 727, 796, 773, 784, 780, 715, 738, 792, 802, 776, 754, 785, 732, 805, 795, 792, 790, 719, 796, 755, 780, 730, 804, 811, 801, 756, 749, 791, 797, 751, 760, 764, 784, 829, 779, 750, 769, 742, 756, 765, 737, 805, 771, 763, 799, 747, 745, 791] payload_rows_per_dst=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] dup_rows_per_dst=[3010, 3088, 2968, 2998, 2928, 2962, 2980, 3030] combine_rows_per_src=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65056 dup_rows=23884 dup_rows_remote=20960 dedup_ratio=0.242961 router_tokens_per_expert=[823, 778, 722, 796, 758, 767, 754, 769, 742, 778, 746, 798, 734, 782, 747, 802, 772, 764, 757, 761, 754, 763, 744, 784, 753, 730, 792, 793, 763, 714, 747, 740, 766, 768, 801, 770, 807, 804, 793, 767, 761, 766, 770, 775, 753, 766, 748, 783, 755, 777, 779, 784, 773, 769, 786, 721, 773, 786, 768, 783, 787, 739, 769, 778] payload_rows_per_dst=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] dup_rows_per_dst=[2942, 3020, 2966, 2950, 3126, 2988, 2924, 2968] combine_rows_per_src=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=344016 combine_payload_bytes=265396224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265740240 ingress_bytes=299999232 link_bytes=299999232 token_bytes=531136464 weight_movement_bytes=34603008 total_nvlink_bytes=565739472 weight_to_token_ratio=0.065149 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128872958e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265764864 dispatch_meta_bytes=344752 combine_payload_bytes=265764864 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266109616 ingress_bytes=334970880 link_bytes=334970880 token_bytes=531874480 weight_movement_bytes=69206016 total_nvlink_bytes=601080496 weight_to_token_ratio=0.130117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260468454e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=345056 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266339296 ingress_bytes=404406272 link_bytes=404406272 token_bytes=532333536 weight_movement_bytes=138412032 total_nvlink_bytes=670745568 weight_to_token_ratio=0.260010 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521748244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65056 dup_rows=23884 dup_rows_remote=20960 dedup_ratio=0.242961 router_tokens_per_expert=[823, 778, 722, 796, 758, 767, 754, 769, 742, 778, 746, 798, 734, 782, 747, 802, 772, 764, 757, 761, 754, 763, 744, 784, 753, 730, 792, 793, 763, 714, 747, 740, 766, 768, 801, 770, 807, 804, 793, 767, 761, 766, 770, 775, 753, 766, 748, 783, 755, 777, 779, 784, 773, 769, 786, 721, 773, 786, 768, 783, 787, 739, 769, 778] payload_rows_per_dst=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] dup_rows_per_dst=[2942, 3020, 2966, 2950, 3126, 2988, 2924, 2968] combine_rows_per_src=[9392, 9238, 9232, 9114, 9426, 9256, 9364, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343728 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452656 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561584 weight_movement_bytes=69206016 total_nvlink_bytes=601767600 weight_to_token_ratio=0.129949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64786 dup_rows=24132 dup_rows_remote=21106 dedup_ratio=0.245483 router_tokens_per_expert=[731, 763, 787, 731, 814, 756, 758, 790, 760, 824, 751, 751, 785, 803, 721, 811, 795, 771, 749, 830, 741, 790, 743, 769, 821, 788, 779, 778, 815, 822, 722, 811, 850, 741, 769, 724, 742, 757, 787, 711, 750, 783, 727, 845, 761, 756, 757, 765, 745, 778, 723, 763, 729, 738, 718, 723, 772, 751, 763, 792, 773, 744, 730, 825] payload_rows_per_dst=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] dup_rows_per_dst=[2962, 3026, 3184, 3246, 2998, 2970, 2772, 2974] combine_rows_per_src=[9298, 9386, 9192, 9426, 9164, 9318, 9062, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64934 dup_rows=24146 dup_rows_remote=21118 dedup_ratio=0.245626 router_tokens_per_expert=[763, 790, 817, 773, 757, 789, 789, 744, 739, 737, 772, 748, 804, 756, 771, 737, 794, 722, 764, 744, 788, 755, 759, 769, 789, 745, 757, 758, 780, 756, 782, 717, 755, 750, 784, 776, 780, 770, 732, 774, 792, 799, 818, 786, 786, 731, 774, 751, 739, 779, 771, 776, 816, 776, 800, 747, 725, 768, 782, 771, 772, 764, 789, 754] payload_rows_per_dst=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] dup_rows_per_dst=[2850, 3082, 2986, 2918, 3036, 3100, 3146, 3028] combine_rows_per_src=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65000 dup_rows=23970 dup_rows_remote=20854 dedup_ratio=0.243835 router_tokens_per_expert=[799, 743, 771, 740, 794, 757, 770, 796, 764, 751, 756, 732, 712, 789, 743, 789, 780, 776, 789, 774, 761, 749, 743, 730, 726, 790, 757, 787, 822, 755, 757, 780, 772, 773, 722, 760, 754, 767, 823, 775, 772, 778, 782, 763, 766, 799, 780, 785, 763, 791, 737, 768, 796, 798, 744, 735, 775, 763, 709, 765, 755, 845, 747, 808] payload_rows_per_dst=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] dup_rows_per_dst=[2934, 2904, 2934, 2984, 3006, 3116, 3044, 3048] combine_rows_per_src=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343744 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083456 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823168 weight_movement_bytes=34603008 total_nvlink_bytes=568426176 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526416 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709072 weight_movement_bytes=34603008 total_nvlink_bytes=567312080 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64994 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[771, 829, 741, 739, 750, 807, 781, 740, 755, 799, 776, 793, 745, 778, 777, 750, 775, 773, 721, 773, 736, 773, 744, 772, 760, 773, 760, 798, 765, 777, 770, 802, 753, 766, 768, 797, 747, 770, 777, 766, 783, 773, 810, 739, 771, 826, 705, 749, 761, 725, 749, 787, 774, 768, 758, 752, 778, 762, 788, 766, 785, 811, 762, 723] payload_rows_per_dst=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] dup_rows_per_dst=[3014, 2972, 2966, 3074, 2944, 2982, 2884, 3100] combine_rows_per_src=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344064 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813440 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282816 weight_movement_bytes=34603008 total_nvlink_bytes=567885824 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65128 dup_rows=23994 dup_rows_remote=21062 dedup_ratio=0.244080 router_tokens_per_expert=[768, 787, 745, 766, 818, 763, 731, 756, 758, 796, 756, 755, 807, 750, 800, 805, 745, 763, 786, 781, 799, 757, 755, 786, 764, 756, 758, 826, 789, 748, 690, 726, 786, 769, 746, 758, 777, 801, 748, 721, 760, 786, 776, 749, 762, 813, 812, 753, 797, 808, 776, 713, 741, 802, 777, 747, 763, 754, 761, 784, 742, 769, 747, 764] payload_rows_per_dst=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] dup_rows_per_dst=[2966, 3102, 3012, 2932, 2978, 3042, 3046, 2916] combine_rows_per_src=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65032 dup_rows=24064 dup_rows_remote=21114 dedup_ratio=0.244792 router_tokens_per_expert=[773, 735, 758, 751, 765, 778, 756, 762, 743, 774, 798, 770, 734, 765, 768, 762, 757, 776, 783, 751, 739, 792, 712, 771, 732, 738, 785, 818, 793, 782, 803, 783, 765, 776, 738, 774, 736, 757, 769, 764, 763, 763, 777, 777, 770, 759, 768, 761, 769, 754, 755, 782, 799, 728, 747, 766, 796, 790, 802, 771, 792, 795, 765, 817] payload_rows_per_dst=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] dup_rows_per_dst=[2926, 2970, 2958, 3164, 2950, 3002, 2986, 3108] combine_rows_per_src=[9230, 9258, 9204, 9304, 9208, 9274, 9214, 9548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65000 dup_rows=23970 dup_rows_remote=20854 dedup_ratio=0.243835 router_tokens_per_expert=[799, 743, 771, 740, 794, 757, 770, 796, 764, 751, 756, 732, 712, 789, 743, 789, 780, 776, 789, 774, 761, 749, 743, 730, 726, 790, 757, 787, 822, 755, 757, 780, 772, 773, 722, 760, 754, 767, 823, 775, 772, 778, 782, 763, 766, 799, 780, 785, 763, 791, 737, 768, 796, 798, 744, 735, 775, 763, 709, 765, 755, 845, 747, 808] payload_rows_per_dst=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] dup_rows_per_dst=[2934, 2904, 2934, 2984, 3006, 3116, 3044, 3048] combine_rows_per_src=[9406, 9168, 9270, 9364, 9286, 9334, 9220, 9286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344064 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813440 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282816 weight_movement_bytes=34603008 total_nvlink_bytes=567885824 weight_to_token_ratio=0.064887 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=65032 dup_rows=24156 dup_rows_remote=21184 dedup_ratio=0.245728 router_tokens_per_expert=[788, 787, 739, 739, 801, 783, 781, 778, 772, 761, 735, 757, 790, 764, 756, 776, 777, 772, 782, 760, 747, 697, 766, 743, 789, 797, 749, 751, 762, 745, 736, 811, 774, 778, 716, 728, 786, 802, 762, 738, 770, 779, 772, 814, 735, 813, 811, 752, 753, 735, 778, 788, 748, 759, 758, 785, 766, 763, 789, 725, 792, 795, 803, 794] payload_rows_per_dst=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] dup_rows_per_dst=[3018, 2990, 2972, 3036, 2942, 3104, 2962, 3132] combine_rows_per_src=[9374, 9232, 9116, 9244, 9226, 9388, 9246, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343832 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559256 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532774680 weight_movement_bytes=34603008 total_nvlink_bytes=567377688 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344760 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109048 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873336 weight_movement_bytes=34603008 total_nvlink_bytes=568476344 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343568 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265707024 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531070480 weight_movement_bytes=34603008 total_nvlink_bytes=565673488 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344208 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313872 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283536 weight_movement_bytes=103809024 total_nvlink_bytes=636092560 weight_to_token_ratio=0.195026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343416 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266583416 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823416 weight_movement_bytes=69206016 total_nvlink_bytes=602029432 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=64934 dup_rows=24146 dup_rows_remote=21118 dedup_ratio=0.245626 router_tokens_per_expert=[763, 790, 817, 773, 757, 789, 789, 744, 739, 737, 772, 748, 804, 756, 771, 737, 794, 722, 764, 744, 788, 755, 759, 769, 789, 745, 757, 758, 780, 756, 782, 717, 755, 750, 784, 776, 780, 770, 732, 774, 792, 799, 818, 786, 786, 731, 774, 751, 739, 779, 771, 776, 816, 776, 800, 747, 725, 768, 782, 771, 772, 764, 789, 754] payload_rows_per_dst=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] dup_rows_per_dst=[2850, 3082, 2986, 2918, 3036, 3100, 3146, 3028] combine_rows_per_src=[9138, 9502, 9204, 9084, 9206, 9372, 9428, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64858 dup_rows=24020 dup_rows_remote=20770 dedup_ratio=0.244344 router_tokens_per_expert=[774, 816, 819, 785, 779, 775, 748, 808, 788, 747, 721, 716, 822, 803, 730, 761, 744, 808, 793, 807, 741, 797, 765, 758, 735, 778, 780, 737, 770, 756, 779, 763, 753, 760, 782, 757, 792, 745, 749, 781, 760, 780, 800, 755, 745, 727, 745, 769, 768, 738, 768, 755, 703, 785, 784, 778, 769, 776, 767, 777, 775, 799, 745, 762] payload_rows_per_dst=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] dup_rows_per_dst=[3250, 2940, 3448, 2864, 3206, 2832, 2896, 2584] combine_rows_per_src=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64910 dup_rows=24222 dup_rows_remote=21022 dedup_ratio=0.246399 router_tokens_per_expert=[728, 780, 776, 821, 774, 764, 786, 767, 742, 770, 736, 751, 795, 782, 824, 760, 740, 806, 742, 747, 747, 790, 746, 759, 800, 744, 772, 780, 775, 757, 800, 758, 802, 773, 793, 746, 728, 745, 737, 774, 758, 777, 747, 761, 785, 741, 773, 756, 757, 782, 797, 753, 772, 763, 761, 812, 791, 800, 734, 807, 739, 718, 765, 786] payload_rows_per_dst=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] dup_rows_per_dst=[3118, 3000, 2912, 3200, 2940, 3050, 2976, 3026] combine_rows_per_src=[9274, 9320, 9242, 9172, 9256, 9146, 9418, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64986 dup_rows=23964 dup_rows_remote=20954 dedup_ratio=0.243774 router_tokens_per_expert=[738, 745, 728, 738, 777, 730, 766, 798, 732, 749, 745, 786, 745, 792, 766, 785, 727, 796, 773, 784, 780, 715, 738, 792, 802, 776, 754, 785, 732, 805, 795, 792, 790, 719, 796, 755, 780, 730, 804, 811, 801, 756, 749, 791, 797, 751, 760, 764, 784, 829, 779, 750, 769, 742, 756, 765, 737, 805, 771, 763, 799, 747, 745, 791] payload_rows_per_dst=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] dup_rows_per_dst=[3010, 3088, 2968, 2998, 2928, 2962, 2980, 3030] combine_rows_per_src=[9354, 9332, 9242, 9210, 9172, 9376, 9368, 9286] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344584 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715656 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086728 weight_movement_bytes=34603008 total_nvlink_bytes=567689736 weight_to_token_ratio=0.064911 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343416 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266583416 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823416 weight_movement_bytes=69206016 total_nvlink_bytes=602029432 weight_to_token_ratio=0.129885 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344864 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715936 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533087008 weight_movement_bytes=34603008 total_nvlink_bytes=567690016 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=343728 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266215088 ingress_bytes=369680384 link_bytes=369680384 token_bytes=532086448 weight_movement_bytes=103809024 total_nvlink_bytes=635895472 weight_to_token_ratio=0.195098 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391077523e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343760 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526416 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709072 weight_movement_bytes=34603008 total_nvlink_bytes=567312080 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64756 dup_rows=24214 dup_rows_remote=21190 dedup_ratio=0.246318 router_tokens_per_expert=[715, 731, 745, 725, 754, 782, 790, 780, 765, 774, 742, 797, 745, 822, 742, 791, 794, 720, 798, 791, 774, 732, 735, 775, 762, 766, 759, 742, 776, 786, 768, 782, 785, 787, 779, 745, 851, 733, 783, 739, 763, 769, 782, 773, 791, 773, 738, 742, 764, 791, 787, 739, 799, 790, 761, 748, 784, 827, 773, 725, 720, 778, 806, 767] payload_rows_per_dst=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] dup_rows_per_dst=[2948, 3010, 2986, 2980, 3110, 2958, 3024, 3198] combine_rows_per_src=[9096, 9346, 9252, 9302, 9294, 9304, 9334, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64804 dup_rows=24212 dup_rows_remote=21280 dedup_ratio=0.246297 router_tokens_per_expert=[781, 782, 775, 774, 746, 789, 791, 727, 746, 775, 755, 796, 686, 790, 780, 767, 803, 796, 752, 781, 777, 782, 768, 766, 768, 767, 738, 740, 827, 752, 779, 725, 778, 779, 770, 767, 753, 753, 811, 758, 813, 795, 766, 772, 738, 782, 735, 772, 758, 777, 748, 801, 736, 739, 767, 793, 757, 725, 761, 751, 754, 798, 767, 797] payload_rows_per_dst=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] dup_rows_per_dst=[2400, 2994, 3122, 3636, 3014, 3180, 2934, 2932] combine_rows_per_src=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64828 dup_rows=24218 dup_rows_remote=21074 dedup_ratio=0.246358 router_tokens_per_expert=[818, 767, 786, 774, 777, 746, 809, 752, 797, 775, 758, 730, 783, 743, 789, 763, 775, 784, 759, 777, 749, 780, 760, 737, 723, 734, 821, 748, 743, 772, 785, 755, 811, 796, 781, 790, 745, 758, 734, 780, 777, 771, 754, 818, 768, 778, 784, 751, 758, 756, 766, 787, 750, 797, 743, 800, 741, 743, 744, 777, 755, 743, 758, 769] payload_rows_per_dst=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] dup_rows_per_dst=[3012, 3018, 3006, 3062, 3028, 3144, 3030, 2918] combine_rows_per_src=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64994 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[771, 829, 741, 739, 750, 807, 781, 740, 755, 799, 776, 793, 745, 778, 777, 750, 775, 773, 721, 773, 736, 773, 744, 772, 760, 773, 760, 798, 765, 777, 770, 802, 753, 766, 768, 797, 747, 770, 777, 766, 783, 773, 810, 739, 771, 826, 705, 749, 761, 725, 749, 787, 774, 768, 758, 752, 778, 762, 788, 766, 785, 811, 762, 723] payload_rows_per_dst=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] dup_rows_per_dst=[3014, 2972, 2966, 3074, 2944, 2982, 2884, 3100] combine_rows_per_src=[9302, 9374, 9168, 9336, 9344, 9330, 9264, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=344208 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266313872 ingress_bytes=369778688 link_bytes=369778688 token_bytes=532283536 weight_movement_bytes=103809024 total_nvlink_bytes=636092560 weight_to_token_ratio=0.195026 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391447433e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=342512 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266000880 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531659248 weight_movement_bytes=69206016 total_nvlink_bytes=600865264 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74466 payload_rows_remote=65232 dup_rows=23838 dup_rows_remote=20874 dedup_ratio=0.242493 router_tokens_per_expert=[763, 815, 789, 793, 741, 786, 757, 807, 778, 762, 785, 811, 751, 735, 729, 748, 788, 765, 801, 769, 803, 815, 780, 746, 729, 714, 768, 745, 712, 742, 743, 740, 788, 758, 770, 823, 749, 772, 734, 714, 769, 800, 791, 809, 734, 757, 748, 722, 780, 771, 761, 777, 761, 789, 719, 747, 804, 840, 806, 751, 783, 807, 731, 777] payload_rows_per_dst=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] dup_rows_per_dst=[3158, 2964, 3018, 2726, 2890, 2978, 2968, 3136] combine_rows_per_src=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65290 dup_rows=23796 dup_rows_remote=20748 dedup_ratio=0.242065 router_tokens_per_expert=[756, 724, 794, 785, 830, 724, 772, 775, 801, 735, 746, 776, 770, 780, 801, 777, 753, 769, 767, 752, 821, 749, 773, 758, 764, 794, 757, 780, 741, 789, 780, 728, 764, 748, 746, 730, 757, 752, 750, 760, 767, 745, 813, 810, 763, 787, 791, 789, 743, 765, 765, 770, 793, 765, 764, 767, 761, 743, 744, 753, 754, 800, 778, 794] payload_rows_per_dst=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] dup_rows_per_dst=[2854, 2982, 2944, 3048, 2884, 3148, 2964, 2972] combine_rows_per_src=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64992 dup_rows=23982 dup_rows_remote=20990 dedup_ratio=0.243958 router_tokens_per_expert=[758, 786, 829, 740, 740, 751, 751, 782, 782, 757, 747, 765, 765, 785, 717, 793, 731, 772, 780, 815, 757, 758, 759, 767, 764, 745, 802, 741, 730, 779, 800, 751, 774, 780, 832, 774, 749, 760, 738, 754, 772, 777, 707, 802, 791, 779, 738, 772, 748, 817, 794, 784, 726, 819, 783, 766, 763, 755, 743, 791, 765, 770, 772, 758] payload_rows_per_dst=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] dup_rows_per_dst=[3018, 3010, 2958, 2928, 2992, 3038, 3050, 2988] combine_rows_per_src=[9256, 9212, 9320, 9296, 9330, 9238, 9424, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64828 dup_rows=24218 dup_rows_remote=21074 dedup_ratio=0.246358 router_tokens_per_expert=[818, 767, 786, 774, 777, 746, 809, 752, 797, 775, 758, 730, 783, 743, 789, 763, 775, 784, 759, 777, 749, 780, 760, 737, 723, 734, 821, 748, 743, 772, 785, 755, 811, 796, 781, 790, 745, 758, 734, 780, 777, 771, 754, 818, 768, 778, 784, 751, 758, 756, 766, 787, 750, 797, 743, 800, 741, 743, 744, 777, 755, 743, 758, 769] payload_rows_per_dst=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] dup_rows_per_dst=[3012, 3018, 3006, 3062, 3028, 3144, 3030, 2918] combine_rows_per_src=[9446, 9258, 9236, 9100, 9362, 9258, 9284, 9142] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65234 dup_rows=23766 dup_rows_remote=20878 dedup_ratio=0.241760 router_tokens_per_expert=[813, 770, 760, 764, 741, 794, 778, 760, 746, 767, 760, 762, 797, 793, 772, 808, 759, 776, 721, 789, 790, 735, 751, 775, 774, 734, 766, 819, 745, 793, 804, 702, 762, 726, 821, 744, 775, 762, 790, 726, 822, 762, 775, 744, 736, 737, 777, 751, 788, 759, 766, 739, 773, 761, 745, 762, 816, 776, 809, 739, 797, 753, 744, 797] payload_rows_per_dst=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] dup_rows_per_dst=[3016, 3014, 2888, 2928, 2898, 2970, 2956, 3096] combine_rows_per_src=[9344, 9396, 9304, 9346, 9314, 9238, 9230, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344424 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267534696 ingress_bytes=336396288 link_bytes=336396288 token_bytes=534724968 weight_movement_bytes=69206016 total_nvlink_bytes=603930984 weight_to_token_ratio=0.129424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265832150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267427840 dispatch_meta_bytes=344152 combine_payload_bytes=267427840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267771992 ingress_bytes=302030848 link_bytes=302030848 token_bytes=535199832 weight_movement_bytes=34603008 total_nvlink_bytes=569802840 weight_to_token_ratio=0.064654 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136517766e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=344336 combine_payload_bytes=265437184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265781520 ingress_bytes=334643200 link_bytes=334643200 token_bytes=531218704 weight_movement_bytes=69206016 total_nvlink_bytes=600424720 weight_to_token_ratio=0.130278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259235421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65128 dup_rows=23994 dup_rows_remote=21062 dedup_ratio=0.244080 router_tokens_per_expert=[768, 787, 745, 766, 818, 763, 731, 756, 758, 796, 756, 755, 807, 750, 800, 805, 745, 763, 786, 781, 799, 757, 755, 786, 764, 756, 758, 826, 789, 748, 690, 726, 786, 769, 746, 758, 777, 801, 748, 721, 760, 786, 776, 749, 762, 813, 812, 753, 797, 808, 776, 713, 741, 802, 777, 747, 763, 754, 761, 784, 742, 769, 747, 764] payload_rows_per_dst=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] dup_rows_per_dst=[2966, 3102, 3012, 2932, 2978, 3042, 3046, 2916] combine_rows_per_src=[9302, 9352, 9332, 9182, 9234, 9380, 9276, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64804 dup_rows=24212 dup_rows_remote=21280 dedup_ratio=0.246297 router_tokens_per_expert=[781, 782, 775, 774, 746, 789, 791, 727, 746, 775, 755, 796, 686, 790, 780, 767, 803, 796, 752, 781, 777, 782, 768, 766, 768, 767, 738, 740, 827, 752, 779, 725, 778, 779, 770, 767, 753, 753, 811, 758, 813, 795, 766, 772, 738, 782, 735, 772, 758, 777, 748, 801, 736, 739, 767, 793, 757, 725, 761, 751, 754, 798, 767, 797] payload_rows_per_dst=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] dup_rows_per_dst=[2400, 2994, 3122, 3636, 3014, 3180, 2934, 2932] combine_rows_per_src=[8506, 9196, 9210, 9980, 9324, 9284, 9304, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64364 dup_rows=24216 dup_rows_remote=20740 dedup_ratio=0.246338 router_tokens_per_expert=[782, 793, 743, 773, 765, 792, 786, 792, 784, 774, 827, 789, 804, 789, 803, 763, 773, 782, 788, 725, 790, 741, 753, 769, 762, 728, 746, 740, 751, 741, 764, 721, 729, 787, 751, 752, 784, 742, 740, 835, 784, 822, 714, 764, 772, 790, 726, 768, 729, 744, 770, 728, 767, 748, 820, 760, 793, 734, 809, 774, 774, 764, 788, 757] payload_rows_per_dst=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] dup_rows_per_dst=[3476, 2956, 2904, 2800, 3114, 2968, 3108, 2890] combine_rows_per_src=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64858 dup_rows=24020 dup_rows_remote=20770 dedup_ratio=0.244344 router_tokens_per_expert=[774, 816, 819, 785, 779, 775, 748, 808, 788, 747, 721, 716, 822, 803, 730, 761, 744, 808, 793, 807, 741, 797, 765, 758, 735, 778, 780, 737, 770, 756, 779, 763, 753, 760, 782, 757, 792, 745, 749, 781, 760, 780, 800, 755, 745, 727, 745, 769, 768, 738, 768, 755, 703, 785, 784, 778, 769, 776, 767, 777, 775, 799, 745, 762] payload_rows_per_dst=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] dup_rows_per_dst=[3250, 2940, 3448, 2864, 3206, 2832, 2896, 2584] combine_rows_per_src=[9426, 9236, 9686, 9278, 9350, 9012, 9262, 9034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343608 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265879096 ingress_bytes=334741504 link_bytes=334741504 token_bytes=531414584 weight_movement_bytes=69206016 total_nvlink_bytes=600620600 weight_to_token_ratio=0.130230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259605331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=343832 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266559256 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532774680 weight_movement_bytes=34603008 total_nvlink_bytes=567377688 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343928 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551160 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758392 weight_movement_bytes=34603008 total_nvlink_bytes=567361400 weight_to_token_ratio=0.064951 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267198464 dispatch_meta_bytes=344448 combine_payload_bytes=267198464 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267542912 ingress_bytes=371007488 link_bytes=371007488 token_bytes=534741376 weight_movement_bytes=103809024 total_nvlink_bytes=638550400 weight_to_token_ratio=0.194129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396071309e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344760 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267109048 ingress_bytes=301367296 link_bytes=301367296 token_bytes=533873336 weight_movement_bytes=34603008 total_nvlink_bytes=568476344 weight_to_token_ratio=0.064815 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134020873e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265658368 dispatch_meta_bytes=342512 combine_payload_bytes=265658368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266000880 ingress_bytes=334864384 link_bytes=334864384 token_bytes=531659248 weight_movement_bytes=69206016 total_nvlink_bytes=600865264 weight_to_token_ratio=0.130170 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260067718e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65148 dup_rows=23826 dup_rows_remote=20942 dedup_ratio=0.242371 router_tokens_per_expert=[751, 780, 770, 775, 785, 775, 764, 768, 828, 729, 758, 788, 728, 809, 784, 766, 739, 786, 805, 764, 771, 777, 800, 731, 802, 776, 795, 772, 775, 737, 735, 719, 783, 727, 772, 740, 760, 723, 790, 797, 769, 759, 700, 777, 771, 784, 784, 747, 784, 748, 790, 717, 766, 740, 786, 776, 751, 809, 780, 819, 771, 770, 745, 775] payload_rows_per_dst=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] dup_rows_per_dst=[3038, 3016, 3064, 2924, 2948, 2926, 2884, 3026] combine_rows_per_src=[9298, 9364, 9282, 9298, 9236, 9256, 9330, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343608 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265879096 ingress_bytes=334741504 link_bytes=334741504 token_bytes=531414584 weight_movement_bytes=69206016 total_nvlink_bytes=600620600 weight_to_token_ratio=0.130230 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259605331e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12257940016 prefetch_bytes=657457152 reduce_bytes=899678208 weight_movement_bytes=1557135360 egress_bytes=6132929072 ingress_bytes=7682146304 link_bytes=7682146304 total_nvlink_bytes=13815075376 weight_to_token_ratio=0.127031 prefetch_share_of_ingress=0.085582 dedup_ratio=0.244379 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=26 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256839047e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65002 dup_rows=24044 dup_rows_remote=20992 dedup_ratio=0.244588 router_tokens_per_expert=[830, 758, 813, 757, 803, 782, 774, 769, 770, 758, 745, 768, 785, 775, 787, 785, 799, 735, 725, 779, 754, 767, 773, 756, 743, 783, 757, 785, 759, 719, 763, 728, 804, 772, 790, 774, 745, 766, 765, 778, 737, 760, 784, 759, 783, 738, 735, 787, 807, 748, 728, 813, 776, 758, 754, 774, 775, 743, 788, 731, 781, 760, 750, 805] payload_rows_per_dst=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] dup_rows_per_dst=[2916, 2976, 2944, 3004, 3052, 2876, 3224, 3052] combine_rows_per_src=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=344336 combine_payload_bytes=265437184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265781520 ingress_bytes=334643200 link_bytes=334643200 token_bytes=531218704 weight_movement_bytes=69206016 total_nvlink_bytes=600424720 weight_to_token_ratio=0.130278 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259235421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263634944 dispatch_meta_bytes=340416 combine_payload_bytes=263634944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263975360 ingress_bytes=332840960 link_bytes=332840960 token_bytes=527610304 weight_movement_bytes=69206016 total_nvlink_bytes=596816320 weight_to_token_ratio=0.131169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252453737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344360 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190568 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036776 weight_movement_bytes=34603008 total_nvlink_bytes=568639784 weight_to_token_ratio=0.064795 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65148 dup_rows=23842 dup_rows_remote=20874 dedup_ratio=0.242533 router_tokens_per_expert=[767, 741, 747, 789, 750, 800, 723, 762, 751, 753, 754, 742, 782, 739, 786, 834, 759, 733, 745, 736, 760, 779, 812, 748, 765, 752, 746, 787, 798, 775, 797, 754, 724, 715, 750, 820, 781, 749, 807, 800, 804, 732, 775, 767, 767, 752, 783, 740, 776, 777, 763, 759, 780, 786, 778, 736, 774, 803, 746, 767, 809, 813, 768, 785] payload_rows_per_dst=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] dup_rows_per_dst=[3004, 2968, 2922, 2972, 2970, 2962, 2992, 3052] combine_rows_per_src=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65370 dup_rows=23596 dup_rows_remote=20616 dedup_ratio=0.240031 router_tokens_per_expert=[785, 799, 771, 796, 761, 765, 745, 799, 810, 804, 821, 719, 787, 774, 737, 779, 777, 769, 768, 772, 722, 756, 760, 758, 734, 759, 799, 802, 734, 744, 781, 749, 751, 736, 788, 787, 744, 778, 809, 785, 838, 741, 765, 763, 741, 796, 791, 724, 769, 728, 720, 784, 758, 756, 757, 762, 752, 794, 790, 768, 788, 760, 736, 757] payload_rows_per_dst=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] dup_rows_per_dst=[3022, 3008, 2978, 2916, 2918, 2980, 2812, 2962] combine_rows_per_src=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74466 payload_rows_remote=65232 dup_rows=23838 dup_rows_remote=20874 dedup_ratio=0.242493 router_tokens_per_expert=[763, 815, 789, 793, 741, 786, 757, 807, 778, 762, 785, 811, 751, 735, 729, 748, 788, 765, 801, 769, 803, 815, 780, 746, 729, 714, 768, 745, 712, 742, 743, 740, 788, 758, 770, 823, 749, 772, 734, 714, 769, 800, 791, 809, 734, 757, 748, 722, 780, 771, 761, 777, 761, 789, 719, 747, 804, 840, 806, 751, 783, 807, 731, 777] payload_rows_per_dst=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] dup_rows_per_dst=[3158, 2964, 3018, 2726, 2890, 2978, 2968, 3136] combine_rows_per_src=[9344, 9234, 9516, 9060, 9326, 9282, 9242, 9462] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=64868 dup_rows=23918 dup_rows_remote=20798 dedup_ratio=0.243306 router_tokens_per_expert=[756, 761, 777, 763, 754, 787, 777, 766, 760, 769, 743, 734, 763, 739, 771, 733, 756, 779, 770, 730, 741, 783, 770, 808, 754, 768, 789, 762, 791, 778, 787, 726, 799, 796, 767, 822, 775, 814, 768, 778, 731, 754, 767, 743, 756, 799, 768, 820, 740, 799, 759, 773, 786, 766, 757, 795, 786, 809, 750, 745, 767, 735, 749, 734] payload_rows_per_dst=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] dup_rows_per_dst=[3028, 2902, 3008, 3050, 3120, 3056, 2968, 2786] combine_rows_per_src=[9254, 9122, 9266, 9260, 9518, 9220, 9382, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344360 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190568 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036776 weight_movement_bytes=34603008 total_nvlink_bytes=568639784 weight_to_token_ratio=0.064795 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20990 dedup_ratio=0.243774 router_tokens_per_expert=[776, 795, 782, 765, 788, 806, 707, 774, 786, 750, 791, 764, 792, 719, 735, 741, 780, 769, 800, 764, 738, 754, 739, 754, 749, 772, 747, 746, 763, 775, 801, 796, 754, 841, 764, 762, 728, 768, 736, 800, 786, 788, 757, 783, 769, 759, 758, 798, 794, 737, 779, 748, 766, 759, 735, 785, 767, 791, 744, 788, 770, 796, 746, 778] payload_rows_per_dst=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] dup_rows_per_dst=[3024, 2966, 2974, 2906, 2966, 3086, 3040, 3002] combine_rows_per_src=[9362, 9190, 9222, 9392, 9340, 9310, 9166, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74708 payload_rows_remote=65370 dup_rows=23596 dup_rows_remote=20616 dedup_ratio=0.240031 router_tokens_per_expert=[785, 799, 771, 796, 761, 765, 745, 799, 810, 804, 821, 719, 787, 774, 737, 779, 777, 769, 768, 772, 722, 756, 760, 758, 734, 759, 799, 802, 734, 744, 781, 749, 751, 736, 788, 787, 744, 778, 809, 785, 838, 741, 765, 763, 741, 796, 791, 724, 769, 728, 720, 784, 758, 756, 757, 762, 752, 794, 790, 768, 788, 760, 736, 757] payload_rows_per_dst=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] dup_rows_per_dst=[3022, 3008, 2978, 2916, 2918, 2980, 2812, 2962] combine_rows_per_src=[9420, 9454, 9186, 9288, 9438, 9338, 9256, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344088 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190296 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036504 weight_movement_bytes=34603008 total_nvlink_bytes=568639512 weight_to_token_ratio=0.064795 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343976 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266592168 ingress_bytes=335454208 link_bytes=335454208 token_bytes=532840360 weight_movement_bytes=69206016 total_nvlink_bytes=602046376 weight_to_token_ratio=0.129881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262287178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65290 dup_rows=23796 dup_rows_remote=20748 dedup_ratio=0.242065 router_tokens_per_expert=[756, 724, 794, 785, 830, 724, 772, 775, 801, 735, 746, 776, 770, 780, 801, 777, 753, 769, 767, 752, 821, 749, 773, 758, 764, 794, 757, 780, 741, 789, 780, 728, 764, 748, 746, 730, 757, 752, 750, 760, 767, 745, 813, 810, 763, 787, 791, 789, 743, 765, 765, 770, 793, 765, 764, 767, 761, 743, 744, 753, 754, 800, 778, 794] payload_rows_per_dst=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] dup_rows_per_dst=[2854, 2982, 2944, 3048, 2884, 3148, 2964, 2972] combine_rows_per_src=[9466, 9390, 9340, 9218, 9130, 9382, 9300, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267755520 dispatch_meta_bytes=343944 combine_payload_bytes=267755520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268099464 ingress_bytes=336961536 link_bytes=336961536 token_bytes=535854984 weight_movement_bytes=69206016 total_nvlink_bytes=605061000 weight_to_token_ratio=0.129151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267959132e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64364 dup_rows=24216 dup_rows_remote=20740 dedup_ratio=0.246338 router_tokens_per_expert=[782, 793, 743, 773, 765, 792, 786, 792, 784, 774, 827, 789, 804, 789, 803, 763, 773, 782, 788, 725, 790, 741, 753, 769, 762, 728, 746, 740, 751, 741, 764, 721, 729, 787, 751, 752, 784, 742, 740, 835, 784, 822, 714, 764, 772, 790, 726, 768, 729, 744, 770, 728, 767, 748, 820, 760, 793, 734, 809, 774, 774, 764, 788, 757] payload_rows_per_dst=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] dup_rows_per_dst=[3476, 2956, 2904, 2800, 3114, 2968, 3108, 2890] combine_rows_per_src=[9724, 9218, 9132, 9106, 9316, 9122, 9238, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65002 dup_rows=24044 dup_rows_remote=20992 dedup_ratio=0.244588 router_tokens_per_expert=[830, 758, 813, 757, 803, 782, 774, 769, 770, 758, 745, 768, 785, 775, 787, 785, 799, 735, 725, 779, 754, 767, 773, 756, 743, 783, 757, 785, 759, 719, 763, 728, 804, 772, 790, 774, 745, 766, 765, 778, 737, 760, 784, 759, 783, 738, 735, 787, 807, 748, 728, 813, 776, 758, 754, 774, 775, 743, 788, 731, 781, 760, 750, 805] payload_rows_per_dst=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] dup_rows_per_dst=[2916, 2976, 2944, 3004, 3052, 2876, 3224, 3052] combine_rows_per_src=[9334, 9370, 9232, 9392, 9292, 9100, 9282, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342664 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266041992 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531741320 weight_movement_bytes=103809024 total_nvlink_bytes=635550344 weight_to_token_ratio=0.195225 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=344424 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267534696 ingress_bytes=336396288 link_bytes=336396288 token_bytes=534724968 weight_movement_bytes=69206016 total_nvlink_bytes=603930984 weight_to_token_ratio=0.129424 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265832150e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344432 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067760 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791088 weight_movement_bytes=34603008 total_nvlink_bytes=568394096 weight_to_token_ratio=0.064825 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64930 dup_rows=24040 dup_rows_remote=21054 dedup_ratio=0.244548 router_tokens_per_expert=[792, 763, 778, 745, 743, 778, 804, 757, 767, 777, 809, 740, 738, 767, 804, 765, 733, 735, 797, 754, 765, 823, 780, 780, 768, 749, 788, 765, 764, 758, 731, 707, 784, 777, 765, 786, 727, 756, 753, 800, 756, 767, 781, 745, 815, 738, 737, 790, 767, 770, 723, 766, 798, 763, 796, 758, 795, 743, 780, 775, 734, 790, 782, 811] payload_rows_per_dst=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] dup_rows_per_dst=[2986, 3158, 3244, 2870, 2986, 2972, 2980, 2844] combine_rows_per_src=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12222339232 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6115113120 ingress_bytes=7387537408 link_bytes=7387537408 total_nvlink_bytes=13502650528 weight_to_token_ratio=0.104752 prefetch_share_of_ingress=0.103047 dedup_ratio=0.244646 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.208639501e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254462064 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6131187824 ingress_bytes=7576600576 link_bytes=7576600576 total_nvlink_bytes=13707788400 weight_to_token_ratio=0.118596 prefetch_share_of_ingress=0.082208 dedup_ratio=0.244229 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239571217e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267427840 dispatch_meta_bytes=344152 combine_payload_bytes=267427840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267771992 ingress_bytes=302030848 link_bytes=302030848 token_bytes=535199832 weight_movement_bytes=34603008 total_nvlink_bytes=569802840 weight_to_token_ratio=0.064654 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136517766e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=263634944 dispatch_meta_bytes=340416 combine_payload_bytes=263634944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=263975360 ingress_bytes=332840960 link_bytes=332840960 token_bytes=527610304 weight_movement_bytes=69206016 total_nvlink_bytes=596816320 weight_to_token_ratio=0.131169 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.252453737e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64962 dup_rows=24068 dup_rows_remote=21024 dedup_ratio=0.244832 router_tokens_per_expert=[785, 733, 746, 773, 757, 771, 746, 762, 767, 751, 785, 785, 795, 749, 771, 735, 784, 797, 736, 770, 777, 746, 791, 780, 732, 845, 770, 804, 755, 738, 811, 756, 815, 760, 709, 756, 744, 758, 788, 715, 723, 756, 832, 751, 762, 793, 740, 808, 769, 783, 766, 778, 765, 772, 739, 787, 794, 752, 741, 741, 784, 763, 790, 815] payload_rows_per_dst=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] dup_rows_per_dst=[2906, 3010, 3086, 3066, 2894, 3008, 3044, 3054] combine_rows_per_src=[9240, 9266, 9276, 9356, 9196, 9322, 9274, 9306] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=267755520 dispatch_meta_bytes=343944 combine_payload_bytes=267755520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=268099464 ingress_bytes=336961536 link_bytes=336961536 token_bytes=535854984 weight_movement_bytes=69206016 total_nvlink_bytes=605061000 weight_to_token_ratio=0.129151 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267959132e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259359688 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133636040 ingress_bytes=7371431936 link_bytes=7371431936 total_nvlink_bytes=13505067976 weight_to_token_ratio=0.101613 prefetch_share_of_ingress=0.103272 dedup_ratio=0.243881 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206004562e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12248722656 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6128315616 ingress_bytes=7400718336 link_bytes=7400718336 total_nvlink_bytes=13529033952 weight_to_token_ratio=0.104526 prefetch_share_of_ingress=0.088837 dedup_ratio=0.244325 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.210795969e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343976 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266592168 ingress_bytes=335454208 link_bytes=335454208 token_bytes=532840360 weight_movement_bytes=69206016 total_nvlink_bytes=602046376 weight_to_token_ratio=0.129881 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262287178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12261245704 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6134579976 ingress_bytes=7614595072 link_bytes=7614595072 total_nvlink_bytes=13749175048 weight_to_token_ratio=0.121352 prefetch_share_of_ingress=0.090886 dedup_ratio=0.244073 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.245787314e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343936 combine_payload_bytes=265953280 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266297216 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250496 weight_movement_bytes=34603008 total_nvlink_bytes=566853504 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254462064 prefetch_bytes=622854144 reduce_bytes=830472192 weight_movement_bytes=1453326336 egress_bytes=6131187824 ingress_bytes=7576600576 link_bytes=7576600576 total_nvlink_bytes=13707788400 weight_to_token_ratio=0.118596 prefetch_share_of_ingress=0.082208 dedup_ratio=0.244229 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=24 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239571217e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12257940016 prefetch_bytes=657457152 reduce_bytes=899678208 weight_movement_bytes=1557135360 egress_bytes=6132929072 ingress_bytes=7682146304 link_bytes=7682146304 total_nvlink_bytes=13815075376 weight_to_token_ratio=0.127031 prefetch_share_of_ingress=0.085582 dedup_ratio=0.244379 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=26 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256839047e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=343944 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266428296 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532512648 weight_movement_bytes=34603008 total_nvlink_bytes=567115656 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12222339232 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6115113120 ingress_bytes=7387537408 link_bytes=7387537408 total_nvlink_bytes=13502650528 weight_to_token_ratio=0.104752 prefetch_share_of_ingress=0.103047 dedup_ratio=0.244646 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.208639501e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12223492848 prefetch_bytes=657457152 reduce_bytes=519045120 weight_movement_bytes=1176502272 egress_bytes=6115693296 ingress_bytes=7284301824 link_bytes=7284646120 total_nvlink_bytes=13399995120 weight_to_token_ratio=0.096249 prefetch_share_of_ingress=0.090257 dedup_ratio=0.244885 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=15 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191805952e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65148 dup_rows=23842 dup_rows_remote=20874 dedup_ratio=0.242533 router_tokens_per_expert=[767, 741, 747, 789, 750, 800, 723, 762, 751, 753, 754, 742, 782, 739, 786, 834, 759, 733, 745, 736, 760, 779, 812, 748, 765, 752, 746, 787, 798, 775, 797, 754, 724, 715, 750, 820, 781, 749, 807, 800, 804, 732, 775, 767, 767, 752, 783, 740, 776, 777, 763, 759, 780, 786, 778, 736, 774, 803, 746, 767, 809, 813, 768, 785] payload_rows_per_dst=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] dup_rows_per_dst=[3004, 2968, 2922, 2972, 2970, 2962, 2992, 3052] combine_rows_per_src=[9154, 9314, 9222, 9376, 9322, 9278, 9318, 9478] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64930 dup_rows=24040 dup_rows_remote=21054 dedup_ratio=0.244548 router_tokens_per_expert=[792, 763, 778, 745, 743, 778, 804, 757, 767, 777, 809, 740, 738, 767, 804, 765, 733, 735, 797, 754, 765, 823, 780, 780, 768, 749, 788, 765, 764, 758, 731, 707, 784, 777, 765, 786, 727, 756, 753, 800, 756, 767, 781, 745, 815, 738, 737, 790, 767, 770, 723, 766, 798, 763, 796, 758, 795, 743, 780, 775, 734, 790, 782, 811] payload_rows_per_dst=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] dup_rows_per_dst=[2986, 3158, 3244, 2870, 2986, 2972, 2980, 2844] combine_rows_per_src=[9334, 9228, 9460, 9190, 9278, 9266, 9302, 9206] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254527248 prefetch_bytes=657457152 reduce_bytes=795869184 weight_movement_bytes=1453326336 egress_bytes=6131220240 ingress_bytes=7576633344 link_bytes=7576633344 total_nvlink_bytes=13707853584 weight_to_token_ratio=0.118595 prefetch_share_of_ingress=0.086774 dedup_ratio=0.244189 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=23 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.239576578e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:56:39] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 795.0 | throughput per GPU (TFLOP/s/GPU): 160.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785748E+00 | loss scale: 1.0 | grad norm: 4.541 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64882 dup_rows=24046 dup_rows_remote=20978 dedup_ratio=0.244609 router_tokens_per_expert=[747, 773, 731, 804, 795, 791, 819, 815, 778, 784, 718, 772, 756, 826, 714, 799, 780, 760, 761, 731, 770, 764, 783, 774, 784, 738, 774, 819, 779, 796, 723, 809, 783, 750, 792, 760, 770, 751, 757, 800, 749, 793, 766, 757, 789, 785, 732, 740, 762, 775, 758, 761, 743, 798, 744, 782, 753, 757, 710, 773, 725, 750, 784, 736] payload_rows_per_dst=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] dup_rows_per_dst=[3140, 3060, 2998, 3068, 3062, 2994, 2922, 2802] combine_rows_per_src=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344088 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267190296 ingress_bytes=301449216 link_bytes=301449216 token_bytes=534036504 weight_movement_bytes=34603008 total_nvlink_bytes=568639512 weight_to_token_ratio=0.064795 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134329131e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343936 combine_payload_bytes=265953280 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266297216 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250496 weight_movement_bytes=34603008 total_nvlink_bytes=566853504 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12259359688 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6133636040 ingress_bytes=7371431936 link_bytes=7371431936 total_nvlink_bytes=13505067976 weight_to_token_ratio=0.101613 prefetch_share_of_ingress=0.103272 dedup_ratio=0.243881 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206004562e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12223492848 prefetch_bytes=657457152 reduce_bytes=519045120 weight_movement_bytes=1176502272 egress_bytes=6115693296 ingress_bytes=7284301824 link_bytes=7284646120 total_nvlink_bytes=13399995120 weight_to_token_ratio=0.096249 prefetch_share_of_ingress=0.090257 dedup_ratio=0.244885 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=15 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029239 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.191805952e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-26 18:08:45] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 796.4 | throughput per GPU (TFLOP/s/GPU): 160.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.785748E+00 | loss scale: 1.0 | grad norm: 4.541 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65166 dup_rows=23810 dup_rows_remote=20706 dedup_ratio=0.242208 router_tokens_per_expert=[798, 794, 761, 726, 797, 753, 775, 790, 766, 795, 743, 753, 735, 791, 770, 724, 769, 766, 718, 841, 779, 794, 762, 726, 770, 800, 818, 781, 763, 777, 819, 752, 751, 814, 748, 765, 766, 732, 774, 738, 746, 780, 752, 750, 720, 774, 780, 806, 759, 802, 790, 762, 780, 760, 763, 730, 757, 765, 744, 741, 795, 802, 761, 739] payload_rows_per_dst=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] dup_rows_per_dst=[3104, 2848, 3002, 3096, 3002, 2890, 2960, 2908] combine_rows_per_src=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64642 dup_rows=24028 dup_rows_remote=20644 dedup_ratio=0.244425 router_tokens_per_expert=[752, 787, 721, 764, 766, 771, 751, 791, 700, 791, 786, 776, 782, 806, 807, 749, 755, 766, 800, 763, 791, 775, 785, 781, 758, 765, 749, 765, 802, 790, 754, 744, 755, 775, 789, 770, 778, 783, 737, 795, 710, 859, 773, 781, 698, 729, 783, 774, 752, 735, 711, 781, 782, 777, 767, 723, 747, 756, 765, 774, 801, 840, 791, 718] payload_rows_per_dst=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] dup_rows_per_dst=[2794, 3010, 3200, 2498, 2972, 3106, 3064, 3384] combine_rows_per_src=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64990 dup_rows=24140 dup_rows_remote=21106 dedup_ratio=0.245565 router_tokens_per_expert=[797, 755, 767, 753, 767, 729, 753, 788, 770, 792, 733, 776, 791, 744, 725, 773, 772, 799, 733, 708, 789, 736, 778, 749, 763, 758, 741, 816, 775, 747, 789, 733, 783, 778, 751, 810, 782, 763, 769, 791, 767, 760, 749, 796, 750, 775, 769, 771, 797, 775, 798, 781, 750, 783, 773, 841, 779, 725, 789, 736, 763, 748, 714, 837] payload_rows_per_dst=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] dup_rows_per_dst=[2996, 3034, 2870, 3040, 3088, 3032, 3076, 3004] combine_rows_per_src=[9222, 9174, 9258, 9204, 9366, 9242, 9520, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65028 dup_rows=24030 dup_rows_remote=21012 dedup_ratio=0.244446 router_tokens_per_expert=[761, 767, 738, 748, 760, 730, 747, 800, 844, 811, 784, 805, 728, 755, 756, 776, 749, 715, 760, 770, 761, 756, 773, 787, 803, 790, 830, 752, 765, 765, 777, 773, 798, 794, 737, 733, 778, 787, 832, 788, 733, 803, 770, 789, 756, 733, 801, 747, 728, 780, 761, 761, 737, 740, 776, 728, 755, 803, 794, 726, 747, 773, 771, 757] payload_rows_per_dst=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] dup_rows_per_dst=[2852, 3134, 2880, 3086, 3212, 3018, 2888, 2960] combine_rows_per_src=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64882 dup_rows=24046 dup_rows_remote=20978 dedup_ratio=0.244609 router_tokens_per_expert=[747, 773, 731, 804, 795, 791, 819, 815, 778, 784, 718, 772, 756, 826, 714, 799, 780, 760, 761, 731, 770, 764, 783, 774, 784, 738, 774, 819, 779, 796, 723, 809, 783, 750, 792, 760, 770, 751, 757, 800, 749, 793, 766, 757, 789, 785, 732, 740, 762, 775, 758, 761, 743, 798, 744, 782, 753, 757, 710, 773, 725, 750, 784, 736] payload_rows_per_dst=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] dup_rows_per_dst=[3140, 3060, 2998, 3068, 3062, 2994, 2922, 2802] combine_rows_per_src=[9410, 9234, 9248, 9376, 9264, 9228, 9324, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64790 dup_rows=24200 dup_rows_remote=21312 dedup_ratio=0.246175 router_tokens_per_expert=[810, 806, 755, 824, 777, 767, 745, 843, 770, 766, 799, 789, 815, 705, 756, 778, 775, 754, 767, 768, 740, 761, 802, 734, 759, 801, 724, 810, 759, 777, 766, 778, 762, 794, 761, 784, 790, 750, 759, 724, 752, 804, 771, 775, 778, 728, 711, 824, 794, 737, 774, 746, 821, 731, 766, 727, 755, 754, 759, 718, 784, 750, 766, 723] payload_rows_per_dst=[9466, 9228, 9314, 9314, 9192, 9260, 9226, 9104] dup_rows_per_dst=[3188, 3128, 2888, 3034, 3056, 3026, 2966, 2914] combine_rows_per_src=[9466, 9228, 9314, 9314, 9192, 9260, 9226, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65028 dup_rows=24030 dup_rows_remote=21012 dedup_ratio=0.244446 router_tokens_per_expert=[761, 767, 738, 748, 760, 730, 747, 800, 844, 811, 784, 805, 728, 755, 756, 776, 749, 715, 760, 770, 761, 756, 773, 787, 803, 790, 830, 752, 765, 765, 777, 773, 798, 794, 737, 733, 778, 787, 832, 788, 733, 803, 770, 789, 756, 733, 801, 747, 728, 780, 761, 761, 737, 740, 776, 728, 755, 803, 794, 726, 747, 773, 771, 757] payload_rows_per_dst=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] dup_rows_per_dst=[2852, 3134, 2880, 3086, 3212, 3018, 2888, 2960] combine_rows_per_src=[9250, 9384, 9262, 9424, 9282, 9246, 9134, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65282 dup_rows=23782 dup_rows_remote=20816 dedup_ratio=0.241923 router_tokens_per_expert=[746, 814, 748, 794, 767, 749, 779, 774, 755, 768, 736, 807, 762, 743, 795, 754, 792, 787, 778, 751, 808, 792, 746, 761, 755, 794, 757, 745, 812, 771, 785, 744, 808, 804, 718, 734, 777, 760, 707, 795, 774, 766, 758, 765, 723, 714, 764, 764, 761, 796, 761, 772, 736, 797, 805, 767, 760, 772, 768, 801, 761, 756, 797, 742] payload_rows_per_dst=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] dup_rows_per_dst=[2956, 2936, 3008, 2952, 2966, 2968, 3008, 2988] combine_rows_per_src=[9386, 9304, 9422, 9374, 9240, 9088, 9382, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64642 dup_rows=24028 dup_rows_remote=20644 dedup_ratio=0.244425 router_tokens_per_expert=[752, 787, 721, 764, 766, 771, 751, 791, 700, 791, 786, 776, 782, 806, 807, 749, 755, 766, 800, 763, 791, 775, 785, 781, 758, 765, 749, 765, 802, 790, 754, 744, 755, 775, 789, 770, 778, 783, 737, 795, 710, 859, 773, 781, 698, 729, 783, 774, 752, 735, 711, 781, 782, 777, 767, 723, 747, 756, 765, 774, 801, 840, 791, 718] payload_rows_per_dst=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] dup_rows_per_dst=[2794, 3010, 3200, 2498, 2972, 3106, 3064, 3384] combine_rows_per_src=[9146, 9324, 9498, 8708, 9216, 9526, 9224, 9634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65050 dup_rows=23902 dup_rows_remote=20976 dedup_ratio=0.243144 router_tokens_per_expert=[680, 768, 785, 754, 783, 788, 765, 710, 758, 756, 804, 726, 772, 811, 742, 753, 802, 726, 746, 754, 767, 749, 760, 770, 756, 785, 842, 815, 821, 747, 762, 805, 741, 732, 785, 771, 772, 775, 749, 781, 783, 755, 803, 803, 740, 741, 816, 736, 799, 750, 761, 723, 797, 802, 761, 746, 775, 792, 744, 808, 779, 711, 749, 810] payload_rows_per_dst=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] dup_rows_per_dst=[2848, 2942, 2874, 3182, 2984, 2996, 2926, 3150] combine_rows_per_src=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=343488 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267263424 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534183360 weight_movement_bytes=69206016 total_nvlink_bytes=603389376 weight_to_token_ratio=0.129555 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341144 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114776 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529888408 weight_movement_bytes=34603008 total_nvlink_bytes=564491416 weight_to_token_ratio=0.065302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344384 combine_payload_bytes=266199040 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266543424 ingress_bytes=370008064 link_bytes=370008064 token_bytes=532742464 weight_movement_bytes=103809024 total_nvlink_bytes=636551488 weight_to_token_ratio=0.194858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392310557e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100112 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531856784 weight_movement_bytes=103809024 total_nvlink_bytes=635665808 weight_to_token_ratio=0.195182 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344160 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698848 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053536 weight_movement_bytes=34603008 total_nvlink_bytes=567656544 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=344408 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265724248 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531104088 weight_movement_bytes=34603008 total_nvlink_bytes=565707096 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344160 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266698848 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053536 weight_movement_bytes=34603008 total_nvlink_bytes=567656544 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344392 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267739464 ingress_bytes=336601088 link_bytes=336601088 token_bytes=535134536 weight_movement_bytes=69206016 total_nvlink_bytes=604340552 weight_to_token_ratio=0.129325 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266602796e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=264773632 dispatch_meta_bytes=341144 combine_payload_bytes=264773632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265114776 ingress_bytes=299376640 link_bytes=299376640 token_bytes=529888408 weight_movement_bytes=34603008 total_nvlink_bytes=564491416 weight_to_token_ratio=0.065302 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126530195e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65166 dup_rows=23810 dup_rows_remote=20706 dedup_ratio=0.242208 router_tokens_per_expert=[798, 794, 761, 726, 797, 753, 775, 790, 766, 795, 743, 753, 735, 791, 770, 724, 769, 766, 718, 841, 779, 794, 762, 726, 770, 800, 818, 781, 763, 777, 819, 752, 751, 814, 748, 765, 766, 732, 774, 738, 746, 780, 752, 750, 720, 774, 780, 806, 759, 802, 790, 762, 780, 760, 763, 730, 757, 765, 744, 741, 795, 802, 761, 739] payload_rows_per_dst=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] dup_rows_per_dst=[3104, 2848, 3002, 3096, 3002, 2890, 2960, 2908] combine_rows_per_src=[9328, 9262, 9308, 9464, 9174, 9326, 9332, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344104 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788904 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233704 weight_movement_bytes=34603008 total_nvlink_bytes=567836712 weight_to_token_ratio=0.064893 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64964 dup_rows=23900 dup_rows_remote=20798 dedup_ratio=0.243123 router_tokens_per_expert=[791, 774, 766, 731, 807, 802, 770, 714, 769, 761, 856, 748, 788, 759, 767, 743, 780, 774, 767, 735, 788, 798, 726, 753, 747, 758, 744, 767, 735, 754, 806, 793, 792, 782, 811, 774, 776, 754, 773, 771, 771, 759, 784, 733, 761, 778, 793, 778, 731, 751, 773, 786, 729, 765, 719, 788, 758, 748, 762, 816, 748, 758, 762, 797] payload_rows_per_dst=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] dup_rows_per_dst=[3102, 3140, 2860, 3202, 2884, 2884, 2902, 2926] combine_rows_per_src=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65002 dup_rows=23942 dup_rows_remote=20890 dedup_ratio=0.243551 router_tokens_per_expert=[774, 803, 772, 757, 789, 778, 813, 813, 782, 755, 773, 777, 767, 777, 736, 784, 760, 714, 798, 789, 749, 761, 779, 747, 739, 810, 773, 732, 794, 785, 770, 787, 753, 783, 750, 783, 748, 751, 758, 815, 728, 764, 798, 754, 728, 742, 755, 745, 765, 763, 777, 736, 775, 761, 779, 778, 799, 734, 799, 765, 729, 773, 787, 740] payload_rows_per_dst=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] dup_rows_per_dst=[2946, 2888, 2894, 2936, 2960, 2900, 3366, 3052] combine_rows_per_src=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65040 dup_rows=23934 dup_rows_remote=20908 dedup_ratio=0.243469 router_tokens_per_expert=[725, 782, 789, 797, 755, 779, 728, 784, 782, 746, 790, 749, 771, 784, 784, 780, 759, 838, 736, 781, 763, 718, 804, 779, 757, 742, 752, 776, 797, 727, 768, 764, 754, 756, 785, 771, 749, 759, 745, 788, 721, 787, 823, 768, 798, 807, 759, 792, 757, 731, 741, 754, 810, 786, 757, 766, 777, 768, 731, 760, 785, 773, 769, 739] payload_rows_per_dst=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] dup_rows_per_dst=[3108, 3012, 3026, 2916, 2934, 3148, 2870, 2920] combine_rows_per_src=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64928 dup_rows=24134 dup_rows_remote=21102 dedup_ratio=0.245504 router_tokens_per_expert=[764, 801, 768, 762, 840, 756, 794, 773, 731, 750, 745, 800, 764, 742, 765, 766, 770, 727, 764, 787, 822, 774, 734, 781, 813, 753, 788, 742, 775, 745, 745, 813, 745, 809, 735, 794, 782, 742, 773, 757, 829, 786, 752, 753, 764, 777, 720, 770, 747, 744, 721, 745, 788, 781, 794, 755, 772, 748, 761, 795, 796, 763, 746, 754] payload_rows_per_dst=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] dup_rows_per_dst=[3190, 2944, 3000, 2924, 3032, 2972, 3034, 3038] combine_rows_per_src=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64722 dup_rows=24196 dup_rows_remote=21152 dedup_ratio=0.246134 router_tokens_per_expert=[784, 804, 784, 780, 767, 732, 730, 770, 802, 770, 784, 758, 815, 751, 781, 754, 818, 776, 714, 808, 757, 768, 773, 776, 757, 778, 759, 767, 747, 757, 710, 758, 735, 768, 744, 758, 799, 798, 799, 747, 759, 785, 793, 772, 807, 765, 798, 803, 779, 769, 770, 765, 731, 727, 742, 753, 795, 796, 771, 742, 698, 759, 754, 782] payload_rows_per_dst=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] dup_rows_per_dst=[3026, 3044, 3094, 2982, 2952, 3224, 2934, 2940] combine_rows_per_src=[9276, 9386, 9286, 9084, 9344, 9340, 9138, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=343488 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267263424 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534183360 weight_movement_bytes=69206016 total_nvlink_bytes=603389376 weight_to_token_ratio=0.129555 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65284 dup_rows=23920 dup_rows_remote=20876 dedup_ratio=0.243327 router_tokens_per_expert=[782, 760, 743, 764, 738, 753, 765, 766, 765, 802, 766, 795, 790, 721, 772, 766, 748, 793, 795, 817, 742, 791, 768, 753, 757, 753, 741, 760, 754, 798, 792, 717, 744, 796, 766, 763, 732, 778, 795, 761, 729, 729, 761, 761, 797, 785, 775, 813, 774, 715, 788, 786, 817, 765, 783, 762, 755, 810, 767, 799, 756, 809, 725, 729] payload_rows_per_dst=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] dup_rows_per_dst=[2952, 2990, 2994, 3044, 2924, 2962, 3028, 3026] combine_rows_per_src=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64950 dup_rows=24240 dup_rows_remote=21324 dedup_ratio=0.246582 router_tokens_per_expert=[790, 784, 785, 728, 812, 685, 766, 776, 774, 789, 810, 742, 781, 735, 757, 780, 795, 723, 782, 818, 821, 756, 786, 792, 751, 716, 783, 804, 731, 747, 790, 819, 753, 811, 774, 755, 783, 746, 772, 798, 772, 755, 737, 751, 734, 787, 756, 723, 758, 739, 739, 773, 803, 770, 774, 781, 755, 761, 782, 742, 768, 760, 782, 750] payload_rows_per_dst=[9240, 9258, 9364, 9268, 9356, 9114, 9266, 9198] dup_rows_per_dst=[3012, 3078, 3182, 3014, 3028, 2916, 3008, 3002] combine_rows_per_src=[9240, 9258, 9364, 9268, 9356, 9114, 9266, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64928 dup_rows=24134 dup_rows_remote=21102 dedup_ratio=0.245504 router_tokens_per_expert=[764, 801, 768, 762, 840, 756, 794, 773, 731, 750, 745, 800, 764, 742, 765, 766, 770, 727, 764, 787, 822, 774, 734, 781, 813, 753, 788, 742, 775, 745, 745, 813, 745, 809, 735, 794, 782, 742, 773, 757, 829, 786, 752, 753, 764, 777, 720, 770, 747, 744, 721, 745, 788, 781, 794, 755, 772, 748, 761, 795, 796, 763, 746, 754] payload_rows_per_dst=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] dup_rows_per_dst=[3190, 2944, 3000, 2924, 3032, 2972, 3034, 3038] combine_rows_per_src=[9326, 9182, 9318, 9424, 9242, 9330, 9116, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65040 dup_rows=23934 dup_rows_remote=20908 dedup_ratio=0.243469 router_tokens_per_expert=[725, 782, 789, 797, 755, 779, 728, 784, 782, 746, 790, 749, 771, 784, 784, 780, 759, 838, 736, 781, 763, 718, 804, 779, 757, 742, 752, 776, 797, 727, 768, 764, 754, 756, 785, 771, 749, 759, 745, 788, 721, 787, 823, 768, 798, 807, 759, 792, 757, 731, 741, 754, 810, 786, 757, 766, 777, 768, 731, 760, 785, 773, 769, 739] payload_rows_per_dst=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] dup_rows_per_dst=[3108, 3012, 3026, 2916, 2934, 3148, 2870, 2920] combine_rows_per_src=[9170, 9360, 9330, 9250, 9280, 9362, 9334, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65002 dup_rows=23942 dup_rows_remote=20890 dedup_ratio=0.243551 router_tokens_per_expert=[774, 803, 772, 757, 789, 778, 813, 813, 782, 755, 773, 777, 767, 777, 736, 784, 760, 714, 798, 789, 749, 761, 779, 747, 739, 810, 773, 732, 794, 785, 770, 787, 753, 783, 750, 783, 748, 751, 758, 815, 728, 764, 798, 754, 728, 742, 755, 745, 765, 763, 777, 736, 775, 761, 779, 778, 799, 734, 799, 765, 729, 773, 787, 740] payload_rows_per_dst=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] dup_rows_per_dst=[2946, 2888, 2894, 2936, 2960, 2900, 3366, 3052] combine_rows_per_src=[9010, 9154, 9198, 9444, 9374, 9278, 9544, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65258 dup_rows=23862 dup_rows_remote=20816 dedup_ratio=0.242737 router_tokens_per_expert=[790, 773, 745, 838, 767, 736, 768, 741, 743, 790, 721, 763, 782, 777, 771, 784, 724, 790, 773, 737, 799, 755, 759, 757, 779, 792, 759, 757, 739, 782, 794, 746, 784, 812, 757, 766, 765, 804, 758, 794, 757, 753, 733, 729, 758, 773, 776, 774, 761, 748, 797, 795, 794, 736, 742, 742, 811, 736, 794, 773, 777, 827, 761, 734] payload_rows_per_dst=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] dup_rows_per_dst=[2992, 2930, 3000, 2976, 3042, 2896, 3046, 2980] combine_rows_per_src=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65284 dup_rows=23920 dup_rows_remote=20876 dedup_ratio=0.243327 router_tokens_per_expert=[782, 760, 743, 764, 738, 753, 765, 766, 765, 802, 766, 795, 790, 721, 772, 766, 748, 793, 795, 817, 742, 791, 768, 753, 757, 753, 741, 760, 754, 798, 792, 717, 744, 796, 766, 763, 732, 778, 795, 761, 729, 729, 761, 761, 797, 785, 775, 813, 774, 715, 788, 786, 817, 765, 783, 762, 755, 810, 767, 799, 756, 809, 725, 729] payload_rows_per_dst=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] dup_rows_per_dst=[2952, 2990, 2994, 3044, 2924, 2962, 3028, 3026] combine_rows_per_src=[9190, 9364, 9420, 9100, 9346, 9338, 9352, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265101312 dispatch_meta_bytes=343496 combine_payload_bytes=265101312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265444808 ingress_bytes=368910336 link_bytes=368910336 token_bytes=530546120 weight_movement_bytes=103809024 total_nvlink_bytes=634355144 weight_to_token_ratio=0.195664 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388179895e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=344640 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267747904 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535151168 weight_movement_bytes=34603008 total_nvlink_bytes=569754176 weight_to_token_ratio=0.064660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=345096 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266380296 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532415496 weight_movement_bytes=34603008 total_nvlink_bytes=567018504 weight_to_token_ratio=0.064992 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344120 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289208 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234296 weight_movement_bytes=69206016 total_nvlink_bytes=601440312 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343792 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266747632 ingress_bytes=370212864 link_bytes=370212864 token_bytes=533151472 weight_movement_bytes=103809024 total_nvlink_bytes=636960496 weight_to_token_ratio=0.194708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393081203e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343568 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591760 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839952 weight_movement_bytes=34603008 total_nvlink_bytes=567442960 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344296 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641064 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937832 weight_movement_bytes=34603008 total_nvlink_bytes=569540840 weight_to_token_ratio=0.064686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64964 dup_rows=23900 dup_rows_remote=20798 dedup_ratio=0.243123 router_tokens_per_expert=[791, 774, 766, 731, 807, 802, 770, 714, 769, 761, 856, 748, 788, 759, 767, 743, 780, 774, 767, 735, 788, 798, 726, 753, 747, 758, 744, 767, 735, 754, 806, 793, 792, 782, 811, 774, 776, 754, 773, 771, 771, 759, 784, 733, 761, 778, 793, 778, 731, 751, 773, 786, 729, 765, 719, 788, 758, 748, 762, 816, 748, 758, 762, 797] payload_rows_per_dst=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] dup_rows_per_dst=[3102, 3140, 2860, 3202, 2884, 2884, 2902, 2926] combine_rows_per_src=[9440, 9242, 9150, 9420, 9168, 9430, 9182, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65040 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[766, 815, 767, 774, 766, 764, 772, 773, 790, 760, 801, 765, 713, 822, 727, 772, 722, 798, 715, 765, 775, 773, 766, 826, 733, 756, 786, 754, 817, 766, 747, 774, 773, 775, 773, 776, 728, 849, 799, 766, 772, 783, 767, 798, 758, 741, 754, 777, 774, 806, 801, 744, 727, 768, 763, 736, 785, 780, 710, 764, 703, 747, 771, 764] payload_rows_per_dst=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] dup_rows_per_dst=[3018, 3022, 2966, 3088, 3114, 3012, 2942, 2824] combine_rows_per_src=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64820 dup_rows=24100 dup_rows_remote=21162 dedup_ratio=0.245158 router_tokens_per_expert=[752, 748, 790, 829, 758, 791, 751, 820, 780, 778, 739, 789, 740, 746, 766, 754, 810, 753, 725, 762, 763, 814, 774, 819, 755, 759, 759, 777, 759, 798, 783, 771, 737, 810, 757, 762, 760, 722, 762, 764, 770, 798, 758, 800, 751, 778, 815, 724, 796, 745, 737, 789, 819, 746, 760, 752, 724, 741, 755, 788, 725, 756, 801, 738] payload_rows_per_dst=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] dup_rows_per_dst=[3130, 2914, 3084, 2938, 2946, 3094, 3032, 2962] combine_rows_per_src=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64814 dup_rows=24278 dup_rows_remote=21306 dedup_ratio=0.246969 router_tokens_per_expert=[792, 769, 768, 739, 830, 787, 793, 791, 742, 749, 782, 765, 821, 749, 798, 741, 798, 746, 777, 740, 734, 778, 777, 744, 782, 778, 784, 720, 788, 694, 751, 752, 739, 785, 739, 742, 801, 751, 790, 775, 753, 752, 769, 773, 747, 783, 776, 739, 773, 808, 775, 774, 738, 804, 762, 776, 803, 738, 777, 779, 778, 772, 756, 766] payload_rows_per_dst=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] dup_rows_per_dst=[3146, 3066, 2994, 2982, 2970, 2972, 3100, 3048] combine_rows_per_src=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65050 dup_rows=23902 dup_rows_remote=20976 dedup_ratio=0.243144 router_tokens_per_expert=[680, 768, 785, 754, 783, 788, 765, 710, 758, 756, 804, 726, 772, 811, 742, 753, 802, 726, 746, 754, 767, 749, 760, 770, 756, 785, 842, 815, 821, 747, 762, 805, 741, 732, 785, 771, 772, 775, 749, 781, 783, 755, 803, 803, 740, 741, 816, 736, 799, 750, 761, 723, 797, 802, 761, 746, 775, 792, 744, 808, 779, 711, 749, 810] payload_rows_per_dst=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] dup_rows_per_dst=[2848, 2942, 2874, 3182, 2984, 2996, 2926, 3150] combine_rows_per_src=[9218, 9302, 9274, 9484, 9228, 9358, 9352, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343048 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266435592 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532528136 weight_movement_bytes=69206016 total_nvlink_bytes=601734152 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65034 dup_rows=23892 dup_rows_remote=20906 dedup_ratio=0.243042 router_tokens_per_expert=[780, 707, 735, 741, 765, 784, 806, 755, 735, 778, 765, 762, 822, 762, 779, 781, 762, 783, 757, 736, 786, 750, 808, 813, 782, 778, 740, 797, 785, 749, 787, 788, 792, 769, 751, 732, 768, 788, 788, 794, 753, 751, 768, 758, 784, 757, 761, 758, 744, 754, 756, 756, 769, 774, 774, 739, 722, 732, 812, 791, 774, 737, 807, 781] payload_rows_per_dst=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] dup_rows_per_dst=[2886, 3020, 3054, 3116, 2986, 2902, 2966, 2962] combine_rows_per_src=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=343568 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266591760 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532839952 weight_movement_bytes=34603008 total_nvlink_bytes=567442960 weight_to_token_ratio=0.064941 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=344120 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266289208 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532234296 weight_movement_bytes=69206016 total_nvlink_bytes=601440312 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265101312 dispatch_meta_bytes=343496 combine_payload_bytes=265101312 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265444808 ingress_bytes=368910336 link_bytes=368910336 token_bytes=530546120 weight_movement_bytes=103809024 total_nvlink_bytes=634355144 weight_to_token_ratio=0.195664 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388179895e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=345096 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266380296 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532415496 weight_movement_bytes=34603008 total_nvlink_bytes=567018504 weight_to_token_ratio=0.064992 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267403264 dispatch_meta_bytes=344640 combine_payload_bytes=267403264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267747904 ingress_bytes=302006272 link_bytes=302006272 token_bytes=535151168 weight_movement_bytes=34603008 total_nvlink_bytes=569754176 weight_to_token_ratio=0.064660 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136425288e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65058 dup_rows=23940 dup_rows_remote=21044 dedup_ratio=0.243530 router_tokens_per_expert=[730, 785, 777, 763, 746, 771, 768, 778, 774, 771, 763, 767, 761, 746, 792, 776, 749, 745, 774, 778, 761, 744, 774, 776, 733, 774, 766, 755, 751, 831, 782, 797, 750, 727, 762, 783, 783, 802, 790, 801, 783, 735, 769, 791, 786, 737, 776, 769, 764, 750, 720, 783, 733, 757, 765, 785, 786, 736, 759, 766, 770, 827, 762, 787] payload_rows_per_dst=[9288, 9262, 9306, 9232, 9348, 9358, 9234, 9336] dup_rows_per_dst=[2948, 3038, 2896, 3146, 3048, 2934, 2880, 3050] combine_rows_per_src=[9288, 9262, 9306, 9232, 9348, 9358, 9234, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74040 payload_rows_remote=64662 dup_rows=24264 dup_rows_remote=21060 dedup_ratio=0.246826 router_tokens_per_expert=[765, 806, 739, 745, 819, 746, 755, 765, 753, 775, 752, 773, 707, 780, 802, 788, 741, 777, 769, 815, 800, 744, 772, 769, 805, 758, 754, 779, 799, 762, 767, 774, 746, 755, 813, 754, 769, 774, 806, 729, 748, 759, 752, 738, 764, 772, 756, 773, 746, 754, 785, 756, 764, 762, 796, 799, 766, 730, 715, 789, 744, 788, 813, 782] payload_rows_per_dst=[9028, 9222, 9172, 9150, 9230, 9180, 9680, 9378] dup_rows_per_dst=[2902, 3038, 2914, 2880, 3000, 3028, 3298, 3204] combine_rows_per_src=[9028, 9222, 9172, 9150, 9230, 9180, 9680, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=65080 dup_rows=24092 dup_rows_remote=21234 dedup_ratio=0.245076 router_tokens_per_expert=[734, 750, 805, 784, 787, 783, 815, 748, 764, 751, 793, 767, 789, 774, 707, 742, 792, 731, 765, 768, 768, 825, 728, 747, 749, 777, 749, 789, 778, 775, 803, 776, 736, 771, 779, 790, 757, 758, 756, 785, 774, 821, 760, 772, 759, 735, 764, 798, 727, 753, 703, 799, 728, 788, 777, 720, 831, 769, 743, 785, 732, 755, 823, 791] payload_rows_per_dst=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] dup_rows_per_dst=[3106, 2906, 3040, 2992, 2958, 3092, 2858, 3140] combine_rows_per_src=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65204 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[753, 778, 763, 750, 747, 797, 764, 733, 742, 791, 726, 772, 740, 787, 746, 762, 773, 797, 793, 790, 788, 737, 757, 750, 783, 755, 783, 743, 772, 804, 752, 811, 786, 853, 743, 732, 765, 737, 731, 793, 764, 780, 753, 811, 838, 823, 794, 776, 737, 762, 722, 758, 737, 801, 743, 812, 714, 791, 781, 768, 736, 739, 750, 783] payload_rows_per_dst=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] dup_rows_per_dst=[2966, 3050, 3036, 2980, 2938, 3016, 3030, 2880] combine_rows_per_src=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343928 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846648 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349368 weight_movement_bytes=69206016 total_nvlink_bytes=600555384 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344016 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747856 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151696 weight_movement_bytes=34603008 total_nvlink_bytes=567754704 weight_to_token_ratio=0.064903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344480 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265822624 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531300768 weight_movement_bytes=69206016 total_nvlink_bytes=600506784 weight_to_token_ratio=0.130258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343760 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723024 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102288 weight_movement_bytes=103809024 total_nvlink_bytes=636911312 weight_to_token_ratio=0.194726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344104 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788904 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233704 weight_movement_bytes=34603008 total_nvlink_bytes=567836712 weight_to_token_ratio=0.064893 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344408 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821976 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299544 weight_movement_bytes=69206016 total_nvlink_bytes=602505560 weight_to_token_ratio=0.129770 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342888 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198440 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053992 weight_movement_bytes=34603008 total_nvlink_bytes=564657000 weight_to_token_ratio=0.065282 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=345256 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266912936 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533480616 weight_movement_bytes=34603008 total_nvlink_bytes=568083624 weight_to_token_ratio=0.064863 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65204 dup_rows=23896 dup_rows_remote=20930 dedup_ratio=0.243083 router_tokens_per_expert=[753, 778, 763, 750, 747, 797, 764, 733, 742, 791, 726, 772, 740, 787, 746, 762, 773, 797, 793, 790, 788, 737, 757, 750, 783, 755, 783, 743, 772, 804, 752, 811, 786, 853, 743, 732, 765, 737, 731, 793, 764, 780, 753, 811, 838, 823, 794, 776, 737, 762, 722, 758, 737, 801, 743, 812, 714, 791, 781, 768, 736, 739, 750, 783] payload_rows_per_dst=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] dup_rows_per_dst=[2966, 3050, 3036, 2980, 2938, 3016, 3030, 2880] combine_rows_per_src=[9204, 9374, 9434, 9426, 9242, 9370, 9114, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65154 dup_rows=23906 dup_rows_remote=20904 dedup_ratio=0.243184 router_tokens_per_expert=[786, 771, 773, 764, 782, 797, 768, 804, 778, 774, 736, 809, 766, 729, 748, 783, 747, 774, 754, 783, 798, 756, 716, 768, 729, 778, 780, 765, 803, 734, 762, 772, 732, 782, 747, 795, 799, 776, 741, 756, 742, 769, 794, 760, 766, 780, 753, 813, 780, 737, 785, 787, 781, 751, 767, 761, 759, 772, 804, 779, 733, 761, 718, 785] payload_rows_per_dst=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] dup_rows_per_dst=[3052, 3030, 2946, 3002, 2998, 2980, 2934, 2964] combine_rows_per_src=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267075584 dispatch_meta_bytes=344536 combine_payload_bytes=267075584 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267420120 ingress_bytes=336281600 link_bytes=336281600 token_bytes=534495704 weight_movement_bytes=69206016 total_nvlink_bytes=603701720 weight_to_token_ratio=0.129479 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265400588e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65034 dup_rows=23892 dup_rows_remote=20906 dedup_ratio=0.243042 router_tokens_per_expert=[780, 707, 735, 741, 765, 784, 806, 755, 735, 778, 765, 762, 822, 762, 779, 781, 762, 783, 757, 736, 786, 750, 808, 813, 782, 778, 740, 797, 785, 749, 787, 788, 792, 769, 751, 732, 768, 788, 788, 794, 753, 751, 768, 758, 784, 757, 761, 758, 744, 754, 756, 756, 769, 774, 774, 739, 722, 732, 812, 791, 774, 737, 807, 781] payload_rows_per_dst=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] dup_rows_per_dst=[2886, 3020, 3054, 3116, 2986, 2902, 2966, 2962] combine_rows_per_src=[9260, 9348, 9336, 9296, 9378, 9278, 9166, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64814 dup_rows=24278 dup_rows_remote=21306 dedup_ratio=0.246969 router_tokens_per_expert=[792, 769, 768, 739, 830, 787, 793, 791, 742, 749, 782, 765, 821, 749, 798, 741, 798, 746, 777, 740, 734, 778, 777, 744, 782, 778, 784, 720, 788, 694, 751, 752, 739, 785, 739, 742, 801, 751, 790, 775, 753, 752, 769, 773, 747, 783, 776, 739, 773, 808, 775, 774, 738, 804, 762, 776, 803, 738, 777, 779, 778, 772, 756, 766] payload_rows_per_dst=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] dup_rows_per_dst=[3146, 3066, 2994, 2982, 2970, 2972, 3100, 3048] combine_rows_per_src=[9392, 9228, 9194, 9116, 9274, 9212, 9320, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65040 dup_rows=23986 dup_rows_remote=20964 dedup_ratio=0.243998 router_tokens_per_expert=[766, 815, 767, 774, 766, 764, 772, 773, 790, 760, 801, 765, 713, 822, 727, 772, 722, 798, 715, 765, 775, 773, 766, 826, 733, 756, 786, 754, 817, 766, 747, 774, 773, 775, 773, 776, 728, 849, 799, 766, 772, 783, 767, 798, 758, 741, 754, 777, 774, 806, 801, 744, 727, 768, 763, 736, 785, 780, 710, 764, 703, 747, 771, 764] payload_rows_per_dst=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] dup_rows_per_dst=[3018, 3022, 2966, 3088, 3114, 3012, 2942, 2824] combine_rows_per_src=[9376, 9278, 9314, 9178, 9364, 9288, 9296, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64820 dup_rows=24100 dup_rows_remote=21162 dedup_ratio=0.245158 router_tokens_per_expert=[752, 748, 790, 829, 758, 791, 751, 820, 780, 778, 739, 789, 740, 746, 766, 754, 810, 753, 725, 762, 763, 814, 774, 819, 755, 759, 759, 777, 759, 798, 783, 771, 737, 810, 757, 762, 760, 722, 762, 764, 770, 798, 758, 800, 751, 778, 815, 724, 796, 745, 737, 789, 819, 746, 760, 752, 724, 741, 755, 788, 725, 756, 801, 738] payload_rows_per_dst=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] dup_rows_per_dst=[3130, 2914, 3084, 2938, 2946, 3094, 3032, 2962] combine_rows_per_src=[9348, 9270, 9356, 9384, 9202, 9294, 9256, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343760 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723024 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102288 weight_movement_bytes=103809024 total_nvlink_bytes=636911312 weight_to_token_ratio=0.194726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=344480 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265822624 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531300768 weight_movement_bytes=69206016 total_nvlink_bytes=600506784 weight_to_token_ratio=0.130258 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64924 dup_rows=24118 dup_rows_remote=21194 dedup_ratio=0.245341 router_tokens_per_expert=[777, 801, 821, 786, 726, 759, 813, 775, 752, 743, 775, 792, 761, 762, 772, 790, 771, 773, 762, 763, 775, 774, 744, 742, 759, 804, 763, 770, 750, 800, 744, 772, 717, 748, 766, 824, 733, 745, 717, 798, 715, 727, 752, 794, 765, 727, 736, 829, 763, 805, 831, 743, 788, 761, 810, 794, 746, 730, 754, 757, 813, 744, 776, 773] payload_rows_per_dst=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] dup_rows_per_dst=[2758, 2822, 2980, 3062, 2990, 3356, 3226, 2924] combine_rows_per_src=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64960 dup_rows=24018 dup_rows_remote=21146 dedup_ratio=0.244324 router_tokens_per_expert=[783, 739, 794, 807, 766, 801, 761, 799, 780, 786, 810, 777, 848, 775, 731, 760, 813, 755, 756, 748, 723, 762, 777, 765, 773, 732, 724, 797, 759, 738, 748, 781, 773, 808, 795, 723, 713, 813, 774, 746, 758, 782, 742, 748, 764, 780, 773, 745, 742, 796, 761, 763, 776, 750, 751, 760, 752, 846, 773, 744, 775, 735, 811, 712] payload_rows_per_dst=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] dup_rows_per_dst=[3112, 3102, 2872, 2928, 3018, 2956, 3030, 3000] combine_rows_per_src=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343928 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846648 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349368 weight_movement_bytes=69206016 total_nvlink_bytes=600555384 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344016 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747856 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151696 weight_movement_bytes=34603008 total_nvlink_bytes=567754704 weight_to_token_ratio=0.064903 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65120 dup_rows=23870 dup_rows_remote=20804 dedup_ratio=0.242818 router_tokens_per_expert=[740, 765, 754, 790, 793, 769, 754, 768, 766, 782, 790, 795, 773, 761, 775, 810, 802, 751, 775, 812, 774, 782, 758, 759, 774, 743, 748, 764, 754, 749, 807, 795, 815, 779, 772, 785, 747, 760, 800, 723, 751, 724, 758, 796, 749, 736, 746, 770, 716, 748, 731, 753, 736, 753, 805, 802, 761, 793, 773, 769, 803, 740, 753, 773] payload_rows_per_dst=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] dup_rows_per_dst=[3066, 2978, 2984, 3032, 2972, 2868, 2950, 3020] combine_rows_per_src=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65258 dup_rows=23862 dup_rows_remote=20816 dedup_ratio=0.242737 router_tokens_per_expert=[790, 773, 745, 838, 767, 736, 768, 741, 743, 790, 721, 763, 782, 777, 771, 784, 724, 790, 773, 737, 799, 755, 759, 757, 779, 792, 759, 757, 739, 782, 794, 746, 784, 812, 757, 766, 765, 804, 758, 794, 757, 753, 733, 729, 758, 773, 776, 774, 761, 748, 797, 795, 794, 736, 742, 742, 811, 736, 794, 773, 777, 827, 761, 734] payload_rows_per_dst=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] dup_rows_per_dst=[2992, 2930, 3000, 2976, 3042, 2896, 3046, 2980] combine_rows_per_src=[9324, 9332, 9188, 9320, 9438, 9210, 9184, 9446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344472 combine_payload_bytes=265928704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266273176 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201880 weight_movement_bytes=69206016 total_nvlink_bytes=601407896 weight_to_token_ratio=0.130037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344424 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266420584 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496744 weight_movement_bytes=69206016 total_nvlink_bytes=601702760 weight_to_token_ratio=0.129965 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=65046 dup_rows=24108 dup_rows_remote=21164 dedup_ratio=0.245239 router_tokens_per_expert=[766, 708, 750, 776, 775, 792, 770, 806, 733, 765, 706, 776, 780, 775, 729, 759, 804, 791, 812, 752, 785, 774, 791, 757, 756, 742, 781, 832, 769, 752, 765, 800, 755, 793, 759, 750, 724, 757, 751, 758, 791, 746, 747, 758, 761, 766, 763, 780, 756, 796, 778, 760, 746, 786, 757, 802, 768, 781, 777, 778, 758, 795, 766, 760] payload_rows_per_dst=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] dup_rows_per_dst=[3058, 2906, 3098, 2952, 2944, 2940, 3094, 3116] combine_rows_per_src=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344296 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267641064 ingress_bytes=301899776 link_bytes=301899776 token_bytes=534937832 weight_movement_bytes=34603008 total_nvlink_bytes=569540840 weight_to_token_ratio=0.064686 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136024552e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65190 dup_rows=23868 dup_rows_remote=20896 dedup_ratio=0.242798 router_tokens_per_expert=[813, 804, 765, 771, 747, 763, 779, 775, 767, 785, 781, 744, 740, 782, 794, 767, 805, 800, 719, 745, 776, 761, 748, 750, 750, 801, 836, 760, 778, 738, 764, 794, 756, 790, 762, 832, 739, 724, 792, 733, 772, 725, 768, 787, 759, 761, 762, 775, 779, 761, 762, 763, 765, 749, 772, 773, 788, 721, 755, 743, 811, 747, 764, 760] payload_rows_per_dst=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] dup_rows_per_dst=[3080, 3020, 2924, 3042, 2998, 2972, 2934, 2898] combine_rows_per_src=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65154 dup_rows=23906 dup_rows_remote=20904 dedup_ratio=0.243184 router_tokens_per_expert=[786, 771, 773, 764, 782, 797, 768, 804, 778, 774, 736, 809, 766, 729, 748, 783, 747, 774, 754, 783, 798, 756, 716, 768, 729, 778, 780, 765, 803, 734, 762, 772, 732, 782, 747, 795, 799, 776, 741, 756, 742, 769, 794, 760, 766, 780, 753, 813, 780, 737, 785, 787, 781, 751, 767, 761, 759, 772, 804, 779, 733, 761, 718, 785] payload_rows_per_dst=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] dup_rows_per_dst=[3052, 3030, 2946, 3002, 2998, 2980, 2934, 2964] combine_rows_per_src=[9438, 9216, 9246, 9244, 9258, 9374, 9364, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65034 dup_rows=23916 dup_rows_remote=20948 dedup_ratio=0.243286 router_tokens_per_expert=[718, 732, 759, 787, 787, 783, 759, 778, 791, 762, 785, 779, 758, 723, 784, 779, 775, 737, 750, 804, 741, 786, 732, 722, 704, 815, 765, 753, 757, 771, 818, 788, 783, 792, 858, 778, 769, 789, 716, 767, 773, 739, 786, 743, 745, 773, 782, 718, 758, 780, 787, 747, 753, 757, 778, 733, 786, 785, 792, 758, 777, 803, 783, 782] payload_rows_per_dst=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] dup_rows_per_dst=[2872, 2968, 2964, 3062, 3082, 2962, 2914, 3092] combine_rows_per_src=[9334, 9354, 9130, 9280, 9422, 9156, 9272, 9440] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65190 dup_rows=23868 dup_rows_remote=20896 dedup_ratio=0.242798 router_tokens_per_expert=[813, 804, 765, 771, 747, 763, 779, 775, 767, 785, 781, 744, 740, 782, 794, 767, 805, 800, 719, 745, 776, 761, 748, 750, 750, 801, 836, 760, 778, 738, 764, 794, 756, 790, 762, 832, 739, 724, 792, 733, 772, 725, 768, 787, 759, 761, 762, 775, 779, 761, 762, 763, 765, 749, 772, 773, 788, 721, 755, 743, 811, 747, 764, 760] payload_rows_per_dst=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] dup_rows_per_dst=[3080, 3020, 2924, 3042, 2998, 2972, 2934, 2898] combine_rows_per_src=[9354, 9300, 9284, 9400, 9258, 9246, 9314, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=65046 dup_rows=24108 dup_rows_remote=21164 dedup_ratio=0.245239 router_tokens_per_expert=[766, 708, 750, 776, 775, 792, 770, 806, 733, 765, 706, 776, 780, 775, 729, 759, 804, 791, 812, 752, 785, 774, 791, 757, 756, 742, 781, 832, 769, 752, 765, 800, 755, 793, 759, 750, 724, 757, 751, 758, 791, 746, 747, 758, 761, 766, 763, 780, 756, 796, 778, 760, 746, 786, 757, 802, 768, 781, 777, 778, 758, 795, 766, 760] payload_rows_per_dst=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] dup_rows_per_dst=[3058, 2906, 3098, 2952, 2944, 2940, 3094, 3116] combine_rows_per_src=[9228, 9140, 9434, 9442, 9150, 9284, 9268, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64960 dup_rows=24018 dup_rows_remote=21146 dedup_ratio=0.244324 router_tokens_per_expert=[783, 739, 794, 807, 766, 801, 761, 799, 780, 786, 810, 777, 848, 775, 731, 760, 813, 755, 756, 748, 723, 762, 777, 765, 773, 732, 724, 797, 759, 738, 748, 781, 773, 808, 795, 723, 713, 813, 774, 746, 758, 782, 742, 748, 764, 780, 773, 745, 742, 796, 761, 763, 776, 750, 751, 760, 752, 846, 773, 744, 775, 735, 811, 712] payload_rows_per_dst=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] dup_rows_per_dst=[3112, 3102, 2872, 2928, 3018, 2956, 3030, 3000] combine_rows_per_src=[9388, 9432, 9326, 9176, 9272, 9228, 9168, 9296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344840 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773256 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201672 weight_movement_bytes=34603008 total_nvlink_bytes=567804680 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344344 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267362584 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380824 weight_movement_bytes=34603008 total_nvlink_bytes=568983832 weight_to_token_ratio=0.064753 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64568 dup_rows=24380 dup_rows_remote=21280 dedup_ratio=0.248006 router_tokens_per_expert=[789, 759, 787, 831, 776, 817, 732, 773, 742, 767, 738, 722, 792, 767, 811, 753, 715, 780, 823, 733, 767, 798, 755, 717, 748, 747, 760, 790, 751, 742, 734, 728, 770, 738, 747, 801, 739, 727, 767, 756, 749, 777, 765, 780, 801, 791, 770, 772, 766, 779, 777, 748, 822, 804, 770, 786, 765, 783, 789, 769, 764, 752, 802, 782] payload_rows_per_dst=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] dup_rows_per_dst=[3150, 2958, 2712, 3248, 2972, 2956, 3284, 3100] combine_rows_per_src=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344232 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267215016 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085800 weight_movement_bytes=34603008 total_nvlink_bytes=568688808 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64924 dup_rows=24118 dup_rows_remote=21194 dedup_ratio=0.245341 router_tokens_per_expert=[777, 801, 821, 786, 726, 759, 813, 775, 752, 743, 775, 792, 761, 762, 772, 790, 771, 773, 762, 763, 775, 774, 744, 742, 759, 804, 763, 770, 750, 800, 744, 772, 717, 748, 766, 824, 733, 745, 717, 798, 715, 727, 752, 794, 765, 727, 736, 829, 763, 805, 831, 743, 788, 761, 810, 794, 746, 730, 754, 757, 813, 744, 776, 773] payload_rows_per_dst=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] dup_rows_per_dst=[2758, 2822, 2980, 3062, 2990, 3356, 3226, 2924] combine_rows_per_src=[8868, 9076, 9166, 9324, 9346, 9624, 9520, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65250 dup_rows=23824 dup_rows_remote=20876 dedup_ratio=0.242350 router_tokens_per_expert=[798, 831, 802, 748, 748, 752, 744, 768, 774, 784, 765, 783, 809, 754, 802, 795, 767, 800, 845, 779, 770, 771, 741, 770, 770, 750, 748, 744, 771, 727, 808, 743, 793, 762, 723, 818, 776, 737, 740, 754, 744, 707, 815, 775, 800, 714, 789, 739, 759, 802, 779, 739, 774, 734, 776, 726, 773, 814, 797, 719, 775, 738, 736, 764] payload_rows_per_dst=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] dup_rows_per_dst=[2928, 3096, 3126, 2964, 2922, 2976, 2948, 2864] combine_rows_per_src=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65052 dup_rows=23818 dup_rows_remote=20794 dedup_ratio=0.242289 router_tokens_per_expert=[823, 750, 752, 752, 780, 780, 734, 783, 761, 803, 764, 766, 788, 780, 774, 735, 772, 775, 753, 761, 766, 781, 810, 811, 767, 732, 769, 733, 773, 749, 747, 802, 760, 737, 767, 783, 738, 773, 803, 780, 775, 760, 707, 766, 778, 787, 764, 781, 781, 804, 775, 765, 750, 729, 725, 752, 750, 788, 761, 752, 770, 778, 826, 761] payload_rows_per_dst=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] dup_rows_per_dst=[2972, 3006, 3024, 2908, 2910, 2992, 2898, 3108] combine_rows_per_src=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343928 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266723192 ingress_bytes=404791296 link_bytes=404791296 token_bytes=533102456 weight_movement_bytes=138412032 total_nvlink_bytes=671514488 weight_to_token_ratio=0.259635 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523197058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267018240 dispatch_meta_bytes=344344 combine_payload_bytes=267018240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267362584 ingress_bytes=301621248 link_bytes=301621248 token_bytes=534380824 weight_movement_bytes=34603008 total_nvlink_bytes=568983832 weight_to_token_ratio=0.064753 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134976474e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344840 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266773256 ingress_bytes=301031424 link_bytes=301031424 token_bytes=533201672 weight_movement_bytes=34603008 total_nvlink_bytes=567804680 weight_to_token_ratio=0.064897 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132757013e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344424 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266420584 ingress_bytes=335282176 link_bytes=335282176 token_bytes=532496744 weight_movement_bytes=69206016 total_nvlink_bytes=601702760 weight_to_token_ratio=0.129965 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261639836e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65120 dup_rows=23870 dup_rows_remote=20804 dedup_ratio=0.242818 router_tokens_per_expert=[740, 765, 754, 790, 793, 769, 754, 768, 766, 782, 790, 795, 773, 761, 775, 810, 802, 751, 775, 812, 774, 782, 758, 759, 774, 743, 748, 764, 754, 749, 807, 795, 815, 779, 772, 785, 747, 760, 800, 723, 751, 724, 758, 796, 749, 736, 746, 770, 716, 748, 731, 753, 736, 753, 805, 802, 761, 793, 773, 769, 803, 740, 753, 773] payload_rows_per_dst=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] dup_rows_per_dst=[3066, 2978, 2984, 3032, 2972, 2868, 2950, 3020] combine_rows_per_src=[9314, 9428, 9464, 9236, 9390, 9192, 9138, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344472 combine_payload_bytes=265928704 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266273176 ingress_bytes=335134720 link_bytes=335134720 token_bytes=532201880 weight_movement_bytes=69206016 total_nvlink_bytes=601407896 weight_to_token_ratio=0.130037 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261084971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344504 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608504 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872504 weight_movement_bytes=34603008 total_nvlink_bytes=569475512 weight_to_token_ratio=0.064694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65126 dup_rows=23918 dup_rows_remote=20970 dedup_ratio=0.243306 router_tokens_per_expert=[746, 778, 793, 701, 771, 701, 793, 766, 781, 773, 766, 760, 729, 794, 772, 780, 792, 779, 768, 744, 770, 742, 774, 752, 833, 761, 748, 740, 756, 750, 741, 775, 784, 772, 778, 779, 784, 757, 761, 830, 747, 763, 792, 751, 763, 755, 782, 791, 791, 751, 735, 761, 745, 776, 730, 775, 770, 799, 806, 799, 818, 758, 758, 762] payload_rows_per_dst=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] dup_rows_per_dst=[2932, 2994, 3002, 2948, 3048, 3022, 2868, 3104] combine_rows_per_src=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65034 dup_rows=24028 dup_rows_remote=21008 dedup_ratio=0.244425 router_tokens_per_expert=[722, 757, 745, 741, 803, 751, 724, 696, 745, 761, 770, 755, 792, 751, 775, 782, 804, 761, 755, 774, 821, 785, 795, 821, 783, 766, 768, 753, 750, 792, 764, 808, 756, 780, 793, 770, 783, 758, 760, 731, 744, 780, 734, 786, 754, 781, 764, 764, 768, 739, 759, 765, 740, 795, 756, 764, 785, 745, 802, 717, 761, 800, 816, 832] payload_rows_per_dst=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] dup_rows_per_dst=[2834, 3020, 3106, 3068, 3014, 2996, 2882, 3108] combine_rows_per_src=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64852 dup_rows=24240 dup_rows_remote=21278 dedup_ratio=0.246582 router_tokens_per_expert=[796, 762, 785, 738, 796, 787, 804, 811, 770, 769, 749, 732, 797, 750, 783, 808, 754, 784, 789, 741, 777, 766, 812, 737, 769, 755, 780, 751, 728, 728, 774, 791, 782, 742, 811, 769, 739, 784, 759, 757, 735, 775, 735, 782, 765, 781, 758, 756, 752, 711, 742, 824, 757, 741, 743, 820, 781, 769, 788, 761, 761, 746, 794, 759] payload_rows_per_dst=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] dup_rows_per_dst=[3208, 3054, 3074, 2946, 2962, 2962, 2984, 3050] combine_rows_per_src=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65084 dup_rows=24130 dup_rows_remote=21120 dedup_ratio=0.245463 router_tokens_per_expert=[804, 743, 806, 779, 740, 813, 730, 747, 758, 747, 750, 778, 774, 787, 736, 786, 752, 764, 762, 799, 794, 792, 782, 737, 796, 820, 743, 803, 764, 753, 739, 785, 767, 775, 771, 747, 756, 770, 705, 759, 761, 762, 795, 707, 803, 736, 767, 792, 773, 761, 815, 733, 781, 769, 729, 774, 765, 765, 769, 797, 807, 726, 787, 765] payload_rows_per_dst=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] dup_rows_per_dst=[2996, 2934, 3136, 3032, 3010, 2986, 2996, 3040] combine_rows_per_src=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65052 dup_rows=23818 dup_rows_remote=20794 dedup_ratio=0.242289 router_tokens_per_expert=[823, 750, 752, 752, 780, 780, 734, 783, 761, 803, 764, 766, 788, 780, 774, 735, 772, 775, 753, 761, 766, 781, 810, 811, 767, 732, 769, 733, 773, 749, 747, 802, 760, 737, 767, 783, 738, 773, 803, 780, 775, 760, 707, 766, 778, 787, 764, 781, 781, 804, 775, 765, 750, 729, 725, 752, 750, 788, 761, 752, 770, 778, 826, 761] payload_rows_per_dst=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] dup_rows_per_dst=[2972, 3006, 3024, 2908, 2910, 2992, 2898, 3108] combine_rows_per_src=[9336, 9336, 9434, 9236, 9372, 9244, 9264, 9264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344384 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100480 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856576 weight_movement_bytes=34603008 total_nvlink_bytes=568459584 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73924 payload_rows_remote=64568 dup_rows=24380 dup_rows_remote=21280 dedup_ratio=0.248006 router_tokens_per_expert=[789, 759, 787, 831, 776, 817, 732, 773, 742, 767, 738, 722, 792, 767, 811, 753, 715, 780, 823, 733, 767, 798, 755, 717, 748, 747, 760, 790, 751, 742, 734, 728, 770, 738, 747, 801, 739, 727, 767, 756, 749, 777, 765, 780, 801, 791, 770, 772, 766, 779, 777, 748, 822, 804, 770, 786, 765, 783, 789, 769, 764, 752, 802, 782] payload_rows_per_dst=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] dup_rows_per_dst=[3150, 2958, 2712, 3248, 2972, 2956, 3284, 3100] combine_rows_per_src=[9378, 9142, 8782, 9474, 9118, 9186, 9488, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64948 dup_rows=24080 dup_rows_remote=21186 dedup_ratio=0.244954 router_tokens_per_expert=[767, 772, 710, 747, 737, 786, 790, 732, 748, 794, 789, 793, 749, 798, 756, 778, 774, 786, 767, 764, 783, 746, 803, 760, 759, 773, 742, 705, 796, 768, 748, 779, 763, 788, 786, 822, 774, 747, 774, 768, 811, 775, 781, 797, 786, 792, 776, 779, 770, 721, 758, 762, 778, 758, 717, 821, 798, 704, 795, 772, 735, 751, 739, 755] payload_rows_per_dst=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] dup_rows_per_dst=[2902, 3130, 3084, 2910, 3052, 3152, 2894, 2956] combine_rows_per_src=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344168 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723432 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102696 weight_movement_bytes=103809024 total_nvlink_bytes=636911720 weight_to_token_ratio=0.194726 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344520 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265978312 ingress_bytes=334839808 link_bytes=334839808 token_bytes=531612104 weight_movement_bytes=69206016 total_nvlink_bytes=600818120 weight_to_token_ratio=0.130181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259975241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249368 weight_movement_bytes=69206016 total_nvlink_bytes=602455384 weight_to_token_ratio=0.129782 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=64934 dup_rows=23924 dup_rows_remote=20842 dedup_ratio=0.243368 router_tokens_per_expert=[702, 748, 742, 770, 782, 809, 758, 744, 804, 737, 785, 784, 746, 771, 740, 797, 774, 780, 801, 739, 776, 783, 759, 802, 735, 751, 726, 776, 802, 771, 780, 741, 782, 803, 754, 767, 776, 809, 733, 753, 786, 769, 783, 741, 742, 785, 771, 760, 788, 735, 790, 754, 792, 795, 779, 792, 792, 801, 726, 734, 769, 749, 754, 773] payload_rows_per_dst=[9446, 9320, 9300, 9284, 9242, 9350, 9432, 9006] dup_rows_per_dst=[3082, 3012, 3128, 2880, 3108, 3060, 3018, 2636] combine_rows_per_src=[9446, 9320, 9300, 9284, 9242, 9350, 9432, 9006] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344816 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928880 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512944 weight_movement_bytes=34603008 total_nvlink_bytes=568115952 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=65080 dup_rows=24092 dup_rows_remote=21234 dedup_ratio=0.245076 router_tokens_per_expert=[734, 750, 805, 784, 787, 783, 815, 748, 764, 751, 793, 767, 789, 774, 707, 742, 792, 731, 765, 768, 768, 825, 728, 747, 749, 777, 749, 789, 778, 775, 803, 776, 736, 771, 779, 790, 757, 758, 756, 785, 774, 821, 760, 772, 759, 735, 764, 798, 727, 753, 703, 799, 728, 788, 777, 720, 831, 769, 743, 785, 732, 755, 823, 791] payload_rows_per_dst=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] dup_rows_per_dst=[3106, 2906, 3040, 2992, 2958, 3092, 2858, 3140] combine_rows_per_src=[9306, 9268, 9208, 9400, 9306, 9274, 9132, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=264470528 dispatch_meta_bytes=343392 combine_payload_bytes=264470528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=264813920 ingress_bytes=368279552 link_bytes=368279552 token_bytes=529284448 weight_movement_bytes=103809024 total_nvlink_bytes=633093472 weight_to_token_ratio=0.196131 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.385806305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344536 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371544 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398552 weight_movement_bytes=34603008 total_nvlink_bytes=567001560 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343384 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266796376 ingress_bytes=335659008 link_bytes=335659008 token_bytes=533249368 weight_movement_bytes=69206016 total_nvlink_bytes=602455384 weight_to_token_ratio=0.129782 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263057824e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266312768 ingress_bytes=300572672 link_bytes=300572672 token_bytes=532282432 weight_movement_bytes=34603008 total_nvlink_bytes=566885440 weight_to_token_ratio=0.065009 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131030767e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=65084 dup_rows=24130 dup_rows_remote=21120 dedup_ratio=0.245463 router_tokens_per_expert=[804, 743, 806, 779, 740, 813, 730, 747, 758, 747, 750, 778, 774, 787, 736, 786, 752, 764, 762, 799, 794, 792, 782, 737, 796, 820, 743, 803, 764, 753, 739, 785, 767, 775, 771, 747, 756, 770, 705, 759, 761, 762, 795, 707, 803, 736, 767, 792, 773, 761, 815, 733, 781, 769, 729, 774, 765, 765, 769, 797, 807, 726, 787, 765] payload_rows_per_dst=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] dup_rows_per_dst=[2996, 2934, 3136, 3032, 3010, 2986, 2996, 3040] combine_rows_per_src=[9328, 9298, 9228, 9374, 9090, 9260, 9274, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=345256 combine_payload_bytes=266567680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266912936 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533480616 weight_movement_bytes=34603008 total_nvlink_bytes=568083624 weight_to_token_ratio=0.064863 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64852 dup_rows=24240 dup_rows_remote=21278 dedup_ratio=0.246582 router_tokens_per_expert=[796, 762, 785, 738, 796, 787, 804, 811, 770, 769, 749, 732, 797, 750, 783, 808, 754, 784, 789, 741, 777, 766, 812, 737, 769, 755, 780, 751, 728, 728, 774, 791, 782, 742, 811, 769, 739, 784, 759, 757, 735, 775, 735, 782, 765, 781, 758, 756, 752, 711, 742, 824, 757, 741, 743, 820, 781, 769, 788, 761, 761, 746, 794, 759] payload_rows_per_dst=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] dup_rows_per_dst=[3208, 3054, 3074, 2946, 2962, 2962, 2984, 3050] combine_rows_per_src=[9350, 9262, 9246, 9206, 9324, 9212, 9196, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65126 dup_rows=23918 dup_rows_remote=20970 dedup_ratio=0.243306 router_tokens_per_expert=[746, 778, 793, 701, 771, 701, 793, 766, 781, 773, 766, 760, 729, 794, 772, 780, 792, 779, 768, 744, 770, 742, 774, 752, 833, 761, 748, 740, 756, 750, 741, 775, 784, 772, 778, 779, 784, 757, 761, 830, 747, 763, 792, 751, 763, 755, 782, 791, 791, 751, 735, 761, 745, 776, 730, 775, 770, 799, 806, 799, 818, 758, 758, 762] payload_rows_per_dst=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] dup_rows_per_dst=[2932, 2994, 3002, 2948, 3048, 3022, 2868, 3104] combine_rows_per_src=[9166, 9316, 9240, 9260, 9442, 9266, 9260, 9436] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65034 dup_rows=24028 dup_rows_remote=21008 dedup_ratio=0.244425 router_tokens_per_expert=[722, 757, 745, 741, 803, 751, 724, 696, 745, 761, 770, 755, 792, 751, 775, 782, 804, 761, 755, 774, 821, 785, 795, 821, 783, 766, 768, 753, 750, 792, 764, 808, 756, 780, 793, 770, 783, 758, 760, 731, 744, 780, 734, 786, 754, 781, 764, 764, 768, 739, 759, 765, 740, 795, 756, 764, 785, 745, 802, 717, 761, 800, 816, 832] payload_rows_per_dst=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] dup_rows_per_dst=[2834, 3020, 3106, 3068, 3014, 2996, 2882, 3108] combine_rows_per_src=[9044, 9242, 9526, 9300, 9248, 9218, 9290, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344816 combine_payload_bytes=266584064 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266928880 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512944 weight_movement_bytes=34603008 total_nvlink_bytes=568115952 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65084 dup_rows=23992 dup_rows_remote=21046 dedup_ratio=0.244059 router_tokens_per_expert=[807, 774, 757, 776, 713, 783, 756, 793, 781, 791, 767, 789, 752, 748, 769, 765, 784, 750, 788, 768, 803, 760, 790, 804, 751, 715, 730, 791, 733, 774, 780, 777, 813, 749, 802, 771, 737, 801, 797, 772, 773, 798, 773, 772, 746, 731, 753, 792, 778, 757, 764, 783, 752, 740, 769, 729, 770, 789, 727, 765, 797, 779, 738, 716] payload_rows_per_dst=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] dup_rows_per_dst=[3116, 2972, 2992, 2976, 3080, 2934, 2976, 2946] combine_rows_per_src=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344520 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265978312 ingress_bytes=334839808 link_bytes=334839808 token_bytes=531612104 weight_movement_bytes=69206016 total_nvlink_bytes=600818120 weight_to_token_ratio=0.130181 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259975241e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344384 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100480 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856576 weight_movement_bytes=34603008 total_nvlink_bytes=568459584 weight_to_token_ratio=0.064817 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64968 dup_rows=24114 dup_rows_remote=21184 dedup_ratio=0.245300 router_tokens_per_expert=[741, 764, 710, 750, 795, 769, 757, 757, 760, 772, 767, 810, 801, 756, 790, 817, 783, 760, 741, 769, 760, 751, 738, 774, 783, 779, 759, 778, 789, 751, 733, 750, 762, 772, 780, 791, 800, 729, 768, 803, 780, 772, 765, 762, 727, 756, 816, 801, 745, 792, 796, 775, 770, 738, 690, 732, 800, 751, 731, 781, 773, 767, 792, 821] payload_rows_per_dst=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] dup_rows_per_dst=[2996, 3094, 2930, 2972, 3190, 2988, 2904, 3040] combine_rows_per_src=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64690 dup_rows=24278 dup_rows_remote=21038 dedup_ratio=0.246969 router_tokens_per_expert=[720, 701, 817, 771, 787, 748, 771, 737, 749, 743, 804, 731, 771, 739, 790, 785, 768, 731, 803, 785, 803, 762, 728, 742, 800, 770, 781, 772, 832, 832, 805, 755, 784, 738, 765, 732, 727, 795, 757, 782, 765, 767, 723, 732, 772, 753, 804, 759, 745, 746, 757, 796, 760, 758, 791, 745, 775, 786, 789, 800, 748, 788, 776, 804] payload_rows_per_dst=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] dup_rows_per_dst=[3240, 3140, 3026, 3052, 2868, 2896, 3258, 2798] combine_rows_per_src=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344168 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266723432 ingress_bytes=370188288 link_bytes=370188288 token_bytes=533102696 weight_movement_bytes=103809024 total_nvlink_bytes=636911720 weight_to_token_ratio=0.194726 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392988725e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344520 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266928584 ingress_bytes=370393088 link_bytes=370393088 token_bytes=533512648 weight_movement_bytes=103809024 total_nvlink_bytes=637321672 weight_to_token_ratio=0.194577 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393759371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65250 dup_rows=23824 dup_rows_remote=20876 dedup_ratio=0.242350 router_tokens_per_expert=[798, 831, 802, 748, 748, 752, 744, 768, 774, 784, 765, 783, 809, 754, 802, 795, 767, 800, 845, 779, 770, 771, 741, 770, 770, 750, 748, 744, 771, 727, 808, 743, 793, 762, 723, 818, 776, 737, 740, 754, 744, 707, 815, 775, 800, 714, 789, 739, 759, 802, 779, 739, 774, 734, 776, 726, 773, 814, 797, 719, 775, 738, 736, 764] payload_rows_per_dst=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] dup_rows_per_dst=[2928, 3096, 3126, 2964, 2922, 2976, 2948, 2864] combine_rows_per_src=[9454, 9436, 9360, 9158, 9284, 9190, 9230, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266453536 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532562464 weight_movement_bytes=34603008 total_nvlink_bytes=567165472 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342912 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313152 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283392 weight_movement_bytes=34603008 total_nvlink_bytes=564886400 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65088 dup_rows=23978 dup_rows_remote=20866 dedup_ratio=0.243917 router_tokens_per_expert=[761, 751, 776, 749, 709, 749, 796, 768, 745, 785, 754, 795, 785, 706, 765, 786, 750, 774, 717, 813, 788, 727, 771, 742, 726, 781, 800, 730, 790, 726, 797, 783, 756, 773, 790, 772, 763, 833, 751, 737, 791, 773, 777, 789, 785, 796, 752, 757, 788, 784, 805, 770, 743, 768, 803, 731, 738, 802, 737, 798, 757, 767, 762, 809] payload_rows_per_dst=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] dup_rows_per_dst=[2846, 2988, 2936, 2908, 3112, 3112, 3110, 2966] combine_rows_per_src=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344504 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267608504 ingress_bytes=301867008 link_bytes=301867008 token_bytes=534872504 weight_movement_bytes=34603008 total_nvlink_bytes=569475512 weight_to_token_ratio=0.064694 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135901249e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73932 payload_rows_remote=64778 dup_rows=24372 dup_rows_remote=21318 dedup_ratio=0.247925 router_tokens_per_expert=[760, 768, 809, 758, 793, 790, 774, 813, 789, 741, 790, 782, 741, 763, 767, 745, 802, 762, 761, 737, 737, 766, 761, 713, 763, 781, 771, 830, 796, 809, 751, 775, 751, 766, 732, 782, 746, 783, 723, 763, 786, 716, 775, 810, 750, 749, 753, 765, 771, 821, 776, 799, 769, 760, 745, 749, 813, 796, 719, 760, 697, 825, 775, 729] payload_rows_per_dst=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] dup_rows_per_dst=[3246, 3104, 2920, 3144, 2824, 3054, 3060, 3020] combine_rows_per_src=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65024 dup_rows=23928 dup_rows_remote=20906 dedup_ratio=0.243408 router_tokens_per_expert=[778, 763, 746, 791, 753, 789, 757, 763, 767, 785, 788, 755, 781, 755, 734, 693, 753, 811, 790, 764, 776, 782, 794, 766, 787, 783, 810, 772, 783, 771, 731, 750, 761, 802, 813, 753, 743, 793, 729, 776, 755, 765, 774, 778, 745, 763, 766, 761, 717, 760, 769, 798, 770, 812, 757, 722, 760, 770, 792, 761, 746, 774, 768, 778] payload_rows_per_dst=[9302, 9204, 9444, 9352, 9334, 9240, 9236, 9264] dup_rows_per_dst=[2978, 2912, 3028, 3022, 3006, 2974, 2974, 3034] combine_rows_per_src=[9302, 9204, 9444, 9352, 9334, 9240, 9236, 9264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343816 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944264 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544712 weight_movement_bytes=69206016 total_nvlink_bytes=602750728 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64828 dup_rows=24306 dup_rows_remote=21304 dedup_ratio=0.247253 router_tokens_per_expert=[781, 775, 757, 769, 794, 766, 800, 780, 744, 804, 775, 771, 750, 756, 747, 739, 803, 759, 732, 747, 791, 762, 771, 783, 791, 817, 729, 737, 703, 727, 756, 736, 797, 757, 749, 783, 788, 768, 792, 753, 777, 772, 747, 810, 747, 756, 750, 728, 711, 786, 775, 760, 821, 763, 778, 807, 736, 787, 789, 775, 808, 754, 762, 814] payload_rows_per_dst=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] dup_rows_per_dst=[3070, 3002, 2982, 2876, 3032, 3080, 3118, 3146] combine_rows_per_src=[9374, 9170, 9314, 9116, 9342, 9094, 9284, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73932 payload_rows_remote=64778 dup_rows=24372 dup_rows_remote=21318 dedup_ratio=0.247925 router_tokens_per_expert=[760, 768, 809, 758, 793, 790, 774, 813, 789, 741, 790, 782, 741, 763, 767, 745, 802, 762, 761, 737, 737, 766, 761, 713, 763, 781, 771, 830, 796, 809, 751, 775, 751, 766, 732, 782, 746, 783, 723, 763, 786, 716, 775, 810, 750, 749, 753, 765, 771, 821, 776, 799, 769, 760, 745, 749, 813, 796, 719, 760, 697, 825, 775, 729] payload_rows_per_dst=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] dup_rows_per_dst=[3246, 3104, 2920, 3144, 2824, 3054, 3060, 3020] combine_rows_per_src=[9284, 9132, 9158, 9408, 9268, 9154, 9320, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64968 dup_rows=24114 dup_rows_remote=21184 dedup_ratio=0.245300 router_tokens_per_expert=[741, 764, 710, 750, 795, 769, 757, 757, 760, 772, 767, 810, 801, 756, 790, 817, 783, 760, 741, 769, 760, 751, 738, 774, 783, 779, 759, 778, 789, 751, 733, 750, 762, 772, 780, 791, 800, 729, 768, 803, 780, 772, 765, 762, 727, 756, 816, 801, 745, 792, 796, 775, 770, 738, 690, 732, 800, 751, 731, 781, 773, 767, 792, 821] payload_rows_per_dst=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] dup_rows_per_dst=[2996, 3094, 2930, 2972, 3190, 2988, 2904, 3040] combine_rows_per_src=[9090, 9452, 9222, 9272, 9220, 9370, 9172, 9392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64786 dup_rows=24018 dup_rows_remote=20926 dedup_ratio=0.244324 router_tokens_per_expert=[766, 759, 736, 797, 760, 786, 736, 729, 727, 753, 767, 783, 751, 787, 755, 742, 718, 786, 761, 740, 774, 782, 786, 756, 737, 771, 762, 754, 786, 758, 756, 803, 772, 768, 805, 798, 795, 800, 729, 749, 800, 772, 747, 756, 781, 734, 774, 765, 800, 790, 755, 779, 774, 798, 753, 726, 801, 781, 779, 771, 819, 764, 760, 793] payload_rows_per_dst=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] dup_rows_per_dst=[2976, 2972, 2954, 3084, 3020, 2926, 2994, 3092] combine_rows_per_src=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344384 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265675072 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005760 weight_movement_bytes=103809024 total_nvlink_bytes=634814784 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343720 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266682024 ingress_bytes=335544320 link_bytes=335544320 token_bytes=533020328 weight_movement_bytes=69206016 total_nvlink_bytes=602226344 weight_to_token_ratio=0.129837 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262626263e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65088 dup_rows=23978 dup_rows_remote=20866 dedup_ratio=0.243917 router_tokens_per_expert=[761, 751, 776, 749, 709, 749, 796, 768, 745, 785, 754, 795, 785, 706, 765, 786, 750, 774, 717, 813, 788, 727, 771, 742, 726, 781, 800, 730, 790, 726, 797, 783, 756, 773, 790, 772, 763, 833, 751, 737, 791, 773, 777, 789, 785, 796, 752, 757, 788, 784, 805, 770, 743, 768, 803, 731, 738, 802, 737, 798, 757, 767, 762, 809] payload_rows_per_dst=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] dup_rows_per_dst=[2846, 2988, 2936, 2908, 3112, 3112, 3110, 2966] combine_rows_per_src=[9272, 9254, 9228, 9358, 9238, 9328, 9274, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65000 dup_rows=24014 dup_rows_remote=20940 dedup_ratio=0.244283 router_tokens_per_expert=[809, 739, 760, 769, 768, 732, 767, 753, 744, 709, 752, 805, 812, 807, 777, 768, 797, 728, 772, 772, 793, 740, 815, 765, 797, 763, 755, 773, 769, 752, 773, 776, 803, 714, 778, 763, 806, 749, 769, 731, 748, 756, 801, 768, 780, 753, 778, 762, 755, 797, 764, 764, 834, 754, 735, 752, 750, 763, 771, 721, 797, 790, 753, 782] payload_rows_per_dst=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] dup_rows_per_dst=[2962, 3084, 3074, 2876, 3046, 2990, 3038, 2944] combine_rows_per_src=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65084 dup_rows=23992 dup_rows_remote=21046 dedup_ratio=0.244059 router_tokens_per_expert=[807, 774, 757, 776, 713, 783, 756, 793, 781, 791, 767, 789, 752, 748, 769, 765, 784, 750, 788, 768, 803, 760, 790, 804, 751, 715, 730, 791, 733, 774, 780, 777, 813, 749, 802, 771, 737, 801, 797, 772, 773, 798, 773, 772, 746, 731, 753, 792, 778, 757, 764, 783, 752, 740, 769, 729, 770, 789, 727, 765, 797, 779, 738, 716] payload_rows_per_dst=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] dup_rows_per_dst=[3116, 2972, 2992, 2976, 3080, 2934, 2976, 2946] combine_rows_per_src=[9202, 9352, 9502, 9126, 9404, 9330, 9168, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=65022 dup_rows=24070 dup_rows_remote=21262 dedup_ratio=0.244853 router_tokens_per_expert=[719, 747, 784, 740, 747, 758, 754, 727, 816, 771, 722, 727, 769, 767, 755, 768, 759, 749, 805, 850, 778, 730, 712, 781, 792, 712, 782, 745, 806, 806, 785, 779, 791, 734, 775, 813, 777, 771, 754, 752, 773, 764, 773, 826, 760, 862, 734, 801, 754, 729, 760, 771, 777, 740, 755, 724, 790, 765, 797, 793, 712, 783, 813, 787] payload_rows_per_dst=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] dup_rows_per_dst=[2838, 2954, 3054, 3082, 3040, 3136, 2808, 3158] combine_rows_per_src=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64744 dup_rows=23988 dup_rows_remote=20746 dedup_ratio=0.244019 router_tokens_per_expert=[763, 798, 786, 827, 815, 779, 763, 806, 727, 725, 796, 772, 793, 772, 791, 746, 764, 754, 784, 767, 762, 742, 775, 760, 774, 780, 780, 752, 774, 744, 780, 710, 806, 731, 775, 757, 788, 772, 749, 766, 760, 763, 785, 761, 767, 757, 747, 760, 734, 772, 740, 778, 753, 764, 750, 737, 775, 801, 765, 750, 822, 774, 755, 777] payload_rows_per_dst=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] dup_rows_per_dst=[3242, 3226, 3004, 3022, 2900, 3010, 2864, 2720] combine_rows_per_src=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=344528 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265880016 ingress_bytes=300138496 link_bytes=300138496 token_bytes=531415504 weight_movement_bytes=34603008 total_nvlink_bytes=566018512 weight_to_token_ratio=0.065115 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129396997e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=344608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266453536 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532562464 weight_movement_bytes=34603008 total_nvlink_bytes=567165472 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64690 dup_rows=24278 dup_rows_remote=21038 dedup_ratio=0.246969 router_tokens_per_expert=[720, 701, 817, 771, 787, 748, 771, 737, 749, 743, 804, 731, 771, 739, 790, 785, 768, 731, 803, 785, 803, 762, 728, 742, 800, 770, 781, 772, 832, 832, 805, 755, 784, 738, 765, 732, 727, 795, 757, 782, 765, 767, 723, 732, 772, 753, 804, 759, 745, 746, 757, 796, 760, 758, 791, 745, 775, 786, 789, 800, 748, 788, 776, 804] payload_rows_per_dst=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] dup_rows_per_dst=[3240, 3140, 3026, 3052, 2868, 2896, 3258, 2798] combine_rows_per_src=[9336, 9338, 9294, 9308, 9052, 9164, 9430, 9104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265330688 dispatch_meta_bytes=344384 combine_payload_bytes=265330688 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265675072 ingress_bytes=369139712 link_bytes=369139712 token_bytes=531005760 weight_movement_bytes=103809024 total_nvlink_bytes=634814784 weight_to_token_ratio=0.195495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389043018e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=343816 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266944264 ingress_bytes=335806464 link_bytes=335806464 token_bytes=533544712 weight_movement_bytes=69206016 total_nvlink_bytes=602750728 weight_to_token_ratio=0.129710 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263612689e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=345136 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266675248 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533005360 weight_movement_bytes=34603008 total_nvlink_bytes=567608368 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344520 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266928584 ingress_bytes=370393088 link_bytes=370393088 token_bytes=533512648 weight_movement_bytes=103809024 total_nvlink_bytes=637321672 weight_to_token_ratio=0.194577 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393759371e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=342912 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313152 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283392 weight_movement_bytes=34603008 total_nvlink_bytes=564886400 weight_to_token_ratio=0.065254 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20866 dedup_ratio=0.243205 router_tokens_per_expert=[776, 712, 739, 755, 759, 738, 815, 778, 785, 793, 750, 755, 804, 768, 749, 797, 727, 795, 790, 750, 784, 783, 782, 752, 763, 803, 793, 760, 779, 773, 783, 736, 775, 779, 746, 762, 746, 773, 766, 780, 709, 767, 811, 783, 768, 782, 772, 777, 783, 736, 794, 737, 751, 747, 792, 762, 798, 787, 781, 781, 732, 785, 704, 760] payload_rows_per_dst=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] dup_rows_per_dst=[2924, 3040, 3000, 3042, 2978, 3042, 2920, 2962] combine_rows_per_src=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65230 dup_rows=23904 dup_rows_remote=20912 dedup_ratio=0.243164 router_tokens_per_expert=[779, 791, 797, 744, 759, 737, 741, 813, 767, 728, 732, 808, 743, 720, 808, 775, 733, 771, 800, 760, 750, 794, 809, 814, 778, 726, 778, 775, 739, 789, 758, 734, 813, 740, 741, 757, 779, 776, 765, 745, 750, 779, 802, 757, 709, 826, 820, 752, 736, 777, 786, 762, 774, 781, 817, 739, 743, 779, 747, 728, 770, 811, 751, 790] payload_rows_per_dst=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] dup_rows_per_dst=[2972, 2992, 3070, 2900, 2942, 3010, 3078, 2940] combine_rows_per_src=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343696 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821264 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533298832 weight_movement_bytes=34603008 total_nvlink_bytes=567901840 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65000 dup_rows=24014 dup_rows_remote=20940 dedup_ratio=0.244283 router_tokens_per_expert=[809, 739, 760, 769, 768, 732, 767, 753, 744, 709, 752, 805, 812, 807, 777, 768, 797, 728, 772, 772, 793, 740, 815, 765, 797, 763, 755, 773, 769, 752, 773, 776, 803, 714, 778, 763, 806, 749, 769, 731, 748, 756, 801, 768, 780, 753, 778, 762, 755, 797, 764, 764, 834, 754, 735, 752, 750, 763, 771, 721, 797, 790, 753, 782] payload_rows_per_dst=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] dup_rows_per_dst=[2962, 3084, 3074, 2876, 3046, 2990, 3038, 2944] combine_rows_per_src=[9232, 9264, 9290, 9440, 9180, 9302, 9272, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342848 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265706304 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531069760 weight_movement_bytes=138412032 total_nvlink_bytes=669481792 weight_to_token_ratio=0.260629 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64948 dup_rows=24080 dup_rows_remote=21186 dedup_ratio=0.244954 router_tokens_per_expert=[767, 772, 710, 747, 737, 786, 790, 732, 748, 794, 789, 793, 749, 798, 756, 778, 774, 786, 767, 764, 783, 746, 803, 760, 759, 773, 742, 705, 796, 768, 748, 779, 763, 788, 786, 822, 774, 747, 774, 768, 811, 775, 781, 797, 786, 792, 776, 779, 770, 721, 758, 762, 778, 758, 717, 821, 798, 704, 795, 772, 735, 751, 739, 755] payload_rows_per_dst=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] dup_rows_per_dst=[2902, 3130, 3084, 2910, 3052, 3152, 2894, 2956] combine_rows_per_src=[9180, 9280, 9282, 9230, 9392, 9442, 9276, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343760 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266583760 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532823760 weight_movement_bytes=34603008 total_nvlink_bytes=567426768 weight_to_token_ratio=0.064943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=341960 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265533384 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530724808 weight_movement_bytes=34603008 total_nvlink_bytes=565327816 weight_to_token_ratio=0.065200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64876 dup_rows=23930 dup_rows_remote=20868 dedup_ratio=0.243429 router_tokens_per_expert=[779, 765, 746, 740, 720, 743, 764, 725, 761, 787, 712, 763, 801, 709, 760, 754, 790, 719, 752, 799, 750, 779, 771, 783, 764, 809, 777, 765, 835, 757, 716, 793, 746, 774, 815, 811, 782, 748, 796, 808, 732, 774, 763, 788, 728, 753, 764, 800, 725, 769, 780, 780, 787, 748, 768, 757, 779, 760, 800, 797, 795, 809, 745, 783] payload_rows_per_dst=[9204, 9096, 9262, 9328, 9498, 9268, 9262, 9456] dup_rows_per_dst=[2760, 2998, 3024, 3104, 3062, 2936, 2966, 3080] combine_rows_per_src=[9204, 9096, 9262, 9328, 9498, 9268, 9262, 9456] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65160 dup_rows=23782 dup_rows_remote=20832 dedup_ratio=0.241923 router_tokens_per_expert=[721, 768, 753, 774, 783, 816, 812, 783, 779, 713, 810, 782, 761, 767, 756, 764, 727, 759, 751, 773, 770, 731, 777, 774, 724, 758, 740, 787, 766, 797, 804, 743, 740, 751, 788, 732, 777, 740, 767, 795, 762, 805, 789, 731, 750, 767, 789, 763, 721, 750, 752, 808, 790, 769, 819, 780, 780, 785, 753, 786, 770, 786, 727, 807] payload_rows_per_dst=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] dup_rows_per_dst=[3026, 2970, 2888, 2978, 2882, 2950, 3016, 3072] combine_rows_per_src=[9394, 9294, 9236, 9260, 9298, 9362, 9362, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74750 payload_rows_remote=65304 dup_rows=23554 dup_rows_remote=20528 dedup_ratio=0.239604 router_tokens_per_expert=[765, 784, 767, 767, 807, 761, 786, 717, 770, 728, 768, 786, 717, 772, 761, 764, 739, 794, 762, 725, 740, 757, 772, 776, 749, 765, 756, 751, 730, 758, 788, 801, 774, 734, 813, 771, 747, 746, 761, 794, 749, 831, 761, 766, 756, 758, 783, 759, 788, 771, 816, 757, 767, 806, 802, 729, 757, 793, 769, 786, 794, 752, 812, 767] payload_rows_per_dst=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] dup_rows_per_dst=[2952, 2878, 2918, 2870, 2944, 3006, 3026, 2960] combine_rows_per_src=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=64786 dup_rows=24018 dup_rows_remote=20926 dedup_ratio=0.244324 router_tokens_per_expert=[766, 759, 736, 797, 760, 786, 736, 729, 727, 753, 767, 783, 751, 787, 755, 742, 718, 786, 761, 740, 774, 782, 786, 756, 737, 771, 762, 754, 786, 758, 756, 803, 772, 768, 805, 798, 795, 800, 729, 749, 800, 772, 747, 756, 781, 734, 774, 765, 800, 790, 755, 779, 774, 798, 753, 726, 801, 781, 779, 771, 819, 764, 760, 793] payload_rows_per_dst=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] dup_rows_per_dst=[2976, 2972, 2954, 3084, 3020, 2926, 2994, 3092] combine_rows_per_src=[9162, 9158, 9252, 9170, 9410, 9316, 9318, 9500] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344568 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526648 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708728 weight_movement_bytes=34603008 total_nvlink_bytes=569311736 weight_to_token_ratio=0.064714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64744 dup_rows=23988 dup_rows_remote=20746 dedup_ratio=0.244019 router_tokens_per_expert=[763, 798, 786, 827, 815, 779, 763, 806, 727, 725, 796, 772, 793, 772, 791, 746, 764, 754, 784, 767, 762, 742, 775, 760, 774, 780, 780, 752, 774, 744, 780, 710, 806, 731, 775, 757, 788, 772, 749, 766, 760, 763, 785, 761, 767, 757, 747, 760, 734, 772, 740, 778, 753, 764, 750, 737, 775, 801, 765, 750, 822, 774, 755, 777] payload_rows_per_dst=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] dup_rows_per_dst=[3242, 3226, 3004, 3022, 2900, 3010, 2864, 2720] combine_rows_per_src=[9572, 9546, 9402, 9312, 9242, 9190, 9192, 8860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20866 dedup_ratio=0.243205 router_tokens_per_expert=[776, 712, 739, 755, 759, 738, 815, 778, 785, 793, 750, 755, 804, 768, 749, 797, 727, 795, 790, 750, 784, 783, 782, 752, 763, 803, 793, 760, 779, 773, 783, 736, 775, 779, 746, 762, 746, 773, 766, 780, 709, 767, 811, 783, 768, 782, 772, 777, 783, 736, 794, 737, 751, 747, 792, 762, 798, 787, 781, 781, 732, 785, 704, 760] payload_rows_per_dst=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] dup_rows_per_dst=[2924, 3040, 3000, 3042, 2978, 3042, 2920, 2962] combine_rows_per_src=[9220, 9362, 9326, 9338, 9276, 9296, 9284, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343760 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266583760 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532823760 weight_movement_bytes=34603008 total_nvlink_bytes=567426768 weight_to_token_ratio=0.064943 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=344536 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266371544 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532398552 weight_movement_bytes=34603008 total_nvlink_bytes=567001560 weight_to_token_ratio=0.064995 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=342976 combine_payload_bytes=265732096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266075072 ingress_bytes=334938112 link_bytes=334938112 token_bytes=531807168 weight_movement_bytes=69206016 total_nvlink_bytes=601013184 weight_to_token_ratio=0.130134 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260345151e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343968 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239328 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134688 weight_movement_bytes=69206016 total_nvlink_bytes=603340704 weight_to_token_ratio=0.129567 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343328 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267828512 ingress_bytes=336691200 link_bytes=336691200 token_bytes=535313696 weight_movement_bytes=69206016 total_nvlink_bytes=604519712 weight_to_token_ratio=0.129281 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266941880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65048 dup_rows=24084 dup_rows_remote=21056 dedup_ratio=0.244995 router_tokens_per_expert=[790, 734, 730, 810, 797, 754, 739, 791, 738, 783, 795, 785, 825, 744, 769, 754, 792, 764, 799, 732, 723, 740, 769, 773, 727, 733, 769, 754, 726, 835, 775, 781, 791, 778, 792, 784, 784, 751, 781, 788, 762, 756, 731, 778, 767, 762, 728, 750, 752, 739, 811, 764, 788, 709, 737, 787, 777, 759, 785, 796, 820, 789, 741, 785] payload_rows_per_dst=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] dup_rows_per_dst=[3022, 2998, 3046, 3028, 3066, 2838, 2960, 3126] combine_rows_per_src=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=342848 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265706304 ingress_bytes=403775488 link_bytes=403775488 token_bytes=531069760 weight_movement_bytes=138412032 total_nvlink_bytes=669481792 weight_to_token_ratio=0.260629 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519374655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=341960 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265533384 ingress_bytes=299794432 link_bytes=299794432 token_bytes=530724808 weight_movement_bytes=34603008 total_nvlink_bytes=565327816 weight_to_token_ratio=0.065200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128102312e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344416 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266781024 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533217632 weight_movement_bytes=103809024 total_nvlink_bytes=637026656 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65120 dup_rows=23952 dup_rows_remote=21002 dedup_ratio=0.243652 router_tokens_per_expert=[772, 762, 750, 741, 848, 755, 738, 797, 772, 799, 749, 800, 765, 762, 723, 721, 780, 772, 790, 797, 734, 728, 789, 762, 791, 777, 797, 810, 739, 764, 845, 765, 699, 771, 755, 711, 785, 739, 769, 775, 761, 787, 741, 724, 773, 799, 765, 779, 835, 764, 812, 790, 769, 778, 749, 774, 751, 800, 754, 777, 769, 730, 676, 797] payload_rows_per_dst=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] dup_rows_per_dst=[3030, 2950, 2962, 3062, 2854, 3006, 3164, 2924] combine_rows_per_src=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65230 dup_rows=23904 dup_rows_remote=20912 dedup_ratio=0.243164 router_tokens_per_expert=[779, 791, 797, 744, 759, 737, 741, 813, 767, 728, 732, 808, 743, 720, 808, 775, 733, 771, 800, 760, 750, 794, 809, 814, 778, 726, 778, 775, 739, 789, 758, 734, 813, 740, 741, 757, 779, 776, 765, 745, 750, 779, 802, 757, 709, 826, 820, 752, 736, 777, 786, 762, 774, 781, 817, 739, 743, 779, 747, 728, 770, 811, 751, 790] payload_rows_per_dst=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] dup_rows_per_dst=[2972, 2992, 3070, 2900, 2942, 3010, 3078, 2940] combine_rows_per_src=[9350, 9170, 9392, 9254, 9290, 9380, 9266, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64860 dup_rows=24140 dup_rows_remote=21008 dedup_ratio=0.245565 router_tokens_per_expert=[792, 760, 779, 746, 745, 791, 738, 781, 758, 801, 790, 779, 768, 775, 787, 794, 721, 751, 800, 760, 756, 774, 791, 750, 785, 769, 818, 771, 741, 776, 798, 758, 759, 725, 768, 725, 716, 792, 754, 734, 732, 776, 747, 753, 748, 734, 799, 786, 781, 741, 809, 794, 775, 797, 766, 779, 785, 765, 745, 740, 780, 768, 781, 795] payload_rows_per_dst=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] dup_rows_per_dst=[2950, 3232, 2998, 2772, 2816, 3188, 3052, 3132] combine_rows_per_src=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343696 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821264 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533298832 weight_movement_bytes=34603008 total_nvlink_bytes=567901840 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=64906 dup_rows=24084 dup_rows_remote=21060 dedup_ratio=0.244995 router_tokens_per_expert=[752, 771, 789, 763, 693, 737, 796, 744, 725, 773, 804, 799, 723, 805, 750, 765, 788, 760, 720, 798, 737, 801, 780, 785, 780, 789, 750, 768, 793, 813, 788, 755, 796, 773, 718, 799, 758, 802, 748, 784, 714, 740, 777, 774, 753, 752, 767, 797, 751, 798, 816, 801, 766, 754, 782, 779, 733, 798, 725, 784, 750, 766, 749, 754] payload_rows_per_dst=[9206, 9294, 9314, 9290, 9352, 9188, 9364, 9212] dup_rows_per_dst=[2884, 2994, 3024, 3182, 3004, 2960, 3130, 2906] combine_rows_per_src=[9206, 9294, 9314, 9290, 9352, 9188, 9364, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65264 dup_rows=23806 dup_rows_remote=20722 dedup_ratio=0.242167 router_tokens_per_expert=[803, 752, 721, 708, 774, 770, 813, 818, 740, 792, 782, 712, 761, 757, 747, 773, 771, 793, 759, 804, 770, 788, 757, 750, 770, 815, 735, 728, 731, 777, 796, 773, 742, 775, 812, 824, 700, 752, 787, 779, 788, 788, 759, 743, 823, 744, 762, 791, 739, 757, 729, 778, 828, 764, 737, 791, 747, 762, 799, 760, 773, 717, 782, 780] payload_rows_per_dst=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] dup_rows_per_dst=[3084, 2962, 2996, 2966, 3172, 3084, 2718, 2824] combine_rows_per_src=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=344568 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267526648 ingress_bytes=301785088 link_bytes=301785088 token_bytes=534708728 weight_movement_bytes=34603008 total_nvlink_bytes=569311736 weight_to_token_ratio=0.064714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135592990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=343472 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266010032 ingress_bytes=334872576 link_bytes=334872576 token_bytes=531676592 weight_movement_bytes=69206016 total_nvlink_bytes=600882608 weight_to_token_ratio=0.130166 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260098544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343864 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198840 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053816 weight_movement_bytes=34603008 total_nvlink_bytes=566656824 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=65022 dup_rows=24070 dup_rows_remote=21262 dedup_ratio=0.244853 router_tokens_per_expert=[719, 747, 784, 740, 747, 758, 754, 727, 816, 771, 722, 727, 769, 767, 755, 768, 759, 749, 805, 850, 778, 730, 712, 781, 792, 712, 782, 745, 806, 806, 785, 779, 791, 734, 775, 813, 777, 771, 754, 752, 773, 764, 773, 826, 760, 862, 734, 801, 754, 729, 760, 771, 777, 740, 755, 724, 790, 765, 797, 793, 712, 783, 813, 787] payload_rows_per_dst=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] dup_rows_per_dst=[2838, 2954, 3054, 3082, 3040, 3136, 2808, 3158] combine_rows_per_src=[9114, 9236, 9274, 9332, 9294, 9450, 9212, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64880 dup_rows=24068 dup_rows_remote=20948 dedup_ratio=0.244832 router_tokens_per_expert=[765, 825, 756, 748, 747, 764, 787, 753, 791, 750, 771, 764, 733, 778, 748, 746, 796, 745, 764, 746, 778, 798, 743, 717, 782, 752, 793, 794, 777, 752, 774, 724, 740, 784, 807, 736, 811, 765, 789, 806, 828, 746, 780, 749, 741, 775, 751, 759, 703, 794, 748, 794, 721, 746, 805, 744, 812, 786, 785, 797, 770, 772, 756, 791] payload_rows_per_dst=[9250, 9260, 9186, 9310, 9356, 9230, 9192, 9452] dup_rows_per_dst=[3040, 2902, 2988, 2986, 3120, 3028, 2918, 3086] combine_rows_per_src=[9250, 9260, 9186, 9310, 9356, 9230, 9192, 9452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64736 dup_rows=24158 dup_rows_remote=21042 dedup_ratio=0.245748 router_tokens_per_expert=[807, 781, 791, 720, 782, 770, 815, 777, 731, 657, 742, 740, 797, 778, 794, 764, 746, 786, 779, 779, 813, 747, 750, 782, 776, 816, 745, 760, 776, 783, 785, 749, 730, 803, 778, 788, 775, 751, 786, 762, 807, 761, 735, 759, 729, 736, 713, 785, 795, 802, 732, 776, 774, 752, 785, 847, 779, 756, 785, 745, 784, 785, 735, 704] payload_rows_per_dst=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] dup_rows_per_dst=[3126, 2930, 3038, 3064, 2994, 2946, 3116, 2944] combine_rows_per_src=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=343944 combine_payload_bytes=267321344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267665288 ingress_bytes=336527360 link_bytes=336527360 token_bytes=534986632 weight_movement_bytes=69206016 total_nvlink_bytes=604192648 weight_to_token_ratio=0.129360 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266325363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65044 dup_rows=23942 dup_rows_remote=20930 dedup_ratio=0.243551 router_tokens_per_expert=[782, 721, 799, 784, 744, 759, 814, 739, 758, 764, 762, 776, 770, 832, 756, 777, 821, 831, 756, 721, 759, 746, 782, 739, 799, 798, 798, 761, 730, 794, 769, 763, 748, 743, 738, 695, 736, 816, 743, 773, 758, 843, 740, 775, 741, 741, 773, 794, 749, 788, 726, 740, 782, 748, 801, 721, 735, 773, 744, 783, 805, 823, 746, 827] payload_rows_per_dst=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] dup_rows_per_dst=[3078, 3024, 3060, 3064, 2778, 3012, 2876, 3050] combine_rows_per_src=[9206, 9366, 9250, 9360, 9206, 9318, 9234, 9422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344488 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076008 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807528 weight_movement_bytes=34603008 total_nvlink_bytes=568410536 weight_to_token_ratio=0.064823 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=64860 dup_rows=24140 dup_rows_remote=21008 dedup_ratio=0.245565 router_tokens_per_expert=[792, 760, 779, 746, 745, 791, 738, 781, 758, 801, 790, 779, 768, 775, 787, 794, 721, 751, 800, 760, 756, 774, 791, 750, 785, 769, 818, 771, 741, 776, 798, 758, 759, 725, 768, 725, 716, 792, 754, 734, 732, 776, 747, 753, 748, 734, 799, 786, 781, 741, 809, 794, 775, 797, 766, 779, 785, 765, 745, 740, 780, 768, 781, 795] payload_rows_per_dst=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] dup_rows_per_dst=[2950, 3232, 2998, 2772, 2816, 3188, 3052, 3132] combine_rows_per_src=[9270, 9272, 9248, 8896, 9130, 9608, 9436, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65264 dup_rows=23806 dup_rows_remote=20722 dedup_ratio=0.242167 router_tokens_per_expert=[803, 752, 721, 708, 774, 770, 813, 818, 740, 792, 782, 712, 761, 757, 747, 773, 771, 793, 759, 804, 770, 788, 757, 750, 770, 815, 735, 728, 731, 777, 796, 773, 742, 775, 812, 824, 700, 752, 787, 779, 788, 788, 759, 743, 823, 744, 762, 791, 739, 757, 729, 778, 828, 764, 737, 791, 747, 762, 799, 760, 773, 717, 782, 780] payload_rows_per_dst=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] dup_rows_per_dst=[3084, 2962, 2996, 2966, 3172, 3084, 2718, 2824] combine_rows_per_src=[9234, 9424, 9388, 9322, 9518, 9312, 9142, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65048 dup_rows=24084 dup_rows_remote=21056 dedup_ratio=0.244995 router_tokens_per_expert=[790, 734, 730, 810, 797, 754, 739, 791, 738, 783, 795, 785, 825, 744, 769, 754, 792, 764, 799, 732, 723, 740, 769, 773, 727, 733, 769, 754, 726, 835, 775, 781, 791, 778, 792, 784, 784, 751, 781, 788, 762, 756, 731, 778, 767, 762, 728, 750, 752, 739, 811, 764, 788, 709, 737, 787, 777, 759, 785, 796, 820, 789, 741, 785] payload_rows_per_dst=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] dup_rows_per_dst=[3022, 2998, 3046, 3028, 3066, 2838, 2960, 3126] combine_rows_per_src=[9268, 9388, 9138, 9172, 9432, 9230, 9214, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=343864 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266198840 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532053816 weight_movement_bytes=34603008 total_nvlink_bytes=566656824 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74654 payload_rows_remote=65222 dup_rows=23650 dup_rows_remote=20610 dedup_ratio=0.240580 router_tokens_per_expert=[785, 782, 725, 741, 751, 753, 751, 782, 794, 764, 725, 742, 799, 731, 786, 754, 788, 779, 790, 779, 750, 808, 778, 734, 775, 805, 795, 758, 801, 763, 789, 750, 802, 793, 759, 770, 788, 755, 779, 752, 778, 740, 758, 809, 765, 799, 787, 770, 765, 780, 749, 756, 779, 733, 756, 716, 770, 735, 780, 751, 736, 749, 811, 775] payload_rows_per_dst=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] dup_rows_per_dst=[2866, 2938, 2970, 3040, 2962, 3012, 2914, 2948] combine_rows_per_src=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=345136 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266675248 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533005360 weight_movement_bytes=34603008 total_nvlink_bytes=567608368 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343312 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091792 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840272 weight_movement_bytes=34603008 total_nvlink_bytes=566443280 weight_to_token_ratio=0.065063 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343112 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265501768 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660424 weight_movement_bytes=103809024 total_nvlink_bytes=634469448 weight_to_token_ratio=0.195622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343896 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764120 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184344 weight_movement_bytes=34603008 total_nvlink_bytes=567787352 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=343472 combine_payload_bytes=265666560 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266010032 ingress_bytes=334872576 link_bytes=334872576 token_bytes=531676592 weight_movement_bytes=69206016 total_nvlink_bytes=600882608 weight_to_token_ratio=0.130166 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260098544e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=343944 combine_payload_bytes=267321344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267665288 ingress_bytes=336527360 link_bytes=336527360 token_bytes=534986632 weight_movement_bytes=69206016 total_nvlink_bytes=604192648 weight_to_token_ratio=0.129360 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266325363e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=343328 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267492640 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534641952 weight_movement_bytes=69206016 total_nvlink_bytes=603847968 weight_to_token_ratio=0.129444 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65154 dup_rows=23870 dup_rows_remote=20970 dedup_ratio=0.242818 router_tokens_per_expert=[784, 753, 746, 796, 780, 781, 786, 824, 739, 763, 667, 772, 767, 786, 779, 817, 797, 758, 755, 798, 742, 766, 760, 806, 724, 804, 833, 745, 774, 712, 701, 792, 767, 752, 724, 769, 772, 798, 742, 792, 780, 749, 800, 765, 744, 800, 770, 785, 778, 766, 737, 784, 793, 759, 774, 773, 742, 741, 788, 780, 773, 761, 758, 729] payload_rows_per_dst=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] dup_rows_per_dst=[3098, 2900, 3016, 2970, 2902, 3018, 3058, 2908] combine_rows_per_src=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65120 dup_rows=23952 dup_rows_remote=21002 dedup_ratio=0.243652 router_tokens_per_expert=[772, 762, 750, 741, 848, 755, 738, 797, 772, 799, 749, 800, 765, 762, 723, 721, 780, 772, 790, 797, 734, 728, 789, 762, 791, 777, 797, 810, 739, 764, 845, 765, 699, 771, 755, 711, 785, 739, 769, 775, 761, 787, 741, 724, 773, 799, 765, 779, 835, 764, 812, 790, 769, 778, 749, 774, 751, 800, 754, 777, 769, 730, 676, 797] payload_rows_per_dst=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] dup_rows_per_dst=[3030, 2950, 2962, 3062, 2854, 3006, 3164, 2924] combine_rows_per_src=[9296, 9232, 9342, 9514, 9154, 9252, 9378, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64820 dup_rows=23972 dup_rows_remote=20824 dedup_ratio=0.243856 router_tokens_per_expert=[781, 761, 750, 778, 780, 764, 778, 708, 727, 745, 780, 764, 749, 761, 798, 802, 768, 795, 805, 795, 721, 771, 775, 748, 792, 805, 821, 768, 827, 747, 781, 734, 766, 764, 735, 760, 814, 762, 783, 780, 795, 774, 705, 721, 819, 802, 771, 780, 796, 773, 815, 788, 704, 741, 743, 784, 774, 706, 766, 768, 797, 721, 707, 759] payload_rows_per_dst=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] dup_rows_per_dst=[2922, 2954, 2738, 3026, 2984, 3068, 3132, 3148] combine_rows_per_src=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344416 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266781024 ingress_bytes=370245632 link_bytes=370245632 token_bytes=533217632 weight_movement_bytes=103809024 total_nvlink_bytes=637026656 weight_to_token_ratio=0.194684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393204506e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74184 payload_rows_remote=65040 dup_rows=24120 dup_rows_remote=21150 dedup_ratio=0.245361 router_tokens_per_expert=[715, 782, 745, 759, 763, 767, 772, 783, 753, 728, 785, 748, 800, 764, 821, 740, 744, 775, 786, 782, 743, 741, 764, 722, 817, 764, 825, 727, 797, 768, 789, 762, 796, 793, 798, 775, 750, 741, 770, 779, 762, 756, 764, 770, 758, 798, 790, 760, 788, 803, 774, 704, 745, 733, 771, 754, 773, 774, 760, 777, 801, 794, 765, 745] payload_rows_per_dst=[9212, 9288, 9144, 9424, 9334, 9360, 9118, 9304] dup_rows_per_dst=[2960, 2990, 2970, 3074, 3070, 2956, 3026, 3074] combine_rows_per_src=[9212, 9288, 9144, 9424, 9334, 9360, 9118, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65160 dup_rows=23924 dup_rows_remote=20916 dedup_ratio=0.243368 router_tokens_per_expert=[746, 748, 738, 748, 747, 774, 803, 782, 775, 780, 776, 747, 784, 820, 776, 794, 800, 780, 779, 722, 776, 768, 794, 809, 744, 775, 758, 770, 723, 751, 740, 743, 802, 694, 753, 771, 771, 745, 783, 795, 781, 727, 795, 738, 757, 757, 838, 735, 803, 750, 792, 756, 739, 835, 777, 758, 754, 756, 735, 776, 758, 796, 781, 774] payload_rows_per_dst=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] dup_rows_per_dst=[2998, 3070, 3104, 2892, 3008, 2892, 3024, 2936] combine_rows_per_src=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64952 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[784, 833, 774, 752, 766, 774, 771, 768, 744, 750, 801, 836, 790, 758, 742, 804, 783, 716, 754, 785, 757, 781, 773, 761, 759, 777, 784, 753, 726, 791, 800, 768, 769, 758, 779, 813, 747, 774, 766, 736, 765, 741, 755, 741, 804, 779, 725, 756, 761, 797, 723, 766, 780, 764, 756, 733, 769, 735, 770, 774, 750, 803, 779, 769] payload_rows_per_dst=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] dup_rows_per_dst=[3036, 3068, 3020, 3044, 3056, 2942, 2956, 3026] combine_rows_per_src=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64882 dup_rows=24116 dup_rows_remote=21102 dedup_ratio=0.245321 router_tokens_per_expert=[814, 746, 791, 746, 772, 791, 765, 794, 749, 757, 767, 808, 758, 739, 754, 794, 801, 772, 784, 733, 777, 787, 760, 765, 761, 704, 785, 786, 713, 808, 759, 788, 748, 798, 758, 692, 785, 796, 770, 802, 795, 736, 763, 758, 746, 764, 816, 782, 757, 788, 758, 775, 780, 767, 774, 727, 756, 755, 730, 790, 741, 796, 734, 787] payload_rows_per_dst=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] dup_rows_per_dst=[3168, 2984, 3054, 3006, 3044, 3014, 2990, 2856] combine_rows_per_src=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344496 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267215280 ingress_bytes=370679808 link_bytes=370679808 token_bytes=534086064 weight_movement_bytes=103809024 total_nvlink_bytes=637895088 weight_to_token_ratio=0.194368 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394838275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64820 dup_rows=23972 dup_rows_remote=20824 dedup_ratio=0.243856 router_tokens_per_expert=[781, 761, 750, 778, 780, 764, 778, 708, 727, 745, 780, 764, 749, 761, 798, 802, 768, 795, 805, 795, 721, 771, 775, 748, 792, 805, 821, 768, 827, 747, 781, 734, 766, 764, 735, 760, 814, 762, 783, 780, 795, 774, 705, 721, 819, 802, 771, 780, 796, 773, 815, 788, 704, 741, 743, 784, 774, 706, 766, 768, 797, 721, 707, 759] payload_rows_per_dst=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] dup_rows_per_dst=[2922, 2954, 2738, 3026, 2984, 3068, 3132, 3148] combine_rows_per_src=[9278, 9246, 8954, 9268, 9370, 9288, 9416, 9512] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65074 dup_rows=23936 dup_rows_remote=20992 dedup_ratio=0.243490 router_tokens_per_expert=[717, 752, 783, 786, 747, 750, 768, 773, 734, 774, 759, 791, 781, 744, 782, 760, 745, 754, 775, 783, 747, 804, 834, 751, 749, 761, 840, 792, 773, 759, 772, 723, 767, 781, 773, 753, 776, 778, 801, 776, 771, 775, 811, 765, 726, 698, 747, 813, 717, 787, 793, 801, 767, 762, 809, 756, 745, 727, 775, 760, 753, 742, 832, 752] payload_rows_per_dst=[9294, 9286, 9432, 9286, 9194, 9222, 9404, 9250] dup_rows_per_dst=[2944, 2964, 3256, 3052, 2914, 2904, 2980, 2922] combine_rows_per_src=[9294, 9286, 9432, 9286, 9194, 9222, 9404, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65058 dup_rows=24040 dup_rows_remote=21026 dedup_ratio=0.244548 router_tokens_per_expert=[761, 771, 778, 749, 736, 715, 791, 824, 795, 813, 781, 748, 767, 774, 797, 781, 759, 790, 772, 816, 724, 813, 742, 740, 799, 770, 783, 701, 760, 735, 744, 818, 774, 754, 788, 833, 772, 725, 733, 714, 777, 749, 792, 762, 779, 789, 749, 775, 777, 738, 752, 738, 715, 743, 792, 783, 774, 808, 780, 788, 755, 773, 753, 771] payload_rows_per_dst=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] dup_rows_per_dst=[2948, 3142, 2956, 3014, 2950, 3092, 2862, 3076] combine_rows_per_src=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344488 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267076008 ingress_bytes=301334528 link_bytes=301334528 token_bytes=533807528 weight_movement_bytes=34603008 total_nvlink_bytes=568410536 weight_to_token_ratio=0.064823 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133897569e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=342576 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265845296 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348016 weight_movement_bytes=34603008 total_nvlink_bytes=565951024 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=344760 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266748600 ingress_bytes=335609856 link_bytes=335609856 token_bytes=533152440 weight_movement_bytes=69206016 total_nvlink_bytes=602358456 weight_to_token_ratio=0.129805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262872869e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74750 payload_rows_remote=65304 dup_rows=23554 dup_rows_remote=20528 dedup_ratio=0.239604 router_tokens_per_expert=[765, 784, 767, 767, 807, 761, 786, 717, 770, 728, 768, 786, 717, 772, 761, 764, 739, 794, 762, 725, 740, 757, 772, 776, 749, 765, 756, 751, 730, 758, 788, 801, 774, 734, 813, 771, 747, 746, 761, 794, 749, 831, 761, 766, 756, 758, 783, 759, 788, 771, 816, 757, 767, 806, 802, 729, 757, 793, 769, 786, 794, 752, 812, 767] payload_rows_per_dst=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] dup_rows_per_dst=[2952, 2878, 2918, 2870, 2944, 3006, 3026, 2960] combine_rows_per_src=[9356, 9254, 9212, 9326, 9336, 9320, 9446, 9500] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65160 dup_rows=23924 dup_rows_remote=20916 dedup_ratio=0.243368 router_tokens_per_expert=[746, 748, 738, 748, 747, 774, 803, 782, 775, 780, 776, 747, 784, 820, 776, 794, 800, 780, 779, 722, 776, 768, 794, 809, 744, 775, 758, 770, 723, 751, 740, 743, 802, 694, 753, 771, 771, 745, 783, 795, 781, 727, 795, 738, 757, 757, 838, 735, 803, 750, 792, 756, 739, 835, 777, 758, 754, 756, 735, 776, 758, 796, 781, 774] payload_rows_per_dst=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] dup_rows_per_dst=[2998, 3070, 3104, 2892, 3008, 2892, 3024, 2936] combine_rows_per_src=[9174, 9434, 9352, 9116, 9220, 9364, 9396, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64882 dup_rows=24116 dup_rows_remote=21102 dedup_ratio=0.245321 router_tokens_per_expert=[814, 746, 791, 746, 772, 791, 765, 794, 749, 757, 767, 808, 758, 739, 754, 794, 801, 772, 784, 733, 777, 787, 760, 765, 761, 704, 785, 786, 713, 808, 759, 788, 748, 798, 758, 692, 785, 796, 770, 802, 795, 736, 763, 758, 746, 764, 816, 782, 757, 788, 758, 775, 780, 767, 774, 727, 756, 755, 730, 790, 741, 796, 734, 787] payload_rows_per_dst=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] dup_rows_per_dst=[3168, 2984, 3054, 3006, 3044, 3014, 2990, 2856] combine_rows_per_src=[9270, 9268, 9304, 9202, 9254, 9306, 9262, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344264 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887368 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430472 weight_movement_bytes=34603008 total_nvlink_bytes=568033480 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74654 payload_rows_remote=65222 dup_rows=23650 dup_rows_remote=20610 dedup_ratio=0.240580 router_tokens_per_expert=[785, 782, 725, 741, 751, 753, 751, 782, 794, 764, 725, 742, 799, 731, 786, 754, 788, 779, 790, 779, 750, 808, 778, 734, 775, 805, 795, 758, 801, 763, 789, 750, 802, 793, 759, 770, 788, 755, 779, 752, 778, 740, 758, 809, 765, 799, 787, 770, 765, 780, 749, 756, 779, 733, 756, 716, 770, 735, 780, 751, 736, 749, 811, 775] payload_rows_per_dst=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] dup_rows_per_dst=[2866, 2938, 2970, 3040, 2962, 3012, 2914, 2948] combine_rows_per_src=[9274, 9252, 9442, 9432, 9434, 9400, 9154, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=343328 combine_payload_bytes=267485184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267828512 ingress_bytes=336691200 link_bytes=336691200 token_bytes=535313696 weight_movement_bytes=69206016 total_nvlink_bytes=604519712 weight_to_token_ratio=0.129281 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266941880e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344304 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239664 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534135024 weight_movement_bytes=69206016 total_nvlink_bytes=603341040 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344576 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266387968 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431360 weight_movement_bytes=34603008 total_nvlink_bytes=567034368 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343936 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100608 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857280 weight_movement_bytes=69206016 total_nvlink_bytes=601063296 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=342576 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265845296 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531348016 weight_movement_bytes=34603008 total_nvlink_bytes=565951024 weight_to_token_ratio=0.065123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344264 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887368 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430472 weight_movement_bytes=34603008 total_nvlink_bytes=568033480 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65240 dup_rows=23944 dup_rows_remote=20994 dedup_ratio=0.243571 router_tokens_per_expert=[788, 779, 711, 770, 763, 807, 785, 763, 748, 774, 763, 760, 775, 748, 748, 719, 774, 770, 771, 814, 755, 790, 785, 815, 783, 785, 742, 756, 784, 751, 741, 802, 767, 788, 782, 775, 781, 812, 779, 780, 786, 743, 734, 770, 772, 775, 768, 737, 749, 781, 739, 754, 730, 754, 724, 749, 727, 797, 764, 767, 797, 788, 778, 786] payload_rows_per_dst=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] dup_rows_per_dst=[2990, 2950, 3162, 2950, 3122, 2938, 2818, 3014] combine_rows_per_src=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65118 dup_rows=23994 dup_rows_remote=21072 dedup_ratio=0.244080 router_tokens_per_expert=[746, 771, 772, 802, 740, 776, 809, 789, 762, 738, 756, 798, 813, 732, 776, 764, 717, 730, 723, 767, 789, 789, 741, 814, 762, 795, 755, 761, 725, 770, 783, 744, 779, 730, 732, 766, 693, 768, 825, 764, 764, 711, 749, 739, 792, 768, 792, 797, 783, 746, 827, 769, 739, 755, 773, 803, 776, 782, 802, 803, 782, 769, 778, 787] payload_rows_per_dst=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] dup_rows_per_dst=[2994, 3000, 3038, 2952, 2922, 3008, 3040, 3040] combine_rows_per_src=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64856 dup_rows=24104 dup_rows_remote=21042 dedup_ratio=0.245199 router_tokens_per_expert=[732, 758, 741, 790, 722, 769, 819, 780, 785, 726, 759, 813, 719, 769, 782, 786, 788, 803, 816, 759, 768, 723, 789, 724, 794, 753, 781, 787, 758, 787, 742, 794, 778, 763, 678, 780, 798, 784, 763, 763, 847, 731, 746, 795, 745, 764, 738, 757, 753, 781, 744, 740, 789, 805, 766, 725, 747, 791, 772, 785, 816, 770, 790, 732] payload_rows_per_dst=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] dup_rows_per_dst=[2684, 2844, 3104, 2988, 3140, 3148, 3134, 3062] combine_rows_per_src=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65154 dup_rows=23870 dup_rows_remote=20970 dedup_ratio=0.242818 router_tokens_per_expert=[784, 753, 746, 796, 780, 781, 786, 824, 739, 763, 667, 772, 767, 786, 779, 817, 797, 758, 755, 798, 742, 766, 760, 806, 724, 804, 833, 745, 774, 712, 701, 792, 767, 752, 724, 769, 772, 798, 742, 792, 780, 749, 800, 765, 744, 800, 770, 785, 778, 766, 737, 784, 793, 759, 774, 773, 742, 741, 788, 780, 773, 761, 758, 729] payload_rows_per_dst=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] dup_rows_per_dst=[3098, 2900, 3016, 2970, 2902, 3018, 3058, 2908] combine_rows_per_src=[9402, 9280, 9348, 9200, 9330, 9368, 9270, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=65048 dup_rows=24196 dup_rows_remote=21142 dedup_ratio=0.246134 router_tokens_per_expert=[830, 750, 759, 738, 761, 767, 792, 791, 734, 774, 795, 792, 767, 740, 782, 777, 795, 795, 754, 718, 763, 731, 757, 744, 781, 773, 745, 779, 745, 816, 775, 779, 759, 785, 817, 759, 750, 775, 737, 745, 756, 790, 727, 800, 750, 780, 788, 799, 768, 774, 761, 794, 763, 792, 834, 768, 736, 778, 731, 717, 723, 797, 715, 785] payload_rows_per_dst=[9344, 9308, 9060, 9344, 9234, 9286, 9424, 9108] dup_rows_per_dst=[3032, 3014, 3054, 3042, 3020, 3094, 3084, 2856] combine_rows_per_src=[9344, 9308, 9060, 9344, 9234, 9286, 9424, 9108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20878 dedup_ratio=0.241821 router_tokens_per_expert=[753, 756, 738, 810, 798, 783, 770, 792, 772, 787, 715, 779, 770, 780, 752, 797, 765, 753, 760, 788, 722, 788, 791, 773, 750, 798, 765, 800, 782, 766, 810, 751, 753, 757, 805, 772, 736, 744, 768, 753, 776, 710, 757, 783, 781, 776, 777, 758, 743, 783, 755, 780, 767, 740, 802, 761, 767, 771, 736, 782, 751, 727, 772, 795] payload_rows_per_dst=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] dup_rows_per_dst=[3120, 3064, 2972, 2998, 2988, 2836, 2894, 2900] combine_rows_per_src=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344936 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267567976 ingress_bytes=336429056 link_bytes=336429056 token_bytes=534791016 weight_movement_bytes=69206016 total_nvlink_bytes=603997032 weight_to_token_ratio=0.129408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64854 dup_rows=24178 dup_rows_remote=21384 dedup_ratio=0.245951 router_tokens_per_expert=[770, 783, 814, 790, 767, 767, 777, 757, 798, 747, 768, 720, 824, 740, 791, 772, 734, 782, 762, 733, 766, 761, 782, 760, 771, 758, 746, 791, 800, 765, 803, 774, 768, 836, 748, 787, 790, 743, 759, 778, 735, 743, 769, 749, 760, 752, 784, 741, 738, 767, 786, 778, 752, 766, 759, 729, 745, 749, 766, 776, 775, 784, 769, 798] payload_rows_per_dst=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] dup_rows_per_dst=[3116, 3086, 2942, 3122, 3120, 2794, 2968, 3030] combine_rows_per_src=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=343328 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267492640 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534641952 weight_movement_bytes=69206016 total_nvlink_bytes=603847968 weight_to_token_ratio=0.129444 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74200 payload_rows_remote=64856 dup_rows=24104 dup_rows_remote=21042 dedup_ratio=0.245199 router_tokens_per_expert=[732, 758, 741, 790, 722, 769, 819, 780, 785, 726, 759, 813, 719, 769, 782, 786, 788, 803, 816, 759, 768, 723, 789, 724, 794, 753, 781, 787, 758, 787, 742, 794, 778, 763, 678, 780, 798, 784, 763, 763, 847, 731, 746, 795, 745, 764, 738, 757, 753, 781, 744, 740, 789, 805, 766, 725, 747, 791, 772, 785, 816, 770, 790, 732] payload_rows_per_dst=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] dup_rows_per_dst=[2684, 2844, 3104, 2988, 3140, 3148, 3134, 3062] combine_rows_per_src=[8926, 9168, 9352, 9176, 9340, 9388, 9506, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64784 dup_rows=24066 dup_rows_remote=21044 dedup_ratio=0.244812 router_tokens_per_expert=[801, 806, 764, 803, 739, 760, 793, 758, 771, 764, 745, 769, 793, 779, 787, 791, 746, 719, 759, 777, 756, 798, 777, 815, 781, 791, 758, 734, 779, 755, 729, 771, 801, 740, 728, 753, 765, 793, 757, 737, 774, 799, 755, 793, 721, 789, 757, 786, 801, 793, 806, 761, 718, 725, 810, 745, 770, 759, 741, 750, 769, 727, 780, 781] payload_rows_per_dst=[9454, 9438, 9314, 9212, 9288, 9246, 9196, 9090] dup_rows_per_dst=[3022, 3048, 2980, 2984, 2860, 3102, 3122, 2948] combine_rows_per_src=[9454, 9438, 9314, 9212, 9288, 9246, 9196, 9090] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65424 dup_rows=23744 dup_rows_remote=20838 dedup_ratio=0.241536 router_tokens_per_expert=[802, 784, 786, 792, 754, 769, 784, 749, 774, 788, 762, 766, 747, 793, 777, 732, 762, 773, 779, 803, 765, 794, 799, 758, 780, 730, 753, 766, 765, 751, 728, 748, 752, 804, 762, 731, 781, 759, 807, 787, 721, 805, 781, 752, 829, 757, 806, 774, 746, 783, 784, 754, 767, 748, 705, 759, 802, 762, 808, 742, 724, 772, 747, 728] payload_rows_per_dst=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] dup_rows_per_dst=[3046, 3028, 2998, 2906, 2892, 3006, 2916, 2952] combine_rows_per_src=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344760 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267068088 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791416 weight_movement_bytes=34603008 total_nvlink_bytes=568394424 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343592 combine_payload_bytes=265650176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265993768 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531643944 weight_movement_bytes=69206016 total_nvlink_bytes=600849960 weight_to_token_ratio=0.130174 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=344760 combine_payload_bytes=266436608 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266781368 ingress_bytes=301039616 link_bytes=301039616 token_bytes=533217976 weight_movement_bytes=34603008 total_nvlink_bytes=567820984 weight_to_token_ratio=0.064895 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132787839e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344496 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267215280 ingress_bytes=370679808 link_bytes=370679808 token_bytes=534086064 weight_movement_bytes=103809024 total_nvlink_bytes=637895088 weight_to_token_ratio=0.194368 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394838275e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344168 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255912 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167656 weight_movement_bytes=34603008 total_nvlink_bytes=568770664 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344952 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986936 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628920 weight_movement_bytes=34603008 total_nvlink_bytes=566231928 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343592 combine_payload_bytes=265650176 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265993768 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531643944 weight_movement_bytes=69206016 total_nvlink_bytes=600849960 weight_to_token_ratio=0.130174 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64736 dup_rows=24158 dup_rows_remote=21042 dedup_ratio=0.245748 router_tokens_per_expert=[807, 781, 791, 720, 782, 770, 815, 777, 731, 657, 742, 740, 797, 778, 794, 764, 746, 786, 779, 779, 813, 747, 750, 782, 776, 816, 745, 760, 776, 783, 785, 749, 730, 803, 778, 788, 775, 751, 786, 762, 807, 761, 735, 759, 729, 736, 713, 785, 795, 802, 732, 776, 774, 752, 785, 847, 779, 756, 785, 745, 784, 785, 735, 704] payload_rows_per_dst=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] dup_rows_per_dst=[3126, 2930, 3038, 3064, 2994, 2946, 3116, 2944] combine_rows_per_src=[9360, 9076, 9326, 9316, 9352, 9104, 9410, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65118 dup_rows=23994 dup_rows_remote=21072 dedup_ratio=0.244080 router_tokens_per_expert=[746, 771, 772, 802, 740, 776, 809, 789, 762, 738, 756, 798, 813, 732, 776, 764, 717, 730, 723, 767, 789, 789, 741, 814, 762, 795, 755, 761, 725, 770, 783, 744, 779, 730, 732, 766, 693, 768, 825, 764, 764, 711, 749, 739, 792, 768, 792, 797, 783, 746, 827, 769, 739, 755, 773, 803, 776, 782, 802, 803, 782, 769, 778, 787] payload_rows_per_dst=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] dup_rows_per_dst=[2994, 3000, 3038, 2952, 2922, 3008, 3040, 3040] combine_rows_per_src=[9416, 9278, 9102, 9238, 9192, 9216, 9350, 9518] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64854 dup_rows=24178 dup_rows_remote=21384 dedup_ratio=0.245951 router_tokens_per_expert=[770, 783, 814, 790, 767, 767, 777, 757, 798, 747, 768, 720, 824, 740, 791, 772, 734, 782, 762, 733, 766, 761, 782, 760, 771, 758, 746, 791, 800, 765, 803, 774, 768, 836, 748, 787, 790, 743, 759, 778, 735, 743, 769, 749, 760, 752, 784, 741, 738, 767, 786, 778, 752, 766, 759, 729, 745, 749, 766, 776, 775, 784, 769, 798] payload_rows_per_dst=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] dup_rows_per_dst=[3116, 3086, 2942, 3122, 3120, 2794, 2968, 3030] combine_rows_per_src=[9334, 9234, 9218, 9294, 9298, 9272, 9182, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343312 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265698576 ingress_bytes=403767296 link_bytes=403767296 token_bytes=531053840 weight_movement_bytes=138412032 total_nvlink_bytes=669465872 weight_to_token_ratio=0.260637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519343829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=345048 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321752 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536298456 weight_movement_bytes=34603008 total_nvlink_bytes=570901464 weight_to_token_ratio=0.064522 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64788 dup_rows=24072 dup_rows_remote=20908 dedup_ratio=0.244873 router_tokens_per_expert=[778, 754, 748, 774, 735, 766, 783, 786, 779, 792, 778, 768, 841, 789, 739, 818, 785, 746, 770, 742, 778, 741, 789, 765, 771, 752, 797, 800, 796, 789, 746, 773, 705, 720, 754, 796, 814, 751, 735, 806, 742, 758, 794, 747, 785, 750, 725, 779, 748, 785, 771, 780, 746, 759, 794, 794, 815, 765, 716, 760, 712, 779, 729, 770] payload_rows_per_dst=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] dup_rows_per_dst=[2930, 3164, 2976, 3100, 2906, 2956, 3066, 2974] combine_rows_per_src=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64786 dup_rows=24192 dup_rows_remote=21142 dedup_ratio=0.246094 router_tokens_per_expert=[787, 721, 749, 813, 768, 759, 773, 811, 770, 767, 733, 746, 772, 761, 738, 784, 736, 748, 725, 797, 753, 744, 799, 769, 745, 761, 778, 811, 765, 753, 767, 763, 769, 780, 785, 785, 770, 780, 740, 779, 743, 756, 766, 808, 790, 746, 792, 791, 772, 767, 764, 800, 797, 740, 764, 763, 758, 791, 738, 730, 771, 765, 807, 779] payload_rows_per_dst=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] dup_rows_per_dst=[3108, 2926, 2964, 3020, 3050, 3094, 3008, 3022] combine_rows_per_src=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=342784 combine_payload_bytes=265371648 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265714432 ingress_bytes=369180672 link_bytes=369180672 token_bytes=531086080 weight_movement_bytes=103809024 total_nvlink_bytes=634895104 weight_to_token_ratio=0.195466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389197147e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65042 dup_rows=23904 dup_rows_remote=20894 dedup_ratio=0.243164 router_tokens_per_expert=[749, 800, 760, 725, 771, 747, 787, 757, 822, 794, 810, 745, 789, 723, 771, 705, 747, 769, 745, 798, 705, 774, 842, 804, 775, 767, 800, 761, 750, 742, 808, 777, 777, 742, 824, 800, 758, 767, 776, 783, 805, 792, 775, 735, 736, 765, 711, 775, 728, 784, 762, 729, 780, 797, 729, 755, 807, 752, 790, 723, 777, 739, 760, 800] payload_rows_per_dst=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] dup_rows_per_dst=[2954, 2962, 3010, 2998, 3104, 2968, 2938, 2970] combine_rows_per_src=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65036 dup_rows=23942 dup_rows_remote=20924 dedup_ratio=0.243551 router_tokens_per_expert=[760, 742, 766, 792, 762, 756, 759, 736, 746, 761, 794, 753, 784, 757, 775, 792, 744, 808, 728, 778, 776, 782, 781, 763, 778, 778, 780, 798, 784, 746, 777, 742, 755, 755, 734, 755, 758, 771, 803, 771, 797, 753, 777, 764, 743, 792, 785, 716, 752, 778, 772, 747, 803, 796, 772, 752, 792, 742, 817, 757, 786, 771, 741, 767] payload_rows_per_dst=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] dup_rows_per_dst=[2958, 3042, 2950, 2994, 2944, 3000, 3018, 3036] combine_rows_per_src=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65140 dup_rows=23840 dup_rows_remote=20886 dedup_ratio=0.242513 router_tokens_per_expert=[816, 755, 771, 742, 794, 757, 769, 823, 793, 728, 794, 780, 773, 769, 826, 733, 717, 732, 748, 810, 730, 755, 780, 745, 759, 797, 752, 772, 811, 757, 763, 792, 776, 775, 739, 833, 786, 784, 795, 759, 790, 744, 765, 793, 715, 779, 765, 788, 741, 756, 784, 717, 753, 753, 744, 723, 755, 728, 767, 789, 778, 787, 776, 772] payload_rows_per_dst=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] dup_rows_per_dst=[3000, 3102, 2850, 3026, 3090, 2954, 2850, 2968] combine_rows_per_src=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65058 dup_rows=24040 dup_rows_remote=21026 dedup_ratio=0.244548 router_tokens_per_expert=[761, 771, 778, 749, 736, 715, 791, 824, 795, 813, 781, 748, 767, 774, 797, 781, 759, 790, 772, 816, 724, 813, 742, 740, 799, 770, 783, 701, 760, 735, 744, 818, 774, 754, 788, 833, 772, 725, 733, 714, 777, 749, 792, 762, 779, 789, 749, 775, 777, 738, 752, 738, 715, 743, 792, 783, 774, 808, 780, 788, 755, 773, 753, 771] payload_rows_per_dst=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] dup_rows_per_dst=[2948, 3142, 2956, 3014, 2950, 3092, 2862, 3076] combine_rows_per_src=[9302, 9370, 9356, 9206, 9236, 9252, 9214, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265158656 dispatch_meta_bytes=343112 combine_payload_bytes=265158656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265501768 ingress_bytes=368967680 link_bytes=368967680 token_bytes=530660424 weight_movement_bytes=103809024 total_nvlink_bytes=634469448 weight_to_token_ratio=0.195622 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388395676e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344760 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267068088 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533791416 weight_movement_bytes=34603008 total_nvlink_bytes=568394424 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=344952 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265986936 ingress_bytes=300244992 link_bytes=300244992 token_bytes=531628920 weight_movement_bytes=34603008 total_nvlink_bytes=566231928 weight_to_token_ratio=0.065089 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129797733e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64668 dup_rows=24278 dup_rows_remote=21032 dedup_ratio=0.246969 router_tokens_per_expert=[770, 761, 763, 738, 748, 741, 755, 865, 730, 786, 753, 799, 744, 704, 834, 717, 763, 746, 751, 711, 744, 796, 773, 787, 765, 735, 746, 781, 755, 770, 785, 785, 779, 815, 758, 760, 805, 769, 780, 739, 791, 749, 777, 790, 794, 756, 817, 791, 749, 760, 768, 782, 810, 769, 792, 805, 739, 739, 739, 725, 785, 799, 746, 774] payload_rows_per_dst=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] dup_rows_per_dst=[2860, 2910, 3210, 3058, 2766, 3068, 3160, 3246] combine_rows_per_src=[8918, 9132, 9436, 9276, 9132, 9316, 9458, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74400 payload_rows_remote=65042 dup_rows=23904 dup_rows_remote=20894 dedup_ratio=0.243164 router_tokens_per_expert=[749, 800, 760, 725, 771, 747, 787, 757, 822, 794, 810, 745, 789, 723, 771, 705, 747, 769, 745, 798, 705, 774, 842, 804, 775, 767, 800, 761, 750, 742, 808, 777, 777, 742, 824, 800, 758, 767, 776, 783, 805, 792, 775, 735, 736, 765, 711, 775, 728, 784, 762, 729, 780, 797, 729, 755, 807, 752, 790, 723, 777, 739, 760, 800] payload_rows_per_dst=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] dup_rows_per_dst=[2954, 2962, 3010, 2998, 3104, 2968, 2938, 2970] combine_rows_per_src=[9238, 9356, 9358, 9362, 9350, 9220, 9190, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65240 dup_rows=23944 dup_rows_remote=20994 dedup_ratio=0.243571 router_tokens_per_expert=[788, 779, 711, 770, 763, 807, 785, 763, 748, 774, 763, 760, 775, 748, 748, 719, 774, 770, 771, 814, 755, 790, 785, 815, 783, 785, 742, 756, 784, 751, 741, 802, 767, 788, 782, 775, 781, 812, 779, 780, 786, 743, 734, 770, 772, 775, 768, 737, 749, 781, 739, 754, 730, 754, 724, 749, 727, 797, 764, 767, 797, 788, 778, 786] payload_rows_per_dst=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] dup_rows_per_dst=[2990, 2950, 3162, 2950, 3122, 2938, 2818, 3014] combine_rows_per_src=[9342, 9120, 9386, 9338, 9406, 9232, 9142, 9394] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343712 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265707168 ingress_bytes=334569472 link_bytes=334569472 token_bytes=531070624 weight_movement_bytes=69206016 total_nvlink_bytes=600276640 weight_to_token_ratio=0.130314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258957988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=64886 dup_rows=23870 dup_rows_remote=20734 dedup_ratio=0.242818 router_tokens_per_expert=[815, 772, 776, 805, 778, 778, 765, 809, 803, 789, 791, 744, 749, 761, 718, 789, 701, 758, 780, 747, 776, 733, 772, 781, 768, 756, 736, 749, 714, 738, 816, 754, 794, 749, 809, 769, 753, 777, 748, 735, 728, 798, 779, 785, 772, 720, 782, 780, 764, 807, 742, 803, 775, 772, 813, 770, 746, 750, 730, 759, 765, 793, 748, 816] payload_rows_per_dst=[9548, 9242, 9106, 9238, 9356, 9270, 9360, 9314] dup_rows_per_dst=[3136, 2958, 2990, 2824, 2994, 2936, 3132, 2900] combine_rows_per_src=[9548, 9242, 9106, 9238, 9356, 9270, 9360, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64978 dup_rows=24020 dup_rows_remote=21082 dedup_ratio=0.244344 router_tokens_per_expert=[781, 739, 756, 767, 801, 806, 763, 805, 831, 794, 718, 764, 775, 748, 792, 764, 783, 762, 781, 748, 780, 728, 800, 791, 797, 761, 789, 714, 753, 740, 800, 768, 759, 762, 768, 747, 768, 746, 730, 801, 762, 753, 775, 755, 784, 749, 760, 755, 774, 738, 795, 796, 767, 756, 761, 797, 745, 767, 764, 721, 818, 770, 766, 744] payload_rows_per_dst=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] dup_rows_per_dst=[3048, 3072, 3072, 2938, 2934, 2990, 3012, 2954] combine_rows_per_src=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=342800 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265222928 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530103056 weight_movement_bytes=34603008 total_nvlink_bytes=564706064 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343744 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266755776 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533167808 weight_movement_bytes=34603008 total_nvlink_bytes=567770816 weight_to_token_ratio=0.064901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343840 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731296 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118752 weight_movement_bytes=34603008 total_nvlink_bytes=567721760 weight_to_token_ratio=0.064907 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267223040 dispatch_meta_bytes=344936 combine_payload_bytes=267223040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267567976 ingress_bytes=336429056 link_bytes=336429056 token_bytes=534791016 weight_movement_bytes=69206016 total_nvlink_bytes=603997032 weight_to_token_ratio=0.129408 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265955453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344104 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267157544 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533970984 weight_movement_bytes=103809024 total_nvlink_bytes=637780008 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=342800 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265222928 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530103056 weight_movement_bytes=34603008 total_nvlink_bytes=564706064 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64930 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[773, 752, 744, 734, 691, 734, 751, 747, 737, 794, 791, 814, 780, 797, 780, 783, 827, 791, 719, 747, 766, 778, 794, 770, 762, 809, 773, 775, 751, 768, 729, 740, 755, 765, 757, 743, 746, 783, 784, 784, 781, 793, 766, 731, 779, 799, 827, 765, 743, 771, 787, 769, 771, 747, 822, 776, 749, 725, 759, 801, 757, 737, 807, 772] payload_rows_per_dst=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] dup_rows_per_dst=[2770, 3118, 2998, 2942, 2922, 3130, 3030, 3030] combine_rows_per_src=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64952 dup_rows=24148 dup_rows_remote=21192 dedup_ratio=0.245646 router_tokens_per_expert=[784, 833, 774, 752, 766, 774, 771, 768, 744, 750, 801, 836, 790, 758, 742, 804, 783, 716, 754, 785, 757, 781, 773, 761, 759, 777, 784, 753, 726, 791, 800, 768, 769, 758, 779, 813, 747, 774, 766, 736, 765, 741, 755, 741, 804, 779, 725, 756, 761, 797, 723, 766, 780, 764, 756, 733, 769, 735, 770, 774, 750, 803, 779, 769] payload_rows_per_dst=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] dup_rows_per_dst=[3036, 3068, 3020, 3044, 3056, 2942, 2956, 3026] combine_rows_per_src=[9408, 9382, 9200, 9272, 9228, 9190, 9204, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64786 dup_rows=24192 dup_rows_remote=21142 dedup_ratio=0.246094 router_tokens_per_expert=[787, 721, 749, 813, 768, 759, 773, 811, 770, 767, 733, 746, 772, 761, 738, 784, 736, 748, 725, 797, 753, 744, 799, 769, 745, 761, 778, 811, 765, 753, 767, 763, 769, 780, 785, 785, 770, 780, 740, 779, 743, 756, 766, 808, 790, 746, 792, 791, 772, 767, 764, 800, 797, 740, 764, 763, 758, 791, 738, 730, 771, 765, 807, 779] payload_rows_per_dst=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] dup_rows_per_dst=[3108, 2926, 2964, 3020, 3050, 3094, 3008, 3022] combine_rows_per_src=[9254, 9216, 9178, 9266, 9326, 9290, 9326, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65140 dup_rows=23840 dup_rows_remote=20886 dedup_ratio=0.242513 router_tokens_per_expert=[816, 755, 771, 742, 794, 757, 769, 823, 793, 728, 794, 780, 773, 769, 826, 733, 717, 732, 748, 810, 730, 755, 780, 745, 759, 797, 752, 772, 811, 757, 763, 792, 776, 775, 739, 833, 786, 784, 795, 759, 790, 744, 765, 793, 715, 779, 765, 788, 741, 756, 784, 717, 753, 753, 744, 723, 755, 728, 767, 789, 778, 787, 776, 772] payload_rows_per_dst=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] dup_rows_per_dst=[3000, 3102, 2850, 3026, 3090, 2954, 2850, 2968] combine_rows_per_src=[9454, 9290, 9184, 9380, 9404, 9324, 9092, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=342480 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266115536 ingress_bytes=334979072 link_bytes=334979072 token_bytes=531888592 weight_movement_bytes=69206016 total_nvlink_bytes=601094608 weight_to_token_ratio=0.130114 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260499280e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344240 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494128 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644016 weight_movement_bytes=34603008 total_nvlink_bytes=567247024 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64948 dup_rows=23910 dup_rows_remote=20960 dedup_ratio=0.243225 router_tokens_per_expert=[773, 734, 783, 801, 752, 786, 777, 805, 790, 783, 763, 756, 778, 790, 815, 777, 735, 782, 769, 735, 733, 829, 774, 809, 774, 729, 781, 772, 795, 749, 768, 781, 783, 749, 733, 780, 809, 786, 764, 794, 745, 777, 777, 766, 709, 721, 755, 737, 729, 757, 756, 741, 761, 749, 751, 766, 758, 753, 749, 806, 780, 766, 787, 780] payload_rows_per_dst=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] dup_rows_per_dst=[3116, 3124, 3064, 3060, 2950, 2746, 2836, 3014] combine_rows_per_src=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343008 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266296288 ingress_bytes=369762304 link_bytes=369762304 token_bytes=532249568 weight_movement_bytes=103809024 total_nvlink_bytes=636058592 weight_to_token_ratio=0.195038 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391385782e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64826 dup_rows=24192 dup_rows_remote=21102 dedup_ratio=0.246094 router_tokens_per_expert=[800, 781, 833, 708, 764, 782, 825, 753, 761, 747, 788, 726, 782, 758, 716, 753, 762, 750, 773, 783, 810, 734, 759, 817, 773, 774, 771, 774, 756, 730, 761, 789, 716, 737, 762, 797, 722, 794, 795, 789, 754, 785, 732, 790, 799, 813, 767, 795, 754, 738, 793, 793, 800, 789, 740, 771, 719, 792, 771, 771, 753, 769, 754, 705] payload_rows_per_dst=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] dup_rows_per_dst=[3098, 2888, 3090, 3068, 3008, 3112, 3038, 2890] combine_rows_per_src=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64992 dup_rows=23892 dup_rows_remote=20880 dedup_ratio=0.243042 router_tokens_per_expert=[782, 793, 787, 774, 790, 763, 792, 814, 772, 755, 781, 749, 767, 793, 775, 761, 737, 759, 772, 742, 752, 764, 715, 726, 748, 787, 763, 745, 765, 708, 782, 714, 704, 742, 798, 860, 803, 732, 750, 754, 737, 783, 775, 794, 772, 792, 799, 764, 772, 765, 802, 729, 757, 808, 790, 779, 808, 712, 770, 736, 779, 751, 791, 817] payload_rows_per_dst=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] dup_rows_per_dst=[3056, 3034, 2774, 2938, 2910, 3012, 3110, 3058] combine_rows_per_src=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73960 payload_rows_remote=64618 dup_rows=24344 dup_rows_remote=21280 dedup_ratio=0.247640 router_tokens_per_expert=[748, 764, 737, 756, 769, 795, 767, 778, 763, 747, 784, 762, 766, 788, 770, 793, 763, 812, 771, 765, 723, 774, 726, 807, 747, 773, 750, 758, 774, 804, 716, 742, 745, 794, 750, 731, 766, 751, 784, 801, 734, 798, 784, 743, 765, 790, 760, 725, 737, 753, 788, 757, 803, 764, 814, 787, 757, 792, 782, 769, 797, 781, 806, 752] payload_rows_per_dst=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] dup_rows_per_dst=[2962, 3074, 3106, 2988, 3030, 2972, 3064, 3148] combine_rows_per_src=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65108 dup_rows=23824 dup_rows_remote=20786 dedup_ratio=0.242350 router_tokens_per_expert=[746, 762, 777, 738, 759, 768, 748, 773, 783, 768, 738, 767, 760, 716, 755, 800, 773, 755, 762, 786, 812, 722, 750, 753, 731, 740, 790, 811, 783, 763, 799, 801, 733, 747, 791, 736, 799, 787, 761, 795, 756, 823, 762, 803, 759, 759, 785, 761, 800, 742, 780, 741, 746, 803, 763, 726, 774, 776, 831, 810, 762, 759, 759, 734] payload_rows_per_dst=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] dup_rows_per_dst=[2874, 2726, 2972, 3206, 3062, 2964, 2982, 3038] combine_rows_per_src=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343632 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266370640 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532397648 weight_movement_bytes=34603008 total_nvlink_bytes=567000656 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65424 dup_rows=23744 dup_rows_remote=20838 dedup_ratio=0.241536 router_tokens_per_expert=[802, 784, 786, 792, 754, 769, 784, 749, 774, 788, 762, 766, 747, 793, 777, 732, 762, 773, 779, 803, 765, 794, 799, 758, 780, 730, 753, 766, 765, 751, 728, 748, 752, 804, 762, 731, 781, 759, 807, 787, 721, 805, 781, 752, 829, 757, 806, 774, 746, 783, 784, 754, 767, 748, 705, 759, 802, 762, 808, 742, 724, 772, 747, 728] payload_rows_per_dst=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] dup_rows_per_dst=[3046, 3028, 2998, 2906, 2892, 3006, 2916, 2952] combine_rows_per_src=[9394, 9250, 9468, 9136, 9474, 9444, 9176, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344576 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266387968 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431360 weight_movement_bytes=34603008 total_nvlink_bytes=567034368 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=343712 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265707168 ingress_bytes=334569472 link_bytes=334569472 token_bytes=531070624 weight_movement_bytes=69206016 total_nvlink_bytes=600276640 weight_to_token_ratio=0.130314 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258957988e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344104 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267157544 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533970984 weight_movement_bytes=103809024 total_nvlink_bytes=637780008 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65108 dup_rows=23824 dup_rows_remote=20786 dedup_ratio=0.242350 router_tokens_per_expert=[746, 762, 777, 738, 759, 768, 748, 773, 783, 768, 738, 767, 760, 716, 755, 800, 773, 755, 762, 786, 812, 722, 750, 753, 731, 740, 790, 811, 783, 763, 799, 801, 733, 747, 791, 736, 799, 787, 761, 795, 756, 823, 762, 803, 759, 759, 785, 761, 800, 742, 780, 741, 746, 803, 763, 726, 774, 776, 831, 810, 762, 759, 759, 734] payload_rows_per_dst=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] dup_rows_per_dst=[2874, 2726, 2972, 3206, 3062, 2964, 2982, 3038] combine_rows_per_src=[9268, 9000, 9254, 9594, 9320, 9452, 9220, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64826 dup_rows=24192 dup_rows_remote=21102 dedup_ratio=0.246094 router_tokens_per_expert=[800, 781, 833, 708, 764, 782, 825, 753, 761, 747, 788, 726, 782, 758, 716, 753, 762, 750, 773, 783, 810, 734, 759, 817, 773, 774, 771, 774, 756, 730, 761, 789, 716, 737, 762, 797, 722, 794, 795, 789, 754, 785, 732, 790, 799, 813, 767, 795, 754, 738, 793, 793, 800, 789, 740, 771, 719, 792, 771, 771, 753, 769, 754, 705] payload_rows_per_dst=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] dup_rows_per_dst=[3098, 2888, 3090, 3068, 3008, 3112, 3038, 2890] combine_rows_per_src=[9394, 9174, 9286, 9188, 9216, 9358, 9318, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=345048 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321752 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536298456 weight_movement_bytes=34603008 total_nvlink_bytes=570901464 weight_to_token_ratio=0.064522 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64788 dup_rows=24072 dup_rows_remote=20908 dedup_ratio=0.244873 router_tokens_per_expert=[778, 754, 748, 774, 735, 766, 783, 786, 779, 792, 778, 768, 841, 789, 739, 818, 785, 746, 770, 742, 778, 741, 789, 765, 771, 752, 797, 800, 796, 789, 746, 773, 705, 720, 754, 796, 814, 751, 735, 806, 742, 758, 794, 747, 785, 750, 725, 779, 748, 785, 771, 780, 746, 759, 794, 794, 815, 765, 716, 760, 712, 779, 729, 770] payload_rows_per_dst=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] dup_rows_per_dst=[2930, 3164, 2976, 3100, 2906, 2956, 3066, 2974] combine_rows_per_src=[9318, 9444, 9256, 9348, 9256, 9204, 9288, 9118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21032 dedup_ratio=0.244344 router_tokens_per_expert=[799, 725, 753, 770, 776, 763, 735, 744, 747, 742, 819, 757, 771, 818, 792, 771, 745, 768, 767, 773, 760, 759, 761, 762, 748, 788, 798, 734, 787, 783, 812, 720, 771, 777, 820, 742, 736, 785, 806, 751, 770, 749, 795, 817, 779, 751, 739, 751, 723, 772, 812, 784, 773, 759, 739, 752, 757, 785, 766, 790, 746, 785, 761, 762] payload_rows_per_dst=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] dup_rows_per_dst=[2988, 3102, 3102, 2990, 3036, 2890, 2944, 2968] combine_rows_per_src=[9154, 9334, 9322, 9350, 9340, 9256, 9232, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65042 dup_rows=23998 dup_rows_remote=21056 dedup_ratio=0.244120 router_tokens_per_expert=[763, 780, 789, 797, 779, 768, 787, 733, 790, 773, 790, 773, 757, 794, 778, 785, 783, 759, 759, 784, 751, 790, 760, 745, 776, 763, 759, 787, 756, 746, 801, 715, 771, 749, 761, 756, 731, 807, 794, 745, 748, 736, 767, 818, 805, 721, 723, 770, 744, 758, 778, 771, 775, 713, 828, 813, 779, 775, 750, 779, 765, 755, 770, 727] payload_rows_per_dst=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] dup_rows_per_dst=[3006, 3042, 2996, 2942, 2930, 2964, 3112, 3006] combine_rows_per_src=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343712 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871008 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398304 weight_movement_bytes=34603008 total_nvlink_bytes=566001312 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343576 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267025944 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708312 weight_movement_bytes=103809024 total_nvlink_bytes=637517336 weight_to_token_ratio=0.194505 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343488 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550720 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532757952 weight_movement_bytes=69206016 total_nvlink_bytes=601963968 weight_to_token_ratio=0.129901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343592 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018920 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694248 weight_movement_bytes=69206016 total_nvlink_bytes=598900264 weight_to_token_ratio=0.130653 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343712 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871008 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398304 weight_movement_bytes=34603008 total_nvlink_bytes=566001312 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=342784 combine_payload_bytes=265371648 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265714432 ingress_bytes=369180672 link_bytes=369180672 token_bytes=531086080 weight_movement_bytes=103809024 total_nvlink_bytes=634895104 weight_to_token_ratio=0.195466 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389197147e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65068 dup_rows=23956 dup_rows_remote=20912 dedup_ratio=0.243693 router_tokens_per_expert=[765, 751, 804, 808, 766, 768, 715, 748, 738, 793, 787, 813, 782, 802, 711, 736, 796, 781, 773, 752, 766, 707, 794, 762, 777, 818, 785, 773, 805, 751, 799, 792, 779, 768, 754, 788, 768, 813, 819, 746, 731, 765, 770, 792, 751, 752, 760, 761, 765, 783, 771, 718, 732, 769, 742, 778, 772, 775, 792, 799, 723, 735, 722, 741] payload_rows_per_dst=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] dup_rows_per_dst=[2946, 3044, 2962, 3136, 3114, 2950, 2858, 2946] combine_rows_per_src=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74532 payload_rows_remote=65164 dup_rows=23772 dup_rows_remote=20878 dedup_ratio=0.241821 router_tokens_per_expert=[753, 756, 738, 810, 798, 783, 770, 792, 772, 787, 715, 779, 770, 780, 752, 797, 765, 753, 760, 788, 722, 788, 791, 773, 750, 798, 765, 800, 782, 766, 810, 751, 753, 757, 805, 772, 736, 744, 768, 753, 776, 710, 757, 783, 781, 776, 777, 758, 743, 783, 755, 780, 767, 740, 802, 761, 767, 771, 736, 782, 751, 727, 772, 795] payload_rows_per_dst=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] dup_rows_per_dst=[3120, 3064, 2972, 2998, 2988, 2836, 2894, 2900] combine_rows_per_src=[9280, 9240, 9308, 9446, 9188, 9400, 9368, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64948 dup_rows=23910 dup_rows_remote=20960 dedup_ratio=0.243225 router_tokens_per_expert=[773, 734, 783, 801, 752, 786, 777, 805, 790, 783, 763, 756, 778, 790, 815, 777, 735, 782, 769, 735, 733, 829, 774, 809, 774, 729, 781, 772, 795, 749, 768, 781, 783, 749, 733, 780, 809, 786, 764, 794, 745, 777, 777, 766, 709, 721, 755, 737, 729, 757, 756, 741, 761, 749, 751, 766, 758, 753, 749, 806, 780, 766, 787, 780] payload_rows_per_dst=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] dup_rows_per_dst=[3116, 3124, 3064, 3060, 2950, 2746, 2836, 3014] combine_rows_per_src=[9306, 9380, 9268, 9238, 9446, 9228, 9184, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64992 dup_rows=23892 dup_rows_remote=20880 dedup_ratio=0.243042 router_tokens_per_expert=[782, 793, 787, 774, 790, 763, 792, 814, 772, 755, 781, 749, 767, 793, 775, 761, 737, 759, 772, 742, 752, 764, 715, 726, 748, 787, 763, 745, 765, 708, 782, 714, 704, 742, 798, 860, 803, 732, 750, 754, 737, 783, 775, 794, 772, 792, 799, 764, 772, 765, 802, 729, 757, 808, 790, 779, 808, 712, 770, 736, 779, 751, 791, 817] payload_rows_per_dst=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] dup_rows_per_dst=[3056, 3034, 2774, 2938, 2910, 3012, 3110, 3058] combine_rows_per_src=[9534, 9272, 9160, 9086, 9376, 9420, 9294, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344648 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267117128 ingress_bytes=301375488 link_bytes=301375488 token_bytes=533889608 weight_movement_bytes=34603008 total_nvlink_bytes=568492616 weight_to_token_ratio=0.064813 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134051699e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344392 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756424 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168456 weight_movement_bytes=34603008 total_nvlink_bytes=567771464 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343920 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862448 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533380976 weight_movement_bytes=103809024 total_nvlink_bytes=637190000 weight_to_token_ratio=0.194625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64860 dup_rows=24144 dup_rows_remote=21174 dedup_ratio=0.245605 router_tokens_per_expert=[768, 775, 803, 741, 766, 784, 780, 753, 782, 728, 714, 778, 769, 793, 818, 785, 750, 764, 797, 818, 769, 816, 796, 719, 735, 755, 756, 784, 770, 784, 772, 763, 766, 750, 753, 757, 820, 761, 748, 780, 748, 753, 787, 766, 741, 800, 733, 795, 741, 748, 778, 722, 798, 751, 795, 759, 730, 743, 822, 753, 788, 738, 752, 791] payload_rows_per_dst=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] dup_rows_per_dst=[3114, 3062, 3106, 2968, 2970, 2986, 2940, 2998] combine_rows_per_src=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64978 dup_rows=24020 dup_rows_remote=21082 dedup_ratio=0.244344 router_tokens_per_expert=[781, 739, 756, 767, 801, 806, 763, 805, 831, 794, 718, 764, 775, 748, 792, 764, 783, 762, 781, 748, 780, 728, 800, 791, 797, 761, 789, 714, 753, 740, 800, 768, 759, 762, 768, 747, 768, 746, 730, 801, 762, 753, 775, 755, 784, 749, 760, 755, 774, 738, 795, 796, 767, 756, 761, 797, 745, 767, 764, 721, 818, 770, 766, 744] payload_rows_per_dst=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] dup_rows_per_dst=[3048, 3072, 3072, 2938, 2934, 2990, 3012, 2954] combine_rows_per_src=[9388, 9300, 9274, 9306, 9228, 9196, 9356, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344168 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255912 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167656 weight_movement_bytes=34603008 total_nvlink_bytes=568770664 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343488 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550720 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532757952 weight_movement_bytes=69206016 total_nvlink_bytes=601963968 weight_to_token_ratio=0.129901 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266027008 dispatch_meta_bytes=343632 combine_payload_bytes=266027008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266370640 ingress_bytes=300630016 link_bytes=300630016 token_bytes=532397648 weight_movement_bytes=34603008 total_nvlink_bytes=567000656 weight_to_token_ratio=0.064995 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131246548e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64452 dup_rows=24316 dup_rows_remote=21104 dedup_ratio=0.247355 router_tokens_per_expert=[784, 775, 755, 761, 805, 734, 765, 786, 760, 749, 754, 782, 835, 775, 795, 744, 802, 752, 725, 758, 733, 762, 757, 798, 798, 753, 760, 793, 762, 724, 767, 784, 797, 729, 792, 733, 725, 782, 739, 757, 800, 769, 748, 713, 762, 745, 751, 808, 800, 796, 784, 766, 794, 788, 757, 775, 740, 764, 757, 777, 811, 788, 752, 766] payload_rows_per_dst=[9254, 9002, 9202, 9300, 8980, 9370, 9344, 9536] dup_rows_per_dst=[3076, 2888, 2972, 2982, 2798, 3152, 3236, 3212] combine_rows_per_src=[9254, 9002, 9202, 9300, 8980, 9370, 9344, 9536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65064 dup_rows=23920 dup_rows_remote=20956 dedup_ratio=0.243327 router_tokens_per_expert=[829, 767, 762, 732, 798, 730, 758, 763, 781, 774, 753, 768, 814, 767, 725, 782, 744, 766, 806, 779, 759, 757, 763, 768, 743, 749, 732, 785, 769, 735, 774, 767, 755, 745, 768, 751, 795, 756, 805, 773, 775, 774, 725, 798, 796, 806, 739, 776, 759, 771, 736, 755, 781, 785, 799, 762, 771, 764, 718, 782, 763, 822, 762, 786] payload_rows_per_dst=[9244, 9350, 9320, 9176, 9286, 9376, 9316, 9316] dup_rows_per_dst=[3034, 2978, 2964, 2932, 3010, 3002, 2980, 3020] combine_rows_per_src=[9244, 9350, 9320, 9176, 9286, 9376, 9316, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65090 dup_rows=24062 dup_rows_remote=21126 dedup_ratio=0.244771 router_tokens_per_expert=[796, 786, 755, 784, 748, 737, 785, 778, 755, 765, 750, 808, 749, 749, 750, 728, 786, 796, 740, 787, 772, 768, 781, 751, 724, 767, 767, 754, 742, 815, 764, 766, 817, 788, 760, 747, 798, 780, 794, 750, 790, 791, 785, 788, 783, 809, 726, 795, 814, 744, 724, 726, 765, 744, 759, 768, 777, 810, 744, 801, 741, 757, 754, 720] payload_rows_per_dst=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] dup_rows_per_dst=[3050, 2932, 3032, 3056, 3040, 3056, 2936, 2960] combine_rows_per_src=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=65036 dup_rows=24150 dup_rows_remote=21152 dedup_ratio=0.245667 router_tokens_per_expert=[742, 775, 743, 788, 775, 780, 704, 780, 738, 785, 770, 809, 808, 754, 753, 805, 772, 768, 794, 789, 771, 738, 714, 791, 783, 755, 735, 770, 774, 746, 764, 749, 785, 766, 782, 767, 791, 772, 755, 772, 786, 750, 751, 746, 738, 769, 760, 758, 781, 812, 733, 785, 804, 790, 794, 792, 781, 754, 725, 795, 760, 753, 722, 801] payload_rows_per_dst=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] dup_rows_per_dst=[2958, 3172, 3062, 2950, 2954, 2998, 3092, 2964] combine_rows_per_src=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64930 dup_rows=23940 dup_rows_remote=20822 dedup_ratio=0.243530 router_tokens_per_expert=[773, 752, 744, 734, 691, 734, 751, 747, 737, 794, 791, 814, 780, 797, 780, 783, 827, 791, 719, 747, 766, 778, 794, 770, 762, 809, 773, 775, 751, 768, 729, 740, 755, 765, 757, 743, 746, 783, 784, 784, 781, 793, 766, 731, 779, 799, 827, 765, 743, 771, 787, 769, 771, 747, 822, 776, 749, 725, 759, 801, 757, 737, 807, 772] payload_rows_per_dst=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] dup_rows_per_dst=[2770, 3118, 2998, 2942, 2922, 3130, 3030, 3030] combine_rows_per_src=[9082, 9434, 9386, 9272, 9312, 9352, 9342, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344240 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266494128 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644016 weight_movement_bytes=34603008 total_nvlink_bytes=567247024 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64904 dup_rows=24046 dup_rows_remote=21108 dedup_ratio=0.244609 router_tokens_per_expert=[809, 787, 755, 768, 762, 741, 746, 732, 738, 772, 748, 811, 765, 789, 785, 742, 753, 689, 779, 749, 804, 775, 808, 739, 752, 753, 781, 711, 786, 787, 814, 762, 803, 715, 778, 784, 760, 777, 744, 778, 800, 753, 760, 832, 768, 744, 814, 762, 742, 796, 741, 784, 755, 717, 820, 789, 799, 758, 772, 792, 821, 734, 718, 750] payload_rows_per_dst=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] dup_rows_per_dst=[2986, 2970, 3000, 2938, 3042, 3116, 2988, 3006] combine_rows_per_src=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=65000 dup_rows=24088 dup_rows_remote=21102 dedup_ratio=0.245036 router_tokens_per_expert=[777, 780, 771, 709, 796, 739, 756, 736, 800, 767, 769, 766, 762, 771, 761, 799, 770, 733, 748, 756, 746, 748, 731, 785, 807, 742, 712, 789, 783, 781, 801, 748, 817, 778, 760, 796, 807, 782, 795, 712, 807, 740, 774, 781, 728, 723, 772, 759, 742, 821, 765, 806, 779, 756, 809, 765, 764, 744, 821, 779, 782, 751, 749, 749] payload_rows_per_dst=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] dup_rows_per_dst=[2986, 3038, 3022, 3110, 3032, 2982, 2930, 2988] combine_rows_per_src=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=344136 combine_payload_bytes=265666560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266010696 ingress_bytes=369475584 link_bytes=369475584 token_bytes=531677256 weight_movement_bytes=103809024 total_nvlink_bytes=635486280 weight_to_token_ratio=0.195248 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390306877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=263995392 dispatch_meta_bytes=342224 combine_payload_bytes=263995392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264337616 ingress_bytes=298598400 link_bytes=298598400 token_bytes=528333008 weight_movement_bytes=34603008 total_nvlink_bytes=562936016 weight_to_token_ratio=0.065495 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123601740e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=65000 dup_rows=24088 dup_rows_remote=21102 dedup_ratio=0.245036 router_tokens_per_expert=[777, 780, 771, 709, 796, 739, 756, 736, 800, 767, 769, 766, 762, 771, 761, 799, 770, 733, 748, 756, 746, 748, 731, 785, 807, 742, 712, 789, 783, 781, 801, 748, 817, 778, 760, 796, 807, 782, 795, 712, 807, 740, 774, 781, 728, 723, 772, 759, 742, 821, 765, 806, 779, 756, 809, 765, 764, 744, 821, 779, 782, 751, 749, 749] payload_rows_per_dst=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] dup_rows_per_dst=[2986, 3038, 3022, 3110, 3032, 2982, 2930, 2988] combine_rows_per_src=[9216, 9358, 9206, 9410, 9382, 9186, 9168, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344080 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846224 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348368 weight_movement_bytes=34603008 total_nvlink_bytes=567951376 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266953504 ingress_bytes=370417664 link_bytes=370417664 token_bytes=533562144 weight_movement_bytes=103809024 total_nvlink_bytes=637371168 weight_to_token_ratio=0.194558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393851849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65036 dup_rows=23942 dup_rows_remote=20924 dedup_ratio=0.243551 router_tokens_per_expert=[760, 742, 766, 792, 762, 756, 759, 736, 746, 761, 794, 753, 784, 757, 775, 792, 744, 808, 728, 778, 776, 782, 781, 763, 778, 778, 780, 798, 784, 746, 777, 742, 755, 755, 734, 755, 758, 771, 803, 771, 797, 753, 777, 764, 743, 792, 785, 716, 752, 778, 772, 747, 803, 796, 772, 752, 792, 742, 817, 757, 786, 771, 741, 767] payload_rows_per_dst=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] dup_rows_per_dst=[2958, 3042, 2950, 2994, 2944, 3000, 3018, 3036] combine_rows_per_src=[9188, 9282, 9370, 9372, 9260, 9254, 9326, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=343008 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266296288 ingress_bytes=369762304 link_bytes=369762304 token_bytes=532249568 weight_movement_bytes=103809024 total_nvlink_bytes=636058592 weight_to_token_ratio=0.195038 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391385782e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344752 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732208 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119664 weight_movement_bytes=34603008 total_nvlink_bytes=567722672 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65098 dup_rows=23968 dup_rows_remote=21028 dedup_ratio=0.243815 router_tokens_per_expert=[743, 737, 728, 784, 824, 795, 833, 787, 752, 761, 754, 758, 706, 772, 798, 788, 757, 737, 781, 811, 766, 773, 783, 749, 804, 767, 790, 787, 780, 755, 761, 782, 743, 745, 748, 775, 761, 769, 748, 756, 765, 770, 762, 744, 773, 769, 763, 734, 769, 768, 772, 779, 792, 761, 761, 756, 735, 785, 798, 791, 759, 734, 817, 747] payload_rows_per_dst=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] dup_rows_per_dst=[3064, 2940, 2992, 3090, 2974, 2952, 3002, 2954] combine_rows_per_src=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344048 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190832 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037616 weight_movement_bytes=34603008 total_nvlink_bytes=566640624 weight_to_token_ratio=0.065039 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=65036 dup_rows=24150 dup_rows_remote=21152 dedup_ratio=0.245667 router_tokens_per_expert=[742, 775, 743, 788, 775, 780, 704, 780, 738, 785, 770, 809, 808, 754, 753, 805, 772, 768, 794, 789, 771, 738, 714, 791, 783, 755, 735, 770, 774, 746, 764, 749, 785, 766, 782, 767, 791, 772, 755, 772, 786, 750, 751, 746, 738, 769, 760, 758, 781, 812, 733, 785, 804, 790, 794, 792, 781, 754, 725, 795, 760, 753, 722, 801] payload_rows_per_dst=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] dup_rows_per_dst=[2958, 3172, 3062, 2950, 2954, 2998, 3092, 2964] combine_rows_per_src=[9216, 9272, 9212, 9202, 9426, 9118, 9490, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64860 dup_rows=24144 dup_rows_remote=21174 dedup_ratio=0.245605 router_tokens_per_expert=[768, 775, 803, 741, 766, 784, 780, 753, 782, 728, 714, 778, 769, 793, 818, 785, 750, 764, 797, 818, 769, 816, 796, 719, 735, 755, 756, 784, 770, 784, 772, 763, 766, 750, 753, 757, 820, 761, 748, 780, 748, 753, 787, 766, 741, 800, 733, 795, 741, 748, 778, 722, 798, 751, 795, 759, 730, 743, 822, 753, 788, 738, 752, 791] payload_rows_per_dst=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] dup_rows_per_dst=[3114, 3062, 3106, 2968, 2970, 2986, 2940, 2998] combine_rows_per_src=[9226, 9272, 9352, 9270, 9300, 9260, 9244, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344408 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266584408 ingress_bytes=300843008 link_bytes=300843008 token_bytes=532824408 weight_movement_bytes=34603008 total_nvlink_bytes=567427416 weight_to_token_ratio=0.064943 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132048019e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344504 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985912 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627320 weight_movement_bytes=34603008 total_nvlink_bytes=568230328 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64844 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[765, 772, 760, 748, 771, 766, 760, 815, 798, 751, 761, 758, 768, 813, 747, 745, 791, 787, 821, 825, 757, 770, 767, 745, 781, 783, 753, 772, 729, 734, 751, 755, 773, 794, 795, 757, 761, 740, 824, 784, 785, 786, 776, 731, 735, 740, 786, 758, 764, 760, 769, 731, 737, 762, 801, 742, 799, 734, 751, 845, 754, 729, 768, 762] payload_rows_per_dst=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] dup_rows_per_dst=[3078, 2990, 3092, 2890, 3036, 2964, 2974, 3016] combine_rows_per_src=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343840 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266731296 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118752 weight_movement_bytes=34603008 total_nvlink_bytes=567721760 weight_to_token_ratio=0.064907 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65042 dup_rows=23998 dup_rows_remote=21056 dedup_ratio=0.244120 router_tokens_per_expert=[763, 780, 789, 797, 779, 768, 787, 733, 790, 773, 790, 773, 757, 794, 778, 785, 783, 759, 759, 784, 751, 790, 760, 745, 776, 763, 759, 787, 756, 746, 801, 715, 771, 749, 761, 756, 731, 807, 794, 745, 748, 736, 767, 818, 805, 721, 723, 770, 744, 758, 778, 771, 775, 713, 828, 813, 779, 775, 750, 779, 765, 755, 770, 727] payload_rows_per_dst=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] dup_rows_per_dst=[3006, 3042, 2996, 2942, 2930, 2964, 3112, 3006] combine_rows_per_src=[9386, 9438, 9266, 9264, 9298, 9212, 9248, 9194] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64834 dup_rows=23926 dup_rows_remote=20780 dedup_ratio=0.243388 router_tokens_per_expert=[802, 810, 754, 725, 780, 814, 793, 746, 767, 746, 760, 794, 794, 710, 714, 736, 816, 771, 759, 757, 758, 740, 744, 813, 755, 779, 760, 792, 778, 768, 750, 796, 774, 742, 812, 793, 770, 727, 784, 769, 778, 793, 762, 755, 730, 737, 775, 729, 798, 784, 757, 773, 795, 753, 768, 797, 739, 763, 749, 777, 774, 772, 801, 741] payload_rows_per_dst=[9138, 9202, 9242, 9410, 9320, 9404, 9118, 9544] dup_rows_per_dst=[3012, 2840, 2946, 3074, 3022, 3012, 2874, 3146] combine_rows_per_src=[9138, 9202, 9242, 9410, 9320, 9404, 9118, 9544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64880 dup_rows=24074 dup_rows_remote=20946 dedup_ratio=0.244893 router_tokens_per_expert=[762, 765, 735, 805, 733, 781, 744, 764, 765, 715, 765, 771, 769, 781, 724, 753, 792, 781, 762, 785, 790, 758, 758, 799, 771, 771, 738, 772, 767, 753, 749, 757, 775, 756, 758, 775, 761, 780, 801, 787, 739, 788, 746, 821, 770, 794, 757, 739, 777, 782, 792, 782, 782, 758, 785, 781, 779, 802, 750, 779, 745, 792, 749, 735] payload_rows_per_dst=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] dup_rows_per_dst=[2990, 2862, 3022, 2910, 3068, 3032, 3128, 3062] combine_rows_per_src=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=344752 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266732208 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533119664 weight_movement_bytes=34603008 total_nvlink_bytes=567722672 weight_to_token_ratio=0.064907 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265666560 dispatch_meta_bytes=344136 combine_payload_bytes=265666560 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266010696 ingress_bytes=369475584 link_bytes=369475584 token_bytes=531677256 weight_movement_bytes=103809024 total_nvlink_bytes=635486280 weight_to_token_ratio=0.195248 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390306877e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65016 dup_rows=23880 dup_rows_remote=20960 dedup_ratio=0.242920 router_tokens_per_expert=[786, 729, 759, 784, 767, 731, 771, 745, 771, 738, 796, 748, 822, 792, 710, 725, 792, 769, 782, 740, 759, 730, 779, 813, 785, 825, 784, 777, 838, 748, 736, 772, 749, 771, 814, 744, 770, 695, 740, 773, 803, 796, 832, 788, 746, 757, 774, 807, 778, 754, 767, 782, 736, 740, 781, 806, 780, 758, 735, 725, 799, 731, 751, 767] payload_rows_per_dst=[9184, 9212, 9408, 9490, 9102, 9550, 9270, 9208] dup_rows_per_dst=[2960, 2992, 2920, 3040, 3010, 3056, 3018, 2884] combine_rows_per_src=[9184, 9212, 9408, 9490, 9102, 9550, 9270, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65180 dup_rows=23876 dup_rows_remote=21110 dedup_ratio=0.242879 router_tokens_per_expert=[743, 822, 762, 783, 752, 723, 764, 763, 814, 792, 769, 772, 760, 770, 805, 812, 783, 728, 777, 730, 796, 740, 796, 781, 713, 758, 792, 760, 760, 768, 750, 733, 769, 765, 788, 793, 749, 750, 808, 785, 702, 768, 793, 733, 752, 754, 755, 750, 768, 792, 747, 773, 765, 844, 759, 801, 769, 774, 779, 754, 747, 769, 749, 777] payload_rows_per_dst=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] dup_rows_per_dst=[2920, 3164, 2924, 2882, 3082, 2766, 3100, 3038] combine_rows_per_src=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64950 dup_rows=23962 dup_rows_remote=20954 dedup_ratio=0.243754 router_tokens_per_expert=[710, 785, 765, 691, 755, 747, 753, 760, 752, 829, 750, 783, 767, 768, 752, 765, 783, 763, 788, 731, 778, 751, 747, 795, 748, 795, 757, 816, 812, 759, 762, 751, 742, 750, 779, 732, 766, 774, 781, 737, 790, 803, 775, 779, 845, 781, 794, 815, 761, 810, 826, 736, 765, 758, 757, 733, 792, 756, 758, 731, 802, 758, 754, 744] payload_rows_per_dst=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] dup_rows_per_dst=[2804, 3030, 3086, 3008, 2930, 3216, 2998, 2890] combine_rows_per_src=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344392 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756424 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168456 weight_movement_bytes=34603008 total_nvlink_bytes=567771464 weight_to_token_ratio=0.064901 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64724 dup_rows=24078 dup_rows_remote=20948 dedup_ratio=0.244934 router_tokens_per_expert=[814, 767, 759, 763, 784, 761, 812, 822, 775, 791, 763, 767, 787, 801, 758, 784, 751, 761, 761, 740, 748, 723, 738, 788, 748, 793, 721, 770, 810, 762, 739, 760, 789, 790, 777, 754, 745, 737, 783, 772, 777, 749, 782, 756, 747, 764, 761, 734, 815, 754, 798, 731, 814, 784, 773, 785, 746, 761, 754, 776, 727, 817, 725, 754] payload_rows_per_dst=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] dup_rows_per_dst=[3130, 3122, 2902, 3026, 3022, 2942, 2964, 2970] combine_rows_per_src=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343392 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265944416 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531545440 weight_movement_bytes=34603008 total_nvlink_bytes=566148448 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342456 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265902520 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531462584 weight_movement_bytes=34603008 total_nvlink_bytes=566065592 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343304 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266091784 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840264 weight_movement_bytes=103809024 total_nvlink_bytes=635649288 weight_to_token_ratio=0.195188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65068 dup_rows=23956 dup_rows_remote=20912 dedup_ratio=0.243693 router_tokens_per_expert=[765, 751, 804, 808, 766, 768, 715, 748, 738, 793, 787, 813, 782, 802, 711, 736, 796, 781, 773, 752, 766, 707, 794, 762, 777, 818, 785, 773, 805, 751, 799, 792, 779, 768, 754, 788, 768, 813, 819, 746, 731, 765, 770, 792, 751, 752, 760, 761, 765, 783, 771, 718, 732, 769, 742, 778, 772, 775, 792, 799, 723, 735, 722, 741] payload_rows_per_dst=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] dup_rows_per_dst=[2946, 3044, 2962, 3136, 3114, 2950, 2858, 2946] combine_rows_per_src=[9304, 9280, 9300, 9464, 9356, 9214, 9258, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=343904 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266649440 ingress_bytes=335511552 link_bytes=335511552 token_bytes=532954976 weight_movement_bytes=69206016 total_nvlink_bytes=602160992 weight_to_token_ratio=0.129853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262502959e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64724 dup_rows=24078 dup_rows_remote=20948 dedup_ratio=0.244934 router_tokens_per_expert=[814, 767, 759, 763, 784, 761, 812, 822, 775, 791, 763, 767, 787, 801, 758, 784, 751, 761, 761, 740, 748, 723, 738, 788, 748, 793, 721, 770, 810, 762, 739, 760, 789, 790, 777, 754, 745, 737, 783, 772, 777, 749, 782, 756, 747, 764, 761, 734, 815, 754, 798, 731, 814, 784, 773, 785, 746, 761, 754, 776, 727, 817, 725, 754] payload_rows_per_dst=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] dup_rows_per_dst=[3130, 3122, 2902, 3026, 3022, 2942, 2964, 2970] combine_rows_per_src=[9502, 9502, 9142, 9180, 9204, 9198, 9348, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73960 payload_rows_remote=64618 dup_rows=24344 dup_rows_remote=21280 dedup_ratio=0.247640 router_tokens_per_expert=[748, 764, 737, 756, 769, 795, 767, 778, 763, 747, 784, 762, 766, 788, 770, 793, 763, 812, 771, 765, 723, 774, 726, 807, 747, 773, 750, 758, 774, 804, 716, 742, 745, 794, 750, 731, 766, 751, 784, 801, 734, 798, 784, 743, 765, 790, 760, 725, 737, 753, 788, 757, 803, 764, 814, 787, 757, 792, 782, 769, 797, 781, 806, 752] payload_rows_per_dst=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] dup_rows_per_dst=[2962, 3074, 3106, 2988, 3030, 2972, 3064, 3148] combine_rows_per_src=[9266, 9272, 9176, 9140, 9214, 9226, 9342, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=343920 combine_payload_bytes=266518528 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266862448 ingress_bytes=370327552 link_bytes=370327552 token_bytes=533380976 weight_movement_bytes=103809024 total_nvlink_bytes=637190000 weight_to_token_ratio=0.194625 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393512764e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65180 dup_rows=23876 dup_rows_remote=21110 dedup_ratio=0.242879 router_tokens_per_expert=[743, 822, 762, 783, 752, 723, 764, 763, 814, 792, 769, 772, 760, 770, 805, 812, 783, 728, 777, 730, 796, 740, 796, 781, 713, 758, 792, 760, 760, 768, 750, 733, 769, 765, 788, 793, 749, 750, 808, 785, 702, 768, 793, 733, 752, 754, 755, 750, 768, 792, 747, 773, 765, 844, 759, 801, 769, 774, 779, 754, 747, 769, 749, 777] payload_rows_per_dst=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] dup_rows_per_dst=[2920, 3164, 2924, 2882, 3082, 2766, 3100, 3038] combine_rows_per_src=[9304, 9424, 9338, 9186, 9332, 9248, 9398, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65060 dup_rows=24132 dup_rows_remote=21138 dedup_ratio=0.245483 router_tokens_per_expert=[775, 782, 786, 781, 765, 819, 818, 735, 745, 747, 737, 743, 791, 733, 803, 754, 825, 753, 738, 779, 787, 741, 781, 757, 751, 732, 749, 771, 751, 759, 759, 782, 763, 724, 800, 804, 819, 764, 783, 737, 775, 755, 736, 764, 769, 763, 760, 739, 765, 801, 766, 822, 751, 738, 777, 798, 722, 786, 765, 784, 759, 787, 798, 749] payload_rows_per_dst=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] dup_rows_per_dst=[3042, 2994, 3004, 2944, 3076, 2946, 3138, 2988] combine_rows_per_src=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=345160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267322440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534299720 weight_movement_bytes=34603008 total_nvlink_bytes=568902728 weight_to_token_ratio=0.064763 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343616 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378816 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414016 weight_movement_bytes=34603008 total_nvlink_bytes=567017024 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64844 dup_rows=24040 dup_rows_remote=21004 dedup_ratio=0.244548 router_tokens_per_expert=[765, 772, 760, 748, 771, 766, 760, 815, 798, 751, 761, 758, 768, 813, 747, 745, 791, 787, 821, 825, 757, 770, 767, 745, 781, 783, 753, 772, 729, 734, 751, 755, 773, 794, 795, 757, 761, 740, 824, 784, 785, 786, 776, 731, 735, 740, 786, 758, 764, 760, 769, 731, 737, 762, 801, 742, 799, 734, 751, 845, 754, 729, 768, 762] payload_rows_per_dst=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] dup_rows_per_dst=[3078, 2990, 3092, 2890, 3036, 2964, 2974, 3016] combine_rows_per_src=[9236, 9292, 9434, 9226, 9420, 9230, 9158, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64904 dup_rows=24046 dup_rows_remote=21108 dedup_ratio=0.244609 router_tokens_per_expert=[809, 787, 755, 768, 762, 741, 746, 732, 738, 772, 748, 811, 765, 789, 785, 742, 753, 689, 779, 749, 804, 775, 808, 739, 752, 753, 781, 711, 786, 787, 814, 762, 803, 715, 778, 784, 760, 777, 744, 778, 800, 753, 760, 832, 768, 744, 814, 762, 742, 796, 741, 784, 755, 717, 820, 789, 799, 758, 772, 792, 821, 734, 718, 750] payload_rows_per_dst=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] dup_rows_per_dst=[2986, 2970, 3000, 2938, 3042, 3116, 2988, 3006] combine_rows_per_src=[9214, 9330, 9192, 9354, 9236, 9350, 9300, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=342688 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265452192 ingress_bytes=334315520 link_bytes=334315520 token_bytes=530561696 weight_movement_bytes=69206016 total_nvlink_bytes=599767712 weight_to_token_ratio=0.130439 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258002387e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344792 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266830552 ingress_bytes=439500800 link_bytes=439500800 token_bytes=533316312 weight_movement_bytes=173015040 total_nvlink_bytes=706331352 weight_to_token_ratio=0.324414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653806128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20874 dedup_ratio=0.242716 router_tokens_per_expert=[799, 767, 759, 759, 799, 809, 802, 780, 789, 771, 759, 751, 730, 755, 730, 821, 768, 724, 789, 749, 735, 779, 792, 787, 781, 744, 781, 748, 756, 764, 766, 762, 757, 736, 809, 747, 776, 789, 751, 782, 775, 799, 794, 741, 748, 723, 751, 786, 745, 742, 779, 835, 759, 764, 782, 787, 750, 736, 777, 763, 797, 758, 768, 741] payload_rows_per_dst=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] dup_rows_per_dst=[3158, 2922, 2996, 2920, 2986, 2936, 3014, 2928] combine_rows_per_src=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74016 payload_rows_remote=64428 dup_rows=24288 dup_rows_remote=21000 dedup_ratio=0.247070 router_tokens_per_expert=[786, 731, 739, 793, 739, 762, 734, 766, 796, 783, 777, 778, 780, 803, 741, 786, 711, 762, 759, 768, 765, 774, 756, 741, 744, 735, 776, 791, 819, 755, 725, 723, 743, 748, 810, 769, 786, 743, 750, 778, 714, 808, 785, 759, 789, 811, 760, 777, 769, 798, 754, 776, 786, 784, 738, 815, 812, 740, 796, 792, 827, 734, 753, 750] payload_rows_per_dst=[9138, 9104, 9206, 9238, 9234, 9044, 9464, 9588] dup_rows_per_dst=[2962, 3002, 2866, 3054, 3032, 2894, 3190, 3288] combine_rows_per_src=[9138, 9104, 9206, 9238, 9234, 9044, 9464, 9588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65210 dup_rows=23774 dup_rows_remote=20784 dedup_ratio=0.241842 router_tokens_per_expert=[778, 798, 743, 753, 792, 730, 702, 776, 820, 767, 757, 792, 778, 807, 769, 772, 775, 728, 782, 738, 751, 761, 711, 768, 807, 723, 744, 760, 819, 771, 773, 763, 811, 775, 765, 750, 798, 777, 795, 796, 765, 780, 748, 813, 751, 778, 765, 772, 762, 750, 764, 792, 803, 806, 718, 760, 795, 742, 743, 747, 719, 743, 771, 790] payload_rows_per_dst=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] dup_rows_per_dst=[2914, 3058, 2856, 2986, 3070, 3016, 2990, 2884] combine_rows_per_src=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=343592 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018920 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529694248 weight_movement_bytes=69206016 total_nvlink_bytes=598900264 weight_to_token_ratio=0.130653 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=345160 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267322440 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534299720 weight_movement_bytes=34603008 total_nvlink_bytes=568902728 weight_to_token_ratio=0.064763 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65042 dup_rows=23974 dup_rows_remote=21010 dedup_ratio=0.243876 router_tokens_per_expert=[743, 780, 806, 739, 740, 794, 756, 761, 796, 754, 821, 766, 810, 731, 780, 760, 742, 789, 808, 771, 706, 734, 779, 797, 771, 766, 776, 778, 764, 764, 795, 731, 727, 731, 763, 766, 815, 760, 793, 773, 778, 820, 833, 734, 767, 839, 767, 751, 756, 725, 740, 732, 762, 764, 761, 774, 783, 784, 712, 767, 777, 768, 734, 788] payload_rows_per_dst=[9318, 9324, 9288, 9312, 9252, 9450, 9118, 9268] dup_rows_per_dst=[2920, 3112, 2964, 2978, 3004, 3128, 2910, 2958] combine_rows_per_src=[9318, 9324, 9288, 9312, 9252, 9450, 9118, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64846 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[800, 742, 753, 739, 786, 760, 763, 771, 782, 775, 750, 807, 755, 773, 781, 753, 743, 776, 736, 773, 768, 783, 790, 774, 745, 758, 787, 772, 808, 790, 807, 756, 786, 748, 727, 757, 786, 738, 756, 748, 767, 736, 761, 782, 748, 811, 777, 749, 789, 797, 781, 776, 772, 747, 772, 793, 728, 776, 780, 739, 768, 794, 792, 715] payload_rows_per_dst=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] dup_rows_per_dst=[3026, 3034, 2994, 3028, 2940, 3018, 3078, 2922] combine_rows_per_src=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64996 dup_rows=24092 dup_rows_remote=21086 dedup_ratio=0.245076 router_tokens_per_expert=[789, 739, 741, 781, 774, 749, 779, 758, 750, 768, 759, 782, 762, 765, 764, 768, 766, 742, 803, 763, 759, 747, 811, 747, 787, 774, 762, 757, 805, 739, 772, 757, 787, 726, 807, 778, 741, 787, 750, 730, 748, 746, 790, 777, 746, 764, 794, 746, 728, 814, 801, 785, 701, 810, 814, 822, 727, 779, 778, 785, 803, 790, 720, 759] payload_rows_per_dst=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] dup_rows_per_dst=[2984, 3022, 3094, 3016, 2996, 3006, 3080, 2894] combine_rows_per_src=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343392 combine_payload_bytes=265601024 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265944416 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531545440 weight_movement_bytes=34603008 total_nvlink_bytes=566148448 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344048 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190832 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532037616 weight_movement_bytes=34603008 total_nvlink_bytes=566640624 weight_to_token_ratio=0.065039 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344040 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141096 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938152 weight_movement_bytes=34603008 total_nvlink_bytes=568541160 weight_to_token_ratio=0.064807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=263897088 dispatch_meta_bytes=341712 combine_payload_bytes=263897088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264238800 ingress_bytes=298500096 link_bytes=298500096 token_bytes=528135888 weight_movement_bytes=34603008 total_nvlink_bytes=562738896 weight_to_token_ratio=0.065519 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123231830e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64662 dup_rows=24098 dup_rows_remote=21042 dedup_ratio=0.245138 router_tokens_per_expert=[762, 745, 778, 785, 770, 826, 784, 817, 752, 788, 769, 753, 787, 769, 789, 801, 769, 796, 737, 754, 760, 725, 745, 771, 757, 847, 794, 757, 801, 692, 736, 808, 764, 769, 755, 801, 785, 752, 747, 743, 757, 788, 784, 704, 754, 800, 719, 746, 782, 767, 767, 792, 753, 755, 829, 740, 762, 769, 740, 788, 778, 731, 741, 766] payload_rows_per_dst=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] dup_rows_per_dst=[3056, 3054, 3240, 3098, 2976, 3022, 2672, 2980] combine_rows_per_src=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343976 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444136 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544296 weight_movement_bytes=69206016 total_nvlink_bytes=603750312 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65098 dup_rows=23968 dup_rows_remote=21028 dedup_ratio=0.243815 router_tokens_per_expert=[743, 737, 728, 784, 824, 795, 833, 787, 752, 761, 754, 758, 706, 772, 798, 788, 757, 737, 781, 811, 766, 773, 783, 749, 804, 767, 790, 787, 780, 755, 761, 782, 743, 745, 748, 775, 761, 769, 748, 756, 765, 770, 762, 744, 773, 769, 763, 734, 769, 768, 772, 779, 792, 761, 761, 756, 735, 785, 798, 791, 759, 734, 817, 747] payload_rows_per_dst=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] dup_rows_per_dst=[3064, 2940, 2992, 3090, 2974, 2952, 3002, 2954] combine_rows_per_src=[9398, 9238, 9322, 9362, 9116, 9208, 9314, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344208 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756240 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168272 weight_movement_bytes=34603008 total_nvlink_bytes=567771280 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64662 dup_rows=24098 dup_rows_remote=21042 dedup_ratio=0.245138 router_tokens_per_expert=[762, 745, 778, 785, 770, 826, 784, 817, 752, 788, 769, 753, 787, 769, 789, 801, 769, 796, 737, 754, 760, 725, 745, 771, 757, 847, 794, 757, 801, 692, 736, 808, 764, 769, 755, 801, 785, 752, 747, 743, 757, 788, 784, 704, 754, 800, 719, 746, 782, 767, 767, 792, 753, 755, 829, 740, 762, 769, 740, 788, 778, 731, 741, 766] payload_rows_per_dst=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] dup_rows_per_dst=[3056, 3054, 3240, 3098, 2976, 3022, 2672, 2980] combine_rows_per_src=[9544, 9298, 9596, 9286, 9256, 9080, 8976, 9170] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344504 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985912 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627320 weight_movement_bytes=34603008 total_nvlink_bytes=568230328 weight_to_token_ratio=0.064845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64996 dup_rows=24092 dup_rows_remote=21086 dedup_ratio=0.245076 router_tokens_per_expert=[789, 739, 741, 781, 774, 749, 779, 758, 750, 768, 759, 782, 762, 765, 764, 768, 766, 742, 803, 763, 759, 747, 811, 747, 787, 774, 762, 757, 805, 739, 772, 757, 787, 726, 807, 778, 741, 787, 750, 730, 748, 746, 790, 777, 746, 764, 794, 746, 728, 814, 801, 785, 701, 810, 814, 822, 727, 779, 778, 785, 803, 790, 720, 759] payload_rows_per_dst=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] dup_rows_per_dst=[2984, 3022, 3094, 3016, 2996, 3006, 3080, 2894] combine_rows_per_src=[9236, 9214, 9182, 9290, 9216, 9216, 9470, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=65090 dup_rows=24062 dup_rows_remote=21126 dedup_ratio=0.244771 router_tokens_per_expert=[796, 786, 755, 784, 748, 737, 785, 778, 755, 765, 750, 808, 749, 749, 750, 728, 786, 796, 740, 787, 772, 768, 781, 751, 724, 767, 767, 754, 742, 815, 764, 766, 817, 788, 760, 747, 798, 780, 794, 750, 790, 791, 785, 788, 783, 809, 726, 795, 814, 744, 724, 726, 765, 744, 759, 768, 777, 810, 744, 801, 741, 757, 754, 720] payload_rows_per_dst=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] dup_rows_per_dst=[3050, 2932, 3032, 3056, 3040, 3056, 2936, 2960] combine_rows_per_src=[9288, 9176, 9330, 9142, 9428, 9478, 9152, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343432 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952648 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531561864 weight_movement_bytes=103809024 total_nvlink_bytes=635370888 weight_to_token_ratio=0.195291 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65108 dup_rows=23980 dup_rows_remote=20918 dedup_ratio=0.243937 router_tokens_per_expert=[771, 716, 753, 782, 767, 764, 802, 789, 772, 811, 770, 751, 731, 770, 770, 764, 796, 757, 844, 744, 781, 765, 782, 783, 756, 818, 738, 743, 731, 784, 796, 756, 807, 767, 767, 816, 800, 760, 807, 760, 808, 789, 734, 754, 748, 771, 735, 752, 756, 777, 756, 747, 755, 781, 742, 784, 762, 761, 730, 749, 738, 747, 735, 800] payload_rows_per_dst=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] dup_rows_per_dst=[3016, 3062, 3100, 2988, 3084, 2940, 2946, 2844] combine_rows_per_src=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344328 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266567944 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532791560 weight_movement_bytes=69206016 total_nvlink_bytes=601997576 weight_to_token_ratio=0.129893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342816 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198368 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053920 weight_movement_bytes=34603008 total_nvlink_bytes=564656928 weight_to_token_ratio=0.065282 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344104 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267026472 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533708840 weight_movement_bytes=69206016 total_nvlink_bytes=602914856 weight_to_token_ratio=0.129670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64932 dup_rows=23966 dup_rows_remote=20936 dedup_ratio=0.243795 router_tokens_per_expert=[750, 762, 793, 758, 739, 758, 754, 791, 795, 787, 799, 765, 775, 745, 808, 764, 812, 756, 765, 761, 800, 733, 760, 788, 760, 820, 756, 727, 777, 783, 756, 777, 734, 783, 774, 776, 768, 763, 808, 812, 786, 745, 768, 746, 759, 727, 750, 765, 780, 781, 738, 805, 765, 741, 763, 754, 746, 778, 772, 743, 782, 757, 746, 763] payload_rows_per_dst=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] dup_rows_per_dst=[2954, 3106, 3078, 2922, 3030, 2918, 3004, 2954] combine_rows_per_src=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65136 dup_rows=23860 dup_rows_remote=20874 dedup_ratio=0.242716 router_tokens_per_expert=[799, 767, 759, 759, 799, 809, 802, 780, 789, 771, 759, 751, 730, 755, 730, 821, 768, 724, 789, 749, 735, 779, 792, 787, 781, 744, 781, 748, 756, 764, 766, 762, 757, 736, 809, 747, 776, 789, 751, 782, 775, 799, 794, 741, 748, 723, 751, 786, 745, 742, 779, 835, 759, 764, 782, 787, 750, 736, 777, 763, 797, 758, 768, 741] payload_rows_per_dst=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] dup_rows_per_dst=[3158, 2922, 2996, 2920, 2986, 2936, 3014, 2928] combine_rows_per_src=[9390, 9290, 9250, 9284, 9308, 9298, 9372, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64950 dup_rows=23962 dup_rows_remote=20954 dedup_ratio=0.243754 router_tokens_per_expert=[710, 785, 765, 691, 755, 747, 753, 760, 752, 829, 750, 783, 767, 768, 752, 765, 783, 763, 788, 731, 778, 751, 747, 795, 748, 795, 757, 816, 812, 759, 762, 751, 742, 750, 779, 732, 766, 774, 781, 737, 790, 803, 775, 779, 845, 781, 794, 815, 761, 810, 826, 736, 765, 758, 757, 733, 792, 756, 758, 731, 802, 758, 754, 744] payload_rows_per_dst=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] dup_rows_per_dst=[2804, 3030, 3086, 3008, 2930, 3216, 2998, 2890] combine_rows_per_src=[9128, 9302, 9186, 9392, 9192, 9548, 9294, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64700 dup_rows=23992 dup_rows_remote=20674 dedup_ratio=0.244059 router_tokens_per_expert=[791, 745, 796, 781, 766, 779, 753, 723, 753, 745, 798, 763, 739, 785, 786, 784, 806, 776, 798, 804, 748, 782, 761, 782, 722, 792, 743, 782, 796, 777, 743, 765, 803, 729, 744, 811, 769, 755, 783, 735, 721, 790, 733, 817, 742, 752, 781, 759, 779, 803, 758, 776, 755, 744, 790, 771, 750, 771, 721, 765, 743, 776, 783, 779] payload_rows_per_dst=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] dup_rows_per_dst=[2870, 2866, 2768, 2914, 3020, 3136, 3100, 3318] combine_rows_per_src=[8978, 9150, 8976, 9216, 9364, 9344, 9672, 9612] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21164 dedup_ratio=0.245117 router_tokens_per_expert=[717, 756, 748, 785, 782, 816, 803, 707, 791, 743, 756, 762, 754, 762, 758, 754, 786, 812, 773, 827, 771, 798, 752, 737, 775, 764, 781, 783, 809, 781, 780, 731, 745, 756, 746, 746, 754, 796, 785, 745, 763, 773, 786, 782, 716, 774, 769, 827, 770, 799, 764, 795, 756, 771, 770, 732, 728, 804, 755, 714, 784, 776, 746, 771] payload_rows_per_dst=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] dup_rows_per_dst=[3116, 2928, 3102, 3072, 2966, 3050, 2932, 2930] combine_rows_per_src=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65008 dup_rows=23870 dup_rows_remote=20772 dedup_ratio=0.242818 router_tokens_per_expert=[791, 780, 718, 789, 766, 757, 805, 760, 767, 780, 791, 716, 765, 714, 742, 743, 755, 815, 787, 794, 772, 782, 769, 788, 776, 810, 762, 752, 817, 759, 809, 776, 799, 777, 773, 772, 753, 775, 750, 770, 775, 794, 768, 717, 739, 830, 775, 734, 758, 735, 757, 743, 773, 747, 786, 753, 717, 811, 757, 772, 789, 768, 729, 749] payload_rows_per_dst=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] dup_rows_per_dst=[3044, 2918, 3098, 3106, 2924, 3024, 2832, 2924] combine_rows_per_src=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264855552 dispatch_meta_bytes=342816 combine_payload_bytes=264855552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265198368 ingress_bytes=299458560 link_bytes=299458560 token_bytes=530053920 weight_movement_bytes=34603008 total_nvlink_bytes=564656928 weight_to_token_ratio=0.065282 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126838453e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344328 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266567944 ingress_bytes=335429632 link_bytes=335429632 token_bytes=532791560 weight_movement_bytes=69206016 total_nvlink_bytes=601997576 weight_to_token_ratio=0.129893 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262194701e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344864 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266953504 ingress_bytes=370417664 link_bytes=370417664 token_bytes=533562144 weight_movement_bytes=103809024 total_nvlink_bytes=637371168 weight_to_token_ratio=0.194558 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393851849e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65056 dup_rows=23938 dup_rows_remote=20976 dedup_ratio=0.243510 router_tokens_per_expert=[781, 770, 757, 770, 728, 763, 852, 805, 790, 746, 756, 725, 740, 733, 813, 738, 739, 824, 798, 740, 736, 742, 745, 825, 756, 749, 791, 808, 785, 772, 750, 725, 741, 785, 777, 755, 747, 797, 724, 758, 789, 737, 779, 754, 786, 766, 776, 773, 744, 764, 772, 788, 765, 750, 761, 793, 794, 747, 770, 785, 748, 773, 787, 815] payload_rows_per_dst=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] dup_rows_per_dst=[3036, 2978, 2988, 2962, 2914, 2964, 2984, 3112] combine_rows_per_src=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65044 dup_rows=24008 dup_rows_remote=21090 dedup_ratio=0.244222 router_tokens_per_expert=[823, 763, 813, 837, 767, 789, 739, 767, 739, 789, 776, 798, 746, 792, 803, 828, 746, 799, 762, 785, 751, 794, 740, 751, 784, 747, 777, 726, 766, 775, 750, 770, 701, 782, 775, 732, 743, 700, 754, 735, 734, 758, 796, 734, 774, 744, 750, 795, 758, 716, 783, 782, 772, 769, 759, 753, 785, 770, 755, 808, 742, 795, 800, 806] payload_rows_per_dst=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] dup_rows_per_dst=[3066, 3118, 2982, 2998, 2856, 2918, 2960, 3110] combine_rows_per_src=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343472 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304944 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266416 weight_movement_bytes=34603008 total_nvlink_bytes=566869424 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64748 dup_rows=24168 dup_rows_remote=20982 dedup_ratio=0.245850 router_tokens_per_expert=[743, 785, 775, 757, 788, 819, 791, 774, 800, 739, 807, 812, 787, 787, 738, 743, 730, 781, 719, 797, 763, 771, 765, 747, 741, 798, 768, 782, 742, 748, 752, 750, 744, 808, 783, 786, 761, 804, 762, 800, 810, 746, 786, 778, 751, 798, 794, 774, 692, 767, 790, 714, 767, 799, 758, 726, 716, 754, 789, 792, 798, 733, 722, 751] payload_rows_per_dst=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] dup_rows_per_dst=[3186, 3122, 2878, 2902, 3160, 3128, 2892, 2900] combine_rows_per_src=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74434 payload_rows_remote=65008 dup_rows=23870 dup_rows_remote=20772 dedup_ratio=0.242818 router_tokens_per_expert=[791, 780, 718, 789, 766, 757, 805, 760, 767, 780, 791, 716, 765, 714, 742, 743, 755, 815, 787, 794, 772, 782, 769, 788, 776, 810, 762, 752, 817, 759, 809, 776, 799, 777, 773, 772, 753, 775, 750, 770, 775, 794, 768, 717, 739, 830, 775, 734, 758, 735, 757, 743, 773, 747, 786, 753, 717, 811, 757, 772, 789, 768, 729, 749] payload_rows_per_dst=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] dup_rows_per_dst=[3044, 2918, 3098, 3106, 2924, 3024, 2832, 2924] combine_rows_per_src=[9288, 9118, 9426, 9416, 9414, 9240, 9272, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344040 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141096 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938152 weight_movement_bytes=34603008 total_nvlink_bytes=568541160 weight_to_token_ratio=0.064807 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=341496 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265352696 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530363896 weight_movement_bytes=34603008 total_nvlink_bytes=564966904 weight_to_token_ratio=0.065244 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343960 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871256 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398552 weight_movement_bytes=34603008 total_nvlink_bytes=566001560 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64978 dup_rows=23892 dup_rows_remote=20866 dedup_ratio=0.243042 router_tokens_per_expert=[790, 760, 753, 756, 825, 807, 737, 762, 732, 795, 753, 759, 843, 792, 791, 765, 800, 719, 765, 750, 771, 746, 795, 753, 820, 751, 803, 755, 719, 757, 780, 738, 753, 735, 759, 749, 782, 789, 763, 773, 784, 759, 781, 784, 742, 754, 767, 756, 765, 699, 776, 767, 797, 787, 771, 774, 797, 777, 786, 716, 716, 744, 825, 783] payload_rows_per_dst=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] dup_rows_per_dst=[3048, 3026, 2912, 2944, 2896, 3012, 3000, 3054] combine_rows_per_src=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343616 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378816 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532414016 weight_movement_bytes=34603008 total_nvlink_bytes=567017024 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=65060 dup_rows=24132 dup_rows_remote=21138 dedup_ratio=0.245483 router_tokens_per_expert=[775, 782, 786, 781, 765, 819, 818, 735, 745, 747, 737, 743, 791, 733, 803, 754, 825, 753, 738, 779, 787, 741, 781, 757, 751, 732, 749, 771, 751, 759, 759, 782, 763, 724, 800, 804, 819, 764, 783, 737, 775, 755, 736, 764, 769, 763, 760, 739, 765, 801, 766, 822, 751, 738, 777, 798, 722, 786, 765, 784, 759, 787, 798, 749] payload_rows_per_dst=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] dup_rows_per_dst=[3042, 2994, 3004, 2944, 3076, 2946, 3138, 2988] combine_rows_per_src=[9480, 9112, 9318, 9164, 9312, 9176, 9298, 9312] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343120 combine_payload_bytes=266272768 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266615888 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532888656 weight_movement_bytes=103809024 total_nvlink_bytes=636697680 weight_to_token_ratio=0.194804 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344128 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813504 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282880 weight_movement_bytes=34603008 total_nvlink_bytes=567885888 weight_to_token_ratio=0.064887 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344536 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764760 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184984 weight_movement_bytes=34603008 total_nvlink_bytes=567787992 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=342920 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265550728 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758536 weight_movement_bytes=69206016 total_nvlink_bytes=599964552 weight_to_token_ratio=0.130391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343376 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266493264 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643152 weight_movement_bytes=34603008 total_nvlink_bytes=567246160 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64782 dup_rows=24112 dup_rows_remote=21120 dedup_ratio=0.245280 router_tokens_per_expert=[807, 769, 835, 774, 748, 791, 779, 797, 739, 703, 757, 764, 830, 798, 769, 743, 790, 769, 795, 779, 791, 737, 713, 753, 758, 779, 741, 712, 796, 790, 813, 785, 778, 778, 760, 765, 762, 768, 785, 805, 766, 754, 703, 795, 775, 748, 772, 754, 763, 771, 770, 766, 749, 766, 720, 775, 762, 747, 736, 748, 769, 822, 765, 751] payload_rows_per_dst=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] dup_rows_per_dst=[3176, 3040, 3018, 3016, 2992, 2906, 2952, 3012] combine_rows_per_src=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64748 dup_rows=24168 dup_rows_remote=20982 dedup_ratio=0.245850 router_tokens_per_expert=[743, 785, 775, 757, 788, 819, 791, 774, 800, 739, 807, 812, 787, 787, 738, 743, 730, 781, 719, 797, 763, 771, 765, 747, 741, 798, 768, 782, 742, 748, 752, 750, 744, 808, 783, 786, 761, 804, 762, 800, 810, 746, 786, 778, 751, 798, 794, 774, 692, 767, 790, 714, 767, 799, 758, 726, 716, 754, 789, 792, 798, 733, 722, 751] payload_rows_per_dst=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] dup_rows_per_dst=[3186, 3122, 2878, 2902, 3160, 3128, 2892, 2900] combine_rows_per_src=[9388, 9322, 9250, 9150, 9336, 9346, 9134, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344792 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266830552 ingress_bytes=439500800 link_bytes=439500800 token_bytes=533316312 weight_movement_bytes=173015040 total_nvlink_bytes=706331352 weight_to_token_ratio=0.324414 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.653806128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65044 dup_rows=24008 dup_rows_remote=21090 dedup_ratio=0.244222 router_tokens_per_expert=[823, 763, 813, 837, 767, 789, 739, 767, 739, 789, 776, 798, 746, 792, 803, 828, 746, 799, 762, 785, 751, 794, 740, 751, 784, 747, 777, 726, 766, 775, 750, 770, 701, 782, 775, 732, 743, 700, 754, 735, 734, 758, 796, 734, 774, 744, 750, 795, 758, 716, 783, 782, 772, 769, 759, 753, 785, 770, 755, 808, 742, 795, 800, 806] payload_rows_per_dst=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] dup_rows_per_dst=[3066, 3118, 2982, 2998, 2856, 2918, 2960, 3110] combine_rows_per_src=[9530, 9424, 9274, 9192, 8988, 9252, 9224, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64880 dup_rows=24074 dup_rows_remote=20946 dedup_ratio=0.244893 router_tokens_per_expert=[762, 765, 735, 805, 733, 781, 744, 764, 765, 715, 765, 771, 769, 781, 724, 753, 792, 781, 762, 785, 790, 758, 758, 799, 771, 771, 738, 772, 767, 753, 749, 757, 775, 756, 758, 775, 761, 780, 801, 787, 739, 788, 746, 821, 770, 794, 757, 739, 777, 782, 792, 782, 782, 758, 785, 781, 779, 802, 750, 779, 745, 792, 749, 735] payload_rows_per_dst=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] dup_rows_per_dst=[2990, 2862, 3022, 2910, 3068, 3032, 3128, 3062] combine_rows_per_src=[9188, 9224, 9428, 9246, 9318, 9276, 9350, 9200] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64904 dup_rows=24098 dup_rows_remote=20918 dedup_ratio=0.245138 router_tokens_per_expert=[789, 784, 753, 806, 795, 786, 740, 756, 773, 793, 784, 755, 767, 763, 752, 750, 816, 760, 787, 769, 773, 765, 729, 768, 728, 745, 732, 795, 762, 751, 738, 749, 782, 820, 771, 799, 779, 795, 783, 725, 739, 732, 767, 762, 758, 772, 759, 813, 717, 762, 753, 768, 767, 797, 782, 751, 756, 820, 768, 801, 778, 767, 748, 748] payload_rows_per_dst=[8982, 9478, 9330, 9216, 9296, 9304, 9298, 9302] dup_rows_per_dst=[2872, 3120, 3024, 2784, 3002, 3082, 3034, 3180] combine_rows_per_src=[8982, 9478, 9330, 9216, 9296, 9304, 9298, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64982 dup_rows=24026 dup_rows_remote=21050 dedup_ratio=0.244405 router_tokens_per_expert=[738, 776, 739, 765, 771, 777, 799, 747, 762, 751, 744, 767, 726, 774, 753, 765, 760, 757, 782, 749, 744, 785, 720, 777, 766, 788, 770, 776, 783, 808, 771, 769, 813, 792, 807, 754, 753, 780, 768, 756, 740, 754, 755, 782, 835, 765, 779, 762, 769, 742, 778, 791, 776, 757, 763, 760, 759, 753, 743, 691, 801, 799, 765, 851] payload_rows_per_dst=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] dup_rows_per_dst=[3004, 2924, 2972, 3100, 3036, 3050, 2976, 2964] combine_rows_per_src=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64932 dup_rows=23966 dup_rows_remote=20936 dedup_ratio=0.243795 router_tokens_per_expert=[750, 762, 793, 758, 739, 758, 754, 791, 795, 787, 799, 765, 775, 745, 808, 764, 812, 756, 765, 761, 800, 733, 760, 788, 760, 820, 756, 727, 777, 783, 756, 777, 734, 783, 774, 776, 768, 763, 808, 812, 786, 745, 768, 746, 759, 727, 750, 765, 780, 781, 738, 805, 765, 741, 763, 754, 746, 778, 772, 743, 782, 757, 746, 763] payload_rows_per_dst=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] dup_rows_per_dst=[2954, 3106, 3078, 2922, 3030, 2918, 3004, 2954] combine_rows_per_src=[9256, 9370, 9272, 9390, 9406, 9174, 9250, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=64846 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[800, 742, 753, 739, 786, 760, 763, 771, 782, 775, 750, 807, 755, 773, 781, 753, 743, 776, 736, 773, 768, 783, 790, 774, 745, 758, 787, 772, 808, 790, 807, 756, 786, 748, 727, 757, 786, 738, 756, 748, 767, 736, 761, 782, 748, 811, 777, 749, 789, 797, 781, 776, 772, 747, 772, 793, 728, 776, 780, 739, 768, 794, 792, 715] payload_rows_per_dst=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] dup_rows_per_dst=[3026, 3034, 2994, 3028, 2940, 3018, 3078, 2922] combine_rows_per_src=[9202, 9318, 9292, 9418, 9152, 9244, 9376, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=342920 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265550728 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758536 weight_movement_bytes=69206016 total_nvlink_bytes=599964552 weight_to_token_ratio=0.130391 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64978 dup_rows=23962 dup_rows_remote=20968 dedup_ratio=0.243754 router_tokens_per_expert=[753, 749, 766, 783, 762, 747, 736, 758, 744, 763, 793, 765, 797, 746, 784, 786, 761, 751, 777, 782, 752, 777, 797, 752, 806, 845, 725, 777, 749, 761, 742, 774, 745, 799, 735, 733, 761, 730, 792, 757, 735, 749, 754, 756, 766, 843, 789, 753, 785, 789, 729, 750, 750, 765, 802, 774, 772, 796, 767, 801, 736, 807, 801, 771] payload_rows_per_dst=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] dup_rows_per_dst=[2870, 3030, 2966, 2994, 2908, 3050, 3046, 3098] combine_rows_per_src=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344536 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764760 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184984 weight_movement_bytes=34603008 total_nvlink_bytes=567787992 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65218 dup_rows=23730 dup_rows_remote=20678 dedup_ratio=0.241394 router_tokens_per_expert=[702, 791, 731, 801, 737, 754, 759, 788, 732, 776, 718, 791, 853, 741, 769, 773, 846, 767, 781, 765, 802, 727, 748, 768, 790, 753, 746, 753, 766, 751, 741, 765, 791, 826, 758, 754, 744, 774, 755, 772, 741, 729, 784, 772, 791, 809, 723, 787, 808, 796, 749, 777, 779, 798, 729, 802, 759, 791, 721, 753, 731, 786, 818, 760] payload_rows_per_dst=[9234, 9318, 9356, 9244, 9380, 9364, 9406, 9272] dup_rows_per_dst=[2892, 2988, 3052, 2886, 2968, 2908, 3070, 2966] combine_rows_per_src=[9234, 9318, 9356, 9244, 9380, 9364, 9406, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343608 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265690680 ingress_bytes=403759104 link_bytes=403759104 token_bytes=531037752 weight_movement_bytes=138412032 total_nvlink_bytes=669449784 weight_to_token_ratio=0.260644 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519313003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64984 dup_rows=24116 dup_rows_remote=21160 dedup_ratio=0.245321 router_tokens_per_expert=[775, 741, 790, 778, 780, 727, 791, 755, 742, 708, 772, 801, 777, 752, 756, 810, 794, 784, 769, 784, 768, 758, 757, 747, 761, 762, 796, 825, 733, 740, 730, 806, 707, 772, 799, 774, 819, 806, 773, 752, 768, 800, 742, 747, 737, 781, 734, 771, 807, 737, 756, 749, 790, 740, 788, 771, 756, 770, 758, 778, 762, 768, 796, 775] payload_rows_per_dst=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] dup_rows_per_dst=[2926, 3024, 3064, 3100, 3062, 2956, 2960, 3024] combine_rows_per_src=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343304 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266091784 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840264 weight_movement_bytes=103809024 total_nvlink_bytes=635649288 weight_to_token_ratio=0.195188 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343288 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190072 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036856 weight_movement_bytes=34603008 total_nvlink_bytes=566639864 weight_to_token_ratio=0.065039 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20786 dedup_ratio=0.243022 router_tokens_per_expert=[763, 819, 735, 716, 785, 820, 772, 787, 770, 783, 751, 696, 791, 796, 786, 793, 710, 732, 814, 763, 797, 777, 783, 785, 731, 752, 729, 746, 781, 757, 758, 782, 793, 793, 810, 780, 769, 713, 836, 756, 769, 800, 786, 763, 798, 763, 789, 749, 821, 720, 776, 754, 776, 761, 750, 763, 797, 754, 778, 713, 749, 737, 716, 760] payload_rows_per_dst=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] dup_rows_per_dst=[3104, 3054, 2992, 3078, 2834, 3080, 2892, 2856] combine_rows_per_src=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343432 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952648 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531561864 weight_movement_bytes=103809024 total_nvlink_bytes=635370888 weight_to_token_ratio=0.195291 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=343472 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266304944 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532266416 weight_movement_bytes=34603008 total_nvlink_bytes=566869424 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344128 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510400 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676672 weight_movement_bytes=34603008 total_nvlink_bytes=567279680 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64790 dup_rows=24112 dup_rows_remote=21032 dedup_ratio=0.245280 router_tokens_per_expert=[779, 805, 742, 768, 740, 733, 788, 760, 705, 762, 811, 831, 790, 776, 763, 803, 770, 769, 750, 808, 769, 754, 806, 715, 772, 768, 769, 779, 811, 736, 821, 792, 757, 837, 805, 816, 789, 739, 763, 774, 745, 760, 764, 749, 730, 799, 749, 723, 759, 731, 789, 779, 798, 740, 759, 752, 778, 721, 764, 759, 732, 732, 728, 787] payload_rows_per_dst=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] dup_rows_per_dst=[2968, 3080, 2988, 3150, 3142, 2932, 2966, 2886] combine_rows_per_src=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343784 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493672 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643560 weight_movement_bytes=69206016 total_nvlink_bytes=601849576 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65108 dup_rows=23980 dup_rows_remote=20918 dedup_ratio=0.243937 router_tokens_per_expert=[771, 716, 753, 782, 767, 764, 802, 789, 772, 811, 770, 751, 731, 770, 770, 764, 796, 757, 844, 744, 781, 765, 782, 783, 756, 818, 738, 743, 731, 784, 796, 756, 807, 767, 767, 816, 800, 760, 807, 760, 808, 789, 734, 754, 748, 771, 735, 752, 756, 777, 756, 747, 755, 781, 742, 784, 762, 761, 730, 749, 738, 747, 735, 800] payload_rows_per_dst=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] dup_rows_per_dst=[3016, 3062, 3100, 2988, 3084, 2940, 2946, 2844] combine_rows_per_src=[9272, 9216, 9404, 9256, 9484, 9242, 9250, 9200] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267132928 dispatch_meta_bytes=343584 combine_payload_bytes=267132928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267476512 ingress_bytes=301735936 link_bytes=301735936 token_bytes=534609440 weight_movement_bytes=34603008 total_nvlink_bytes=569212448 weight_to_token_ratio=0.064726 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135408035e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344576 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266519040 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693504 weight_movement_bytes=34603008 total_nvlink_bytes=567296512 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343640 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267091544 ingress_bytes=405159936 link_bytes=405159936 token_bytes=533839448 weight_movement_bytes=138412032 total_nvlink_bytes=672251480 weight_to_token_ratio=0.259277 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524584221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343288 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723128 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531102968 weight_movement_bytes=69206016 total_nvlink_bytes=600308984 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20878 dedup_ratio=0.243693 router_tokens_per_expert=[772, 795, 785, 760, 760, 797, 748, 742, 768, 775, 754, 786, 771, 770, 758, 765, 768, 766, 723, 739, 791, 767, 798, 779, 778, 768, 781, 819, 761, 751, 783, 768, 812, 755, 742, 789, 788, 780, 741, 813, 747, 754, 753, 798, 768, 752, 740, 749, 758, 815, 737, 778, 777, 765, 828, 752, 746, 745, 742, 747, 763, 774, 783, 715] payload_rows_per_dst=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] dup_rows_per_dst=[3014, 2918, 2996, 3122, 3078, 2900, 3020, 2908] combine_rows_per_src=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=65124 dup_rows=23890 dup_rows_remote=20786 dedup_ratio=0.243022 router_tokens_per_expert=[763, 819, 735, 716, 785, 820, 772, 787, 770, 783, 751, 696, 791, 796, 786, 793, 710, 732, 814, 763, 797, 777, 783, 785, 731, 752, 729, 746, 781, 757, 758, 782, 793, 793, 810, 780, 769, 713, 836, 756, 769, 800, 786, 763, 798, 763, 789, 749, 821, 720, 776, 754, 776, 761, 750, 763, 797, 754, 778, 713, 749, 737, 716, 760] payload_rows_per_dst=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] dup_rows_per_dst=[3104, 3054, 2992, 3078, 2834, 3080, 2892, 2856] combine_rows_per_src=[9290, 9278, 9330, 9462, 9198, 9354, 9350, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344104 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267026472 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533708840 weight_movement_bytes=69206016 total_nvlink_bytes=602914856 weight_to_token_ratio=0.129670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64984 dup_rows=24116 dup_rows_remote=21160 dedup_ratio=0.245321 router_tokens_per_expert=[775, 741, 790, 778, 780, 727, 791, 755, 742, 708, 772, 801, 777, 752, 756, 810, 794, 784, 769, 784, 768, 758, 757, 747, 761, 762, 796, 825, 733, 740, 730, 806, 707, 772, 799, 774, 819, 806, 773, 752, 768, 800, 742, 747, 737, 781, 734, 771, 807, 737, 756, 749, 790, 740, 788, 771, 756, 770, 758, 778, 762, 768, 796, 775] payload_rows_per_dst=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] dup_rows_per_dst=[2926, 3024, 3064, 3100, 3062, 2956, 2960, 3024] combine_rows_per_src=[9348, 9212, 9258, 9206, 9342, 9204, 9316, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65210 dup_rows=23774 dup_rows_remote=20784 dedup_ratio=0.241842 router_tokens_per_expert=[778, 798, 743, 753, 792, 730, 702, 776, 820, 767, 757, 792, 778, 807, 769, 772, 775, 728, 782, 738, 751, 761, 711, 768, 807, 723, 744, 760, 819, 771, 773, 763, 811, 775, 765, 750, 798, 777, 795, 796, 765, 780, 748, 813, 751, 778, 765, 772, 762, 750, 764, 792, 803, 806, 718, 760, 795, 742, 743, 747, 719, 743, 771, 790] payload_rows_per_dst=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] dup_rows_per_dst=[2914, 3058, 2856, 2986, 3070, 3016, 2990, 2884] combine_rows_per_src=[9230, 9466, 9172, 9334, 9464, 9328, 9320, 9216] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64254 dup_rows=24306 dup_rows_remote=20772 dedup_ratio=0.247253 router_tokens_per_expert=[766, 780, 784, 779, 782, 766, 781, 757, 744, 730, 775, 833, 778, 771, 743, 794, 789, 763, 762, 789, 701, 789, 791, 771, 736, 745, 725, 747, 750, 729, 780, 749, 793, 814, 722, 737, 772, 785, 771, 738, 736, 744, 793, 765, 809, 774, 799, 775, 777, 776, 797, 792, 797, 780, 776, 781, 779, 742, 746, 782, 703, 748, 784, 786] payload_rows_per_dst=[9182, 9078, 9312, 8854, 8980, 9450, 9398, 9744] dup_rows_per_dst=[3010, 2930, 3010, 2610, 2790, 3268, 3154, 3534] combine_rows_per_src=[9182, 9078, 9312, 8854, 8980, 9450, 9398, 9744] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65154 dup_rows=23834 dup_rows_remote=20944 dedup_ratio=0.242452 router_tokens_per_expert=[790, 786, 766, 777, 813, 786, 749, 788, 774, 740, 762, 779, 753, 811, 781, 783, 722, 760, 783, 737, 789, 720, 797, 765, 777, 787, 766, 768, 766, 789, 789, 795, 755, 801, 801, 743, 720, 747, 784, 787, 783, 765, 710, 751, 763, 769, 809, 790, 715, 800, 766, 750, 762, 777, 789, 744, 753, 763, 758, 762, 761, 758, 745, 723] payload_rows_per_dst=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] dup_rows_per_dst=[3054, 2938, 2930, 3082, 3090, 2954, 2890, 2896] combine_rows_per_src=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343640 combine_payload_bytes=266747904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267091544 ingress_bytes=405159936 link_bytes=405159936 token_bytes=533839448 weight_movement_bytes=138412032 total_nvlink_bytes=672251480 weight_to_token_ratio=0.259277 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524584221e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74366 payload_rows_remote=65056 dup_rows=23938 dup_rows_remote=20976 dedup_ratio=0.243510 router_tokens_per_expert=[781, 770, 757, 770, 728, 763, 852, 805, 790, 746, 756, 725, 740, 733, 813, 738, 739, 824, 798, 740, 736, 742, 745, 825, 756, 749, 791, 808, 785, 772, 750, 725, 741, 785, 777, 755, 747, 797, 724, 758, 789, 737, 779, 754, 786, 766, 776, 773, 744, 764, 772, 788, 765, 750, 761, 793, 794, 747, 770, 785, 748, 773, 787, 815] payload_rows_per_dst=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] dup_rows_per_dst=[3036, 2978, 2988, 2962, 2914, 2964, 2984, 3112] combine_rows_per_src=[9416, 9104, 9310, 9310, 9254, 9356, 9290, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64782 dup_rows=24112 dup_rows_remote=21120 dedup_ratio=0.245280 router_tokens_per_expert=[807, 769, 835, 774, 748, 791, 779, 797, 739, 703, 757, 764, 830, 798, 769, 743, 790, 769, 795, 779, 791, 737, 713, 753, 758, 779, 741, 712, 796, 790, 813, 785, 778, 778, 760, 765, 762, 768, 785, 805, 766, 754, 703, 795, 775, 748, 772, 754, 763, 771, 770, 766, 749, 766, 720, 775, 762, 747, 736, 748, 769, 822, 765, 751] payload_rows_per_dst=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] dup_rows_per_dst=[3176, 3040, 3018, 3016, 2992, 2906, 2952, 3012] combine_rows_per_src=[9424, 9166, 9236, 9332, 9410, 9228, 9208, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64824 dup_rows=24226 dup_rows_remote=21260 dedup_ratio=0.246440 router_tokens_per_expert=[755, 820, 750, 762, 775, 813, 766, 753, 729, 734, 777, 760, 802, 817, 770, 763, 797, 748, 729, 765, 787, 795, 737, 843, 786, 795, 748, 737, 728, 813, 762, 741, 771, 747, 813, 745, 770, 776, 723, 799, 746, 766, 740, 799, 724, 798, 763, 773, 769, 816, 787, 730, 787, 748, 759, 763, 747, 787, 750, 762, 775, 737, 782, 743] payload_rows_per_dst=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] dup_rows_per_dst=[3084, 3014, 3076, 2966, 3022, 2958, 3118, 2988] combine_rows_per_src=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74158 payload_rows_remote=65076 dup_rows=24146 dup_rows_remote=21116 dedup_ratio=0.245626 router_tokens_per_expert=[751, 768, 794, 767, 757, 765, 760, 793, 777, 791, 732, 742, 763, 808, 771, 760, 775, 768, 767, 783, 723, 699, 780, 761, 837, 763, 775, 778, 794, 784, 805, 779, 755, 774, 770, 781, 758, 812, 799, 769, 749, 765, 735, 789, 800, 798, 726, 716, 764, 796, 746, 711, 702, 825, 783, 737, 749, 715, 774, 775, 754, 814, 747, 794] payload_rows_per_dst=[9270, 9374, 9082, 9498, 9310, 9234, 9164, 9226] dup_rows_per_dst=[3040, 2914, 3030, 3132, 3126, 2922, 2964, 3018] combine_rows_per_src=[9270, 9374, 9082, 9498, 9310, 9234, 9164, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343456 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526112 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532708768 weight_movement_bytes=69206016 total_nvlink_bytes=601914784 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65228 dup_rows=23872 dup_rows_remote=20952 dedup_ratio=0.242839 router_tokens_per_expert=[775, 767, 778, 805, 732, 764, 729, 749, 742, 776, 773, 771, 743, 723, 822, 775, 772, 784, 811, 745, 777, 804, 769, 766, 780, 752, 729, 761, 750, 812, 738, 782, 789, 784, 788, 740, 743, 778, 767, 774, 737, 762, 745, 748, 763, 816, 740, 751, 729, 745, 784, 765, 822, 771, 802, 806, 767, 763, 811, 740, 758, 763, 769, 776] payload_rows_per_dst=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] dup_rows_per_dst=[2992, 2956, 3088, 2946, 2930, 2920, 3000, 3040] combine_rows_per_src=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=344576 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266519040 ingress_bytes=300777472 link_bytes=300777472 token_bytes=532693504 weight_movement_bytes=34603008 total_nvlink_bytes=567296512 weight_to_token_ratio=0.064959 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131801413e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343976 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444136 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544296 weight_movement_bytes=69206016 total_nvlink_bytes=603750312 weight_to_token_ratio=0.129467 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=263184384 dispatch_meta_bytes=340104 combine_payload_bytes=263184384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263524488 ingress_bytes=297787392 link_bytes=297787392 token_bytes=526708872 weight_movement_bytes=34603008 total_nvlink_bytes=561311880 weight_to_token_ratio=0.065697 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.120549982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65020 dup_rows=24004 dup_rows_remote=20846 dedup_ratio=0.244181 router_tokens_per_expert=[783, 764, 729, 764, 774, 817, 784, 804, 716, 767, 797, 790, 755, 750, 788, 732, 803, 757, 781, 784, 792, 775, 733, 788, 767, 732, 776, 783, 746, 800, 768, 756, 757, 744, 786, 784, 818, 756, 768, 783, 782, 782, 747, 835, 756, 750, 784, 852, 745, 759, 746, 765, 773, 784, 740, 747, 732, 721, 758, 732, 752, 750, 756, 753] payload_rows_per_dst=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] dup_rows_per_dst=[3158, 3018, 3052, 3030, 3014, 3102, 2820, 2810] combine_rows_per_src=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344392 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267215176 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085960 weight_movement_bytes=34603008 total_nvlink_bytes=568688968 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65142 dup_rows=23906 dup_rows_remote=20816 dedup_ratio=0.243184 router_tokens_per_expert=[805, 787, 750, 749, 794, 787, 765, 743, 776, 733, 787, 737, 794, 779, 779, 788, 744, 774, 765, 707, 725, 714, 765, 790, 769, 743, 771, 806, 703, 714, 785, 766, 753, 742, 823, 788, 788, 800, 807, 768, 780, 787, 772, 713, 784, 796, 748, 804, 765, 785, 766, 760, 740, 795, 772, 753, 762, 752, 774, 802, 802, 801, 775, 701] payload_rows_per_dst=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] dup_rows_per_dst=[3004, 3090, 2872, 2912, 3142, 2984, 2902, 3000] combine_rows_per_src=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=344336 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265863440 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531382544 weight_movement_bytes=34603008 total_nvlink_bytes=565985552 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=64978 dup_rows=23892 dup_rows_remote=20866 dedup_ratio=0.243042 router_tokens_per_expert=[790, 760, 753, 756, 825, 807, 737, 762, 732, 795, 753, 759, 843, 792, 791, 765, 800, 719, 765, 750, 771, 746, 795, 753, 820, 751, 803, 755, 719, 757, 780, 738, 753, 735, 759, 749, 782, 789, 763, 773, 784, 759, 781, 784, 742, 754, 767, 756, 765, 699, 776, 767, 797, 787, 771, 774, 797, 777, 786, 716, 716, 744, 825, 783] payload_rows_per_dst=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] dup_rows_per_dst=[3048, 3026, 2912, 2944, 2896, 3012, 3000, 3054] combine_rows_per_src=[9332, 9434, 9286, 9302, 9310, 9242, 9272, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266469376 dispatch_meta_bytes=344128 combine_payload_bytes=266469376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266813504 ingress_bytes=301072384 link_bytes=301072384 token_bytes=533282880 weight_movement_bytes=34603008 total_nvlink_bytes=567885888 weight_to_token_ratio=0.064887 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132911143e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343608 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265690680 ingress_bytes=403759104 link_bytes=403759104 token_bytes=531037752 weight_movement_bytes=138412032 total_nvlink_bytes=669449784 weight_to_token_ratio=0.260644 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519313003e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344768 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266896064 ingress_bytes=301154304 link_bytes=301154304 token_bytes=533447360 weight_movement_bytes=34603008 total_nvlink_bytes=568050368 weight_to_token_ratio=0.064867 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133219401e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344720 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518608 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692496 weight_movement_bytes=34603008 total_nvlink_bytes=569295504 weight_to_token_ratio=0.064716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343464 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266665384 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987304 weight_movement_bytes=69206016 total_nvlink_bytes=602193320 weight_to_token_ratio=0.129846 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343832 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165464 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987096 weight_movement_bytes=34603008 total_nvlink_bytes=568590104 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65026 dup_rows=24030 dup_rows_remote=20966 dedup_ratio=0.244446 router_tokens_per_expert=[776, 772, 764, 742, 743, 744, 815, 842, 735, 753, 764, 764, 778, 716, 768, 782, 776, 744, 796, 784, 764, 742, 823, 770, 773, 735, 705, 784, 743, 719, 767, 791, 790, 749, 743, 803, 754, 752, 774, 791, 790, 754, 797, 787, 735, 753, 801, 709, 799, 791, 778, 787, 764, 799, 773, 743, 747, 744, 797, 785, 796, 748, 754, 791] payload_rows_per_dst=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] dup_rows_per_dst=[3064, 2868, 3028, 2924, 3064, 2994, 3068, 3020] combine_rows_per_src=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=65020 dup_rows=24004 dup_rows_remote=20846 dedup_ratio=0.244181 router_tokens_per_expert=[783, 764, 729, 764, 774, 817, 784, 804, 716, 767, 797, 790, 755, 750, 788, 732, 803, 757, 781, 784, 792, 775, 733, 788, 767, 732, 776, 783, 746, 800, 768, 756, 757, 744, 786, 784, 818, 756, 768, 783, 782, 782, 747, 835, 756, 750, 784, 852, 745, 759, 746, 765, 773, 784, 740, 747, 732, 721, 758, 732, 752, 750, 756, 753] payload_rows_per_dst=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] dup_rows_per_dst=[3158, 3018, 3052, 3030, 3014, 3102, 2820, 2810] combine_rows_per_src=[9280, 9242, 9374, 9226, 9378, 9474, 9228, 9098] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343376 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266493264 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532643152 weight_movement_bytes=34603008 total_nvlink_bytes=567246160 weight_to_token_ratio=0.064965 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65228 dup_rows=23872 dup_rows_remote=20952 dedup_ratio=0.242839 router_tokens_per_expert=[775, 767, 778, 805, 732, 764, 729, 749, 742, 776, 773, 771, 743, 723, 822, 775, 772, 784, 811, 745, 777, 804, 769, 766, 780, 752, 729, 761, 750, 812, 738, 782, 789, 784, 788, 740, 743, 778, 767, 774, 737, 762, 745, 748, 763, 816, 740, 751, 729, 745, 784, 765, 822, 771, 802, 806, 767, 763, 811, 740, 758, 763, 769, 776] payload_rows_per_dst=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] dup_rows_per_dst=[2992, 2956, 3088, 2946, 2930, 2920, 3000, 3040] combine_rows_per_src=[9206, 9294, 9368, 9262, 9396, 9204, 9448, 9254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21164 dedup_ratio=0.245117 router_tokens_per_expert=[717, 756, 748, 785, 782, 816, 803, 707, 791, 743, 756, 762, 754, 762, 758, 754, 786, 812, 773, 827, 771, 798, 752, 737, 775, 764, 781, 783, 809, 781, 780, 731, 745, 756, 746, 746, 754, 796, 785, 745, 763, 773, 786, 782, 716, 774, 769, 827, 770, 799, 764, 795, 756, 771, 770, 732, 728, 804, 755, 714, 784, 776, 746, 771] payload_rows_per_dst=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] dup_rows_per_dst=[3116, 2928, 3102, 3072, 2966, 3050, 2932, 2930] combine_rows_per_src=[9112, 9232, 9410, 9336, 9180, 9330, 9382, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74530 payload_rows_remote=65096 dup_rows=23774 dup_rows_remote=20696 dedup_ratio=0.241842 router_tokens_per_expert=[758, 774, 782, 772, 708, 782, 754, 796, 797, 788, 792, 731, 765, 726, 800, 788, 753, 763, 761, 724, 746, 770, 793, 789, 805, 756, 759, 690, 789, 814, 762, 758, 792, 801, 788, 747, 772, 766, 722, 804, 766, 726, 763, 795, 727, 753, 742, 767, 782, 793, 790, 753, 792, 767, 722, 788, 793, 767, 786, 779, 810, 789, 721, 744] payload_rows_per_dst=[9272, 9396, 9332, 9274, 9186, 9298, 9338, 9434] dup_rows_per_dst=[2980, 2978, 2866, 2992, 2838, 3006, 3036, 3078] combine_rows_per_src=[9272, 9396, 9332, 9274, 9186, 9298, 9338, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64996 dup_rows=24008 dup_rows_remote=20856 dedup_ratio=0.244222 router_tokens_per_expert=[747, 779, 772, 753, 753, 752, 787, 720, 805, 764, 756, 791, 804, 731, 735, 744, 738, 795, 777, 786, 762, 826, 779, 754, 732, 807, 759, 737, 771, 800, 736, 758, 785, 746, 770, 752, 725, 778, 758, 773, 830, 723, 751, 744, 820, 771, 757, 738, 765, 739, 807, 780, 728, 826, 786, 795, 770, 745, 817, 743, 766, 797, 781, 776] payload_rows_per_dst=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] dup_rows_per_dst=[2936, 2998, 3044, 2950, 2874, 3004, 3152, 3050] combine_rows_per_src=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65178 dup_rows=23758 dup_rows_remote=20678 dedup_ratio=0.241679 router_tokens_per_expert=[746, 767, 737, 752, 725, 759, 766, 778, 716, 779, 745, 776, 733, 780, 801, 766, 771, 766, 796, 736, 753, 817, 789, 760, 777, 777, 774, 781, 760, 784, 800, 771, 788, 790, 728, 729, 781, 750, 790, 775, 787, 780, 793, 736, 733, 811, 762, 785, 781, 784, 756, 817, 814, 776, 812, 718, 744, 769, 751, 763, 753, 713, 775, 770] payload_rows_per_dst=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] dup_rows_per_dst=[2874, 2912, 2976, 3080, 2966, 2972, 3096, 2882] combine_rows_per_src=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343968 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266690464 ingress_bytes=404758528 link_bytes=404758528 token_bytes=533036960 weight_movement_bytes=138412032 total_nvlink_bytes=671448992 weight_to_token_ratio=0.259667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523073755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343464 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266665384 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987304 weight_movement_bytes=69206016 total_nvlink_bytes=602193320 weight_to_token_ratio=0.129846 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=64978 dup_rows=23962 dup_rows_remote=20968 dedup_ratio=0.243754 router_tokens_per_expert=[753, 749, 766, 783, 762, 747, 736, 758, 744, 763, 793, 765, 797, 746, 784, 786, 761, 751, 777, 782, 752, 777, 797, 752, 806, 845, 725, 777, 749, 761, 742, 774, 745, 799, 735, 733, 761, 730, 792, 757, 735, 749, 754, 756, 766, 843, 789, 753, 785, 789, 729, 750, 750, 765, 802, 774, 772, 796, 767, 801, 736, 807, 801, 771] payload_rows_per_dst=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] dup_rows_per_dst=[2870, 3030, 2966, 2994, 2908, 3050, 3046, 3098] combine_rows_per_src=[9238, 9326, 9332, 9364, 9196, 9240, 9242, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64986 dup_rows=23956 dup_rows_remote=20878 dedup_ratio=0.243693 router_tokens_per_expert=[772, 795, 785, 760, 760, 797, 748, 742, 768, 775, 754, 786, 771, 770, 758, 765, 768, 766, 723, 739, 791, 767, 798, 779, 778, 768, 781, 819, 761, 751, 783, 768, 812, 755, 742, 789, 788, 780, 741, 813, 747, 754, 753, 798, 768, 752, 740, 749, 758, 815, 737, 778, 777, 765, 828, 752, 746, 745, 742, 747, 763, 774, 783, 715] payload_rows_per_dst=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] dup_rows_per_dst=[3014, 2918, 2996, 3122, 3078, 2900, 3020, 2908] combine_rows_per_src=[9304, 9376, 9266, 9296, 9362, 9222, 9400, 9122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65012 dup_rows=24098 dup_rows_remote=21064 dedup_ratio=0.245138 router_tokens_per_expert=[781, 742, 731, 784, 759, 773, 786, 759, 758, 772, 729, 746, 793, 761, 734, 761, 754, 741, 741, 759, 802, 729, 808, 780, 804, 780, 776, 751, 726, 800, 807, 757, 805, 743, 777, 766, 791, 788, 757, 750, 784, 804, 784, 748, 778, 740, 768, 824, 722, 785, 799, 765, 817, 741, 815, 717, 741, 787, 799, 753, 776, 756, 771, 717] payload_rows_per_dst=[9152, 9298, 9194, 9356, 9298, 9348, 9294, 9266] dup_rows_per_dst=[3078, 2810, 3034, 3046, 3056, 3112, 3028, 2934] combine_rows_per_src=[9152, 9298, 9194, 9356, 9298, 9348, 9294, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65050 dup_rows=23842 dup_rows_remote=20908 dedup_ratio=0.242533 router_tokens_per_expert=[778, 744, 739, 784, 744, 763, 750, 806, 742, 766, 767, 706, 702, 781, 775, 815, 771, 778, 803, 747, 781, 790, 762, 739, 776, 796, 769, 755, 765, 792, 803, 749, 773, 760, 783, 805, 732, 787, 805, 727, 789, 780, 781, 794, 721, 766, 757, 785, 746, 762, 812, 819, 734, 768, 744, 777, 737, 781, 751, 775, 743, 756, 777, 787] payload_rows_per_dst=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] dup_rows_per_dst=[2948, 2930, 3044, 3020, 3044, 2934, 2998, 2924] combine_rows_per_src=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343168 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266976384 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533609600 weight_movement_bytes=34603008 total_nvlink_bytes=568212608 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64946 dup_rows=24070 dup_rows_remote=21022 dedup_ratio=0.244853 router_tokens_per_expert=[771, 774, 770, 750, 796, 763, 749, 715, 777, 746, 741, 757, 779, 780, 768, 714, 793, 755, 792, 784, 798, 821, 767, 727, 766, 793, 795, 761, 777, 755, 783, 789, 765, 818, 770, 753, 800, 719, 800, 812, 745, 780, 756, 756, 762, 756, 761, 728, 781, 777, 797, 715, 796, 798, 774, 747, 744, 744, 768, 780, 760, 784, 774, 726] payload_rows_per_dst=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] dup_rows_per_dst=[3048, 3266, 2972, 3216, 2978, 2848, 2844, 2898] combine_rows_per_src=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343408 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567024 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532790640 weight_movement_bytes=103809024 total_nvlink_bytes=636599664 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267173888 dispatch_meta_bytes=344720 combine_payload_bytes=267173888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267518608 ingress_bytes=301776896 link_bytes=301776896 token_bytes=534692496 weight_movement_bytes=34603008 total_nvlink_bytes=569295504 weight_to_token_ratio=0.064716 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135562165e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343960 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871256 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398552 weight_movement_bytes=34603008 total_nvlink_bytes=566001560 weight_to_token_ratio=0.065117 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65068 dup_rows=24032 dup_rows_remote=21042 dedup_ratio=0.244466 router_tokens_per_expert=[756, 719, 780, 741, 780, 836, 760, 745, 805, 729, 742, 778, 780, 754, 778, 731, 772, 782, 740, 757, 787, 756, 769, 766, 751, 809, 808, 747, 779, 762, 772, 747, 745, 716, 765, 823, 786, 735, 790, 811, 731, 786, 837, 804, 740, 804, 796, 735, 731, 758, 743, 801, 794, 814, 783, 733, 717, 755, 743, 747, 764, 797, 776, 774] payload_rows_per_dst=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] dup_rows_per_dst=[3030, 2990, 3084, 3002, 3004, 3074, 2940, 2908] combine_rows_per_src=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=343424 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267312512 ingress_bytes=336175104 link_bytes=336175104 token_bytes=534281600 weight_movement_bytes=69206016 total_nvlink_bytes=603487616 weight_to_token_ratio=0.129531 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264999852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266633216 dispatch_meta_bytes=343168 combine_payload_bytes=266633216 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266976384 ingress_bytes=301236224 link_bytes=301236224 token_bytes=533609600 weight_movement_bytes=34603008 total_nvlink_bytes=568212608 weight_to_token_ratio=0.064847 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133527659e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64790 dup_rows=24112 dup_rows_remote=21032 dedup_ratio=0.245280 router_tokens_per_expert=[779, 805, 742, 768, 740, 733, 788, 760, 705, 762, 811, 831, 790, 776, 763, 803, 770, 769, 750, 808, 769, 754, 806, 715, 772, 768, 769, 779, 811, 736, 821, 792, 757, 837, 805, 816, 789, 739, 763, 774, 745, 760, 764, 749, 730, 799, 749, 723, 759, 731, 789, 779, 798, 740, 759, 752, 778, 721, 764, 759, 732, 732, 728, 787] payload_rows_per_dst=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] dup_rows_per_dst=[2968, 3080, 2988, 3150, 3142, 2932, 2966, 2886] combine_rows_per_src=[9262, 9402, 9294, 9346, 9418, 9106, 9248, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343784 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266493672 ingress_bytes=335355904 link_bytes=335355904 token_bytes=532643560 weight_movement_bytes=69206016 total_nvlink_bytes=601849576 weight_to_token_ratio=0.129929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261917268e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=344304 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266633456 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532922608 weight_movement_bytes=69206016 total_nvlink_bytes=602128624 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343832 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788632 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233432 weight_movement_bytes=34603008 total_nvlink_bytes=567836440 weight_to_token_ratio=0.064893 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343872 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362688 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381504 weight_movement_bytes=34603008 total_nvlink_bytes=566984512 weight_to_token_ratio=0.064997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344440 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266862968 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381496 weight_movement_bytes=69206016 total_nvlink_bytes=602587512 weight_to_token_ratio=0.129750 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64822 dup_rows=24092 dup_rows_remote=20986 dedup_ratio=0.245076 router_tokens_per_expert=[784, 755, 745, 746, 785, 803, 784, 746, 765, 767, 823, 794, 804, 719, 780, 757, 730, 809, 790, 779, 757, 794, 766, 805, 730, 741, 763, 833, 789, 745, 786, 738, 805, 732, 778, 810, 718, 776, 832, 797, 782, 727, 810, 781, 696, 785, 766, 755, 723, 719, 804, 722, 692, 752, 797, 781, 781, 759, 748, 764, 761, 746, 810, 731] payload_rows_per_dst=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] dup_rows_per_dst=[3062, 3050, 3100, 2994, 3106, 3028, 2804, 2948] combine_rows_per_src=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=343456 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266526112 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532708768 weight_movement_bytes=69206016 total_nvlink_bytes=601914784 weight_to_token_ratio=0.129913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343288 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723128 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531102968 weight_movement_bytes=69206016 total_nvlink_bytes=600308984 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64946 dup_rows=24070 dup_rows_remote=21022 dedup_ratio=0.244853 router_tokens_per_expert=[771, 774, 770, 750, 796, 763, 749, 715, 777, 746, 741, 757, 779, 780, 768, 714, 793, 755, 792, 784, 798, 821, 767, 727, 766, 793, 795, 761, 777, 755, 783, 789, 765, 818, 770, 753, 800, 719, 800, 812, 745, 780, 756, 756, 762, 756, 761, 728, 781, 777, 797, 715, 796, 798, 774, 747, 744, 744, 768, 780, 760, 784, 774, 726] payload_rows_per_dst=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] dup_rows_per_dst=[3048, 3266, 2972, 3216, 2978, 2848, 2844, 2898] combine_rows_per_src=[9288, 9418, 9362, 9442, 9292, 9240, 8930, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65050 dup_rows=23842 dup_rows_remote=20908 dedup_ratio=0.242533 router_tokens_per_expert=[778, 744, 739, 784, 744, 763, 750, 806, 742, 766, 767, 706, 702, 781, 775, 815, 771, 778, 803, 747, 781, 790, 762, 739, 776, 796, 769, 755, 765, 792, 803, 749, 773, 760, 783, 805, 732, 787, 805, 727, 789, 780, 781, 794, 721, 766, 757, 785, 746, 762, 812, 819, 734, 768, 744, 777, 737, 781, 751, 775, 743, 756, 777, 787] payload_rows_per_dst=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] dup_rows_per_dst=[2948, 2930, 3044, 3020, 3044, 2934, 2998, 2924] combine_rows_per_src=[9268, 9178, 9298, 9390, 9300, 9412, 9326, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=64982 dup_rows=24026 dup_rows_remote=21050 dedup_ratio=0.244405 router_tokens_per_expert=[738, 776, 739, 765, 771, 777, 799, 747, 762, 751, 744, 767, 726, 774, 753, 765, 760, 757, 782, 749, 744, 785, 720, 777, 766, 788, 770, 776, 783, 808, 771, 769, 813, 792, 807, 754, 753, 780, 768, 756, 740, 754, 755, 782, 835, 765, 779, 762, 769, 742, 778, 791, 776, 757, 763, 760, 759, 753, 743, 691, 801, 799, 765, 851] payload_rows_per_dst=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] dup_rows_per_dst=[3004, 2924, 2972, 3100, 3036, 3050, 2976, 2964] combine_rows_per_src=[9220, 9160, 9176, 9362, 9410, 9294, 9296, 9360] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=64916 dup_rows=23948 dup_rows_remote=20768 dedup_ratio=0.243612 router_tokens_per_expert=[787, 794, 817, 745, 769, 824, 817, 716, 765, 744, 746, 787, 775, 724, 773, 763, 784, 768, 730, 792, 762, 771, 796, 784, 773, 700, 764, 755, 740, 764, 726, 799, 768, 763, 782, 784, 740, 780, 765, 761, 761, 764, 777, 778, 761, 767, 754, 793, 806, 761, 751, 756, 776, 746, 719, 771, 742, 763, 814, 738, 749, 805, 807, 796] payload_rows_per_dst=[9364, 9230, 9126, 9070, 9288, 9256, 9582, 9440] dup_rows_per_dst=[3006, 2880, 3036, 2792, 3030, 2814, 3210, 3180] combine_rows_per_src=[9364, 9230, 9126, 9070, 9288, 9256, 9582, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65024 dup_rows=24098 dup_rows_remote=21126 dedup_ratio=0.245138 router_tokens_per_expert=[715, 784, 726, 747, 757, 839, 779, 763, 767, 744, 752, 756, 788, 697, 765, 780, 786, 778, 791, 760, 734, 758, 816, 779, 792, 735, 795, 727, 780, 748, 772, 775, 801, 797, 762, 788, 804, 759, 759, 818, 742, 796, 750, 789, 828, 760, 761, 826, 789, 784, 755, 791, 716, 745, 755, 742, 771, 768, 741, 755, 756, 751, 756, 752] payload_rows_per_dst=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] dup_rows_per_dst=[3042, 2874, 3090, 3020, 3136, 3072, 2972, 2892] combine_rows_per_src=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343872 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266362688 ingress_bytes=300621824 link_bytes=300621824 token_bytes=532381504 weight_movement_bytes=34603008 total_nvlink_bytes=566984512 weight_to_token_ratio=0.064997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131215722e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65160 dup_rows=23820 dup_rows_remote=20966 dedup_ratio=0.242310 router_tokens_per_expert=[781, 804, 772, 718, 757, 723, 803, 749, 784, 757, 758, 758, 754, 743, 759, 745, 776, 748, 799, 769, 777, 742, 785, 781, 789, 749, 710, 788, 727, 753, 771, 802, 747, 782, 817, 809, 786, 757, 771, 741, 779, 755, 730, 748, 791, 732, 758, 740, 754, 826, 755, 773, 768, 797, 795, 729, 813, 814, 771, 782, 777, 833, 762, 729] payload_rows_per_dst=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] dup_rows_per_dst=[2990, 2964, 3032, 2854, 3018, 2856, 2984, 3122] combine_rows_per_src=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343232 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265854144 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531365056 weight_movement_bytes=69206016 total_nvlink_bytes=600571072 weight_to_token_ratio=0.130242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=342736 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266238672 ingress_bytes=300498944 link_bytes=300498944 token_bytes=532134608 weight_movement_bytes=34603008 total_nvlink_bytes=566737616 weight_to_token_ratio=0.065027 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130753334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65094 dup_rows=23966 dup_rows_remote=21108 dedup_ratio=0.243795 router_tokens_per_expert=[802, 795, 735, 760, 778, 782, 765, 793, 764, 787, 731, 768, 767, 794, 785, 790, 773, 759, 757, 790, 725, 743, 785, 789, 727, 742, 795, 785, 770, 760, 803, 771, 748, 742, 826, 750, 766, 783, 738, 759, 784, 728, 751, 797, 715, 806, 742, 728, 742, 748, 799, 745, 783, 764, 770, 794, 786, 780, 778, 778, 780, 762, 777, 733] payload_rows_per_dst=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] dup_rows_per_dst=[2960, 3074, 3014, 2976, 3058, 2858, 3060, 2966] combine_rows_per_src=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=65076 dup_rows=24072 dup_rows_remote=21168 dedup_ratio=0.244873 router_tokens_per_expert=[798, 732, 752, 778, 808, 778, 760, 727, 742, 842, 782, 782, 745, 781, 830, 767, 788, 762, 760, 732, 730, 735, 781, 742, 779, 743, 778, 775, 815, 770, 796, 817, 731, 724, 773, 732, 788, 732, 759, 806, 726, 762, 767, 806, 808, 733, 746, 784, 788, 768, 747, 754, 754, 781, 726, 722, 748, 824, 763, 773, 807, 735, 824, 754] payload_rows_per_dst=[9302, 9388, 9156, 9354, 9222, 9296, 9180, 9334] dup_rows_per_dst=[2964, 3154, 2904, 3192, 2868, 2968, 2900, 3122] combine_rows_per_src=[9302, 9388, 9156, 9354, 9222, 9296, 9180, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64834 dup_rows=24166 dup_rows_remote=21032 dedup_ratio=0.245829 router_tokens_per_expert=[785, 807, 805, 734, 743, 747, 785, 768, 799, 771, 773, 785, 747, 788, 740, 786, 820, 768, 777, 722, 789, 802, 737, 756, 753, 713, 736, 812, 755, 787, 813, 778, 736, 726, 745, 773, 774, 748, 772, 769, 751, 761, 769, 820, 766, 791, 745, 757, 729, 756, 763, 714, 809, 755, 751, 812, 746, 804, 770, 762, 792, 755, 798, 752] payload_rows_per_dst=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] dup_rows_per_dst=[3134, 3020, 3006, 3040, 2912, 3048, 2964, 3042] combine_rows_per_src=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344600 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682904 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533021208 weight_movement_bytes=34603008 total_nvlink_bytes=567624216 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64898 dup_rows=24088 dup_rows_remote=20978 dedup_ratio=0.245036 router_tokens_per_expert=[744, 743, 764, 784, 759, 766, 802, 753, 815, 786, 756, 775, 749, 784, 801, 748, 766, 721, 798, 739, 760, 760, 810, 766, 792, 781, 727, 770, 753, 796, 819, 784, 757, 768, 790, 734, 779, 787, 777, 754, 717, 763, 745, 778, 736, 717, 772, 784, 792, 815, 759, 770, 781, 772, 778, 729, 766, 783, 810, 770, 731, 747, 771, 749] payload_rows_per_dst=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] dup_rows_per_dst=[2988, 3110, 3032, 2988, 3018, 2878, 3070, 3004] combine_rows_per_src=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64824 dup_rows=24226 dup_rows_remote=21260 dedup_ratio=0.246440 router_tokens_per_expert=[755, 820, 750, 762, 775, 813, 766, 753, 729, 734, 777, 760, 802, 817, 770, 763, 797, 748, 729, 765, 787, 795, 737, 843, 786, 795, 748, 737, 728, 813, 762, 741, 771, 747, 813, 745, 770, 776, 723, 799, 746, 766, 740, 799, 724, 798, 763, 773, 769, 816, 787, 730, 787, 748, 759, 763, 747, 787, 750, 762, 775, 737, 782, 743] payload_rows_per_dst=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] dup_rows_per_dst=[3084, 3014, 3076, 2966, 3022, 2958, 3118, 2988] combine_rows_per_src=[9304, 9290, 9326, 9254, 9266, 9260, 9200, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343832 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266788632 ingress_bytes=301047808 link_bytes=301047808 token_bytes=533233432 weight_movement_bytes=34603008 total_nvlink_bytes=567836440 weight_to_token_ratio=0.064893 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132818665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65026 dup_rows=24030 dup_rows_remote=20966 dedup_ratio=0.244446 router_tokens_per_expert=[776, 772, 764, 742, 743, 744, 815, 842, 735, 753, 764, 764, 778, 716, 768, 782, 776, 744, 796, 784, 764, 742, 823, 770, 773, 735, 705, 784, 743, 719, 767, 791, 790, 749, 743, 803, 754, 752, 774, 791, 790, 754, 797, 787, 735, 753, 801, 709, 799, 791, 778, 787, 764, 799, 773, 743, 747, 744, 797, 785, 796, 748, 754, 791] payload_rows_per_dst=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] dup_rows_per_dst=[3064, 2868, 3028, 2924, 3064, 2994, 3068, 3020] combine_rows_per_src=[9332, 9252, 9370, 9110, 9248, 9258, 9400, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344128 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510400 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676672 weight_movement_bytes=34603008 total_nvlink_bytes=567279680 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265895936 dispatch_meta_bytes=342736 combine_payload_bytes=265895936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266238672 ingress_bytes=300498944 link_bytes=300498944 token_bytes=532134608 weight_movement_bytes=34603008 total_nvlink_bytes=566737616 weight_to_token_ratio=0.065027 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130753334e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344504 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267239864 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135224 weight_movement_bytes=34603008 total_nvlink_bytes=568738232 weight_to_token_ratio=0.064783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344808 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969832 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594856 weight_movement_bytes=69206016 total_nvlink_bytes=602800872 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65142 dup_rows=23906 dup_rows_remote=20816 dedup_ratio=0.243184 router_tokens_per_expert=[805, 787, 750, 749, 794, 787, 765, 743, 776, 733, 787, 737, 794, 779, 779, 788, 744, 774, 765, 707, 725, 714, 765, 790, 769, 743, 771, 806, 703, 714, 785, 766, 753, 742, 823, 788, 788, 800, 807, 768, 780, 787, 772, 713, 784, 796, 748, 804, 765, 785, 766, 760, 740, 795, 772, 753, 762, 752, 774, 802, 802, 801, 775, 701] payload_rows_per_dst=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] dup_rows_per_dst=[3004, 3090, 2872, 2912, 3142, 2984, 2902, 3000] combine_rows_per_src=[9356, 9256, 9096, 9202, 9396, 9384, 9370, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344976 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266896272 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533447568 weight_movement_bytes=69206016 total_nvlink_bytes=602653584 weight_to_token_ratio=0.129733 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=343464 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265903528 ingress_bytes=334766080 link_bytes=334766080 token_bytes=531463592 weight_movement_bytes=69206016 total_nvlink_bytes=600669608 weight_to_token_ratio=0.130218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259697808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343504 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266165712 ingress_bytes=335028224 link_bytes=335028224 token_bytes=531987920 weight_movement_bytes=69206016 total_nvlink_bytes=601193936 weight_to_token_ratio=0.130089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260684235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65110 dup_rows=24016 dup_rows_remote=21128 dedup_ratio=0.244303 router_tokens_per_expert=[767, 731, 753, 777, 750, 756, 749, 796, 748, 760, 777, 770, 803, 807, 782, 778, 737, 795, 786, 753, 776, 751, 758, 713, 783, 756, 773, 815, 746, 754, 786, 812, 729, 771, 733, 788, 758, 741, 772, 741, 745, 772, 772, 772, 766, 786, 742, 778, 770, 772, 779, 788, 763, 777, 747, 767, 780, 811, 774, 786, 766, 758, 762, 788] payload_rows_per_dst=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] dup_rows_per_dst=[2808, 3112, 2938, 3074, 2888, 3050, 3052, 3094] combine_rows_per_src=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=344336 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265863440 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531382544 weight_movement_bytes=34603008 total_nvlink_bytes=565985552 weight_to_token_ratio=0.065119 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343968 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266690464 ingress_bytes=404758528 link_bytes=404758528 token_bytes=533036960 weight_movement_bytes=138412032 total_nvlink_bytes=671448992 weight_to_token_ratio=0.259667 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523073755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343832 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165464 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987096 weight_movement_bytes=34603008 total_nvlink_bytes=568590104 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64834 dup_rows=24166 dup_rows_remote=21032 dedup_ratio=0.245829 router_tokens_per_expert=[785, 807, 805, 734, 743, 747, 785, 768, 799, 771, 773, 785, 747, 788, 740, 786, 820, 768, 777, 722, 789, 802, 737, 756, 753, 713, 736, 812, 755, 787, 813, 778, 736, 726, 745, 773, 774, 748, 772, 769, 751, 761, 769, 820, 766, 791, 745, 757, 729, 756, 763, 714, 809, 755, 751, 812, 746, 804, 770, 762, 792, 755, 798, 752] payload_rows_per_dst=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] dup_rows_per_dst=[3134, 3020, 3006, 3040, 2912, 3048, 2964, 3042] combine_rows_per_src=[9304, 9358, 9336, 9254, 9084, 9272, 9214, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65094 dup_rows=23966 dup_rows_remote=21108 dedup_ratio=0.243795 router_tokens_per_expert=[802, 795, 735, 760, 778, 782, 765, 793, 764, 787, 731, 768, 767, 794, 785, 790, 773, 759, 757, 790, 725, 743, 785, 789, 727, 742, 795, 785, 770, 760, 803, 771, 748, 742, 826, 750, 766, 783, 738, 759, 784, 728, 751, 797, 715, 806, 742, 728, 742, 748, 799, 745, 783, 764, 770, 794, 786, 780, 778, 778, 780, 762, 777, 733] payload_rows_per_dst=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] dup_rows_per_dst=[2960, 3074, 3014, 2976, 3058, 2858, 3060, 2966] combine_rows_per_src=[9460, 9298, 9228, 9330, 9166, 9244, 9230, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74012 payload_rows_remote=64750 dup_rows=24292 dup_rows_remote=21216 dedup_ratio=0.247111 router_tokens_per_expert=[726, 784, 761, 814, 740, 756, 769, 808, 774, 766, 782, 781, 721, 764, 802, 803, 743, 805, 774, 729, 722, 794, 783, 729, 774, 782, 785, 786, 816, 772, 772, 773, 801, 733, 760, 768, 762, 799, 749, 791, 717, 770, 777, 808, 781, 749, 772, 730, 712, 777, 772, 705, 750, 768, 785, 792, 782, 786, 755, 784, 738, 759, 749, 781] payload_rows_per_dst=[9154, 9154, 9158, 9502, 9200, 9398, 9184, 9262] dup_rows_per_dst=[3010, 2832, 3000, 3146, 3126, 3210, 2892, 3076] combine_rows_per_src=[9154, 9154, 9158, 9502, 9200, 9398, 9184, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65186 dup_rows=23948 dup_rows_remote=21116 dedup_ratio=0.243612 router_tokens_per_expert=[780, 816, 812, 765, 787, 796, 781, 773, 771, 774, 774, 748, 759, 830, 747, 744, 740, 760, 781, 772, 752, 791, 788, 745, 764, 792, 782, 762, 745, 691, 817, 680, 761, 809, 789, 794, 763, 766, 771, 749, 739, 776, 789, 729, 774, 750, 758, 782, 747, 756, 728, 778, 730, 789, 756, 717, 769, 802, 772, 781, 765, 780, 807, 757] payload_rows_per_dst=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] dup_rows_per_dst=[3044, 3098, 2954, 2952, 3044, 2968, 2832, 3056] combine_rows_per_src=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65154 dup_rows=23834 dup_rows_remote=20944 dedup_ratio=0.242452 router_tokens_per_expert=[790, 786, 766, 777, 813, 786, 749, 788, 774, 740, 762, 779, 753, 811, 781, 783, 722, 760, 783, 737, 789, 720, 797, 765, 777, 787, 766, 768, 766, 789, 789, 795, 755, 801, 801, 743, 720, 747, 784, 787, 783, 765, 710, 751, 763, 769, 809, 790, 715, 800, 766, 750, 762, 777, 789, 744, 753, 763, 758, 762, 761, 758, 745, 723] payload_rows_per_dst=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] dup_rows_per_dst=[3054, 2938, 2930, 3082, 3090, 2954, 2890, 2896] combine_rows_per_src=[9456, 9428, 9216, 9392, 9186, 9326, 9316, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=343464 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265903528 ingress_bytes=334766080 link_bytes=334766080 token_bytes=531463592 weight_movement_bytes=69206016 total_nvlink_bytes=600669608 weight_to_token_ratio=0.130218 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259697808e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64878 dup_rows=24166 dup_rows_remote=21138 dedup_ratio=0.245829 router_tokens_per_expert=[773, 763, 765, 688, 778, 760, 742, 794, 733, 782, 811, 775, 800, 767, 767, 840, 791, 791, 775, 791, 789, 742, 811, 799, 805, 755, 758, 775, 757, 770, 742, 782, 775, 781, 787, 764, 769, 759, 740, 723, 773, 814, 721, 787, 779, 731, 736, 778, 806, 743, 754, 747, 766, 732, 777, 765, 700, 823, 751, 795, 764, 774, 756, 711] payload_rows_per_dst=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] dup_rows_per_dst=[2948, 3056, 3154, 3028, 3026, 3042, 2984, 2928] combine_rows_per_src=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344952 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267035512 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533726072 weight_movement_bytes=34603008 total_nvlink_bytes=568329080 weight_to_token_ratio=0.064833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343864 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265559864 ingress_bytes=369025024 link_bytes=369025024 token_bytes=530775864 weight_movement_bytes=103809024 total_nvlink_bytes=634584888 weight_to_token_ratio=0.195580 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388611456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65142 dup_rows=23806 dup_rows_remote=20694 dedup_ratio=0.242167 router_tokens_per_expert=[752, 778, 822, 781, 710, 793, 765, 812, 780, 771, 770, 793, 732, 771, 749, 711, 791, 795, 750, 761, 751, 789, 794, 765, 740, 735, 795, 723, 745, 718, 773, 732, 772, 791, 727, 761, 754, 798, 767, 756, 821, 770, 754, 792, 773, 753, 759, 812, 759, 821, 768, 771, 731, 760, 765, 804, 751, 771, 768, 756, 791, 769, 801, 759] payload_rows_per_dst=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] dup_rows_per_dst=[2980, 2896, 3050, 2802, 3004, 3112, 3020, 2942] combine_rows_per_src=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74608 payload_rows_remote=65496 dup_rows=23696 dup_rows_remote=20906 dedup_ratio=0.241048 router_tokens_per_expert=[746, 778, 748, 776, 774, 812, 805, 757, 775, 791, 719, 811, 757, 758, 790, 718, 782, 771, 716, 757, 714, 767, 700, 744, 778, 767, 726, 758, 776, 785, 788, 757, 806, 786, 784, 775, 755, 764, 743, 792, 768, 794, 782, 765, 772, 776, 721, 773, 825, 797, 765, 757, 800, 773, 817, 758, 810, 727, 776, 743, 787, 752, 752, 756] payload_rows_per_dst=[9314, 9308, 9112, 9372, 9388, 9338, 9466, 9310] dup_rows_per_dst=[3078, 2930, 2790, 2898, 3022, 2964, 3118, 2896] combine_rows_per_src=[9314, 9308, 9112, 9372, 9388, 9338, 9466, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345208 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267347064 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348920 weight_movement_bytes=103809024 total_nvlink_bytes=638157944 weight_to_token_ratio=0.194272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64970 dup_rows=24198 dup_rows_remote=21234 dedup_ratio=0.246155 router_tokens_per_expert=[772, 776, 751, 748, 750, 744, 731, 778, 792, 782, 808, 788, 747, 778, 759, 780, 804, 780, 843, 735, 768, 769, 692, 785, 775, 775, 787, 782, 771, 769, 766, 738, 695, 755, 768, 777, 748, 700, 796, 766, 751, 800, 795, 784, 796, 816, 729, 717, 729, 823, 767, 774, 759, 777, 797, 745, 783, 769, 755, 801, 734, 772, 788, 763] payload_rows_per_dst=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] dup_rows_per_dst=[2964, 3108, 3358, 3042, 2852, 3028, 3044, 2802] combine_rows_per_src=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64818 dup_rows=24066 dup_rows_remote=21058 dedup_ratio=0.244812 router_tokens_per_expert=[783, 790, 798, 812, 770, 747, 778, 759, 734, 768, 772, 783, 796, 804, 759, 798, 757, 753, 758, 759, 733, 779, 836, 747, 785, 746, 760, 793, 790, 758, 759, 773, 764, 697, 761, 757, 791, 784, 791, 730, 737, 776, 766, 749, 797, 755, 745, 806, 745, 747, 762, 750, 770, 791, 767, 758, 769, 748, 753, 755, 734, 836, 775, 749] payload_rows_per_dst=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] dup_rows_per_dst=[3092, 3008, 2960, 3070, 2974, 2996, 2958, 3008] combine_rows_per_src=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344808 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969832 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594856 weight_movement_bytes=69206016 total_nvlink_bytes=602800872 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65178 dup_rows=23758 dup_rows_remote=20678 dedup_ratio=0.241679 router_tokens_per_expert=[746, 767, 737, 752, 725, 759, 766, 778, 716, 779, 745, 776, 733, 780, 801, 766, 771, 766, 796, 736, 753, 817, 789, 760, 777, 777, 774, 781, 760, 784, 800, 771, 788, 790, 728, 729, 781, 750, 790, 775, 787, 780, 793, 736, 733, 811, 762, 785, 781, 784, 756, 817, 814, 776, 812, 718, 744, 769, 751, 763, 753, 713, 775, 770] payload_rows_per_dst=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] dup_rows_per_dst=[2874, 2912, 2976, 3080, 2966, 2972, 3096, 2882] combine_rows_per_src=[9186, 9280, 9400, 9368, 9296, 9402, 9420, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64822 dup_rows=24092 dup_rows_remote=20986 dedup_ratio=0.245076 router_tokens_per_expert=[784, 755, 745, 746, 785, 803, 784, 746, 765, 767, 823, 794, 804, 719, 780, 757, 730, 809, 790, 779, 757, 794, 766, 805, 730, 741, 763, 833, 789, 745, 786, 738, 805, 732, 778, 810, 718, 776, 832, 797, 782, 727, 810, 781, 696, 785, 766, 755, 723, 719, 804, 722, 692, 752, 797, 781, 781, 759, 748, 764, 761, 746, 810, 731] payload_rows_per_dst=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] dup_rows_per_dst=[3062, 3050, 3100, 2994, 3106, 3028, 2804, 2948] combine_rows_per_src=[9234, 9368, 9360, 9256, 9390, 9176, 9176, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343864 combine_payload_bytes=265216000 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265559864 ingress_bytes=369025024 link_bytes=369025024 token_bytes=530775864 weight_movement_bytes=103809024 total_nvlink_bytes=634584888 weight_to_token_ratio=0.195580 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388611456e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344064 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084352 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824640 weight_movement_bytes=34603008 total_nvlink_bytes=566427648 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343344 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267164976 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533986608 weight_movement_bytes=34603008 total_nvlink_bytes=568589616 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=65068 dup_rows=24032 dup_rows_remote=21042 dedup_ratio=0.244466 router_tokens_per_expert=[756, 719, 780, 741, 780, 836, 760, 745, 805, 729, 742, 778, 780, 754, 778, 731, 772, 782, 740, 757, 787, 756, 769, 766, 751, 809, 808, 747, 779, 762, 772, 747, 745, 716, 765, 823, 786, 735, 790, 811, 731, 786, 837, 804, 740, 804, 796, 735, 731, 758, 743, 801, 794, 814, 783, 733, 717, 755, 743, 747, 764, 797, 776, 774] payload_rows_per_dst=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] dup_rows_per_dst=[3030, 2990, 3084, 3002, 3004, 3074, 2940, 2908] combine_rows_per_src=[9204, 9204, 9174, 9348, 9338, 9392, 9374, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=344392 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267215176 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085960 weight_movement_bytes=34603008 total_nvlink_bytes=568688968 weight_to_token_ratio=0.064789 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=268271616 dispatch_meta_bytes=345608 combine_payload_bytes=268271616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268617224 ingress_bytes=302874624 link_bytes=302874624 token_bytes=536888840 weight_movement_bytes=34603008 total_nvlink_bytes=571491848 weight_to_token_ratio=0.064451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.139692827e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344816 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266461936 ingress_bytes=335323136 link_bytes=335323136 token_bytes=532579056 weight_movement_bytes=69206016 total_nvlink_bytes=601785072 weight_to_token_ratio=0.129945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261793965e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343504 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265838032 ingress_bytes=403906560 link_bytes=403906560 token_bytes=531332560 weight_movement_bytes=138412032 total_nvlink_bytes=669744592 weight_to_token_ratio=0.260500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519867868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65420 dup_rows=23764 dup_rows_remote=20844 dedup_ratio=0.241740 router_tokens_per_expert=[740, 730, 777, 764, 804, 770, 775, 769, 778, 795, 797, 747, 797, 770, 768, 788, 750, 762, 788, 797, 774, 755, 718, 766, 763, 758, 798, 749, 798, 759, 750, 750, 780, 730, 691, 770, 795, 734, 731, 789, 807, 792, 782, 758, 801, 784, 807, 828, 836, 750, 708, 715, 819, 728, 724, 742, 798, 728, 816, 742, 773, 743, 750, 797] payload_rows_per_dst=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] dup_rows_per_dst=[2944, 3076, 2904, 2982, 2920, 3138, 2880, 2920] combine_rows_per_src=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=343424 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267312512 ingress_bytes=336175104 link_bytes=336175104 token_bytes=534281600 weight_movement_bytes=69206016 total_nvlink_bytes=603487616 weight_to_token_ratio=0.129531 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264999852e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343232 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265854144 ingress_bytes=334716928 link_bytes=334716928 token_bytes=531365056 weight_movement_bytes=69206016 total_nvlink_bytes=600571072 weight_to_token_ratio=0.130242 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259512853e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266518528 dispatch_meta_bytes=344440 combine_payload_bytes=266518528 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266862968 ingress_bytes=335724544 link_bytes=335724544 token_bytes=533381496 weight_movement_bytes=69206016 total_nvlink_bytes=602587512 weight_to_token_ratio=0.129750 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263304431e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64970 dup_rows=24198 dup_rows_remote=21234 dedup_ratio=0.246155 router_tokens_per_expert=[772, 776, 751, 748, 750, 744, 731, 778, 792, 782, 808, 788, 747, 778, 759, 780, 804, 780, 843, 735, 768, 769, 692, 785, 775, 775, 787, 782, 771, 769, 766, 738, 695, 755, 768, 777, 748, 700, 796, 766, 751, 800, 795, 784, 796, 816, 729, 717, 729, 823, 767, 774, 759, 777, 797, 745, 783, 769, 755, 801, 734, 772, 788, 763] payload_rows_per_dst=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] dup_rows_per_dst=[2964, 3108, 3358, 3042, 2852, 3028, 3044, 2802] combine_rows_per_src=[9136, 9530, 9504, 9284, 9282, 9348, 9128, 8894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74498 payload_rows_remote=65142 dup_rows=23806 dup_rows_remote=20694 dedup_ratio=0.242167 router_tokens_per_expert=[752, 778, 822, 781, 710, 793, 765, 812, 780, 771, 770, 793, 732, 771, 749, 711, 791, 795, 750, 761, 751, 789, 794, 765, 740, 735, 795, 723, 745, 718, 773, 732, 772, 791, 727, 761, 754, 798, 767, 756, 821, 770, 754, 792, 773, 753, 759, 812, 759, 821, 768, 771, 731, 760, 765, 804, 751, 771, 768, 756, 791, 769, 801, 759] payload_rows_per_dst=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] dup_rows_per_dst=[2980, 2896, 3050, 2802, 3004, 3112, 3020, 2942] combine_rows_per_src=[9446, 9258, 9342, 9120, 9248, 9356, 9338, 9390] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64802 dup_rows=24208 dup_rows_remote=21208 dedup_ratio=0.246257 router_tokens_per_expert=[788, 783, 738, 797, 804, 790, 769, 754, 760, 793, 734, 781, 780, 744, 824, 726, 781, 772, 750, 777, 777, 779, 759, 772, 798, 782, 761, 738, 742, 774, 768, 780, 751, 769, 788, 745, 719, 805, 795, 775, 787, 725, 773, 785, 743, 815, 771, 775, 779, 735, 752, 762, 760, 768, 726, 729, 787, 751, 743, 771, 762, 753, 797, 781] payload_rows_per_dst=[9292, 9044, 9190, 9402, 9196, 9048, 9630, 9294] dup_rows_per_dst=[3154, 2824, 2872, 3204, 3098, 2678, 3378, 3000] combine_rows_per_src=[9292, 9044, 9190, 9402, 9196, 9048, 9630, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65134 dup_rows=24008 dup_rows_remote=21044 dedup_ratio=0.244222 router_tokens_per_expert=[764, 738, 756, 826, 761, 738, 767, 758, 768, 748, 777, 800, 730, 795, 784, 757, 770, 787, 745, 784, 813, 749, 734, 784, 779, 799, 777, 809, 787, 764, 739, 757, 747, 766, 749, 751, 797, 769, 775, 786, 786, 744, 764, 768, 778, 783, 724, 763, 737, 763, 807, 736, 762, 770, 711, 777, 764, 798, 749, 787, 770, 800, 727, 800] payload_rows_per_dst=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] dup_rows_per_dst=[2946, 2990, 3056, 3050, 3008, 2974, 2964, 3020] combine_rows_per_src=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64996 dup_rows=24008 dup_rows_remote=20856 dedup_ratio=0.244222 router_tokens_per_expert=[747, 779, 772, 753, 753, 752, 787, 720, 805, 764, 756, 791, 804, 731, 735, 744, 738, 795, 777, 786, 762, 826, 779, 754, 732, 807, 759, 737, 771, 800, 736, 758, 785, 746, 770, 752, 725, 778, 758, 773, 830, 723, 751, 744, 820, 771, 757, 738, 765, 739, 807, 780, 728, 826, 786, 795, 770, 745, 817, 743, 766, 797, 781, 776] payload_rows_per_dst=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] dup_rows_per_dst=[2936, 2998, 3044, 2950, 2874, 3004, 3152, 3050] combine_rows_per_src=[9190, 9262, 9390, 9250, 9300, 9264, 9300, 9340] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65150 dup_rows=23922 dup_rows_remote=20948 dedup_ratio=0.243347 router_tokens_per_expert=[779, 730, 788, 752, 772, 729, 810, 804, 735, 782, 715, 762, 717, 744, 793, 754, 824, 809, 711, 798, 833, 771, 740, 778, 806, 722, 771, 773, 729, 760, 752, 790, 783, 790, 790, 759, 816, 760, 757, 746, 763, 750, 773, 752, 753, 754, 796, 790, 742, 753, 772, 758, 818, 775, 797, 762, 780, 755, 783, 790, 738, 727, 775, 762] payload_rows_per_dst=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] dup_rows_per_dst=[2996, 2794, 3104, 2974, 3086, 2984, 3014, 2970] combine_rows_per_src=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267960320 dispatch_meta_bytes=345056 combine_payload_bytes=267960320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268305376 ingress_bytes=302563328 link_bytes=302563328 token_bytes=536265696 weight_movement_bytes=34603008 total_nvlink_bytes=570868704 weight_to_token_ratio=0.064526 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138521445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344040 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265773032 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531202024 weight_movement_bytes=69206016 total_nvlink_bytes=600408040 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64870 dup_rows=23952 dup_rows_remote=20964 dedup_ratio=0.243652 router_tokens_per_expert=[790, 747, 762, 775, 753, 766, 784, 784, 784, 739, 802, 765, 794, 766, 771, 753, 776, 749, 825, 818, 768, 749, 753, 743, 772, 776, 777, 720, 732, 742, 774, 756, 785, 815, 760, 756, 756, 804, 760, 726, 786, 788, 788, 802, 752, 763, 781, 775, 783, 751, 719, 768, 764, 768, 802, 734, 717, 785, 827, 742, 735, 782, 778, 735] payload_rows_per_dst=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] dup_rows_per_dst=[3016, 3062, 3044, 2932, 3070, 2988, 2914, 2926] combine_rows_per_src=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65138 dup_rows=23962 dup_rows_remote=20994 dedup_ratio=0.243754 router_tokens_per_expert=[713, 751, 782, 805, 745, 710, 780, 773, 809, 762, 738, 791, 744, 718, 778, 751, 774, 773, 801, 759, 786, 786, 774, 738, 818, 741, 710, 767, 769, 712, 738, 750, 734, 814, 740, 763, 769, 790, 784, 843, 766, 796, 783, 778, 769, 769, 730, 763, 753, 825, 789, 772, 741, 783, 792, 762, 756, 745, 801, 818, 777, 796, 756, 749] payload_rows_per_dst=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] dup_rows_per_dst=[2968, 2904, 3000, 2816, 3154, 3030, 3094, 2996] combine_rows_per_src=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344816 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266461936 ingress_bytes=335323136 link_bytes=335323136 token_bytes=532579056 weight_movement_bytes=69206016 total_nvlink_bytes=601785072 weight_to_token_ratio=0.129945 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261793965e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65150 dup_rows=23944 dup_rows_remote=21030 dedup_ratio=0.243571 router_tokens_per_expert=[774, 712, 722, 753, 859, 784, 777, 764, 788, 751, 770, 714, 782, 770, 746, 771, 766, 768, 735, 753, 713, 763, 789, 775, 750, 768, 776, 788, 789, 740, 758, 782, 767, 763, 739, 782, 805, 776, 775, 767, 780, 748, 737, 753, 770, 755, 813, 825, 765, 745, 732, 780, 727, 760, 793, 775, 801, 746, 796, 771, 827, 814, 743, 772] payload_rows_per_dst=[9318, 9214, 9210, 9294, 9350, 9406, 9214, 9354] dup_rows_per_dst=[2972, 2970, 2914, 3008, 2998, 2956, 2940, 3186] combine_rows_per_src=[9318, 9214, 9210, 9294, 9350, 9406, 9214, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344712 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133576 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922440 weight_movement_bytes=34603008 total_nvlink_bytes=568525448 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65058 dup_rows=23944 dup_rows_remote=21062 dedup_ratio=0.243571 router_tokens_per_expert=[744, 763, 769, 788, 776, 787, 729, 783, 734, 777, 746, 782, 741, 743, 783, 786, 763, 744, 799, 787, 755, 718, 760, 778, 745, 793, 729, 834, 722, 796, 800, 791, 803, 756, 765, 760, 753, 756, 719, 781, 809, 814, 710, 783, 786, 764, 729, 810, 744, 747, 786, 781, 796, 757, 766, 735, 742, 770, 747, 779, 815, 761, 769, 814] payload_rows_per_dst=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] dup_rows_per_dst=[3096, 2882, 2974, 3034, 2968, 3072, 2924, 2994] combine_rows_per_src=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344392 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267198792 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534053192 weight_movement_bytes=34603008 total_nvlink_bytes=568656200 weight_to_token_ratio=0.064793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343344 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267164976 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533986608 weight_movement_bytes=34603008 total_nvlink_bytes=568589616 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=344040 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265773032 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531202024 weight_movement_bytes=69206016 total_nvlink_bytes=600408040 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65160 dup_rows=23820 dup_rows_remote=20966 dedup_ratio=0.242310 router_tokens_per_expert=[781, 804, 772, 718, 757, 723, 803, 749, 784, 757, 758, 758, 754, 743, 759, 745, 776, 748, 799, 769, 777, 742, 785, 781, 789, 749, 710, 788, 727, 753, 771, 802, 747, 782, 817, 809, 786, 757, 771, 741, 779, 755, 730, 748, 791, 732, 758, 740, 754, 826, 755, 773, 768, 797, 795, 729, 813, 814, 771, 782, 777, 833, 762, 729] payload_rows_per_dst=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] dup_rows_per_dst=[2990, 2964, 3032, 2854, 3018, 2856, 2984, 3122] combine_rows_per_src=[9224, 9152, 9322, 9324, 9402, 9210, 9410, 9440] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343336 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050856 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758376 weight_movement_bytes=69206016 total_nvlink_bytes=600964392 weight_to_token_ratio=0.130146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344528 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149776 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955024 weight_movement_bytes=34603008 total_nvlink_bytes=568558032 weight_to_token_ratio=0.064805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65110 dup_rows=24016 dup_rows_remote=21128 dedup_ratio=0.244303 router_tokens_per_expert=[767, 731, 753, 777, 750, 756, 749, 796, 748, 760, 777, 770, 803, 807, 782, 778, 737, 795, 786, 753, 776, 751, 758, 713, 783, 756, 773, 815, 746, 754, 786, 812, 729, 771, 733, 788, 758, 741, 772, 741, 745, 772, 772, 772, 766, 786, 742, 778, 770, 772, 779, 788, 763, 777, 747, 767, 780, 811, 774, 786, 766, 758, 762, 788] payload_rows_per_dst=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] dup_rows_per_dst=[2808, 3112, 2938, 3074, 2888, 3050, 3052, 3094] combine_rows_per_src=[9350, 9338, 9200, 9376, 9178, 9216, 9274, 9356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343408 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567024 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532790640 weight_movement_bytes=103809024 total_nvlink_bytes=636599664 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64898 dup_rows=24088 dup_rows_remote=20978 dedup_ratio=0.245036 router_tokens_per_expert=[744, 743, 764, 784, 759, 766, 802, 753, 815, 786, 756, 775, 749, 784, 801, 748, 766, 721, 798, 739, 760, 760, 810, 766, 792, 781, 727, 770, 753, 796, 819, 784, 757, 768, 790, 734, 779, 787, 777, 754, 717, 763, 745, 778, 736, 717, 772, 784, 792, 815, 759, 770, 781, 772, 778, 729, 766, 783, 810, 770, 731, 747, 771, 749] payload_rows_per_dst=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] dup_rows_per_dst=[2988, 3110, 3032, 2988, 3018, 2878, 3070, 3004] combine_rows_per_src=[9242, 9318, 9208, 9456, 9274, 9146, 9322, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344720 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267199120 ingress_bytes=336060416 link_bytes=336060416 token_bytes=534053520 weight_movement_bytes=69206016 total_nvlink_bytes=603259536 weight_to_token_ratio=0.129586 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264568290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263505976 prefetch_bytes=761266176 reduce_bytes=449839104 weight_movement_bytes=1211105280 egress_bytes=6135709752 ingress_bytes=7338901504 link_bytes=7338901504 total_nvlink_bytes=13474611256 weight_to_token_ratio=0.098757 prefetch_share_of_ingress=0.103730 dedup_ratio=0.243743 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=13 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200682414e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822048 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299616 weight_movement_bytes=69206016 total_nvlink_bytes=602505632 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64810 dup_rows=24224 dup_rows_remote=21298 dedup_ratio=0.246419 router_tokens_per_expert=[732, 729, 722, 746, 745, 799, 757, 761, 770, 821, 766, 766, 808, 748, 760, 793, 757, 757, 766, 781, 804, 766, 782, 738, 783, 777, 831, 784, 760, 749, 769, 757, 748, 784, 766, 755, 745, 743, 748, 809, 801, 749, 796, 776, 780, 777, 790, 740, 738, 789, 770, 774, 756, 789, 738, 783, 754, 746, 805, 741, 774, 728, 800, 776] payload_rows_per_dst=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] dup_rows_per_dst=[2866, 3166, 3028, 3134, 2926, 3118, 3032, 2954] combine_rows_per_src=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344952 combine_payload_bytes=266690560 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267035512 ingress_bytes=301293568 link_bytes=301293568 token_bytes=533726072 weight_movement_bytes=34603008 total_nvlink_bytes=568329080 weight_to_token_ratio=0.064833 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133743440e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=344504 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267239864 ingress_bytes=301498368 link_bytes=301498368 token_bytes=534135224 weight_movement_bytes=34603008 total_nvlink_bytes=568738232 weight_to_token_ratio=0.064783 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134514086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=343504 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266165712 ingress_bytes=335028224 link_bytes=335028224 token_bytes=531987920 weight_movement_bytes=69206016 total_nvlink_bytes=601193936 weight_to_token_ratio=0.130089 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260684235e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65138 dup_rows=23962 dup_rows_remote=20994 dedup_ratio=0.243754 router_tokens_per_expert=[713, 751, 782, 805, 745, 710, 780, 773, 809, 762, 738, 791, 744, 718, 778, 751, 774, 773, 801, 759, 786, 786, 774, 738, 818, 741, 710, 767, 769, 712, 738, 750, 734, 814, 740, 763, 769, 790, 784, 843, 766, 796, 783, 778, 769, 769, 730, 763, 753, 825, 789, 772, 741, 783, 792, 762, 756, 745, 801, 818, 777, 796, 756, 749] payload_rows_per_dst=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] dup_rows_per_dst=[2968, 2904, 3000, 2816, 3154, 3030, 3094, 2996] combine_rows_per_src=[9204, 9224, 9382, 9194, 9518, 9278, 9340, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64870 dup_rows=23952 dup_rows_remote=20964 dedup_ratio=0.243652 router_tokens_per_expert=[790, 747, 762, 775, 753, 766, 784, 784, 784, 739, 802, 765, 794, 766, 771, 753, 776, 749, 825, 818, 768, 749, 753, 743, 772, 776, 777, 720, 732, 742, 774, 756, 785, 815, 760, 756, 756, 804, 760, 726, 786, 788, 788, 802, 752, 763, 781, 775, 783, 751, 719, 768, 764, 768, 802, 734, 717, 785, 827, 742, 735, 782, 778, 735] payload_rows_per_dst=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] dup_rows_per_dst=[3016, 3062, 3044, 2932, 3070, 2988, 2914, 2926] combine_rows_per_src=[9306, 9286, 9318, 9166, 9254, 9482, 9264, 9276] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65108 dup_rows=23708 dup_rows_remote=20658 dedup_ratio=0.241170 router_tokens_per_expert=[786, 845, 769, 811, 769, 792, 750, 740, 786, 745, 810, 731, 797, 771, 807, 831, 715, 787, 738, 751, 790, 770, 776, 819, 729, 781, 763, 751, 822, 727, 784, 734, 730, 782, 746, 742, 788, 791, 770, 747, 773, 804, 758, 750, 732, 713, 800, 725, 729, 805, 802, 726, 756, 767, 822, 764, 792, 747, 795, 782, 733, 744, 737, 723] payload_rows_per_dst=[9448, 9204, 9362, 8900, 9646, 9200, 9348, 9488] dup_rows_per_dst=[3076, 2920, 2906, 2640, 3188, 2934, 2994, 3050] combine_rows_per_src=[9448, 9204, 9362, 8900, 9646, 9200, 9348, 9488] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65020 dup_rows=23976 dup_rows_remote=21008 dedup_ratio=0.243896 router_tokens_per_expert=[810, 763, 764, 758, 781, 783, 791, 710, 769, 758, 751, 788, 806, 772, 773, 791, 764, 759, 760, 766, 780, 761, 733, 757, 748, 784, 742, 762, 785, 783, 756, 789, 708, 800, 799, 769, 766, 741, 773, 791, 791, 765, 784, 784, 751, 766, 770, 749, 717, 799, 750, 758, 803, 801, 758, 752, 785, 759, 743, 765, 793, 731, 764, 770] payload_rows_per_dst=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] dup_rows_per_dst=[3044, 2982, 2976, 2948, 3002, 3042, 2968, 3014] combine_rows_per_src=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256558176 prefetch_bytes=588251136 reduce_bytes=1245708288 weight_movement_bytes=1833959424 egress_bytes=6132235360 ingress_bytes=7958282240 link_bytes=7958282240 total_nvlink_bytes=14090517600 weight_to_token_ratio=0.149631 prefetch_share_of_ingress=0.073917 dedup_ratio=0.244287 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=36 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.302016321e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65084 dup_rows=24056 dup_rows_remote=21126 dedup_ratio=0.244710 router_tokens_per_expert=[814, 741, 761, 747, 769, 766, 713, 746, 750, 788, 772, 785, 777, 746, 782, 796, 766, 801, 774, 763, 788, 739, 737, 792, 769, 752, 785, 852, 774, 762, 746, 836, 732, 808, 757, 765, 785, 769, 710, 760, 738, 798, 741, 754, 810, 784, 725, 697, 798, 759, 745, 776, 734, 721, 756, 805, 802, 753, 750, 771, 782, 802, 770, 806] payload_rows_per_dst=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] dup_rows_per_dst=[2910, 3082, 2986, 3204, 2864, 2930, 3006, 3074] combine_rows_per_src=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=344432 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265806192 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531267952 weight_movement_bytes=34603008 total_nvlink_bytes=565870960 weight_to_token_ratio=0.065133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65024 dup_rows=24098 dup_rows_remote=21126 dedup_ratio=0.245138 router_tokens_per_expert=[715, 784, 726, 747, 757, 839, 779, 763, 767, 744, 752, 756, 788, 697, 765, 780, 786, 778, 791, 760, 734, 758, 816, 779, 792, 735, 795, 727, 780, 748, 772, 775, 801, 797, 762, 788, 804, 759, 759, 818, 742, 796, 750, 789, 828, 760, 761, 826, 789, 784, 755, 791, 716, 745, 755, 742, 771, 768, 741, 755, 756, 751, 756, 752] payload_rows_per_dst=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] dup_rows_per_dst=[3042, 2874, 3090, 3020, 3136, 3072, 2972, 2892] combine_rows_per_src=[9178, 9224, 9314, 9228, 9440, 9432, 9182, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344528 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149776 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955024 weight_movement_bytes=34603008 total_nvlink_bytes=568558032 weight_to_token_ratio=0.064805 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65054 dup_rows=24048 dup_rows_remote=21086 dedup_ratio=0.244629 router_tokens_per_expert=[794, 809, 790, 751, 729, 775, 753, 788, 801, 750, 761, 773, 780, 729, 756, 721, 771, 768, 718, 745, 795, 785, 746, 754, 820, 799, 810, 783, 787, 739, 761, 762, 764, 727, 757, 728, 756, 765, 775, 739, 842, 761, 819, 783, 775, 785, 766, 713, 769, 771, 789, 783, 771, 771, 799, 783, 769, 746, 754, 737, 793, 754, 739, 766] payload_rows_per_dst=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] dup_rows_per_dst=[3018, 2914, 2962, 3144, 2882, 3030, 3114, 2984] combine_rows_per_src=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343336 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050856 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758376 weight_movement_bytes=69206016 total_nvlink_bytes=600964392 weight_to_token_ratio=0.130146 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343064 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267025432 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533707800 weight_movement_bytes=34603008 total_nvlink_bytes=568310808 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65420 dup_rows=23764 dup_rows_remote=20844 dedup_ratio=0.241740 router_tokens_per_expert=[740, 730, 777, 764, 804, 770, 775, 769, 778, 795, 797, 747, 797, 770, 768, 788, 750, 762, 788, 797, 774, 755, 718, 766, 763, 758, 798, 749, 798, 759, 750, 750, 780, 730, 691, 770, 795, 734, 731, 789, 807, 792, 782, 758, 801, 784, 807, 828, 836, 750, 708, 715, 819, 728, 724, 742, 798, 728, 816, 742, 773, 743, 750, 797] payload_rows_per_dst=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] dup_rows_per_dst=[2944, 3076, 2904, 2982, 2920, 3138, 2880, 2920] combine_rows_per_src=[9314, 9404, 9316, 9268, 9120, 9580, 9164, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74138 payload_rows_remote=64878 dup_rows=24166 dup_rows_remote=21138 dedup_ratio=0.245829 router_tokens_per_expert=[773, 763, 765, 688, 778, 760, 742, 794, 733, 782, 811, 775, 800, 767, 767, 840, 791, 791, 775, 791, 789, 742, 811, 799, 805, 755, 758, 775, 757, 770, 742, 782, 775, 781, 787, 764, 769, 759, 740, 723, 773, 814, 721, 787, 779, 731, 736, 778, 806, 743, 754, 747, 766, 732, 777, 765, 700, 823, 751, 795, 764, 774, 756, 711] payload_rows_per_dst=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] dup_rows_per_dst=[2948, 3056, 3154, 3028, 3026, 3042, 2984, 2928] combine_rows_per_src=[9178, 9494, 9424, 9260, 9170, 9196, 9196, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344600 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682904 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533021208 weight_movement_bytes=34603008 total_nvlink_bytes=567624216 weight_to_token_ratio=0.064919 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64818 dup_rows=24066 dup_rows_remote=21058 dedup_ratio=0.244812 router_tokens_per_expert=[783, 790, 798, 812, 770, 747, 778, 759, 734, 768, 772, 783, 796, 804, 759, 798, 757, 753, 758, 759, 733, 779, 836, 747, 785, 746, 760, 793, 790, 758, 759, 773, 764, 697, 761, 757, 791, 784, 791, 730, 737, 776, 766, 749, 797, 755, 745, 806, 745, 747, 762, 750, 770, 791, 767, 758, 769, 748, 753, 755, 734, 836, 775, 749] payload_rows_per_dst=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] dup_rows_per_dst=[3092, 3008, 2960, 3070, 2974, 2996, 2958, 3008] combine_rows_per_src=[9382, 9420, 9284, 9258, 9176, 9266, 9222, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344560 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805744 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266928 weight_movement_bytes=69206016 total_nvlink_bytes=602472944 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12215802008 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6111844504 ingress_bytes=7418871808 link_bytes=7418871808 total_nvlink_bytes=13530716312 weight_to_token_ratio=0.107640 prefetch_share_of_ingress=0.093284 dedup_ratio=0.244988 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.213765972e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=267960320 dispatch_meta_bytes=345056 combine_payload_bytes=267960320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268305376 ingress_bytes=302563328 link_bytes=302563328 token_bytes=536265696 weight_movement_bytes=34603008 total_nvlink_bytes=570868704 weight_to_token_ratio=0.064526 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138521445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265740288 dispatch_meta_bytes=344064 combine_payload_bytes=265740288 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266084352 ingress_bytes=300343296 link_bytes=300343296 token_bytes=531824640 weight_movement_bytes=34603008 total_nvlink_bytes=566427648 weight_to_token_ratio=0.065065 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130167643e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64790 dup_rows=24136 dup_rows_remote=21134 dedup_ratio=0.245524 router_tokens_per_expert=[775, 761, 781, 782, 773, 758, 775, 745, 782, 818, 768, 770, 744, 775, 754, 799, 783, 747, 773, 772, 807, 778, 755, 758, 803, 782, 787, 752, 761, 748, 777, 763, 803, 719, 792, 801, 732, 781, 702, 769, 779, 755, 793, 763, 762, 801, 757, 758, 773, 809, 761, 750, 787, 740, 760, 751, 836, 739, 728, 723, 766, 743, 724, 789] payload_rows_per_dst=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] dup_rows_per_dst=[3002, 3144, 3112, 2986, 2964, 2992, 3018, 2918] combine_rows_per_src=[9378, 9434, 9242, 9360, 9234, 9186, 9156, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=65054 dup_rows=24048 dup_rows_remote=21086 dedup_ratio=0.244629 router_tokens_per_expert=[794, 809, 790, 751, 729, 775, 753, 788, 801, 750, 761, 773, 780, 729, 756, 721, 771, 768, 718, 745, 795, 785, 746, 754, 820, 799, 810, 783, 787, 739, 761, 762, 764, 727, 757, 728, 756, 765, 775, 739, 842, 761, 819, 783, 775, 785, 766, 713, 769, 771, 789, 783, 771, 771, 799, 783, 769, 746, 754, 737, 793, 754, 739, 766] payload_rows_per_dst=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] dup_rows_per_dst=[3018, 2914, 2962, 3144, 2882, 3030, 3114, 2984] combine_rows_per_src=[9360, 9228, 9202, 9378, 9140, 9458, 9358, 9132] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666032 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987952 weight_movement_bytes=69206016 total_nvlink_bytes=602193968 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343504 combine_payload_bytes=265494528 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265838032 ingress_bytes=403906560 link_bytes=403906560 token_bytes=531332560 weight_movement_bytes=138412032 total_nvlink_bytes=669744592 weight_to_token_ratio=0.260500 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519867868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262674320 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6135295888 ingress_bytes=7373086720 link_bytes=7373086720 total_nvlink_bytes=13508382608 weight_to_token_ratio=0.101585 prefetch_share_of_ingress=0.103249 dedup_ratio=0.244102 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206275293e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=65084 dup_rows=24056 dup_rows_remote=21126 dedup_ratio=0.244710 router_tokens_per_expert=[814, 741, 761, 747, 769, 766, 713, 746, 750, 788, 772, 785, 777, 746, 782, 796, 766, 801, 774, 763, 788, 739, 737, 792, 769, 752, 785, 852, 774, 762, 746, 836, 732, 808, 757, 765, 785, 769, 710, 760, 738, 798, 741, 754, 810, 784, 725, 697, 798, 759, 745, 776, 734, 721, 756, 805, 802, 753, 750, 771, 782, 802, 770, 806] payload_rows_per_dst=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] dup_rows_per_dst=[2910, 3082, 2986, 3204, 2864, 2930, 3006, 3074] combine_rows_per_src=[9204, 9310, 9334, 9348, 9308, 9164, 9182, 9398] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65186 dup_rows=23948 dup_rows_remote=21116 dedup_ratio=0.243612 router_tokens_per_expert=[780, 816, 812, 765, 787, 796, 781, 773, 771, 774, 774, 748, 759, 830, 747, 744, 740, 760, 781, 772, 752, 791, 788, 745, 764, 792, 782, 762, 745, 691, 817, 680, 761, 809, 789, 794, 763, 766, 771, 749, 739, 776, 789, 729, 774, 750, 758, 782, 747, 756, 728, 778, 730, 789, 756, 717, 769, 802, 772, 781, 765, 780, 807, 757] payload_rows_per_dst=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] dup_rows_per_dst=[3044, 3098, 2954, 2952, 3044, 2968, 2832, 3056] combine_rows_per_src=[9576, 9196, 9304, 9114, 9360, 9226, 9170, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343696 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723536 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103376 weight_movement_bytes=34603008 total_nvlink_bytes=565706384 weight_to_token_ratio=0.065153 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344840 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266928904 ingress_bytes=335790080 link_bytes=335790080 token_bytes=533512968 weight_movement_bytes=69206016 total_nvlink_bytes=602718984 weight_to_token_ratio=0.129718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263551038e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65150 dup_rows=23922 dup_rows_remote=20948 dedup_ratio=0.243347 router_tokens_per_expert=[779, 730, 788, 752, 772, 729, 810, 804, 735, 782, 715, 762, 717, 744, 793, 754, 824, 809, 711, 798, 833, 771, 740, 778, 806, 722, 771, 773, 729, 760, 752, 790, 783, 790, 790, 759, 816, 760, 757, 746, 763, 750, 773, 752, 753, 754, 796, 790, 742, 753, 772, 758, 818, 775, 797, 762, 780, 755, 783, 790, 738, 727, 775, 762] payload_rows_per_dst=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] dup_rows_per_dst=[2996, 2794, 3104, 2974, 3086, 2984, 3014, 2970] combine_rows_per_src=[9332, 9210, 9424, 9232, 9316, 9278, 9340, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64810 dup_rows=24224 dup_rows_remote=21298 dedup_ratio=0.246419 router_tokens_per_expert=[732, 729, 722, 746, 745, 799, 757, 761, 770, 821, 766, 766, 808, 748, 760, 793, 757, 757, 766, 781, 804, 766, 782, 738, 783, 777, 831, 784, 760, 749, 769, 757, 748, 784, 766, 755, 745, 743, 748, 809, 801, 749, 796, 776, 780, 777, 790, 740, 738, 789, 770, 774, 756, 789, 738, 783, 754, 746, 805, 741, 774, 728, 800, 776] payload_rows_per_dst=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] dup_rows_per_dst=[2866, 3166, 3028, 3134, 2926, 3118, 3032, 2954] combine_rows_per_src=[9116, 9298, 9274, 9286, 9270, 9300, 9242, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246061576 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6126981640 ingress_bytes=7503200256 link_bytes=7503200256 total_nvlink_bytes=13630181896 weight_to_token_ratio=0.113026 prefetch_share_of_ingress=0.083012 dedup_ratio=0.244239 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227562544e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=345208 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267347064 ingress_bytes=370810880 link_bytes=370810880 token_bytes=534348920 weight_movement_bytes=103809024 total_nvlink_bytes=638157944 weight_to_token_ratio=0.194272 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395331489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65058 dup_rows=23944 dup_rows_remote=21062 dedup_ratio=0.243571 router_tokens_per_expert=[744, 763, 769, 788, 776, 787, 729, 783, 734, 777, 746, 782, 741, 743, 783, 786, 763, 744, 799, 787, 755, 718, 760, 778, 745, 793, 729, 834, 722, 796, 800, 791, 803, 756, 765, 760, 753, 756, 719, 781, 809, 814, 710, 783, 786, 764, 729, 810, 744, 747, 786, 781, 796, 757, 766, 735, 742, 770, 747, 779, 815, 761, 769, 814] payload_rows_per_dst=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] dup_rows_per_dst=[3096, 2882, 2974, 3034, 2968, 3072, 2924, 2994] combine_rows_per_src=[9182, 9302, 9234, 9386, 9218, 9338, 9300, 9400] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258170824 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133045192 ingress_bytes=7370833920 link_bytes=7370833920 total_nvlink_bytes=13503879112 weight_to_token_ratio=0.101623 prefetch_share_of_ingress=0.098586 dedup_ratio=0.244368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205906723e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=344392 combine_payload_bytes=266854400 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267198792 ingress_bytes=301457408 link_bytes=301457408 token_bytes=534053192 weight_movement_bytes=34603008 total_nvlink_bytes=568656200 weight_to_token_ratio=0.064793 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134359957e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=344432 combine_payload_bytes=265461760 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265806192 ingress_bytes=300064768 link_bytes=300064768 token_bytes=531267952 weight_movement_bytes=34603008 total_nvlink_bytes=565870960 weight_to_token_ratio=0.065133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129119565e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822048 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299616 weight_movement_bytes=69206016 total_nvlink_bytes=602505632 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263505976 prefetch_bytes=761266176 reduce_bytes=449839104 weight_movement_bytes=1211105280 egress_bytes=6135709752 ingress_bytes=7338901504 link_bytes=7338901504 total_nvlink_bytes=13474611256 weight_to_token_ratio=0.098757 prefetch_share_of_ingress=0.103730 dedup_ratio=0.243743 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=13 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029565 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200682414e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12254608120 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6131260152 ingress_bytes=7542071296 link_bytes=7542071296 total_nvlink_bytes=13673331448 weight_to_token_ratio=0.115771 prefetch_share_of_ingress=0.087172 dedup_ratio=0.244249 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.233922048e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12215802008 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6111844504 ingress_bytes=7418871808 link_bytes=7418871808 total_nvlink_bytes=13530716312 weight_to_token_ratio=0.107640 prefetch_share_of_ingress=0.093284 dedup_ratio=0.244988 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.213765972e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343696 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265723536 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531103376 weight_movement_bytes=34603008 total_nvlink_bytes=565706384 weight_to_token_ratio=0.065153 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344560 combine_payload_bytes=266461184 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266805744 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533266928 weight_movement_bytes=69206016 total_nvlink_bytes=602472944 weight_to_token_ratio=0.129777 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65134 dup_rows=24008 dup_rows_remote=21044 dedup_ratio=0.244222 router_tokens_per_expert=[764, 738, 756, 826, 761, 738, 767, 758, 768, 748, 777, 800, 730, 795, 784, 757, 770, 787, 745, 784, 813, 749, 734, 784, 779, 799, 777, 809, 787, 764, 739, 757, 747, 766, 749, 751, 797, 769, 775, 786, 786, 744, 764, 768, 778, 783, 724, 763, 737, 763, 807, 736, 762, 770, 711, 777, 764, 798, 749, 787, 770, 800, 727, 800] payload_rows_per_dst=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] dup_rows_per_dst=[2946, 2990, 3056, 3050, 3008, 2974, 2964, 3020] combine_rows_per_src=[9270, 9328, 9276, 9372, 9272, 9246, 9162, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12256558176 prefetch_bytes=588251136 reduce_bytes=1245708288 weight_movement_bytes=1833959424 egress_bytes=6132235360 ingress_bytes=7958282240 link_bytes=7958282240 total_nvlink_bytes=14090517600 weight_to_token_ratio=0.149631 prefetch_share_of_ingress=0.073917 dedup_ratio=0.244287 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=36 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.302016321e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344712 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267133576 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533922440 weight_movement_bytes=34603008 total_nvlink_bytes=568525448 weight_to_token_ratio=0.064809 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65020 dup_rows=23976 dup_rows_remote=21008 dedup_ratio=0.243896 router_tokens_per_expert=[810, 763, 764, 758, 781, 783, 791, 710, 769, 758, 751, 788, 806, 772, 773, 791, 764, 759, 760, 766, 780, 761, 733, 757, 748, 784, 742, 762, 785, 783, 756, 789, 708, 800, 799, 769, 766, 741, 773, 791, 791, 765, 784, 784, 751, 766, 770, 749, 717, 799, 750, 758, 803, 801, 758, 752, 785, 759, 743, 765, 793, 731, 764, 770] payload_rows_per_dst=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] dup_rows_per_dst=[3044, 2982, 2976, 2948, 3002, 3042, 2968, 3014] combine_rows_per_src=[9276, 9434, 9184, 9350, 9292, 9278, 9308, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266666032 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987952 weight_movement_bytes=69206016 total_nvlink_bytes=602193968 weight_to_token_ratio=0.129845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262493752 prefetch_bytes=657457152 reduce_bytes=622854144 weight_movement_bytes=1280311296 egress_bytes=6135205432 ingress_bytes=7407599616 link_bytes=7407599616 total_nvlink_bytes=13542805048 weight_to_token_ratio=0.104409 prefetch_share_of_ingress=0.088754 dedup_ratio=0.243987 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=18 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028913 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211921782e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258170824 prefetch_bytes=726663168 reduce_bytes=519045120 weight_movement_bytes=1245708288 egress_bytes=6133045192 ingress_bytes=7370833920 link_bytes=7370833920 total_nvlink_bytes=13503879112 weight_to_token_ratio=0.101623 prefetch_share_of_ingress=0.098586 dedup_ratio=0.244368 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=15 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.205906723e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262674320 prefetch_bytes=761266176 reduce_bytes=484442112 weight_movement_bytes=1245708288 egress_bytes=6135295888 ingress_bytes=7373086720 link_bytes=7373086720 total_nvlink_bytes=13508382608 weight_to_token_ratio=0.101585 prefetch_share_of_ingress=0.103249 dedup_ratio=0.244102 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=14 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.206275293e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12246061576 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6126981640 ingress_bytes=7503200256 link_bytes=7503200256 total_nvlink_bytes=13630181896 weight_to_token_ratio=0.113026 prefetch_share_of_ingress=0.083012 dedup_ratio=0.244239 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227562544e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none - [2026-08-26 16:56:40] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 793.2 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.619553E+00 | loss scale: 1.0 | grad norm: 2.956 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65128 dup_rows=23864 dup_rows_remote=20864 dedup_ratio=0.242757 router_tokens_per_expert=[763, 781, 809, 747, 784, 777, 761, 767, 755, 793, 712, 784, 802, 782, 775, 753, 738, 790, 804, 800, 767, 765, 799, 753, 792, 810, 779, 739, 690, 772, 761, 751, 752, 760, 731, 754, 797, 736, 746, 746, 745, 760, 741, 773, 830, 769, 728, 840, 756, 764, 799, 748, 777, 767, 717, 751, 763, 728, 781, 769, 775, 794, 804, 796] payload_rows_per_dst=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] dup_rows_per_dst=[3068, 3000, 3060, 2904, 2838, 3016, 2904, 3074] combine_rows_per_src=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21020 dedup_ratio=0.244283 router_tokens_per_expert=[748, 775, 729, 790, 783, 824, 766, 779, 781, 774, 788, 740, 773, 761, 766, 788, 788, 793, 785, 788, 767, 772, 766, 793, 729, 788, 827, 754, 785, 780, 741, 803, 753, 750, 800, 734, 745, 773, 759, 760, 737, 791, 752, 764, 764, 757, 737, 749, 803, 776, 714, 832, 776, 728, 776, 793, 740, 749, 726, 727, 793, 747, 770, 753] payload_rows_per_dst=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] dup_rows_per_dst=[2994, 3242, 3274, 3044, 2936, 2886, 2934, 2704] combine_rows_per_src=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74556 payload_rows_remote=65236 dup_rows=23748 dup_rows_remote=20826 dedup_ratio=0.241577 router_tokens_per_expert=[813, 769, 782, 764, 783, 747, 775, 788, 711, 755, 759, 795, 775, 780, 777, 788, 744, 728, 787, 753, 760, 758, 735, 782, 741, 797, 730, 769, 749, 778, 786, 771, 792, 714, 769, 756, 728, 718, 776, 783, 727, 763, 741, 779, 783, 757, 777, 743, 841, 810, 762, 802, 767, 810, 759, 774, 724, 800, 817, 726, 790, 836, 713, 786] payload_rows_per_dst=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] dup_rows_per_dst=[3036, 2958, 2904, 2922, 2796, 2942, 3120, 3070] combine_rows_per_src=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] + [2026-08-26 18:08:46] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 791.4 | throughput per GPU (TFLOP/s/GPU): 161.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.619553E+00 | loss scale: 1.0 | grad norm: 2.956 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64870 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[739, 733, 796, 737, 745, 817, 757, 781, 769, 726, 764, 736, 746, 790, 763, 786, 803, 754, 800, 790, 743, 784, 778, 771, 771, 783, 769, 812, 738, 779, 776, 729, 823, 798, 776, 726, 724, 776, 764, 773, 784, 747, 767, 772, 766, 746, 766, 765, 732, 703, 794, 768, 774, 804, 851, 759, 763, 752, 749, 797, 722, 791, 793, 762] payload_rows_per_dst=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] dup_rows_per_dst=[3000, 2980, 3080, 3062, 3026, 2912, 2988, 3020] combine_rows_per_src=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65196 dup_rows=23908 dup_rows_remote=20818 dedup_ratio=0.243205 router_tokens_per_expert=[785, 785, 789, 772, 763, 798, 769, 761, 766, 819, 750, 810, 779, 787, 746, 761, 787, 756, 730, 724, 751, 795, 749, 767, 781, 762, 751, 728, 719, 760, 832, 778, 784, 786, 778, 729, 778, 762, 777, 751, 760, 779, 814, 747, 786, 765, 771, 685, 759, 744, 818, 742, 753, 767, 731, 746, 770, 764, 791, 784, 771, 792, 782, 776] payload_rows_per_dst=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] dup_rows_per_dst=[3060, 3154, 2862, 2972, 3090, 2846, 2892, 3032] combine_rows_per_src=[9384, 9282, 9256, 9250, 9200, 9368, 9228, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64842 dup_rows=24222 dup_rows_remote=21246 dedup_ratio=0.246399 router_tokens_per_expert=[787, 728, 753, 727, 802, 788, 782, 746, 819, 790, 763, 716, 779, 714, 799, 762, 820, 753, 750, 802, 699, 779, 782, 816, 765, 758, 758, 754, 745, 749, 765, 744, 791, 716, 789, 766, 761, 804, 751, 754, 789, 744, 771, 782, 753, 756, 816, 751, 767, 773, 751, 727, 769, 787, 801, 733, 739, 744, 811, 821, 746, 815, 777, 803] payload_rows_per_dst=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] dup_rows_per_dst=[2980, 3042, 3118, 2918, 3016, 3084, 2976, 3088] combine_rows_per_src=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65128 dup_rows=23864 dup_rows_remote=20864 dedup_ratio=0.242757 router_tokens_per_expert=[763, 781, 809, 747, 784, 777, 761, 767, 755, 793, 712, 784, 802, 782, 775, 753, 738, 790, 804, 800, 767, 765, 799, 753, 792, 810, 779, 739, 690, 772, 761, 751, 752, 760, 731, 754, 797, 736, 746, 746, 745, 760, 741, 773, 830, 769, 728, 840, 756, 764, 799, 748, 777, 767, 717, 751, 763, 728, 781, 769, 775, 794, 804, 796] payload_rows_per_dst=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] dup_rows_per_dst=[3068, 3000, 3060, 2904, 2838, 3016, 2904, 3074] combine_rows_per_src=[9310, 9312, 9372, 9284, 9206, 9356, 9254, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65192 dup_rows=23842 dup_rows_remote=20932 dedup_ratio=0.242533 router_tokens_per_expert=[787, 755, 755, 760, 741, 761, 790, 776, 769, 779, 793, 745, 781, 779, 739, 790, 746, 783, 734, 766, 720, 750, 818, 798, 744, 804, 762, 796, 765, 814, 765, 708, 758, 760, 798, 784, 745, 755, 763, 767, 730, 784, 806, 733, 785, 778, 719, 755, 775, 782, 782, 769, 807, 788, 771, 729, 807, 771, 752, 778, 788, 711, 780, 769] payload_rows_per_dst=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] dup_rows_per_dst=[2908, 3040, 2910, 3054, 2964, 2910, 3038, 3018] combine_rows_per_src=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21146 dedup_ratio=0.245138 router_tokens_per_expert=[763, 751, 772, 740, 765, 743, 770, 749, 752, 754, 765, 816, 773, 815, 772, 795, 793, 784, 794, 789, 749, 789, 717, 794, 765, 763, 774, 784, 749, 780, 809, 784, 780, 725, 766, 768, 827, 768, 761, 756, 760, 773, 748, 781, 747, 700, 763, 809, 785, 782, 800, 778, 752, 719, 770, 757, 771, 793, 732, 764, 754, 761, 748, 742] payload_rows_per_dst=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] dup_rows_per_dst=[2874, 2814, 2830, 3090, 3348, 3192, 2998, 2952] combine_rows_per_src=[9092, 9066, 9004, 9442, 9558, 9554, 9250, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74556 payload_rows_remote=65236 dup_rows=23748 dup_rows_remote=20826 dedup_ratio=0.241577 router_tokens_per_expert=[813, 769, 782, 764, 783, 747, 775, 788, 711, 755, 759, 795, 775, 780, 777, 788, 744, 728, 787, 753, 760, 758, 735, 782, 741, 797, 730, 769, 749, 778, 786, 771, 792, 714, 769, 756, 728, 718, 776, 783, 727, 763, 741, 779, 783, 757, 777, 743, 841, 810, 762, 802, 767, 810, 759, 774, 724, 800, 817, 726, 790, 836, 713, 786] payload_rows_per_dst=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] dup_rows_per_dst=[3036, 2958, 2904, 2922, 2796, 2942, 3120, 3070] combine_rows_per_src=[9406, 9322, 9190, 9320, 9276, 9198, 9530, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64896 dup_rows=24014 dup_rows_remote=21020 dedup_ratio=0.244283 router_tokens_per_expert=[748, 775, 729, 790, 783, 824, 766, 779, 781, 774, 788, 740, 773, 761, 766, 788, 788, 793, 785, 788, 767, 772, 766, 793, 729, 788, 827, 754, 785, 780, 741, 803, 753, 750, 800, 734, 745, 773, 759, 760, 737, 791, 752, 764, 764, 757, 737, 749, 803, 776, 714, 832, 776, 728, 776, 793, 740, 749, 726, 727, 793, 747, 770, 753] payload_rows_per_dst=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] dup_rows_per_dst=[2994, 3242, 3274, 3044, 2936, 2886, 2934, 2704] combine_rows_per_src=[9394, 9360, 9474, 9370, 9212, 9216, 9202, 9062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344248 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550904 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757560 weight_movement_bytes=34603008 total_nvlink_bytes=569360568 weight_to_token_ratio=0.064708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=343968 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108256 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872544 weight_movement_bytes=69206016 total_nvlink_bytes=603078560 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343664 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266157680 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971696 weight_movement_bytes=69206016 total_nvlink_bytes=601177712 weight_to_token_ratio=0.130093 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343432 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050952 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758472 weight_movement_bytes=69206016 total_nvlink_bytes=600964488 weight_to_token_ratio=0.130146 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=344056 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267386872 ingress_bytes=301645824 link_bytes=301645824 token_bytes=534429688 weight_movement_bytes=34603008 total_nvlink_bytes=569032696 weight_to_token_ratio=0.064748 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135068951e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65192 dup_rows=23842 dup_rows_remote=20932 dedup_ratio=0.242533 router_tokens_per_expert=[787, 755, 755, 760, 741, 761, 790, 776, 769, 779, 793, 745, 781, 779, 739, 790, 746, 783, 734, 766, 720, 750, 818, 798, 744, 804, 762, 796, 765, 814, 765, 708, 758, 760, 798, 784, 745, 755, 763, 767, 730, 784, 806, 733, 785, 778, 719, 755, 775, 782, 782, 769, 807, 788, 771, 729, 807, 771, 752, 778, 788, 711, 780, 769] payload_rows_per_dst=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] dup_rows_per_dst=[2908, 3040, 2910, 3054, 2964, 2910, 3038, 3018] combine_rows_per_src=[9342, 9310, 9320, 9262, 9296, 9270, 9368, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344352 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265937184 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531530016 weight_movement_bytes=103809024 total_nvlink_bytes=635339040 weight_to_token_ratio=0.195302 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344448 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266445184 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545920 weight_movement_bytes=69206016 total_nvlink_bytes=601751936 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64870 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[739, 733, 796, 737, 745, 817, 757, 781, 769, 726, 764, 736, 746, 790, 763, 786, 803, 754, 800, 790, 743, 784, 778, 771, 771, 783, 769, 812, 738, 779, 776, 729, 823, 798, 776, 726, 724, 776, 764, 773, 784, 747, 767, 772, 766, 746, 766, 765, 732, 703, 794, 768, 774, 804, 851, 759, 763, 752, 749, 797, 722, 791, 793, 762] payload_rows_per_dst=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] dup_rows_per_dst=[3000, 2980, 3080, 3062, 3026, 2912, 2988, 3020] combine_rows_per_src=[9210, 9180, 9366, 9252, 9294, 9314, 9382, 9238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=343968 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108256 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872544 weight_movement_bytes=69206016 total_nvlink_bytes=603078560 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267026432 dispatch_meta_bytes=344496 combine_payload_bytes=267026432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267370928 ingress_bytes=301629440 link_bytes=301629440 token_bytes=534397360 weight_movement_bytes=34603008 total_nvlink_bytes=569000368 weight_to_token_ratio=0.064751 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135007300e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64952 dup_rows=23964 dup_rows_remote=20938 dedup_ratio=0.243774 router_tokens_per_expert=[763, 797, 741, 751, 768, 797, 816, 773, 789, 812, 791, 742, 776, 759, 745, 760, 733, 767, 770, 741, 736, 765, 729, 815, 807, 780, 784, 770, 778, 787, 736, 765, 754, 732, 757, 736, 768, 768, 807, 718, 791, 765, 787, 798, 740, 756, 771, 753, 773, 761, 777, 792, 734, 767, 782, 783, 761, 762, 764, 797, 755, 777, 774, 749] payload_rows_per_dst=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] dup_rows_per_dst=[3054, 2936, 2960, 3026, 2854, 2982, 3206, 2946] combine_rows_per_src=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265707520 dispatch_meta_bytes=343432 combine_payload_bytes=265707520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266050952 ingress_bytes=334913536 link_bytes=334913536 token_bytes=531758472 weight_movement_bytes=69206016 total_nvlink_bytes=600964488 weight_to_token_ratio=0.130146 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260252673e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65126 dup_rows=23886 dup_rows_remote=20900 dedup_ratio=0.242981 router_tokens_per_expert=[815, 752, 802, 753, 772, 770, 831, 732, 812, 735, 763, 801, 750, 698, 800, 780, 744, 814, 801, 767, 769, 800, 775, 781, 814, 741, 780, 702, 788, 738, 750, 789, 801, 754, 777, 754, 755, 772, 782, 766, 782, 756, 772, 766, 757, 819, 696, 762, 764, 770, 769, 767, 767, 760, 757, 790, 769, 730, 744, 784, 717, 736, 752, 786] payload_rows_per_dst=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] dup_rows_per_dst=[3094, 2986, 3056, 2920, 3032, 3006, 2980, 2812] combine_rows_per_src=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344448 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266445184 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545920 weight_movement_bytes=69206016 total_nvlink_bytes=601751936 weight_to_token_ratio=0.129953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344248 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550904 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757560 weight_movement_bytes=34603008 total_nvlink_bytes=569360568 weight_to_token_ratio=0.064708 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343664 combine_payload_bytes=265814016 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266157680 ingress_bytes=335020032 link_bytes=335020032 token_bytes=531971696 weight_movement_bytes=69206016 total_nvlink_bytes=601177712 weight_to_token_ratio=0.130093 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260653409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64912 dup_rows=24164 dup_rows_remote=21224 dedup_ratio=0.245809 router_tokens_per_expert=[833, 783, 731, 743, 795, 804, 804, 794, 738, 748, 800, 757, 737, 799, 766, 721, 761, 770, 708, 749, 747, 796, 774, 779, 769, 778, 761, 789, 750, 761, 766, 757, 771, 788, 774, 833, 761, 802, 709, 698, 776, 755, 767, 765, 770, 793, 744, 771, 753, 725, 776, 747, 759, 780, 807, 789, 789, 742, 773, 827, 735, 764, 771, 770] payload_rows_per_dst=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] dup_rows_per_dst=[3116, 2948, 2940, 3000, 3048, 3002, 3040, 3070] combine_rows_per_src=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65044 dup_rows=23936 dup_rows_remote=20964 dedup_ratio=0.243490 router_tokens_per_expert=[761, 780, 775, 770, 769, 755, 802, 739, 759, 788, 788, 737, 777, 744, 802, 752, 760, 769, 725, 792, 741, 776, 759, 782, 791, 782, 808, 783, 735, 793, 795, 771, 723, 761, 766, 799, 744, 797, 792, 766, 774, 729, 756, 785, 754, 730, 779, 757, 772, 740, 756, 805, 785, 729, 797, 724, 760, 741, 753, 785, 701, 817, 811, 804] payload_rows_per_dst=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] dup_rows_per_dst=[3034, 2928, 3052, 3128, 2972, 2890, 2932, 3000] combine_rows_per_src=[9268, 9366, 9156, 9388, 9324, 9238, 9284, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64982 dup_rows=23974 dup_rows_remote=20950 dedup_ratio=0.243876 router_tokens_per_expert=[763, 792, 760, 766, 791, 740, 786, 756, 793, 787, 788, 783, 780, 759, 786, 770, 782, 812, 729, 743, 783, 784, 703, 704, 789, 757, 729, 729, 769, 776, 757, 773, 770, 752, 725, 742, 775, 788, 748, 772, 792, 749, 751, 775, 732, 753, 771, 791, 788, 778, 773, 772, 819, 773, 771, 821, 803, 767, 768, 752, 734, 772, 801, 755] payload_rows_per_dst=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] dup_rows_per_dst=[3024, 3022, 2838, 2922, 2970, 3036, 3126, 3036] combine_rows_per_src=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=65032 dup_rows=24068 dup_rows_remote=21190 dedup_ratio=0.244832 router_tokens_per_expert=[806, 786, 783, 741, 746, 729, 749, 763, 807, 802, 830, 745, 722, 758, 767, 779, 801, 811, 804, 744, 778, 772, 735, 763, 718, 744, 742, 741, 741, 790, 793, 759, 764, 792, 762, 750, 756, 747, 798, 809, 795, 782, 730, 792, 798, 786, 764, 759, 758, 729, 774, 781, 757, 725, 773, 744, 794, 781, 778, 770, 769, 742, 787, 757] payload_rows_per_dst=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] dup_rows_per_dst=[2954, 3116, 3020, 2862, 3084, 3108, 2878, 3046] combine_rows_per_src=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64842 dup_rows=24222 dup_rows_remote=21246 dedup_ratio=0.246399 router_tokens_per_expert=[787, 728, 753, 727, 802, 788, 782, 746, 819, 790, 763, 716, 779, 714, 799, 762, 820, 753, 750, 802, 699, 779, 782, 816, 765, 758, 758, 754, 745, 749, 765, 744, 791, 716, 789, 766, 761, 804, 751, 754, 789, 744, 771, 782, 753, 756, 816, 751, 767, 773, 751, 727, 769, 787, 801, 733, 739, 744, 811, 821, 746, 815, 777, 803] payload_rows_per_dst=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] dup_rows_per_dst=[2980, 3042, 3118, 2918, 3016, 3084, 2976, 3088] combine_rows_per_src=[9246, 9242, 9284, 9158, 9248, 9240, 9240, 9424] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65126 dup_rows=23886 dup_rows_remote=20900 dedup_ratio=0.242981 router_tokens_per_expert=[815, 752, 802, 753, 772, 770, 831, 732, 812, 735, 763, 801, 750, 698, 800, 780, 744, 814, 801, 767, 769, 800, 775, 781, 814, 741, 780, 702, 788, 738, 750, 789, 801, 754, 777, 754, 755, 772, 782, 766, 782, 756, 772, 766, 757, 819, 696, 762, 764, 770, 769, 767, 767, 760, 757, 790, 769, 730, 744, 784, 717, 736, 752, 786] payload_rows_per_dst=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] dup_rows_per_dst=[3094, 2986, 3056, 2920, 3032, 3006, 2980, 2812] combine_rows_per_src=[9360, 9292, 9446, 9284, 9290, 9214, 9308, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65292 dup_rows=23832 dup_rows_remote=20830 dedup_ratio=0.242432 router_tokens_per_expert=[768, 726, 735, 749, 751, 806, 743, 767, 767, 761, 795, 789, 736, 764, 769, 778, 788, 821, 749, 746, 752, 755, 874, 784, 746, 797, 734, 752, 772, 789, 779, 767, 748, 776, 743, 744, 733, 752, 814, 783, 762, 779, 764, 748, 746, 783, 748, 761, 740, 752, 777, 744, 791, 766, 784, 831, 743, 733, 762, 768, 798, 822, 800, 748] payload_rows_per_dst=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] dup_rows_per_dst=[2830, 2946, 3126, 2948, 2952, 3002, 3042, 2986] combine_rows_per_src=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64952 dup_rows=23964 dup_rows_remote=20938 dedup_ratio=0.243774 router_tokens_per_expert=[763, 797, 741, 751, 768, 797, 816, 773, 789, 812, 791, 742, 776, 759, 745, 760, 733, 767, 770, 741, 736, 765, 729, 815, 807, 780, 784, 770, 778, 787, 736, 765, 754, 732, 757, 736, 768, 768, 807, 718, 791, 765, 787, 798, 740, 756, 771, 753, 773, 761, 777, 792, 734, 767, 782, 783, 761, 762, 764, 797, 755, 777, 774, 749] payload_rows_per_dst=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] dup_rows_per_dst=[3054, 2936, 2960, 3026, 2854, 2982, 3206, 2946] combine_rows_per_src=[9358, 9412, 9152, 9388, 9226, 9340, 9132, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64972 dup_rows=24066 dup_rows_remote=21036 dedup_ratio=0.244812 router_tokens_per_expert=[764, 749, 824, 753, 771, 761, 754, 673, 751, 760, 763, 707, 780, 762, 799, 797, 765, 754, 791, 739, 792, 789, 768, 812, 780, 751, 767, 766, 767, 784, 780, 745, 768, 783, 775, 766, 764, 776, 775, 810, 787, 813, 780, 792, 785, 797, 767, 755, 749, 760, 724, 761, 743, 765, 817, 776, 752, 741, 762, 763, 776, 726, 756, 770] payload_rows_per_dst=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] dup_rows_per_dst=[2840, 2976, 2980, 3090, 3122, 3050, 2978, 3030] combine_rows_per_src=[9054, 9046, 9300, 9332, 9526, 9480, 9234, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64982 dup_rows=23974 dup_rows_remote=20950 dedup_ratio=0.243876 router_tokens_per_expert=[763, 792, 760, 766, 791, 740, 786, 756, 793, 787, 788, 783, 780, 759, 786, 770, 782, 812, 729, 743, 783, 784, 703, 704, 789, 757, 729, 729, 769, 776, 757, 773, 770, 752, 725, 742, 775, 788, 748, 772, 792, 749, 751, 775, 732, 753, 771, 791, 788, 778, 773, 772, 819, 773, 771, 821, 803, 767, 768, 752, 734, 772, 801, 755] payload_rows_per_dst=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] dup_rows_per_dst=[3024, 3022, 2838, 2922, 2970, 3036, 3126, 3036] combine_rows_per_src=[9348, 9470, 9242, 9236, 9174, 9192, 9400, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344544 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266224096 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103648 weight_movement_bytes=69206016 total_nvlink_bytes=601309664 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344032 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764256 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184480 weight_movement_bytes=34603008 total_nvlink_bytes=567787488 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344104 combine_payload_bytes=266756096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267100200 ingress_bytes=335962112 link_bytes=335962112 token_bytes=533856296 weight_movement_bytes=69206016 total_nvlink_bytes=603062312 weight_to_token_ratio=0.129634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264198380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344488 combine_payload_bytes=267436032 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267780520 ingress_bytes=302039040 link_bytes=302039040 token_bytes=535216552 weight_movement_bytes=34603008 total_nvlink_bytes=569819560 weight_to_token_ratio=0.064652 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136548591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=343560 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266386952 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532430344 weight_movement_bytes=34603008 total_nvlink_bytes=567033352 weight_to_token_ratio=0.064991 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344032 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764256 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184480 weight_movement_bytes=34603008 total_nvlink_bytes=567787488 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344104 combine_payload_bytes=266756096 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267100200 ingress_bytes=335962112 link_bytes=335962112 token_bytes=533856296 weight_movement_bytes=69206016 total_nvlink_bytes=603062312 weight_to_token_ratio=0.129634 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264198380e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344888 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266715960 ingress_bytes=335577088 link_bytes=335577088 token_bytes=533087032 weight_movement_bytes=69206016 total_nvlink_bytes=602293048 weight_to_token_ratio=0.129821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262749566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343728 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510000 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676272 weight_movement_bytes=69206016 total_nvlink_bytes=601882288 weight_to_token_ratio=0.129921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=344032 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469344 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594656 weight_movement_bytes=34603008 total_nvlink_bytes=567197664 weight_to_token_ratio=0.064971 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64912 dup_rows=24164 dup_rows_remote=21224 dedup_ratio=0.245809 router_tokens_per_expert=[833, 783, 731, 743, 795, 804, 804, 794, 738, 748, 800, 757, 737, 799, 766, 721, 761, 770, 708, 749, 747, 796, 774, 779, 769, 778, 761, 789, 750, 761, 766, 757, 771, 788, 774, 833, 761, 802, 709, 698, 776, 755, 767, 765, 770, 793, 744, 771, 753, 725, 776, 747, 759, 780, 807, 789, 789, 742, 773, 827, 735, 764, 771, 770] payload_rows_per_dst=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] dup_rows_per_dst=[3116, 2948, 2940, 3000, 3048, 3002, 3040, 3070] combine_rows_per_src=[9458, 9184, 9228, 9262, 9224, 9280, 9232, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65292 dup_rows=23832 dup_rows_remote=20830 dedup_ratio=0.242432 router_tokens_per_expert=[768, 726, 735, 749, 751, 806, 743, 767, 767, 761, 795, 789, 736, 764, 769, 778, 788, 821, 749, 746, 752, 755, 874, 784, 746, 797, 734, 752, 772, 789, 779, 767, 748, 776, 743, 744, 733, 752, 814, 783, 762, 779, 764, 748, 746, 783, 748, 761, 740, 752, 777, 744, 791, 766, 784, 831, 743, 733, 762, 768, 798, 822, 800, 748] payload_rows_per_dst=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] dup_rows_per_dst=[2830, 2946, 3126, 2948, 2952, 3002, 3042, 2986] combine_rows_per_src=[9260, 9372, 9412, 9324, 9234, 9180, 9328, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343728 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510000 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676272 weight_movement_bytes=69206016 total_nvlink_bytes=601882288 weight_to_token_ratio=0.129921 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=344352 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265937184 ingress_bytes=369401856 link_bytes=369401856 token_bytes=531530016 weight_movement_bytes=103809024 total_nvlink_bytes=635339040 weight_to_token_ratio=0.195302 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390029445e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64936 dup_rows=23992 dup_rows_remote=20948 dedup_ratio=0.244059 router_tokens_per_expert=[757, 774, 764, 773, 734, 760, 750, 724, 730, 839, 794, 762, 755, 755, 774, 778, 771, 798, 776, 787, 760, 767, 757, 794, 720, 761, 774, 765, 794, 790, 781, 818, 782, 750, 772, 790, 755, 795, 774, 754, 767, 731, 767, 812, 732, 774, 795, 752, 744, 746, 703, 762, 758, 808, 799, 737, 750, 806, 781, 797, 787, 734, 768, 734] payload_rows_per_dst=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] dup_rows_per_dst=[2866, 3078, 3044, 3094, 2962, 3002, 2970, 2976] combine_rows_per_src=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74420 payload_rows_remote=65276 dup_rows=23884 dup_rows_remote=20906 dedup_ratio=0.242961 router_tokens_per_expert=[748, 750, 814, 778, 775, 789, 795, 750, 739, 764, 768, 789, 795, 763, 805, 805, 786, 743, 711, 785, 781, 779, 732, 756, 809, 768, 784, 793, 696, 782, 804, 703, 776, 756, 739, 726, 826, 763, 762, 713, 789, 802, 764, 769, 753, 760, 786, 763, 732, 719, 820, 786, 791, 781, 781, 679, 777, 768, 754, 771, 810, 750, 774, 773] payload_rows_per_dst=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] dup_rows_per_dst=[3058, 2960, 2952, 2974, 2978, 3058, 2910, 2994] combine_rows_per_src=[9340, 9496, 9194, 9304, 9144, 9314, 9268, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=64894 dup_rows=23960 dup_rows_remote=20904 dedup_ratio=0.243734 router_tokens_per_expert=[780, 791, 777, 730, 773, 816, 765, 816, 790, 788, 723, 742, 732, 769, 771, 754, 781, 747, 767, 781, 767, 718, 825, 770, 807, 740, 819, 805, 734, 792, 756, 800, 755, 730, 772, 776, 737, 762, 726, 742, 802, 764, 732, 761, 804, 777, 804, 795, 726, 835, 796, 799, 749, 743, 728, 786, 732, 744, 787, 722, 737, 755, 777, 771] payload_rows_per_dst=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] dup_rows_per_dst=[3134, 2892, 2988, 3056, 2878, 3032, 2994, 2986] combine_rows_per_src=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=65156 dup_rows=23900 dup_rows_remote=20836 dedup_ratio=0.243123 router_tokens_per_expert=[787, 801, 785, 727, 747, 806, 744, 741, 784, 754, 740, 764, 820, 759, 768, 767, 779, 797, 740, 767, 795, 763, 741, 724, 732, 718, 801, 762, 754, 828, 735, 769, 744, 749, 809, 755, 748, 712, 796, 767, 745, 746, 764, 800, 766, 736, 798, 762, 754, 749, 752, 806, 787, 776, 806, 735, 750, 789, 806, 763, 795, 816, 782, 790] payload_rows_per_dst=[9348, 9248, 9268, 9262, 9110, 9312, 9378, 9478] dup_rows_per_dst=[2928, 3064, 2944, 2936, 3050, 2922, 2952, 3104] combine_rows_per_src=[9348, 9248, 9268, 9262, 9110, 9312, 9378, 9478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[747, 804, 759, 761, 711, 777, 753, 762, 817, 735, 786, 759, 813, 764, 763, 769, 745, 742, 797, 778, 766, 842, 760, 761, 749, 839, 809, 746, 779, 780, 772, 753, 771, 829, 754, 744, 774, 701, 782, 774, 781, 755, 757, 716, 786, 798, 742, 768, 762, 729, 760, 786, 755, 762, 748, 824, 794, 783, 762, 751, 767, 749, 765, 725] payload_rows_per_dst=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] dup_rows_per_dst=[2958, 3064, 3050, 3130, 2926, 2924, 2914, 2942] combine_rows_per_src=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344544 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266224096 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103648 weight_movement_bytes=69206016 total_nvlink_bytes=601309664 weight_to_token_ratio=0.130061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=267436032 dispatch_meta_bytes=344488 combine_payload_bytes=267436032 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267780520 ingress_bytes=302039040 link_bytes=302039040 token_bytes=535216552 weight_movement_bytes=34603008 total_nvlink_bytes=569819560 weight_to_token_ratio=0.064652 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136548591e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64946 dup_rows=24002 dup_rows_remote=20996 dedup_ratio=0.244161 router_tokens_per_expert=[755, 763, 790, 783, 711, 750, 758, 762, 762, 771, 806, 734, 817, 774, 779, 727, 799, 770, 758, 758, 779, 810, 769, 768, 791, 773, 734, 790, 733, 764, 735, 787, 775, 774, 760, 781, 777, 739, 755, 700, 781, 773, 757, 786, 776, 746, 743, 819, 804, 783, 804, 780, 796, 778, 779, 742, 793, 777, 748, 757, 730, 768, 716, 795] payload_rows_per_dst=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] dup_rows_per_dst=[2970, 2978, 3034, 3034, 2966, 3006, 3094, 2920] combine_rows_per_src=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=64894 dup_rows=23960 dup_rows_remote=20904 dedup_ratio=0.243734 router_tokens_per_expert=[780, 791, 777, 730, 773, 816, 765, 816, 790, 788, 723, 742, 732, 769, 771, 754, 781, 747, 767, 781, 767, 718, 825, 770, 807, 740, 819, 805, 734, 792, 756, 800, 755, 730, 772, 776, 737, 762, 726, 742, 802, 764, 732, 761, 804, 777, 804, 795, 726, 835, 796, 799, 749, 743, 728, 786, 732, 744, 787, 722, 737, 755, 777, 771] payload_rows_per_dst=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] dup_rows_per_dst=[3134, 2892, 2988, 3056, 2878, 3032, 2994, 2986] combine_rows_per_src=[9362, 9246, 9324, 9450, 9122, 9446, 9330, 9064] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64840 dup_rows=24022 dup_rows_remote=21010 dedup_ratio=0.244364 router_tokens_per_expert=[763, 829, 753, 768, 761, 800, 795, 749, 745, 770, 781, 782, 784, 770, 759, 790, 769, 764, 764, 727, 739, 798, 775, 758, 747, 761, 765, 777, 783, 776, 766, 771, 787, 825, 766, 776, 775, 757, 741, 802, 749, 784, 756, 756, 767, 724, 853, 728, 804, 756, 747, 773, 761, 808, 736, 735, 777, 781, 763, 748, 734, 707, 777, 760] payload_rows_per_dst=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] dup_rows_per_dst=[3012, 3246, 3034, 2992, 3066, 2868, 2922, 2882] combine_rows_per_src=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74178 payload_rows_remote=64804 dup_rows=24126 dup_rows_remote=20950 dedup_ratio=0.245422 router_tokens_per_expert=[769, 773, 756, 727, 800, 780, 822, 731, 731, 736, 781, 827, 781, 782, 742, 814, 753, 773, 780, 775, 749, 802, 754, 772, 757, 779, 760, 769, 829, 799, 740, 739, 764, 758, 726, 802, 760, 743, 795, 738, 767, 774, 756, 750, 738, 778, 806, 786, 776, 794, 735, 809, 729, 766, 779, 773, 753, 778, 763, 667, 765, 805, 796, 741] payload_rows_per_dst=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] dup_rows_per_dst=[3030, 2722, 3050, 2866, 3284, 3046, 2952, 3176] combine_rows_per_src=[9286, 8998, 9190, 9090, 9556, 9302, 9382, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74282 payload_rows_remote=64840 dup_rows=24022 dup_rows_remote=21010 dedup_ratio=0.244364 router_tokens_per_expert=[763, 829, 753, 768, 761, 800, 795, 749, 745, 770, 781, 782, 784, 770, 759, 790, 769, 764, 764, 727, 739, 798, 775, 758, 747, 761, 765, 777, 783, 776, 766, 771, 787, 825, 766, 776, 775, 757, 741, 802, 749, 784, 756, 756, 767, 724, 853, 728, 804, 756, 747, 773, 761, 808, 736, 735, 777, 781, 763, 748, 734, 707, 777, 760] payload_rows_per_dst=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] dup_rows_per_dst=[3012, 3246, 3034, 2992, 3066, 2868, 2922, 2882] combine_rows_per_src=[9442, 9306, 9136, 9300, 9392, 9176, 9318, 9212] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343536 combine_payload_bytes=265977856 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266321392 ingress_bytes=404389888 link_bytes=404389888 token_bytes=532299248 weight_movement_bytes=138412032 total_nvlink_bytes=670711280 weight_to_token_ratio=0.260027 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521686592e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=65032 dup_rows=24068 dup_rows_remote=21190 dedup_ratio=0.244832 router_tokens_per_expert=[806, 786, 783, 741, 746, 729, 749, 763, 807, 802, 830, 745, 722, 758, 767, 779, 801, 811, 804, 744, 778, 772, 735, 763, 718, 744, 742, 741, 741, 790, 793, 759, 764, 792, 762, 750, 756, 747, 798, 809, 795, 782, 730, 792, 798, 786, 764, 759, 758, 729, 774, 781, 757, 725, 773, 744, 794, 781, 778, 770, 769, 742, 787, 757] payload_rows_per_dst=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] dup_rows_per_dst=[2954, 3116, 3020, 2862, 3084, 3108, 2878, 3046] combine_rows_per_src=[9252, 9304, 9396, 9194, 9272, 9304, 9204, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344728 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267715224 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085720 weight_movement_bytes=34603008 total_nvlink_bytes=569688728 weight_to_token_ratio=0.064668 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343192 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266149016 ingress_bytes=335011840 link_bytes=335011840 token_bytes=531954840 weight_movement_bytes=69206016 total_nvlink_bytes=601160856 weight_to_token_ratio=0.130098 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260622583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343968 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267222944 ingress_bytes=336084992 link_bytes=336084992 token_bytes=534101920 weight_movement_bytes=69206016 total_nvlink_bytes=603307936 weight_to_token_ratio=0.129575 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264660768e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344208 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821776 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299344 weight_movement_bytes=34603008 total_nvlink_bytes=567902352 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343400 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928040 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531512680 weight_movement_bytes=34603008 total_nvlink_bytes=566115688 weight_to_token_ratio=0.065103 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343768 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362584 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381400 weight_movement_bytes=69206016 total_nvlink_bytes=601587416 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343192 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266149016 ingress_bytes=335011840 link_bytes=335011840 token_bytes=531954840 weight_movement_bytes=69206016 total_nvlink_bytes=601160856 weight_to_token_ratio=0.130098 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260622583e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265437184 dispatch_meta_bytes=343016 combine_payload_bytes=265437184 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265780200 ingress_bytes=300040192 link_bytes=300040192 token_bytes=531217384 weight_movement_bytes=34603008 total_nvlink_bytes=565820392 weight_to_token_ratio=0.065139 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129027087e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64946 dup_rows=24002 dup_rows_remote=20996 dedup_ratio=0.244161 router_tokens_per_expert=[755, 763, 790, 783, 711, 750, 758, 762, 762, 771, 806, 734, 817, 774, 779, 727, 799, 770, 758, 758, 779, 810, 769, 768, 791, 773, 734, 790, 733, 764, 735, 787, 775, 774, 760, 781, 777, 739, 755, 700, 781, 773, 757, 786, 776, 746, 743, 819, 804, 783, 804, 780, 796, 778, 779, 742, 793, 777, 748, 757, 730, 768, 716, 795] payload_rows_per_dst=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] dup_rows_per_dst=[2970, 2978, 3034, 3034, 2966, 3006, 3094, 2920] combine_rows_per_src=[9174, 9362, 9388, 9180, 9156, 9356, 9438, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=64936 dup_rows=23992 dup_rows_remote=20948 dedup_ratio=0.244059 router_tokens_per_expert=[757, 774, 764, 773, 734, 760, 750, 724, 730, 839, 794, 762, 755, 755, 774, 778, 771, 798, 776, 787, 760, 767, 757, 794, 720, 761, 774, 765, 794, 790, 781, 818, 782, 750, 772, 790, 755, 795, 774, 754, 767, 731, 767, 812, 732, 774, 795, 752, 744, 746, 703, 762, 758, 808, 799, 737, 750, 806, 781, 797, 787, 734, 768, 734] payload_rows_per_dst=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] dup_rows_per_dst=[2866, 3078, 3044, 3094, 2962, 3002, 2970, 2976] combine_rows_per_src=[9206, 9296, 9376, 9312, 9382, 9258, 9144, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343400 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928040 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531512680 weight_movement_bytes=34603008 total_nvlink_bytes=566115688 weight_to_token_ratio=0.065103 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344888 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266715960 ingress_bytes=335577088 link_bytes=335577088 token_bytes=533087032 weight_movement_bytes=69206016 total_nvlink_bytes=602293048 weight_to_token_ratio=0.129821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262749566e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65074 dup_rows=23930 dup_rows_remote=20886 dedup_ratio=0.243429 router_tokens_per_expert=[800, 796, 781, 797, 749, 758, 755, 746, 801, 759, 713, 758, 724, 773, 797, 798, 761, 772, 818, 769, 744, 783, 731, 794, 759, 744, 784, 780, 760, 808, 779, 771, 746, 760, 757, 812, 779, 744, 744, 799, 761, 824, 775, 772, 789, 746, 762, 803, 736, 731, 750, 787, 739, 751, 800, 730, 724, 764, 732, 770, 750, 804, 804, 745] payload_rows_per_dst=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] dup_rows_per_dst=[3046, 3016, 3044, 2956, 3004, 3048, 2890, 2926] combine_rows_per_src=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65224 dup_rows=23866 dup_rows_remote=20810 dedup_ratio=0.242778 router_tokens_per_expert=[787, 792, 805, 793, 778, 773, 777, 790, 730, 742, 778, 763, 754, 801, 796, 723, 817, 731, 778, 773, 794, 765, 767, 782, 763, 744, 789, 753, 808, 741, 823, 730, 826, 743, 774, 777, 739, 728, 773, 775, 754, 778, 765, 756, 811, 770, 760, 746, 781, 767, 737, 748, 782, 743, 727, 721, 730, 795, 740, 797, 738, 822, 733, 776] payload_rows_per_dst=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] dup_rows_per_dst=[3106, 2896, 3042, 2950, 3056, 3008, 2884, 2924] combine_rows_per_src=[9484, 9278, 9372, 9352, 9214, 9272, 9128, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21038 dedup_ratio=0.244344 router_tokens_per_expert=[722, 762, 790, 793, 784, 750, 751, 801, 744, 751, 760, 796, 744, 763, 733, 785, 787, 785, 815, 818, 783, 780, 806, 763, 812, 723, 739, 770, 775, 822, 735, 692, 785, 768, 770, 744, 823, 779, 767, 761, 762, 734, 815, 761, 755, 806, 773, 799, 704, 741, 769, 790, 756, 750, 727, 759, 777, 755, 720, 802, 746, 749, 760, 811] payload_rows_per_dst=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] dup_rows_per_dst=[2988, 2992, 3242, 2982, 3012, 3090, 2874, 2840] combine_rows_per_src=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65038 dup_rows=24046 dup_rows_remote=21098 dedup_ratio=0.244609 router_tokens_per_expert=[749, 708, 738, 762, 742, 774, 763, 808, 772, 778, 729, 776, 794, 713, 760, 762, 709, 793, 779, 788, 797, 796, 721, 758, 799, 755, 773, 772, 769, 731, 769, 748, 711, 784, 776, 826, 714, 731, 814, 801, 808, 801, 755, 796, 805, 753, 728, 775, 802, 762, 800, 821, 752, 773, 821, 779, 753, 785, 752, 754, 764, 775, 777, 719] payload_rows_per_dst=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] dup_rows_per_dst=[2874, 2948, 2954, 3004, 3104, 3102, 3136, 2924] combine_rows_per_src=[9214, 9220, 9328, 9228, 9210, 9340, 9484, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64972 dup_rows=24006 dup_rows_remote=20966 dedup_ratio=0.244202 router_tokens_per_expert=[794, 751, 769, 777, 779, 726, 791, 814, 812, 783, 760, 718, 742, 753, 742, 763, 758, 786, 759, 756, 789, 743, 796, 775, 765, 777, 785, 827, 793, 791, 766, 743, 751, 779, 733, 744, 725, 748, 733, 793, 794, 735, 805, 731, 757, 783, 709, 767, 773, 759, 756, 775, 816, 756, 796, 752, 738, 803, 753, 832, 764, 770, 781, 758] payload_rows_per_dst=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] dup_rows_per_dst=[3056, 3008, 3050, 3094, 2840, 2862, 3040, 3056] combine_rows_per_src=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=343768 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362584 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381400 weight_movement_bytes=69206016 total_nvlink_bytes=601587416 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64980 dup_rows=24082 dup_rows_remote=21076 dedup_ratio=0.244975 router_tokens_per_expert=[737, 793, 757, 753, 757, 735, 749, 788, 741, 743, 809, 785, 758, 770, 738, 750, 762, 779, 754, 800, 744, 717, 797, 758, 802, 768, 805, 762, 776, 778, 764, 777, 814, 757, 774, 728, 741, 787, 765, 763, 748, 808, 763, 796, 824, 800, 780, 762, 727, 780, 755, 797, 758, 764, 762, 786, 773, 770, 746, 722, 771, 789, 760, 776] payload_rows_per_dst=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] dup_rows_per_dst=[3006, 3348, 3010, 2984, 2974, 2786, 2990, 2984] combine_rows_per_src=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=65144 dup_rows=23680 dup_rows_remote=20586 dedup_ratio=0.240885 router_tokens_per_expert=[797, 727, 824, 746, 724, 756, 771, 740, 711, 827, 756, 754, 771, 741, 745, 748, 768, 784, 751, 763, 753, 792, 741, 758, 766, 784, 731, 761, 793, 765, 793, 793, 766, 744, 757, 803, 784, 771, 779, 761, 779, 808, 769, 770, 790, 791, 764, 816, 738, 732, 768, 789, 797, 829, 735, 748, 781, 778, 793, 772, 776, 739, 785, 706] payload_rows_per_dst=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] dup_rows_per_dst=[2900, 2866, 2854, 3074, 3066, 3094, 2940, 2886] combine_rows_per_src=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=65130 dup_rows=24020 dup_rows_remote=21038 dedup_ratio=0.244344 router_tokens_per_expert=[722, 762, 790, 793, 784, 750, 751, 801, 744, 751, 760, 796, 744, 763, 733, 785, 787, 785, 815, 818, 783, 780, 806, 763, 812, 723, 739, 770, 775, 822, 735, 692, 785, 768, 770, 744, 823, 779, 767, 761, 762, 734, 815, 761, 755, 806, 773, 799, 704, 741, 769, 790, 756, 750, 727, 759, 777, 755, 720, 802, 746, 749, 760, 811] payload_rows_per_dst=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] dup_rows_per_dst=[2988, 2992, 3242, 2982, 3012, 3090, 2874, 2840] combine_rows_per_src=[9318, 9160, 9432, 9154, 9382, 9320, 9118, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64732 dup_rows=24164 dup_rows_remote=20986 dedup_ratio=0.245809 router_tokens_per_expert=[739, 796, 752, 788, 761, 812, 742, 790, 826, 783, 756, 783, 772, 804, 748, 706, 797, 791, 793, 751, 793, 787, 725, 741, 762, 762, 738, 734, 776, 756, 754, 738, 765, 792, 754, 743, 820, 715, 785, 793, 760, 751, 765, 774, 776, 779, 760, 742, 789, 759, 791, 756, 746, 755, 753, 792, 727, 775, 797, 794, 782, 740, 795, 771] payload_rows_per_dst=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] dup_rows_per_dst=[2888, 2980, 2806, 2924, 3112, 3048, 3228, 3178] combine_rows_per_src=[9248, 9376, 8986, 9116, 9332, 9166, 9508, 9408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343536 combine_payload_bytes=265977856 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266321392 ingress_bytes=404389888 link_bytes=404389888 token_bytes=532299248 weight_movement_bytes=138412032 total_nvlink_bytes=670711280 weight_to_token_ratio=0.260027 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521686592e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64980 dup_rows=24082 dup_rows_remote=21076 dedup_ratio=0.244975 router_tokens_per_expert=[737, 793, 757, 753, 757, 735, 749, 788, 741, 743, 809, 785, 758, 770, 738, 750, 762, 779, 754, 800, 744, 717, 797, 758, 802, 768, 805, 762, 776, 778, 764, 777, 814, 757, 774, 728, 741, 787, 765, 763, 748, 808, 763, 796, 824, 800, 780, 762, 727, 780, 755, 797, 758, 764, 762, 786, 773, 770, 746, 722, 771, 789, 760, 776] payload_rows_per_dst=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] dup_rows_per_dst=[3006, 3348, 3010, 2984, 2974, 2786, 2990, 2984] combine_rows_per_src=[9242, 9608, 9244, 9374, 9284, 8972, 9268, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344672 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267117152 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533889632 weight_movement_bytes=69206016 total_nvlink_bytes=603095648 weight_to_token_ratio=0.129626 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343840 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266886944 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430048 weight_movement_bytes=103809024 total_nvlink_bytes=637239072 weight_to_token_ratio=0.194607 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344136 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267501640 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659144 weight_movement_bytes=34603008 total_nvlink_bytes=569262152 weight_to_token_ratio=0.064720 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344224 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502304 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660384 weight_movement_bytes=69206016 total_nvlink_bytes=601866400 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65058 dup_rows=23908 dup_rows_remote=20994 dedup_ratio=0.243205 router_tokens_per_expert=[747, 804, 759, 761, 711, 777, 753, 762, 817, 735, 786, 759, 813, 764, 763, 769, 745, 742, 797, 778, 766, 842, 760, 761, 749, 839, 809, 746, 779, 780, 772, 753, 771, 829, 754, 744, 774, 701, 782, 774, 781, 755, 757, 716, 786, 798, 742, 768, 762, 729, 760, 786, 755, 762, 748, 824, 794, 783, 762, 751, 767, 749, 765, 725] payload_rows_per_dst=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] dup_rows_per_dst=[2958, 3064, 3050, 3130, 2926, 2924, 2914, 2942] combine_rows_per_src=[9190, 9348, 9332, 9324, 9332, 9282, 9338, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344544 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266740192 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135840 weight_movement_bytes=34603008 total_nvlink_bytes=567738848 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469064 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594376 weight_movement_bytes=34603008 total_nvlink_bytes=567197384 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=342920 combine_payload_bytes=266829824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267172744 ingress_bytes=370638848 link_bytes=370638848 token_bytes=534002568 weight_movement_bytes=103809024 total_nvlink_bytes=637811592 weight_to_token_ratio=0.194398 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394684146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344672 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267117152 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533889632 weight_movement_bytes=69206016 total_nvlink_bytes=603095648 weight_to_token_ratio=0.129626 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=342872 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265485144 ingress_bytes=299745280 link_bytes=299745280 token_bytes=530627416 weight_movement_bytes=34603008 total_nvlink_bytes=565230424 weight_to_token_ratio=0.065211 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127917357e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74624 payload_rows_remote=65144 dup_rows=23680 dup_rows_remote=20586 dedup_ratio=0.240885 router_tokens_per_expert=[797, 727, 824, 746, 724, 756, 771, 740, 711, 827, 756, 754, 771, 741, 745, 748, 768, 784, 751, 763, 753, 792, 741, 758, 766, 784, 731, 761, 793, 765, 793, 793, 766, 744, 757, 803, 784, 771, 779, 761, 779, 808, 769, 770, 790, 791, 764, 816, 738, 732, 768, 789, 797, 829, 735, 748, 781, 778, 793, 772, 776, 739, 785, 706] payload_rows_per_dst=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] dup_rows_per_dst=[2900, 2866, 2854, 3074, 3066, 3094, 2940, 2886] combine_rows_per_src=[9270, 9240, 9366, 9298, 9264, 9480, 9332, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65074 dup_rows=23930 dup_rows_remote=20886 dedup_ratio=0.243429 router_tokens_per_expert=[800, 796, 781, 797, 749, 758, 755, 746, 801, 759, 713, 758, 724, 773, 797, 798, 761, 772, 818, 769, 744, 783, 731, 794, 759, 744, 784, 780, 760, 808, 779, 771, 746, 760, 757, 812, 779, 744, 744, 799, 761, 824, 775, 772, 789, 746, 762, 803, 736, 731, 750, 787, 739, 751, 800, 730, 724, 764, 732, 770, 750, 804, 804, 745] payload_rows_per_dst=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] dup_rows_per_dst=[3046, 3016, 3044, 2956, 3004, 3048, 2890, 2926] combine_rows_per_src=[9318, 9230, 9300, 9414, 9278, 9416, 9158, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64756 dup_rows=24268 dup_rows_remote=21190 dedup_ratio=0.246867 router_tokens_per_expert=[748, 753, 737, 779, 782, 728, 809, 757, 747, 773, 779, 761, 780, 790, 765, 775, 748, 752, 741, 765, 770, 768, 741, 741, 780, 747, 758, 772, 778, 774, 786, 784, 799, 789, 796, 770, 748, 776, 772, 788, 775, 707, 802, 767, 776, 789, 745, 743, 809, 764, 781, 770, 705, 769, 790, 793, 773, 781, 775, 731, 797, 769, 757, 778] payload_rows_per_dst=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] dup_rows_per_dst=[2958, 3014, 2968, 3078, 3096, 2986, 3120, 3048] combine_rows_per_src=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=344224 combine_payload_bytes=266158080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266502304 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532660384 weight_movement_bytes=69206016 total_nvlink_bytes=601866400 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344208 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266821776 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299344 weight_movement_bytes=34603008 total_nvlink_bytes=567902352 weight_to_token_ratio=0.064885 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65226 dup_rows=23776 dup_rows_remote=20872 dedup_ratio=0.241862 router_tokens_per_expert=[801, 731, 792, 783, 791, 756, 743, 802, 738, 793, 702, 746, 763, 815, 781, 810, 776, 780, 748, 791, 776, 755, 733, 744, 744, 809, 769, 783, 742, 768, 756, 787, 730, 784, 795, 755, 791, 764, 722, 777, 763, 745, 762, 786, 724, 783, 770, 782, 752, 853, 783, 772, 759, 800, 773, 740, 757, 741, 747, 774, 794, 747, 764, 755] payload_rows_per_dst=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] dup_rows_per_dst=[3014, 3046, 2904, 3012, 2914, 2934, 3094, 2858] combine_rows_per_src=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65044 dup_rows=23918 dup_rows_remote=20904 dedup_ratio=0.243306 router_tokens_per_expert=[746, 750, 745, 770, 811, 752, 794, 757, 680, 811, 780, 788, 748, 781, 715, 749, 741, 745, 745, 724, 762, 781, 781, 809, 726, 766, 752, 767, 786, 735, 748, 772, 789, 800, 779, 758, 777, 735, 764, 776, 783, 788, 739, 795, 785, 756, 774, 761, 797, 766, 786, 793, 777, 793, 764, 786, 784, 781, 754, 814, 777, 769, 786, 749] payload_rows_per_dst=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] dup_rows_per_dst=[2980, 2852, 2968, 2836, 3014, 3102, 3102, 3064] combine_rows_per_src=[9270, 9252, 9208, 9268, 9342, 9260, 9422, 9364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=342920 combine_payload_bytes=266829824 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267172744 ingress_bytes=370638848 link_bytes=370638848 token_bytes=534002568 weight_movement_bytes=103809024 total_nvlink_bytes=637811592 weight_to_token_ratio=0.194398 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394684146e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65212 dup_rows=23840 dup_rows_remote=20792 dedup_ratio=0.242513 router_tokens_per_expert=[804, 751, 820, 790, 745, 764, 763, 757, 775, 759, 760, 782, 767, 782, 779, 746, 813, 754, 741, 770, 752, 785, 741, 744, 751, 785, 737, 798, 780, 763, 793, 754, 753, 751, 749, 760, 745, 715, 807, 772, 747, 785, 770, 752, 740, 752, 751, 768, 798, 781, 770, 758, 764, 753, 812, 707, 852, 800, 778, 748, 761, 795, 790, 763] payload_rows_per_dst=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] dup_rows_per_dst=[2992, 3048, 2874, 2998, 2922, 2972, 2866, 3168] combine_rows_per_src=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65178 dup_rows=23854 dup_rows_remote=20826 dedup_ratio=0.242655 router_tokens_per_expert=[723, 762, 763, 777, 813, 756, 791, 802, 745, 721, 759, 816, 747, 760, 803, 793, 734, 786, 792, 832, 796, 727, 778, 739, 752, 747, 785, 783, 757, 769, 795, 745, 767, 776, 792, 738, 748, 756, 783, 760, 762, 762, 777, 787, 738, 783, 771, 770, 759, 742, 737, 739, 737, 752, 783, 771, 763, 778, 757, 751, 809, 795, 790, 771] payload_rows_per_dst=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] dup_rows_per_dst=[3044, 3078, 2950, 2958, 2944, 3028, 2850, 3002] combine_rows_per_src=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64756 dup_rows=24268 dup_rows_remote=21190 dedup_ratio=0.246867 router_tokens_per_expert=[748, 753, 737, 779, 782, 728, 809, 757, 747, 773, 779, 761, 780, 790, 765, 775, 748, 752, 741, 765, 770, 768, 741, 741, 780, 747, 758, 772, 778, 774, 786, 784, 799, 789, 796, 770, 748, 776, 772, 788, 775, 707, 802, 767, 776, 789, 745, 743, 809, 764, 781, 770, 705, 769, 790, 793, 773, 781, 775, 731, 797, 769, 757, 778] payload_rows_per_dst=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] dup_rows_per_dst=[2958, 3014, 2968, 3078, 3096, 2986, 3120, 3048] combine_rows_per_src=[9228, 9326, 9084, 9280, 9380, 9222, 9242, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65110 dup_rows=23976 dup_rows_remote=20946 dedup_ratio=0.243896 router_tokens_per_expert=[700, 779, 768, 761, 770, 788, 786, 772, 755, 751, 763, 753, 777, 725, 762, 777, 755, 781, 764, 791, 783, 735, 719, 728, 809, 715, 752, 802, 753, 758, 870, 738, 802, 692, 763, 785, 742, 787, 785, 788, 798, 766, 753, 734, 849, 801, 766, 755, 753, 784, 719, 743, 755, 813, 833, 768, 788, 774, 764, 757, 769, 747, 778, 801] payload_rows_per_dst=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] dup_rows_per_dst=[3030, 2874, 3236, 3092, 3136, 3118, 2768, 2722] combine_rows_per_src=[9218, 9252, 9510, 9558, 9376, 9326, 9088, 9000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74150 payload_rows_remote=64864 dup_rows=24154 dup_rows_remote=21072 dedup_ratio=0.245707 router_tokens_per_expert=[751, 766, 779, 797, 790, 819, 813, 757, 769, 766, 769, 753, 740, 751, 728, 768, 772, 764, 777, 758, 712, 780, 799, 797, 755, 779, 724, 806, 748, 720, 734, 758, 782, 754, 773, 758, 765, 742, 791, 778, 761, 749, 735, 727, 756, 775, 764, 768, 765, 863, 793, 784, 772, 764, 810, 750, 767, 761, 791, 783, 743, 764, 787, 778] payload_rows_per_dst=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] dup_rows_per_dst=[2968, 3072, 3030, 2878, 2964, 2956, 3204, 3082] combine_rows_per_src=[9126, 9350, 9288, 9170, 9322, 9146, 9462, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65140 dup_rows=23936 dup_rows_remote=21058 dedup_ratio=0.243490 router_tokens_per_expert=[778, 732, 780, 804, 771, 810, 731, 777, 742, 773, 779, 823, 741, 790, 782, 784, 806, 776, 733, 755, 755, 759, 752, 783, 762, 758, 802, 809, 764, 781, 735, 709, 776, 761, 805, 736, 747, 729, 743, 750, 768, 795, 762, 840, 785, 765, 743, 806, 752, 768, 777, 765, 753, 797, 735, 706, 775, 746, 763, 762, 787, 780, 755, 784] payload_rows_per_dst=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] dup_rows_per_dst=[2990, 3000, 3050, 3034, 2980, 3002, 2878, 3002] combine_rows_per_src=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65212 dup_rows=23840 dup_rows_remote=20792 dedup_ratio=0.242513 router_tokens_per_expert=[804, 751, 820, 790, 745, 764, 763, 757, 775, 759, 760, 782, 767, 782, 779, 746, 813, 754, 741, 770, 752, 785, 741, 744, 751, 785, 737, 798, 780, 763, 793, 754, 753, 751, 749, 760, 745, 715, 807, 772, 747, 785, 770, 752, 740, 752, 751, 768, 798, 781, 770, 758, 764, 753, 812, 707, 852, 800, 778, 748, 761, 795, 790, 763] payload_rows_per_dst=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] dup_rows_per_dst=[2992, 3048, 2874, 2998, 2922, 2972, 2866, 3168] combine_rows_per_src=[9396, 9252, 9326, 9324, 9182, 9158, 9420, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343840 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266886944 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430048 weight_movement_bytes=103809024 total_nvlink_bytes=637239072 weight_to_token_ratio=0.194607 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344392 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267510088 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675784 weight_movement_bytes=34603008 total_nvlink_bytes=569278792 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64972 dup_rows=24006 dup_rows_remote=20966 dedup_ratio=0.244202 router_tokens_per_expert=[794, 751, 769, 777, 779, 726, 791, 814, 812, 783, 760, 718, 742, 753, 742, 763, 758, 786, 759, 756, 789, 743, 796, 775, 765, 777, 785, 827, 793, 791, 766, 743, 751, 779, 733, 744, 725, 748, 733, 793, 794, 735, 805, 731, 757, 783, 709, 767, 773, 759, 756, 775, 816, 756, 796, 752, 738, 803, 753, 832, 764, 770, 781, 758] payload_rows_per_dst=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] dup_rows_per_dst=[3056, 3008, 3050, 3094, 2840, 2862, 3040, 3056] combine_rows_per_src=[9346, 9138, 9274, 9400, 9172, 9300, 9326, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343792 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764016 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184240 weight_movement_bytes=34603008 total_nvlink_bytes=567787248 weight_to_token_ratio=0.064899 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344224 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267034784 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725344 weight_movement_bytes=138412032 total_nvlink_bytes=672137376 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343744 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026688 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709632 weight_movement_bytes=69206016 total_nvlink_bytes=600915648 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344792 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267158232 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533971672 weight_movement_bytes=103809024 total_nvlink_bytes=637780696 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=343784 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584360 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530824936 weight_movement_bytes=34603008 total_nvlink_bytes=565427944 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267108352 dispatch_meta_bytes=344016 combine_payload_bytes=267108352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267452368 ingress_bytes=301711360 link_bytes=301711360 token_bytes=534560720 weight_movement_bytes=34603008 total_nvlink_bytes=569163728 weight_to_token_ratio=0.064732 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135315558e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65178 dup_rows=23854 dup_rows_remote=20826 dedup_ratio=0.242655 router_tokens_per_expert=[723, 762, 763, 777, 813, 756, 791, 802, 745, 721, 759, 816, 747, 760, 803, 793, 734, 786, 792, 832, 796, 727, 778, 739, 752, 747, 785, 783, 757, 769, 795, 745, 767, 776, 792, 738, 748, 756, 783, 760, 762, 762, 777, 787, 738, 783, 771, 770, 759, 742, 737, 739, 737, 752, 783, 771, 763, 778, 757, 751, 809, 795, 790, 771] payload_rows_per_dst=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] dup_rows_per_dst=[3044, 3078, 2950, 2958, 2944, 3028, 2850, 3002] combine_rows_per_src=[9330, 9210, 9418, 9308, 9296, 9272, 9190, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344016 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313104 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282192 weight_movement_bytes=34603008 total_nvlink_bytes=568885200 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343744 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026688 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709632 weight_movement_bytes=69206016 total_nvlink_bytes=600915648 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266690560 dispatch_meta_bytes=344224 combine_payload_bytes=266690560 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267034784 ingress_bytes=405102592 link_bytes=405102592 token_bytes=533725344 weight_movement_bytes=138412032 total_nvlink_bytes=672137376 weight_to_token_ratio=0.259332 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524368440e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343752 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469064 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594376 weight_movement_bytes=34603008 total_nvlink_bytes=567197384 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64986 dup_rows=24162 dup_rows_remote=21220 dedup_ratio=0.245789 router_tokens_per_expert=[801, 785, 789, 779, 756, 784, 777, 750, 772, 762, 743, 799, 715, 767, 755, 756, 707, 766, 748, 724, 798, 768, 740, 798, 749, 781, 764, 786, 787, 744, 738, 776, 767, 762, 772, 798, 806, 794, 741, 772, 813, 743, 772, 788, 776, 745, 799, 783, 748, 760, 687, 798, 798, 742, 794, 707, 751, 782, 800, 821, 781, 740, 794, 754] payload_rows_per_dst=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] dup_rows_per_dst=[3096, 2912, 2942, 2980, 3154, 3112, 2924, 3042] combine_rows_per_src=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65162 dup_rows=23930 dup_rows_remote=20950 dedup_ratio=0.243429 router_tokens_per_expert=[720, 747, 743, 772, 790, 746, 768, 800, 735, 777, 758, 780, 803, 719, 721, 816, 762, 762, 793, 806, 790, 781, 785, 773, 752, 793, 778, 719, 775, 780, 762, 737, 795, 778, 761, 784, 755, 803, 772, 764, 765, 767, 745, 743, 762, 791, 746, 771, 823, 747, 765, 829, 782, 740, 787, 756, 754, 744, 767, 792, 769, 773, 737, 742] payload_rows_per_dst=[9258, 9230, 9448, 9212, 9366, 9266, 9366, 9228] dup_rows_per_dst=[2914, 2988, 3056, 2980, 3058, 2914, 3092, 2928] combine_rows_per_src=[9258, 9230, 9448, 9212, 9366, 9266, 9366, 9228] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=65030 dup_rows=24014 dup_rows_remote=21036 dedup_ratio=0.244283 router_tokens_per_expert=[717, 746, 784, 770, 768, 781, 757, 763, 793, 742, 735, 717, 712, 746, 786, 772, 746, 764, 826, 794, 714, 758, 767, 731, 793, 772, 745, 786, 791, 807, 772, 810, 766, 721, 780, 816, 802, 771, 746, 717, 799, 781, 799, 738, 775, 772, 767, 792, 760, 763, 796, 789, 805, 783, 807, 697, 773, 749, 741, 784, 781, 794, 767, 756] payload_rows_per_dst=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] dup_rows_per_dst=[2924, 2860, 2928, 3246, 2978, 3038, 3080, 2960] combine_rows_per_src=[9248, 9146, 9272, 9306, 9260, 9408, 9320, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65226 dup_rows=23776 dup_rows_remote=20872 dedup_ratio=0.241862 router_tokens_per_expert=[801, 731, 792, 783, 791, 756, 743, 802, 738, 793, 702, 746, 763, 815, 781, 810, 776, 780, 748, 791, 776, 755, 733, 744, 744, 809, 769, 783, 742, 768, 756, 787, 730, 784, 795, 755, 791, 764, 722, 777, 763, 745, 762, 786, 724, 783, 770, 782, 752, 853, 783, 772, 759, 800, 773, 740, 757, 741, 747, 774, 794, 747, 764, 755] payload_rows_per_dst=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] dup_rows_per_dst=[3014, 3046, 2904, 3012, 2914, 2934, 3094, 2858] combine_rows_per_src=[9384, 9250, 9302, 9304, 9322, 9296, 9370, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344016 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313104 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282192 weight_movement_bytes=34603008 total_nvlink_bytes=568885200 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64676 dup_rows=24074 dup_rows_remote=20730 dedup_ratio=0.244893 router_tokens_per_expert=[823, 756, 755, 777, 754, 828, 779, 745, 772, 747, 788, 748, 797, 783, 750, 767, 784, 761, 795, 757, 764, 766, 781, 780, 750, 735, 740, 761, 733, 757, 755, 760, 769, 743, 764, 769, 753, 784, 799, 777, 745, 755, 729, 772, 780, 748, 773, 757, 769, 803, 740, 746, 789, 748, 789, 786, 727, 784, 823, 798, 771, 814, 743, 757] payload_rows_per_dst=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] dup_rows_per_dst=[3344, 2962, 3044, 2924, 3072, 2810, 2918, 3000] combine_rows_per_src=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65058 dup_rows=24006 dup_rows_remote=21026 dedup_ratio=0.244202 router_tokens_per_expert=[797, 804, 773, 822, 788, 801, 771, 769, 713, 776, 758, 706, 820, 747, 770, 754, 773, 756, 754, 741, 760, 763, 827, 727, 740, 788, 752, 760, 823, 781, 776, 735, 784, 753, 761, 780, 765, 797, 746, 793, 730, 769, 730, 799, 771, 771, 789, 751, 740, 837, 766, 742, 728, 771, 748, 798, 788, 733, 722, 783, 813, 753, 758, 758] payload_rows_per_dst=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] dup_rows_per_dst=[3160, 2840, 2912, 3010, 3130, 2980, 2978, 2996] combine_rows_per_src=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65126 dup_rows=24034 dup_rows_remote=21052 dedup_ratio=0.244486 router_tokens_per_expert=[778, 795, 824, 806, 765, 763, 761, 740, 765, 786, 765, 785, 726, 768, 724, 744, 772, 767, 810, 800, 757, 766, 806, 761, 727, 795, 761, 758, 712, 796, 825, 747, 765, 766, 772, 767, 763, 770, 763, 726, 708, 751, 821, 782, 772, 804, 761, 792, 718, 720, 714, 788, 810, 783, 730, 775, 766, 805, 767, 807, 762, 717, 766, 786] payload_rows_per_dst=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] dup_rows_per_dst=[3072, 2982, 3166, 2974, 2944, 3010, 2908, 2978] combine_rows_per_src=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344824 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266527480 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532710136 weight_movement_bytes=69206016 total_nvlink_bytes=601916152 weight_to_token_ratio=0.129913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64882 dup_rows=24138 dup_rows_remote=21086 dedup_ratio=0.245544 router_tokens_per_expert=[742, 756, 762, 775, 795, 709, 783, 780, 782, 759, 753, 761, 755, 761, 742, 770, 772, 770, 797, 746, 816, 737, 746, 771, 786, 791, 801, 776, 729, 804, 739, 787, 800, 752, 780, 732, 727, 814, 742, 719, 793, 779, 785, 745, 755, 777, 713, 801, 818, 777, 729, 776, 813, 806, 740, 796, 779, 693, 788, 732, 789, 771, 793, 785] payload_rows_per_dst=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] dup_rows_per_dst=[2452, 3478, 2902, 2990, 3070, 3042, 3152, 3052] combine_rows_per_src=[8830, 9610, 9244, 9360, 9226, 9254, 9358, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=65126 dup_rows=24034 dup_rows_remote=21052 dedup_ratio=0.244486 router_tokens_per_expert=[778, 795, 824, 806, 765, 763, 761, 740, 765, 786, 765, 785, 726, 768, 724, 744, 772, 767, 810, 800, 757, 766, 806, 761, 727, 795, 761, 758, 712, 796, 825, 747, 765, 766, 772, 767, 763, 770, 763, 726, 708, 751, 821, 782, 772, 804, 761, 792, 718, 720, 714, 788, 810, 783, 730, 775, 766, 805, 767, 807, 762, 717, 766, 786] payload_rows_per_dst=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] dup_rows_per_dst=[3072, 2982, 3166, 2974, 2944, 3010, 2908, 2978] combine_rows_per_src=[9392, 9144, 9312, 9268, 9240, 9372, 9168, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64696 dup_rows=24230 dup_rows_remote=21168 dedup_ratio=0.246480 router_tokens_per_expert=[739, 762, 812, 797, 759, 777, 724, 742, 791, 760, 813, 741, 736, 782, 758, 787, 753, 778, 741, 774, 778, 766, 773, 773, 794, 781, 788, 776, 764, 759, 778, 764, 748, 783, 747, 750, 772, 727, 790, 715, 724, 777, 767, 785, 739, 767, 790, 786, 771, 791, 751, 760, 811, 795, 781, 760, 788, 738, 773, 747, 772, 784, 772, 771] payload_rows_per_dst=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] dup_rows_per_dst=[2964, 3090, 3038, 3036, 2928, 3010, 3062, 3102] combine_rows_per_src=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64676 dup_rows=24074 dup_rows_remote=20730 dedup_ratio=0.244893 router_tokens_per_expert=[823, 756, 755, 777, 754, 828, 779, 745, 772, 747, 788, 748, 797, 783, 750, 767, 784, 761, 795, 757, 764, 766, 781, 780, 750, 735, 740, 761, 733, 757, 755, 760, 769, 743, 764, 769, 753, 784, 799, 777, 745, 755, 729, 772, 780, 748, 773, 757, 769, 803, 740, 746, 789, 748, 789, 786, 727, 784, 823, 798, 771, 814, 743, 757] payload_rows_per_dst=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] dup_rows_per_dst=[3344, 2962, 3044, 2924, 3072, 2810, 2918, 3000] combine_rows_per_src=[9554, 9342, 9360, 9076, 9244, 9262, 9174, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65140 dup_rows=23936 dup_rows_remote=21058 dedup_ratio=0.243490 router_tokens_per_expert=[778, 732, 780, 804, 771, 810, 731, 777, 742, 773, 779, 823, 741, 790, 782, 784, 806, 776, 733, 755, 755, 759, 752, 783, 762, 758, 802, 809, 764, 781, 735, 709, 776, 761, 805, 736, 747, 729, 743, 750, 768, 795, 762, 840, 785, 765, 743, 806, 752, 768, 777, 765, 753, 797, 735, 706, 775, 746, 763, 762, 787, 780, 755, 784] payload_rows_per_dst=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] dup_rows_per_dst=[2990, 3000, 3050, 3034, 2980, 3002, 2878, 3002] combine_rows_per_src=[9376, 9428, 9188, 9206, 9114, 9526, 9228, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=344448 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267248000 ingress_bytes=336109568 link_bytes=336109568 token_bytes=534151552 weight_movement_bytes=69206016 total_nvlink_bytes=603357568 weight_to_token_ratio=0.129563 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264753245e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344264 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707144 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070024 weight_movement_bytes=34603008 total_nvlink_bytes=567673032 weight_to_token_ratio=0.064913 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344392 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267510088 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675784 weight_movement_bytes=34603008 total_nvlink_bytes=569278792 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=341624 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265254520 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530167416 weight_movement_bytes=69206016 total_nvlink_bytes=599373432 weight_to_token_ratio=0.130536 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343872 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100544 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857216 weight_movement_bytes=69206016 total_nvlink_bytes=601063232 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266756096 dispatch_meta_bytes=344712 combine_payload_bytes=266756096 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267100808 ingress_bytes=301359104 link_bytes=301359104 token_bytes=533856904 weight_movement_bytes=34603008 total_nvlink_bytes=568459912 weight_to_token_ratio=0.064817 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133990047e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65058 dup_rows=24006 dup_rows_remote=21026 dedup_ratio=0.244202 router_tokens_per_expert=[797, 804, 773, 822, 788, 801, 771, 769, 713, 776, 758, 706, 820, 747, 770, 754, 773, 756, 754, 741, 760, 763, 827, 727, 740, 788, 752, 760, 823, 781, 776, 735, 784, 753, 761, 780, 765, 797, 746, 793, 730, 769, 730, 799, 771, 771, 789, 751, 740, 837, 766, 742, 728, 771, 748, 798, 788, 733, 722, 783, 813, 753, 758, 758] payload_rows_per_dst=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] dup_rows_per_dst=[3160, 2840, 2912, 3010, 3130, 2980, 2978, 2996] combine_rows_per_src=[9490, 9248, 9290, 9300, 9228, 9240, 9282, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65274 dup_rows=23860 dup_rows_remote=20956 dedup_ratio=0.242716 router_tokens_per_expert=[780, 741, 800, 728, 777, 760, 763, 782, 772, 734, 763, 784, 769, 745, 785, 755, 799, 739, 807, 776, 791, 759, 815, 755, 732, 787, 711, 730, 816, 763, 781, 783, 740, 765, 757, 730, 752, 738, 775, 780, 764, 779, 782, 797, 770, 773, 752, 771, 727, 780, 761, 745, 775, 767, 770, 761, 783, 769, 756, 796, 817, 791, 773, 774] payload_rows_per_dst=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] dup_rows_per_dst=[2990, 2942, 3012, 2856, 2904, 3018, 2948, 3190] combine_rows_per_src=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343872 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266100544 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857216 weight_movement_bytes=69206016 total_nvlink_bytes=601063232 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=341624 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265254520 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530167416 weight_movement_bytes=69206016 total_nvlink_bytes=599373432 weight_to_token_ratio=0.130536 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344792 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267158232 ingress_bytes=370622464 link_bytes=370622464 token_bytes=533971672 weight_movement_bytes=103809024 total_nvlink_bytes=637780696 weight_to_token_ratio=0.194409 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394622494e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65208 dup_rows=23808 dup_rows_remote=20784 dedup_ratio=0.242188 router_tokens_per_expert=[784, 728, 756, 779, 762, 736, 775, 758, 733, 832, 771, 745, 744, 772, 746, 814, 770, 733, 765, 814, 796, 734, 788, 756, 779, 742, 729, 734, 800, 799, 781, 762, 790, 771, 749, 746, 741, 781, 791, 722, 798, 772, 738, 762, 806, 781, 750, 813, 737, 763, 763, 765, 777, 790, 744, 778, 817, 791, 803, 743, 749, 725, 782, 797] payload_rows_per_dst=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] dup_rows_per_dst=[2934, 2988, 3024, 2936, 2924, 3118, 2902, 2982] combine_rows_per_src=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65148 dup_rows=23822 dup_rows_remote=20936 dedup_ratio=0.242330 router_tokens_per_expert=[740, 771, 745, 787, 747, 793, 790, 819, 782, 804, 756, 733, 757, 787, 757, 785, 718, 776, 769, 768, 745, 791, 777, 757, 797, 772, 781, 762, 772, 724, 728, 774, 772, 747, 787, 775, 745, 758, 751, 793, 752, 726, 797, 777, 773, 746, 800, 760, 765, 753, 854, 766, 740, 758, 783, 772, 777, 789, 784, 783, 773, 725, 765, 742] payload_rows_per_dst=[9378, 9294, 9234, 9334, 9332, 9278, 9356, 9276] dup_rows_per_dst=[3006, 3028, 2968, 2886, 2924, 2984, 3026, 3000] combine_rows_per_src=[9378, 9294, 9234, 9334, 9332, 9278, 9356, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65072 dup_rows=23986 dup_rows_remote=21006 dedup_ratio=0.243998 router_tokens_per_expert=[795, 765, 780, 754, 724, 760, 758, 762, 746, 758, 765, 769, 800, 804, 762, 805, 815, 762, 825, 772, 748, 809, 764, 765, 777, 721, 771, 748, 746, 744, 774, 776, 748, 776, 771, 705, 805, 801, 808, 775, 821, 757, 733, 744, 738, 783, 778, 716, 707, 740, 722, 726, 764, 751, 822, 764, 786, 804, 802, 767, 807, 787, 755, 765] payload_rows_per_dst=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] dup_rows_per_dst=[2980, 3114, 3282, 2870, 3060, 2852, 2902, 2926] combine_rows_per_src=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65250 dup_rows=23788 dup_rows_remote=20830 dedup_ratio=0.241984 router_tokens_per_expert=[787, 780, 811, 750, 768, 723, 801, 789, 735, 760, 786, 760, 754, 774, 746, 753, 756, 732, 760, 754, 772, 754, 745, 762, 802, 786, 760, 797, 777, 739, 734, 811, 761, 741, 758, 729, 765, 795, 794, 794, 808, 764, 790, 753, 748, 754, 753, 742, 774, 760, 713, 763, 802, 761, 816, 790, 787, 740, 806, 777, 828, 809, 710, 749] payload_rows_per_dst=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] dup_rows_per_dst=[3042, 2906, 2872, 3036, 2950, 2958, 3014, 3010] combine_rows_per_src=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65274 dup_rows=23860 dup_rows_remote=20956 dedup_ratio=0.242716 router_tokens_per_expert=[780, 741, 800, 728, 777, 760, 763, 782, 772, 734, 763, 784, 769, 745, 785, 755, 799, 739, 807, 776, 791, 759, 815, 755, 732, 787, 711, 730, 816, 763, 781, 783, 740, 765, 757, 730, 752, 738, 775, 780, 764, 779, 782, 797, 770, 773, 752, 771, 727, 780, 761, 745, 775, 767, 770, 761, 783, 769, 756, 796, 817, 791, 773, 774] payload_rows_per_dst=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] dup_rows_per_dst=[2990, 2942, 3012, 2856, 2904, 3018, 2948, 3190] combine_rows_per_src=[9272, 9272, 9470, 9350, 9170, 9358, 9224, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343968 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267435936 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534527904 weight_movement_bytes=34603008 total_nvlink_bytes=569130912 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64938 dup_rows=23902 dup_rows_remote=20844 dedup_ratio=0.243144 router_tokens_per_expert=[761, 790, 717, 751, 762, 779, 745, 812, 781, 776, 816, 808, 720, 809, 775, 776, 746, 715, 740, 757, 772, 768, 770, 782, 766, 792, 808, 820, 711, 788, 779, 763, 762, 720, 773, 763, 717, 776, 751, 773, 791, 838, 763, 762, 734, 742, 784, 780, 742, 793, 818, 717, 799, 781, 766, 744, 779, 768, 744, 740, 760, 785, 786, 746] payload_rows_per_dst=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] dup_rows_per_dst=[3036, 3058, 2950, 3066, 2858, 3038, 2960, 2936] combine_rows_per_src=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64956 dup_rows=24012 dup_rows_remote=21078 dedup_ratio=0.244263 router_tokens_per_expert=[769, 760, 787, 796, 780, 756, 775, 825, 761, 752, 827, 773, 736, 756, 760, 779, 755, 771, 766, 748, 768, 758, 755, 762, 760, 804, 765, 750, 783, 736, 839, 771, 794, 789, 711, 782, 753, 764, 747, 750, 783, 751, 790, 766, 764, 806, 757, 757, 830, 745, 767, 758, 727, 729, 763, 751, 763, 750, 775, 780, 766, 803, 752, 746] payload_rows_per_dst=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] dup_rows_per_dst=[2914, 2624, 2956, 3036, 3350, 3040, 3158, 2934] combine_rows_per_src=[9232, 8798, 9210, 9346, 9696, 9342, 9332, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64986 dup_rows=24162 dup_rows_remote=21220 dedup_ratio=0.245789 router_tokens_per_expert=[801, 785, 789, 779, 756, 784, 777, 750, 772, 762, 743, 799, 715, 767, 755, 756, 707, 766, 748, 724, 798, 768, 740, 798, 749, 781, 764, 786, 787, 744, 738, 776, 767, 762, 772, 798, 806, 794, 741, 772, 813, 743, 772, 788, 776, 745, 799, 783, 748, 760, 687, 798, 798, 742, 794, 707, 751, 782, 800, 821, 781, 740, 794, 754] payload_rows_per_dst=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] dup_rows_per_dst=[3096, 2912, 2942, 2980, 3154, 3112, 2924, 3042] combine_rows_per_src=[9346, 9226, 9156, 9270, 9270, 9326, 9144, 9404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344336 combine_payload_bytes=266477568 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266821904 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299472 weight_movement_bytes=69206016 total_nvlink_bytes=602505488 weight_to_token_ratio=0.129770 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=64938 dup_rows=23902 dup_rows_remote=20844 dedup_ratio=0.243144 router_tokens_per_expert=[761, 790, 717, 751, 762, 779, 745, 812, 781, 776, 816, 808, 720, 809, 775, 776, 746, 715, 740, 757, 772, 768, 770, 782, 766, 792, 808, 820, 711, 788, 779, 763, 762, 720, 773, 763, 717, 776, 751, 773, 791, 838, 763, 762, 734, 742, 784, 780, 742, 793, 818, 717, 799, 781, 766, 744, 779, 768, 744, 740, 760, 785, 786, 746] payload_rows_per_dst=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] dup_rows_per_dst=[3036, 3058, 2950, 3066, 2858, 3038, 2960, 2936] combine_rows_per_src=[9198, 9464, 9150, 9388, 9212, 9350, 9360, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65014 dup_rows=23948 dup_rows_remote=20934 dedup_ratio=0.243612 router_tokens_per_expert=[763, 756, 795, 777, 763, 812, 797, 776, 852, 783, 808, 799, 803, 809, 761, 788, 737, 799, 762, 753, 736, 764, 797, 746, 746, 760, 752, 762, 708, 801, 800, 802, 742, 764, 727, 770, 745, 710, 775, 779, 826, 778, 742, 776, 712, 770, 770, 747, 808, 742, 783, 802, 752, 765, 733, 793, 743, 750, 755, 792, 710, 711, 747, 766] payload_rows_per_dst=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] dup_rows_per_dst=[3150, 3212, 2976, 2970, 2768, 2948, 3014, 2910] combine_rows_per_src=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65072 dup_rows=23986 dup_rows_remote=21006 dedup_ratio=0.243998 router_tokens_per_expert=[795, 765, 780, 754, 724, 760, 758, 762, 746, 758, 765, 769, 800, 804, 762, 805, 815, 762, 825, 772, 748, 809, 764, 765, 777, 721, 771, 748, 746, 744, 774, 776, 748, 776, 771, 705, 805, 801, 808, 775, 821, 757, 733, 744, 738, 783, 778, 716, 707, 740, 722, 726, 764, 751, 822, 764, 786, 804, 802, 767, 807, 787, 755, 765] payload_rows_per_dst=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] dup_rows_per_dst=[2980, 3114, 3282, 2870, 3060, 2852, 2902, 2926] combine_rows_per_src=[9246, 9454, 9502, 9246, 9318, 9138, 9090, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344336 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190544 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036752 weight_movement_bytes=69206016 total_nvlink_bytes=603242768 weight_to_token_ratio=0.129590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74074 payload_rows_remote=64696 dup_rows=24230 dup_rows_remote=21168 dedup_ratio=0.246480 router_tokens_per_expert=[739, 762, 812, 797, 759, 777, 724, 742, 791, 760, 813, 741, 736, 782, 758, 787, 753, 778, 741, 774, 778, 766, 773, 773, 794, 781, 788, 776, 764, 759, 778, 764, 748, 783, 747, 750, 772, 727, 790, 715, 724, 777, 767, 785, 739, 767, 790, 786, 771, 791, 751, 760, 811, 795, 781, 760, 788, 738, 773, 747, 772, 784, 772, 771] payload_rows_per_dst=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] dup_rows_per_dst=[2964, 3090, 3038, 3036, 2928, 3010, 3062, 3102] combine_rows_per_src=[9260, 9246, 9234, 9372, 9136, 9260, 9378, 9188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344320 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267608320 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872320 weight_movement_bytes=69206016 total_nvlink_bytes=604078336 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344920 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267707224 ingress_bytes=301965312 link_bytes=301965312 token_bytes=535069528 weight_movement_bytes=34603008 total_nvlink_bytes=569672536 weight_to_token_ratio=0.064670 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136271159e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266846208 dispatch_meta_bytes=344336 combine_payload_bytes=266846208 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267190544 ingress_bytes=336052224 link_bytes=336052224 token_bytes=534036752 weight_movement_bytes=69206016 total_nvlink_bytes=603242768 weight_to_token_ratio=0.129590 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264537464e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344312 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879224 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414136 weight_movement_bytes=69206016 total_nvlink_bytes=602620152 weight_to_token_ratio=0.129742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344136 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266403912 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463688 weight_movement_bytes=103809024 total_nvlink_bytes=636272712 weight_to_token_ratio=0.194960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344824 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266527480 ingress_bytes=335388672 link_bytes=335388672 token_bytes=532710136 weight_movement_bytes=69206016 total_nvlink_bytes=601916152 weight_to_token_ratio=0.129913 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262040572e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265986048 dispatch_meta_bytes=343128 combine_payload_bytes=265986048 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266329176 ingress_bytes=335192064 link_bytes=335192064 token_bytes=532315224 weight_movement_bytes=69206016 total_nvlink_bytes=601521240 weight_to_token_ratio=0.130009 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261300752e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343792 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641136 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938480 weight_movement_bytes=34603008 total_nvlink_bytes=567541488 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65250 dup_rows=23788 dup_rows_remote=20830 dedup_ratio=0.241984 router_tokens_per_expert=[787, 780, 811, 750, 768, 723, 801, 789, 735, 760, 786, 760, 754, 774, 746, 753, 756, 732, 760, 754, 772, 754, 745, 762, 802, 786, 760, 797, 777, 739, 734, 811, 761, 741, 758, 729, 765, 795, 794, 794, 808, 764, 790, 753, 748, 754, 753, 742, 774, 760, 713, 763, 802, 761, 816, 790, 787, 740, 806, 777, 828, 809, 710, 749] payload_rows_per_dst=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] dup_rows_per_dst=[3042, 2906, 2872, 3036, 2950, 2958, 3014, 3010] combine_rows_per_src=[9376, 9230, 9198, 9376, 9324, 9266, 9344, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64912 dup_rows=24240 dup_rows_remote=21166 dedup_ratio=0.246582 router_tokens_per_expert=[769, 761, 781, 800, 777, 736, 782, 769, 760, 744, 798, 770, 787, 805, 764, 731, 753, 805, 764, 794, 776, 763, 766, 806, 783, 739, 773, 719, 734, 792, 734, 768, 765, 758, 775, 778, 805, 735, 741, 756, 812, 814, 747, 760, 808, 774, 771, 765, 751, 760, 796, 783, 701, 790, 743, 739, 778, 731, 782, 775, 742, 753, 758, 803] payload_rows_per_dst=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] dup_rows_per_dst=[3042, 3116, 3018, 2890, 3074, 3108, 2996, 2996] combine_rows_per_src=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=344136 combine_payload_bytes=266059776 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266403912 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463688 weight_movement_bytes=103809024 total_nvlink_bytes=636272712 weight_to_token_ratio=0.194960 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266534912 dispatch_meta_bytes=344312 combine_payload_bytes=266534912 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266879224 ingress_bytes=335740928 link_bytes=335740928 token_bytes=533414136 weight_movement_bytes=69206016 total_nvlink_bytes=602620152 weight_to_token_ratio=0.129742 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263366083e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65184 dup_rows=23912 dup_rows_remote=21020 dedup_ratio=0.243245 router_tokens_per_expert=[770, 766, 765, 765, 770, 764, 779, 768, 794, 721, 743, 768, 796, 741, 746, 750, 747, 769, 764, 766, 735, 754, 762, 753, 796, 776, 821, 784, 717, 754, 794, 778, 719, 737, 769, 765, 796, 790, 779, 749, 749, 841, 784, 796, 804, 732, 770, 774, 738, 763, 750, 718, 797, 731, 768, 751, 801, 816, 760, 759, 777, 803, 815, 775] payload_rows_per_dst=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] dup_rows_per_dst=[3036, 2898, 2892, 3056, 2932, 3134, 2872, 3092] combine_rows_per_src=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=264994816 dispatch_meta_bytes=343456 combine_payload_bytes=264994816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265338272 ingress_bytes=334200832 link_bytes=334200832 token_bytes=530333088 weight_movement_bytes=69206016 total_nvlink_bytes=599539104 weight_to_token_ratio=0.130495 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257570825e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74514 payload_rows_remote=65318 dup_rows=23790 dup_rows_remote=20940 dedup_ratio=0.242004 router_tokens_per_expert=[790, 768, 765, 746, 759, 809, 752, 765, 745, 777, 804, 757, 797, 752, 784, 738, 748, 771, 752, 803, 748, 731, 767, 818, 743, 741, 770, 750, 768, 760, 745, 746, 799, 798, 804, 771, 799, 737, 773, 827, 801, 763, 773, 780, 771, 800, 732, 786, 740, 725, 759, 749, 781, 777, 762, 770, 806, 754, 757, 739, 757, 718, 787, 788] payload_rows_per_dst=[9342, 9306, 9276, 9196, 9398, 9412, 9274, 9310] dup_rows_per_dst=[2966, 3002, 3000, 2850, 3218, 3000, 2852, 2902] combine_rows_per_src=[9342, 9306, 9276, 9196, 9398, 9412, 9274, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74758 payload_rows_remote=65358 dup_rows=23546 dup_rows_remote=20596 dedup_ratio=0.239522 router_tokens_per_expert=[767, 767, 763, 749, 757, 736, 789, 775, 775, 762, 800, 795, 793, 754, 747, 744, 785, 709, 740, 803, 746, 783, 734, 765, 784, 791, 735, 792, 764, 783, 738, 758, 769, 727, 738, 767, 803, 771, 814, 739, 723, 753, 782, 795, 752, 765, 786, 774, 767, 743, 754, 738, 757, 888, 783, 781, 758, 822, 758, 758, 774, 820, 741, 769] payload_rows_per_dst=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] dup_rows_per_dst=[2950, 3078, 3112, 2964, 2948, 2620, 2978, 2896] combine_rows_per_src=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344816 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338480 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332144 weight_movement_bytes=34603008 total_nvlink_bytes=568935152 weight_to_token_ratio=0.064759 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65122 dup_rows=23892 dup_rows_remote=20924 dedup_ratio=0.243042 router_tokens_per_expert=[763, 762, 755, 764, 780, 748, 786, 788, 761, 735, 761, 799, 766, 776, 769, 785, 767, 740, 750, 758, 747, 754, 750, 752, 744, 802, 774, 747, 746, 779, 744, 770, 785, 777, 745, 760, 776, 783, 793, 760, 729, 759, 788, 742, 773, 794, 765, 779, 799, 764, 825, 782, 761, 767, 754, 795, 781, 738, 792, 734, 812, 724, 784, 810] payload_rows_per_dst=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] dup_rows_per_dst=[2980, 3016, 2834, 2974, 3092, 2968, 3020, 3008] combine_rows_per_src=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74064 payload_rows_remote=64912 dup_rows=24240 dup_rows_remote=21166 dedup_ratio=0.246582 router_tokens_per_expert=[769, 761, 781, 800, 777, 736, 782, 769, 760, 744, 798, 770, 787, 805, 764, 731, 753, 805, 764, 794, 776, 763, 766, 806, 783, 739, 773, 719, 734, 792, 734, 768, 765, 758, 775, 778, 805, 735, 741, 756, 812, 814, 747, 760, 808, 774, 771, 765, 751, 760, 796, 783, 701, 790, 743, 739, 778, 731, 782, 775, 742, 753, 758, 803] payload_rows_per_dst=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] dup_rows_per_dst=[3042, 3116, 3018, 2890, 3074, 3108, 2996, 2996] combine_rows_per_src=[9308, 9202, 9436, 9194, 9152, 9394, 9130, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64896 dup_rows=24042 dup_rows_remote=21024 dedup_ratio=0.244568 router_tokens_per_expert=[761, 745, 789, 730, 771, 754, 782, 756, 801, 769, 775, 784, 774, 711, 782, 796, 751, 777, 785, 771, 735, 772, 766, 769, 758, 783, 743, 789, 781, 751, 771, 796, 748, 738, 718, 787, 771, 767, 742, 776, 769, 762, 732, 794, 768, 776, 770, 840, 809, 808, 799, 709, 732, 743, 779, 745, 806, 774, 748, 805, 701, 799, 806, 753] payload_rows_per_dst=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] dup_rows_per_dst=[2972, 3018, 2996, 3054, 2846, 3094, 2990, 3072] combine_rows_per_src=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65054 dup_rows=23908 dup_rows_remote=20988 dedup_ratio=0.243205 router_tokens_per_expert=[797, 753, 745, 783, 794, 723, 714, 801, 756, 780, 743, 770, 752, 732, 764, 750, 771, 752, 722, 815, 764, 746, 749, 758, 735, 770, 776, 812, 801, 744, 811, 793, 778, 765, 789, 739, 757, 813, 757, 808, 749, 757, 772, 792, 785, 754, 756, 764, 793, 759, 759, 765, 794, 782, 780, 778, 762, 742, 762, 787, 762, 763, 767, 786] payload_rows_per_dst=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] dup_rows_per_dst=[3000, 2894, 2844, 3006, 2950, 3086, 3208, 2920] combine_rows_per_src=[9220, 9200, 9310, 9350, 9108, 9300, 9566, 9342] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64896 dup_rows=24042 dup_rows_remote=21024 dedup_ratio=0.244568 router_tokens_per_expert=[761, 745, 789, 730, 771, 754, 782, 756, 801, 769, 775, 784, 774, 711, 782, 796, 751, 777, 785, 771, 735, 772, 766, 769, 758, 783, 743, 789, 781, 751, 771, 796, 748, 738, 718, 787, 771, 767, 742, 776, 769, 762, 732, 794, 768, 776, 770, 840, 809, 808, 799, 709, 732, 743, 779, 745, 806, 774, 748, 805, 701, 799, 806, 753] payload_rows_per_dst=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] dup_rows_per_dst=[2972, 3018, 2996, 3054, 2846, 3094, 2990, 3072] combine_rows_per_src=[9204, 9366, 9256, 9290, 9248, 9328, 9258, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65208 dup_rows=23808 dup_rows_remote=20784 dedup_ratio=0.242188 router_tokens_per_expert=[784, 728, 756, 779, 762, 736, 775, 758, 733, 832, 771, 745, 744, 772, 746, 814, 770, 733, 765, 814, 796, 734, 788, 756, 779, 742, 729, 734, 800, 799, 781, 762, 790, 771, 749, 746, 741, 781, 791, 722, 798, 772, 738, 762, 806, 781, 750, 813, 737, 763, 763, 765, 777, 790, 744, 778, 817, 791, 803, 743, 749, 725, 782, 797] payload_rows_per_dst=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] dup_rows_per_dst=[2934, 2988, 3024, 2936, 2924, 3118, 2902, 2982] combine_rows_per_src=[9222, 9326, 9288, 9316, 9258, 9322, 9332, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64990 dup_rows=24106 dup_rows_remote=21232 dedup_ratio=0.245219 router_tokens_per_expert=[792, 774, 777, 762, 769, 741, 791, 751, 800, 742, 697, 724, 790, 812, 759, 765, 766, 775, 758, 798, 754, 748, 783, 764, 762, 816, 765, 735, 807, 798, 767, 796, 786, 781, 781, 774, 804, 797, 777, 804, 739, 760, 758, 732, 792, 769, 725, 812, 780, 728, 703, 767, 768, 774, 735, 786, 814, 746, 759, 722, 746, 756, 749, 790] payload_rows_per_dst=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] dup_rows_per_dst=[3036, 2986, 3040, 3174, 3080, 2982, 2874, 2934] combine_rows_per_src=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267264000 dispatch_meta_bytes=344320 combine_payload_bytes=267264000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267608320 ingress_bytes=336470016 link_bytes=336470016 token_bytes=534872320 weight_movement_bytes=69206016 total_nvlink_bytes=604078336 weight_to_token_ratio=0.129388 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266109582e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74758 payload_rows_remote=65358 dup_rows=23546 dup_rows_remote=20596 dedup_ratio=0.239522 router_tokens_per_expert=[767, 767, 763, 749, 757, 736, 789, 775, 775, 762, 800, 795, 793, 754, 747, 744, 785, 709, 740, 803, 746, 783, 734, 765, 784, 791, 735, 792, 764, 783, 738, 758, 769, 727, 738, 767, 803, 771, 814, 739, 723, 753, 782, 795, 752, 765, 786, 774, 767, 743, 754, 738, 757, 888, 783, 781, 758, 822, 758, 758, 774, 820, 741, 769] payload_rows_per_dst=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] dup_rows_per_dst=[2950, 3078, 3112, 2964, 2948, 2620, 2978, 2896] combine_rows_per_src=[9400, 9522, 9628, 9366, 9188, 9030, 9444, 9180] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344312 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266223864 ingress_bytes=369688576 link_bytes=369688576 token_bytes=532103416 weight_movement_bytes=103809024 total_nvlink_bytes=635912440 weight_to_token_ratio=0.195092 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391108349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267542528 dispatch_meta_bytes=345032 combine_payload_bytes=267542528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267887560 ingress_bytes=336748544 link_bytes=336748544 token_bytes=535430088 weight_movement_bytes=69206016 total_nvlink_bytes=604636104 weight_to_token_ratio=0.129253 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267157661e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=343816 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268050184 ingress_bytes=302309376 link_bytes=302309376 token_bytes=535756552 weight_movement_bytes=34603008 total_nvlink_bytes=570359560 weight_to_token_ratio=0.064587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137565844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344168 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805352 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266536 weight_movement_bytes=103809024 total_nvlink_bytes=637075560 weight_to_token_ratio=0.194666 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344184 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083896 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823608 weight_movement_bytes=69206016 total_nvlink_bytes=603029624 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=344312 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266223864 ingress_bytes=369688576 link_bytes=369688576 token_bytes=532103416 weight_movement_bytes=103809024 total_nvlink_bytes=635912440 weight_to_token_ratio=0.195092 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391108349e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343680 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157696 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971712 weight_movement_bytes=34603008 total_nvlink_bytes=566574720 weight_to_token_ratio=0.065047 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344888 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543928 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742968 weight_movement_bytes=34603008 total_nvlink_bytes=567345976 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343968 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267435936 ingress_bytes=301694976 link_bytes=301694976 token_bytes=534527904 weight_movement_bytes=34603008 total_nvlink_bytes=569130912 weight_to_token_ratio=0.064736 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135253906e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65302 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[754, 779, 764, 773, 733, 763, 787, 757, 745, 742, 782, 792, 773, 806, 720, 805, 771, 792, 762, 793, 787, 762, 770, 756, 741, 791, 744, 792, 741, 813, 772, 799, 686, 734, 738, 762, 787, 750, 761, 744, 738, 802, 756, 754, 743, 725, 805, 804, 787, 767, 746, 749, 801, 820, 757, 767, 769, 743, 789, 826, 778, 775, 753, 775] payload_rows_per_dst=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] dup_rows_per_dst=[2944, 2964, 3042, 3020, 2902, 2968, 3032, 3108] combine_rows_per_src=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65122 dup_rows=23892 dup_rows_remote=20924 dedup_ratio=0.243042 router_tokens_per_expert=[763, 762, 755, 764, 780, 748, 786, 788, 761, 735, 761, 799, 766, 776, 769, 785, 767, 740, 750, 758, 747, 754, 750, 752, 744, 802, 774, 747, 746, 779, 744, 770, 785, 777, 745, 760, 776, 783, 793, 760, 729, 759, 788, 742, 773, 794, 765, 779, 799, 764, 825, 782, 761, 767, 754, 795, 781, 738, 792, 734, 812, 724, 784, 810] payload_rows_per_dst=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] dup_rows_per_dst=[2980, 3016, 2834, 2974, 3092, 2968, 3020, 3008] combine_rows_per_src=[9312, 9288, 9202, 9238, 9266, 9290, 9474, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64854 dup_rows=23882 dup_rows_remote=20656 dedup_ratio=0.242940 router_tokens_per_expert=[781, 787, 747, 803, 796, 772, 772, 773, 746, 734, 780, 771, 760, 804, 795, 779, 791, 751, 785, 762, 740, 722, 777, 782, 760, 761, 788, 798, 746, 741, 752, 748, 778, 747, 815, 767, 764, 769, 741, 751, 785, 712, 751, 749, 761, 765, 777, 789, 757, 770, 777, 759, 764, 770, 814, 783, 781, 769, 773, 776, 786, 710, 789, 749] payload_rows_per_dst=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] dup_rows_per_dst=[3226, 3010, 3030, 2898, 2992, 2888, 2968, 2870] combine_rows_per_src=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=344168 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266805352 ingress_bytes=370270208 link_bytes=370270208 token_bytes=533266536 weight_movement_bytes=103809024 total_nvlink_bytes=637075560 weight_to_token_ratio=0.194666 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393296984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65014 dup_rows=23948 dup_rows_remote=20934 dedup_ratio=0.243612 router_tokens_per_expert=[763, 756, 795, 777, 763, 812, 797, 776, 852, 783, 808, 799, 803, 809, 761, 788, 737, 799, 762, 753, 736, 764, 797, 746, 746, 760, 752, 762, 708, 801, 800, 802, 742, 764, 727, 770, 745, 710, 775, 779, 826, 778, 742, 776, 712, 770, 770, 747, 808, 742, 783, 802, 752, 765, 733, 793, 743, 750, 755, 792, 710, 711, 747, 766] payload_rows_per_dst=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] dup_rows_per_dst=[3150, 3212, 2976, 2970, 2768, 2948, 3014, 2910] combine_rows_per_src=[9328, 9594, 9212, 9292, 9256, 9294, 9342, 9038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=267706368 dispatch_meta_bytes=343816 combine_payload_bytes=267706368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268050184 ingress_bytes=302309376 link_bytes=302309376 token_bytes=535756552 weight_movement_bytes=34603008 total_nvlink_bytes=570359560 weight_to_token_ratio=0.064587 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137565844e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65142 dup_rows=23852 dup_rows_remote=20926 dedup_ratio=0.242635 router_tokens_per_expert=[791, 800, 731, 781, 754, 820, 752, 735, 774, 825, 753, 795, 771, 746, 780, 771, 741, 776, 784, 719, 782, 789, 801, 726, 734, 778, 813, 725, 746, 769, 783, 745, 734, 781, 770, 756, 777, 761, 768, 756, 756, 771, 712, 802, 798, 793, 810, 760, 760, 749, 788, 742, 797, 806, 763, 770, 737, 707, 754, 836, 753, 735, 813, 747] payload_rows_per_dst=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] dup_rows_per_dst=[2998, 3176, 2926, 2982, 2952, 3010, 2948, 2860] combine_rows_per_src=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343792 combine_payload_bytes=266297344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266641136 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938480 weight_movement_bytes=34603008 total_nvlink_bytes=567541488 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344272 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165904 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987536 weight_movement_bytes=34603008 total_nvlink_bytes=568590544 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65078 dup_rows=23922 dup_rows_remote=20734 dedup_ratio=0.243347 router_tokens_per_expert=[808, 767, 801, 752, 774, 764, 727, 784, 775, 809, 783, 762, 773, 789, 785, 787, 778, 767, 729, 778, 763, 749, 743, 753, 771, 779, 769, 768, 820, 783, 769, 787, 775, 729, 792, 760, 747, 757, 759, 784, 780, 775, 804, 751, 766, 770, 782, 740, 749, 742, 761, 723, 750, 792, 771, 716, 764, 756, 798, 776, 765, 797, 701, 774] payload_rows_per_dst=[9282, 9478, 9224, 9304, 9332, 9290, 9158, 9314] dup_rows_per_dst=[3072, 3048, 2896, 3188, 2874, 3046, 2850, 2948] combine_rows_per_src=[9282, 9478, 9224, 9304, 9332, 9290, 9158, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64880 dup_rows=24124 dup_rows_remote=21092 dedup_ratio=0.245402 router_tokens_per_expert=[805, 795, 757, 763, 760, 721, 751, 717, 801, 813, 775, 756, 795, 766, 751, 780, 716, 776, 777, 751, 730, 753, 731, 794, 776, 768, 755, 748, 759, 810, 734, 777, 787, 801, 721, 812, 756, 779, 756, 803, 782, 764, 754, 742, 796, 786, 753, 769, 790, 737, 793, 776, 764, 773, 754, 777, 776, 751, 788, 776, 774, 749, 775, 777] payload_rows_per_dst=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] dup_rows_per_dst=[2480, 2662, 3226, 3034, 3606, 3020, 3064, 3032] combine_rows_per_src=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65124 dup_rows=23692 dup_rows_remote=20688 dedup_ratio=0.241007 router_tokens_per_expert=[762, 768, 777, 790, 754, 785, 762, 782, 802, 792, 798, 742, 763, 722, 794, 720, 773, 757, 744, 779, 769, 765, 783, 732, 787, 809, 799, 701, 760, 759, 792, 773, 751, 758, 774, 780, 737, 784, 789, 766, 798, 747, 782, 811, 803, 800, 758, 747, 780, 790, 752, 751, 781, 752, 764, 747, 745, 787, 733, 749, 779, 747, 783, 732] payload_rows_per_dst=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] dup_rows_per_dst=[3010, 2954, 2902, 2970, 3002, 3004, 3012, 2838] combine_rows_per_src=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64928 dup_rows=24050 dup_rows_remote=20976 dedup_ratio=0.244649 router_tokens_per_expert=[769, 787, 761, 794, 804, 793, 810, 756, 730, 781, 775, 793, 750, 780, 789, 802, 738, 755, 766, 790, 744, 738, 745, 768, 802, 775, 776, 774, 735, 769, 786, 787, 764, 749, 737, 714, 769, 736, 745, 728, 776, 761, 735, 752, 764, 784, 765, 771, 785, 774, 768, 779, 787, 740, 755, 788, 785, 753, 777, 777, 775, 743, 787, 807] payload_rows_per_dst=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] dup_rows_per_dst=[3116, 3074, 2886, 3158, 2760, 3006, 3050, 3000] combine_rows_per_src=[9432, 9326, 9202, 9250, 9124, 9210, 9302, 9408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=65302 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[754, 779, 764, 773, 733, 763, 787, 757, 745, 742, 782, 792, 773, 806, 720, 805, 771, 792, 762, 793, 787, 762, 770, 756, 741, 791, 744, 792, 741, 813, 772, 799, 686, 734, 738, 762, 787, 750, 761, 744, 738, 802, 756, 754, 743, 725, 805, 804, 787, 767, 746, 749, 801, 820, 757, 767, 769, 743, 789, 826, 778, 775, 753, 775] payload_rows_per_dst=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] dup_rows_per_dst=[2944, 2964, 3042, 3020, 2902, 2968, 3032, 3108] combine_rows_per_src=[9276, 9366, 9344, 9366, 9022, 9286, 9356, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64820 dup_rows=24174 dup_rows_remote=21158 dedup_ratio=0.245911 router_tokens_per_expert=[714, 769, 788, 750, 768, 776, 807, 797, 725, 822, 759, 702, 825, 733, 762, 797, 774, 767, 783, 775, 767, 727, 752, 764, 780, 809, 751, 729, 734, 776, 772, 779, 749, 771, 817, 769, 781, 723, 765, 727, 776, 749, 767, 790, 791, 762, 769, 773, 772, 752, 757, 775, 819, 749, 779, 760, 754, 787, 803, 760, 775, 760, 793, 745] payload_rows_per_dst=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] dup_rows_per_dst=[3010, 2990, 2972, 3028, 2986, 3102, 3016, 3070] combine_rows_per_src=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=345520 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267822512 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535299504 weight_movement_bytes=34603008 total_nvlink_bytes=569902512 weight_to_token_ratio=0.064642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74392 payload_rows_remote=65184 dup_rows=23912 dup_rows_remote=21020 dedup_ratio=0.243245 router_tokens_per_expert=[770, 766, 765, 765, 770, 764, 779, 768, 794, 721, 743, 768, 796, 741, 746, 750, 747, 769, 764, 766, 735, 754, 762, 753, 796, 776, 821, 784, 717, 754, 794, 778, 719, 737, 769, 765, 796, 790, 779, 749, 749, 841, 784, 796, 804, 732, 770, 774, 738, 763, 750, 718, 797, 731, 768, 751, 801, 816, 760, 759, 777, 803, 815, 775] payload_rows_per_dst=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] dup_rows_per_dst=[3036, 2898, 2892, 3056, 2932, 3134, 2872, 3092] combine_rows_per_src=[9258, 9220, 9208, 9384, 9276, 9366, 9160, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64880 dup_rows=24124 dup_rows_remote=21092 dedup_ratio=0.245402 router_tokens_per_expert=[805, 795, 757, 763, 760, 721, 751, 717, 801, 813, 775, 756, 795, 766, 751, 780, 716, 776, 777, 751, 730, 753, 731, 794, 776, 768, 755, 748, 759, 810, 734, 777, 787, 801, 721, 812, 756, 779, 756, 803, 782, 764, 754, 742, 796, 786, 753, 769, 790, 737, 793, 776, 764, 773, 754, 777, 776, 751, 788, 776, 774, 749, 775, 777] payload_rows_per_dst=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] dup_rows_per_dst=[2480, 2662, 3226, 3034, 3606, 3020, 3064, 3032] combine_rows_per_src=[8882, 8904, 9606, 9220, 9732, 9240, 9296, 9300] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344184 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083896 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823608 weight_movement_bytes=69206016 total_nvlink_bytes=603029624 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=342040 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265984024 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531626008 weight_movement_bytes=69206016 total_nvlink_bytes=600832024 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=64854 dup_rows=23882 dup_rows_remote=20656 dedup_ratio=0.242940 router_tokens_per_expert=[781, 787, 747, 803, 796, 772, 772, 773, 746, 734, 780, 771, 760, 804, 795, 779, 791, 751, 785, 762, 740, 722, 777, 782, 760, 761, 788, 798, 746, 741, 752, 748, 778, 747, 815, 767, 764, 769, 741, 751, 785, 712, 751, 749, 761, 765, 777, 789, 757, 770, 777, 759, 764, 770, 814, 783, 781, 769, 773, 776, 786, 710, 789, 749] payload_rows_per_dst=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] dup_rows_per_dst=[3226, 3010, 3030, 2898, 2992, 2888, 2968, 2870] combine_rows_per_src=[9568, 9526, 9190, 9060, 9272, 9100, 9318, 9388] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343248 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266902736 ingress_bytes=335765504 link_bytes=335765504 token_bytes=533462224 weight_movement_bytes=69206016 total_nvlink_bytes=602668240 weight_to_token_ratio=0.129730 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263458560e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343888 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266092368 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840848 weight_movement_bytes=34603008 total_nvlink_bytes=566443856 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343248 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267091152 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839056 weight_movement_bytes=103809024 total_nvlink_bytes=637648080 weight_to_token_ratio=0.194458 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343616 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288704 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233792 weight_movement_bytes=69206016 total_nvlink_bytes=601439808 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343912 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846632 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349352 weight_movement_bytes=69206016 total_nvlink_bytes=600555368 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=344816 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267338480 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332144 weight_movement_bytes=34603008 total_nvlink_bytes=568935152 weight_to_token_ratio=0.064759 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267476992 dispatch_meta_bytes=345520 combine_payload_bytes=267476992 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267822512 ingress_bytes=302080000 link_bytes=302080000 token_bytes=535299504 weight_movement_bytes=34603008 total_nvlink_bytes=569902512 weight_to_token_ratio=0.064642 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136702721e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343888 combine_payload_bytes=265748480 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266092368 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840848 weight_movement_bytes=34603008 total_nvlink_bytes=566443856 weight_to_token_ratio=0.065063 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265641984 dispatch_meta_bytes=342040 combine_payload_bytes=265641984 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265984024 ingress_bytes=334848000 link_bytes=334848000 token_bytes=531626008 weight_movement_bytes=69206016 total_nvlink_bytes=600832024 weight_to_token_ratio=0.130178 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260006067e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65394 dup_rows=23764 dup_rows_remote=20866 dedup_ratio=0.241740 router_tokens_per_expert=[745, 789, 768, 812, 759, 785, 748, 740, 775, 758, 760, 765, 803, 782, 745, 734, 792, 782, 713, 764, 762, 748, 715, 746, 733, 777, 788, 752, 753, 797, 752, 826, 770, 791, 739, 761, 799, 794, 799, 755, 764, 779, 801, 790, 781, 797, 741, 737, 783, 757, 806, 761, 774, 755, 761, 709, 749, 789, 798, 741, 777, 734, 836, 756] payload_rows_per_dst=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] dup_rows_per_dst=[2952, 2964, 2898, 2962, 2942, 3034, 2974, 3038] combine_rows_per_src=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74198 payload_rows_remote=64990 dup_rows=24106 dup_rows_remote=21232 dedup_ratio=0.245219 router_tokens_per_expert=[792, 774, 777, 762, 769, 741, 791, 751, 800, 742, 697, 724, 790, 812, 759, 765, 766, 775, 758, 798, 754, 748, 783, 764, 762, 816, 765, 735, 807, 798, 767, 796, 786, 781, 781, 774, 804, 797, 777, 804, 739, 760, 758, 732, 792, 769, 725, 812, 780, 728, 703, 767, 768, 774, 735, 786, 814, 746, 759, 722, 746, 756, 749, 790] payload_rows_per_dst=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] dup_rows_per_dst=[3036, 2986, 3040, 3174, 3080, 2982, 2874, 2934] combine_rows_per_src=[9278, 9192, 9252, 9318, 9528, 9192, 9208, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267853824 dispatch_meta_bytes=345040 combine_payload_bytes=267853824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268198864 ingress_bytes=302456832 link_bytes=302456832 token_bytes=536052688 weight_movement_bytes=34603008 total_nvlink_bytes=570655696 weight_to_token_ratio=0.064552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138120709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65074 dup_rows=24042 dup_rows_remote=21044 dedup_ratio=0.244568 router_tokens_per_expert=[802, 749, 743, 756, 762, 789, 732, 766, 757, 802, 747, 772, 775, 758, 774, 768, 760, 764, 739, 744, 753, 749, 800, 760, 775, 771, 710, 755, 770, 771, 796, 745, 766, 761, 788, 807, 765, 772, 769, 770, 761, 747, 835, 768, 744, 786, 778, 816, 767, 771, 768, 742, 784, 792, 791, 761, 799, 774, 772, 752, 764, 753, 764, 751] payload_rows_per_dst=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] dup_rows_per_dst=[2994, 3056, 2884, 2998, 3024, 3020, 3040, 3026] combine_rows_per_src=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64946 dup_rows=24046 dup_rows_remote=21060 dedup_ratio=0.244609 router_tokens_per_expert=[789, 757, 764, 797, 807, 735, 778, 759, 753, 745, 787, 803, 776, 819, 764, 726, 736, 763, 774, 806, 803, 756, 762, 793, 790, 743, 781, 772, 805, 768, 804, 723, 733, 715, 728, 767, 724, 802, 761, 753, 826, 734, 743, 750, 736, 777, 775, 808, 757, 783, 757, 773, 758, 770, 788, 789, 774, 763, 759, 786, 773, 736, 731, 785] payload_rows_per_dst=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] dup_rows_per_dst=[3026, 3072, 3070, 3068, 2790, 2986, 3038, 2996] combine_rows_per_src=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344888 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543928 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532742968 weight_movement_bytes=34603008 total_nvlink_bytes=567345976 weight_to_token_ratio=0.064953 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65208 dup_rows=23780 dup_rows_remote=20760 dedup_ratio=0.241903 router_tokens_per_expert=[755, 760, 763, 747, 813, 778, 717, 734, 748, 799, 807, 819, 704, 726, 788, 777, 766, 789, 772, 753, 819, 726, 783, 779, 752, 760, 729, 729, 802, 782, 760, 754, 762, 793, 776, 790, 781, 721, 759, 798, 766, 744, 747, 772, 745, 773, 813, 817, 818, 678, 805, 785, 768, 791, 715, 754, 794, 773, 754, 831, 725, 740, 772, 802] payload_rows_per_dst=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] dup_rows_per_dst=[2876, 3020, 3012, 2980, 2990, 3030, 2880, 2992] combine_rows_per_src=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64880 dup_rows=24034 dup_rows_remote=20988 dedup_ratio=0.244486 router_tokens_per_expert=[709, 730, 782, 791, 741, 794, 786, 772, 711, 775, 754, 758, 773, 751, 782, 772, 775, 775, 810, 791, 757, 748, 755, 795, 727, 752, 754, 763, 773, 771, 770, 782, 794, 784, 733, 777, 792, 803, 783, 752, 768, 779, 742, 811, 742, 761, 771, 789, 819, 765, 782, 722, 763, 785, 799, 763, 803, 735, 729, 770, 804, 767, 760, 726] payload_rows_per_dst=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] dup_rows_per_dst=[2976, 2976, 3130, 2928, 3046, 2996, 3084, 2898] combine_rows_per_src=[9234, 9176, 9282, 9256, 9390, 9330, 9312, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65064 dup_rows=23962 dup_rows_remote=20876 dedup_ratio=0.243754 router_tokens_per_expert=[821, 753, 730, 769, 741, 721, 764, 778, 782, 806, 783, 776, 776, 754, 802, 788, 746, 731, 812, 768, 753, 740, 789, 752, 727, 748, 737, 805, 754, 760, 756, 736, 753, 765, 759, 762, 739, 814, 724, 751, 765, 774, 749, 790, 773, 795, 763, 751, 815, 789, 735, 804, 769, 754, 830, 789, 750, 775, 767, 761, 812, 768, 805, 744] payload_rows_per_dst=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] dup_rows_per_dst=[2934, 3124, 2986, 2882, 2854, 2918, 3178, 3086] combine_rows_per_src=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64868 dup_rows=23926 dup_rows_remote=20866 dedup_ratio=0.243388 router_tokens_per_expert=[754, 754, 797, 813, 762, 773, 728, 786, 769, 748, 782, 754, 800, 803, 754, 778, 846, 780, 740, 761, 752, 762, 789, 804, 760, 745, 769, 693, 818, 748, 737, 812, 772, 747, 793, 777, 740, 785, 724, 732, 737, 811, 706, 763, 785, 729, 700, 779, 769, 798, 762, 772, 802, 761, 763, 778, 802, 775, 809, 777, 748, 769, 754, 762] payload_rows_per_dst=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] dup_rows_per_dst=[3060, 2916, 3084, 3292, 3054, 2860, 2732, 2928] combine_rows_per_src=[9510, 9226, 9384, 9570, 9394, 9230, 8980, 9084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344472 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887576 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430680 weight_movement_bytes=34603008 total_nvlink_bytes=568033688 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344024 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362840 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381656 weight_movement_bytes=69206016 total_nvlink_bytes=601587672 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65064 dup_rows=23962 dup_rows_remote=20876 dedup_ratio=0.243754 router_tokens_per_expert=[821, 753, 730, 769, 741, 721, 764, 778, 782, 806, 783, 776, 776, 754, 802, 788, 746, 731, 812, 768, 753, 740, 789, 752, 727, 748, 737, 805, 754, 760, 756, 736, 753, 765, 759, 762, 739, 814, 724, 751, 765, 774, 749, 790, 773, 795, 763, 751, 815, 789, 735, 804, 769, 754, 830, 789, 750, 775, 767, 761, 812, 768, 805, 744] payload_rows_per_dst=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] dup_rows_per_dst=[2934, 3124, 2986, 2882, 2854, 2918, 3178, 3086] combine_rows_per_src=[9220, 9370, 9236, 9164, 9280, 9226, 9568, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=65074 dup_rows=24042 dup_rows_remote=21044 dedup_ratio=0.244568 router_tokens_per_expert=[802, 749, 743, 756, 762, 789, 732, 766, 757, 802, 747, 772, 775, 758, 774, 768, 760, 764, 739, 744, 753, 749, 800, 760, 775, 771, 710, 755, 770, 771, 796, 745, 766, 761, 788, 807, 765, 772, 769, 770, 761, 747, 835, 768, 744, 786, 778, 816, 767, 771, 768, 742, 784, 792, 791, 761, 799, 774, 772, 752, 764, 753, 764, 751] payload_rows_per_dst=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] dup_rows_per_dst=[2994, 3056, 2884, 2998, 3024, 3020, 3040, 3026] combine_rows_per_src=[9204, 9250, 9254, 9188, 9372, 9450, 9312, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74524 payload_rows_remote=65208 dup_rows=23780 dup_rows_remote=20760 dedup_ratio=0.241903 router_tokens_per_expert=[755, 760, 763, 747, 813, 778, 717, 734, 748, 799, 807, 819, 704, 726, 788, 777, 766, 789, 772, 753, 819, 726, 783, 779, 752, 760, 729, 729, 802, 782, 760, 754, 762, 793, 776, 790, 781, 721, 759, 798, 766, 744, 747, 772, 745, 773, 813, 817, 818, 678, 805, 785, 768, 791, 715, 754, 794, 773, 754, 831, 725, 740, 772, 802] payload_rows_per_dst=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] dup_rows_per_dst=[2876, 3020, 3012, 2980, 2990, 3030, 2880, 2992] combine_rows_per_src=[9258, 9316, 9362, 9156, 9370, 9324, 9348, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74612 payload_rows_remote=65124 dup_rows=23692 dup_rows_remote=20688 dedup_ratio=0.241007 router_tokens_per_expert=[762, 768, 777, 790, 754, 785, 762, 782, 802, 792, 798, 742, 763, 722, 794, 720, 773, 757, 744, 779, 769, 765, 783, 732, 787, 809, 799, 701, 760, 759, 792, 773, 751, 758, 774, 780, 737, 784, 789, 766, 798, 747, 782, 811, 803, 800, 758, 747, 780, 790, 752, 751, 781, 752, 764, 747, 745, 787, 733, 749, 779, 747, 783, 732] payload_rows_per_dst=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] dup_rows_per_dst=[3010, 2954, 2902, 2970, 3002, 3004, 3012, 2838] combine_rows_per_src=[9350, 9312, 9302, 9390, 9276, 9488, 9222, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343872 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435840 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527808 weight_movement_bytes=103809024 total_nvlink_bytes=638336832 weight_to_token_ratio=0.194207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343472 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266091952 ingress_bytes=300351488 link_bytes=300351488 token_bytes=531840432 weight_movement_bytes=34603008 total_nvlink_bytes=566443440 weight_to_token_ratio=0.065063 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130198469e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74646 payload_rows_remote=65332 dup_rows=23658 dup_rows_remote=20664 dedup_ratio=0.240662 router_tokens_per_expert=[736, 754, 806, 786, 750, 804, 791, 760, 726, 765, 745, 789, 739, 795, 783, 800, 787, 770, 783, 752, 746, 768, 778, 746, 745, 777, 758, 788, 722, 720, 790, 781, 709, 758, 741, 836, 741, 760, 737, 792, 769, 756, 737, 765, 782, 772, 761, 779, 794, 788, 762, 732, 775, 786, 788, 729, 819, 764, 770, 795, 775, 810, 765, 765] payload_rows_per_dst=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] dup_rows_per_dst=[2972, 2928, 3020, 2856, 2936, 2978, 2994, 2974] combine_rows_per_src=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65142 dup_rows=23852 dup_rows_remote=20926 dedup_ratio=0.242635 router_tokens_per_expert=[791, 800, 731, 781, 754, 820, 752, 735, 774, 825, 753, 795, 771, 746, 780, 771, 741, 776, 784, 719, 782, 789, 801, 726, 734, 778, 813, 725, 746, 769, 783, 745, 734, 781, 770, 756, 777, 761, 768, 756, 756, 771, 712, 802, 798, 793, 810, 760, 760, 749, 788, 742, 797, 806, 763, 770, 737, 707, 754, 836, 753, 735, 813, 747] payload_rows_per_dst=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] dup_rows_per_dst=[2998, 3176, 2926, 2982, 2952, 3010, 2948, 2860] combine_rows_per_src=[9330, 9254, 9310, 9204, 9254, 9394, 9402, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=342936 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042264 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741592 weight_movement_bytes=34603008 total_nvlink_bytes=566344600 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343872 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435840 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527808 weight_movement_bytes=103809024 total_nvlink_bytes=638336832 weight_to_token_ratio=0.194207 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64836 dup_rows=24206 dup_rows_remote=21112 dedup_ratio=0.246236 router_tokens_per_expert=[752, 754, 772, 762, 798, 789, 739, 799, 792, 800, 766, 791, 741, 720, 771, 772, 731, 818, 732, 771, 799, 761, 772, 794, 760, 806, 759, 756, 712, 757, 787, 769, 839, 753, 754, 752, 739, 733, 795, 768, 777, 782, 743, 764, 781, 795, 744, 772, 763, 772, 758, 755, 738, 746, 781, 760, 773, 723, 775, 770, 787, 795, 778, 785] payload_rows_per_dst=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] dup_rows_per_dst=[3010, 2988, 3094, 3000, 3034, 3040, 2938, 3102] combine_rows_per_src=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343760 combine_payload_bytes=266502144 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266845904 ingress_bytes=370311168 link_bytes=370311168 token_bytes=533348048 weight_movement_bytes=103809024 total_nvlink_bytes=637157072 weight_to_token_ratio=0.194637 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393451113e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344472 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266887576 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533430680 weight_movement_bytes=34603008 total_nvlink_bytes=568033688 weight_to_token_ratio=0.064869 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343248 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267091152 ingress_bytes=370556928 link_bytes=370556928 token_bytes=533839056 weight_movement_bytes=103809024 total_nvlink_bytes=637648080 weight_to_token_ratio=0.194458 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394375888e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=343984 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267943856 ingress_bytes=336805888 link_bytes=336805888 token_bytes=535543728 weight_movement_bytes=69206016 total_nvlink_bytes=604749744 weight_to_token_ratio=0.129226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267373441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344272 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267165904 ingress_bytes=301424640 link_bytes=301424640 token_bytes=533987536 weight_movement_bytes=34603008 total_nvlink_bytes=568590544 weight_to_token_ratio=0.064801 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134236654e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64820 dup_rows=24174 dup_rows_remote=21158 dedup_ratio=0.245911 router_tokens_per_expert=[714, 769, 788, 750, 768, 776, 807, 797, 725, 822, 759, 702, 825, 733, 762, 797, 774, 767, 783, 775, 767, 727, 752, 764, 780, 809, 751, 729, 734, 776, 772, 779, 749, 771, 817, 769, 781, 723, 765, 727, 776, 749, 767, 790, 791, 762, 769, 773, 772, 752, 757, 775, 819, 749, 779, 760, 754, 787, 803, 760, 775, 760, 793, 745] payload_rows_per_dst=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] dup_rows_per_dst=[3010, 2990, 2972, 3028, 2986, 3102, 3016, 3070] combine_rows_per_src=[9328, 9260, 9246, 9232, 9218, 9252, 9310, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=343792 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912048 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480304 weight_movement_bytes=69206016 total_nvlink_bytes=600686320 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65130 dup_rows=23874 dup_rows_remote=20796 dedup_ratio=0.242859 router_tokens_per_expert=[737, 792, 805, 786, 772, 776, 775, 756, 769, 772, 767, 782, 760, 763, 772, 707, 769, 773, 769, 790, 740, 794, 779, 752, 720, 738, 812, 734, 826, 761, 810, 788, 789, 742, 744, 733, 757, 798, 793, 791, 760, 760, 770, 765, 727, 790, 749, 735, 757, 745, 787, 777, 775, 760, 758, 786, 728, 755, 757, 778, 793, 797, 806, 744] payload_rows_per_dst=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] dup_rows_per_dst=[3112, 2896, 3018, 3078, 2912, 2878, 2964, 3016] combine_rows_per_src=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=20910 dedup_ratio=0.244649 router_tokens_per_expert=[778, 798, 719, 790, 815, 754, 759, 721, 741, 811, 752, 696, 787, 739, 775, 744, 766, 765, 792, 780, 736, 732, 769, 762, 751, 786, 807, 745, 832, 810, 810, 729, 755, 790, 771, 749, 789, 795, 790, 775, 776, 776, 787, 797, 793, 761, 766, 787, 832, 795, 749, 718, 756, 774, 754, 732, 787, 725, 755, 739, 780, 730, 751, 767] payload_rows_per_dst=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] dup_rows_per_dst=[2952, 2938, 2970, 3068, 3130, 3140, 2930, 2922] combine_rows_per_src=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64972 dup_rows=24016 dup_rows_remote=20906 dedup_ratio=0.244303 router_tokens_per_expert=[783, 799, 792, 778, 739, 744, 731, 758, 761, 764, 791, 777, 766, 784, 759, 811, 778, 749, 740, 772, 774, 706, 764, 764, 753, 729, 811, 791, 774, 771, 721, 768, 743, 735, 719, 789, 789, 760, 769, 728, 812, 767, 771, 802, 805, 789, 774, 762, 754, 787, 785, 779, 778, 759, 744, 736, 812, 757, 726, 808, 771, 736, 808, 796] payload_rows_per_dst=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] dup_rows_per_dst=[2972, 3110, 2944, 2884, 2924, 3056, 3016, 3110] combine_rows_per_src=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343912 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846632 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349352 weight_movement_bytes=69206016 total_nvlink_bytes=600555368 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64950 dup_rows=24046 dup_rows_remote=20904 dedup_ratio=0.244609 router_tokens_per_expert=[757, 746, 754, 704, 795, 806, 767, 748, 748, 728, 740, 765, 753, 777, 776, 780, 805, 811, 796, 758, 767, 780, 785, 817, 763, 778, 791, 786, 760, 776, 824, 712, 776, 749, 790, 753, 773, 780, 808, 796, 754, 744, 778, 749, 803, 733, 768, 765, 785, 756, 730, 772, 738, 747, 768, 832, 773, 740, 790, 760, 746, 738, 744, 761] payload_rows_per_dst=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] dup_rows_per_dst=[2906, 2898, 3194, 3014, 3142, 2964, 2980, 2948] combine_rows_per_src=[9248, 9236, 9444, 9366, 9308, 9224, 9276, 9156] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64882 dup_rows=24114 dup_rows_remote=21104 dedup_ratio=0.245300 router_tokens_per_expert=[813, 792, 765, 794, 806, 711, 755, 711, 823, 752, 772, 789, 781, 724, 707, 804, 751, 767, 763, 807, 775, 751, 806, 744, 751, 819, 755, 757, 736, 711, 743, 786, 786, 796, 803, 805, 755, 800, 772, 808, 757, 810, 757, 795, 725, 735, 742, 743, 763, 733, 788, 799, 771, 761, 732, 798, 795, 743, 780, 718, 761, 778, 774, 748] payload_rows_per_dst=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] dup_rows_per_dst=[3010, 3184, 2994, 3072, 3062, 2782, 3090, 2920] combine_rows_per_src=[9308, 9464, 9314, 9226, 9402, 9002, 9200, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=64972 dup_rows=24016 dup_rows_remote=20906 dedup_ratio=0.244303 router_tokens_per_expert=[783, 799, 792, 778, 739, 744, 731, 758, 761, 764, 791, 777, 766, 784, 759, 811, 778, 749, 740, 772, 774, 706, 764, 764, 753, 729, 811, 791, 774, 771, 721, 768, 743, 735, 719, 789, 789, 760, 769, 728, 812, 767, 771, 802, 805, 789, 774, 762, 754, 787, 785, 779, 778, 759, 744, 736, 812, 757, 726, 808, 771, 736, 808, 796] payload_rows_per_dst=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] dup_rows_per_dst=[2972, 3110, 2944, 2884, 2924, 3056, 3016, 3110] combine_rows_per_src=[9276, 9316, 9150, 9352, 9140, 9508, 9228, 9318] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64670 dup_rows=24238 dup_rows_remote=21172 dedup_ratio=0.246562 router_tokens_per_expert=[759, 802, 776, 762, 795, 770, 806, 775, 757, 734, 790, 759, 757, 752, 741, 722, 771, 745, 766, 794, 736, 802, 785, 755, 761, 834, 781, 732, 788, 751, 791, 798, 774, 775, 750, 753, 699, 768, 775, 762, 798, 780, 753, 811, 771, 819, 759, 807, 753, 761, 753, 751, 749, 816, 706, 739, 791, 776, 786, 779, 744, 746, 749, 752] payload_rows_per_dst=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] dup_rows_per_dst=[2928, 2916, 2990, 2814, 3034, 3202, 3288, 3066] combine_rows_per_src=[9288, 9108, 9318, 9076, 9078, 9452, 9350, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65130 dup_rows=23874 dup_rows_remote=20796 dedup_ratio=0.242859 router_tokens_per_expert=[737, 792, 805, 786, 772, 776, 775, 756, 769, 772, 767, 782, 760, 763, 772, 707, 769, 773, 769, 790, 740, 794, 779, 752, 720, 738, 812, 734, 826, 761, 810, 788, 789, 742, 744, 733, 757, 798, 793, 791, 760, 760, 770, 765, 727, 790, 749, 735, 757, 745, 787, 777, 775, 760, 758, 786, 728, 755, 757, 778, 793, 797, 806, 744] payload_rows_per_dst=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] dup_rows_per_dst=[3112, 2896, 3018, 3078, 2912, 2878, 2964, 3016] combine_rows_per_src=[9286, 9288, 9314, 9300, 9382, 9234, 9326, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343416 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378616 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413816 weight_movement_bytes=69206016 total_nvlink_bytes=601619832 weight_to_token_ratio=0.129985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65144 dup_rows=23866 dup_rows_remote=20940 dedup_ratio=0.242778 router_tokens_per_expert=[813, 745, 742, 764, 801, 788, 776, 757, 742, 789, 796, 750, 782, 798, 830, 759, 723, 766, 759, 747, 782, 747, 766, 742, 753, 783, 771, 780, 785, 778, 776, 759, 776, 798, 787, 741, 755, 777, 748, 731, 829, 743, 798, 746, 725, 735, 772, 752, 789, 763, 757, 747, 754, 763, 751, 786, 795, 745, 753, 784, 755, 795, 766, 787] payload_rows_per_dst=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] dup_rows_per_dst=[3050, 3008, 2894, 3062, 2972, 2954, 2926, 3000] combine_rows_per_src=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343272 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206440 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069608 weight_movement_bytes=69206016 total_nvlink_bytes=601275624 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343704 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267116184 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533888664 weight_movement_bytes=69206016 total_nvlink_bytes=603094680 weight_to_token_ratio=0.129626 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64946 dup_rows=24046 dup_rows_remote=21060 dedup_ratio=0.244609 router_tokens_per_expert=[789, 757, 764, 797, 807, 735, 778, 759, 753, 745, 787, 803, 776, 819, 764, 726, 736, 763, 774, 806, 803, 756, 762, 793, 790, 743, 781, 772, 805, 768, 804, 723, 733, 715, 728, 767, 724, 802, 761, 753, 826, 734, 743, 750, 736, 777, 775, 808, 757, 783, 757, 773, 758, 770, 788, 789, 774, 763, 759, 786, 773, 736, 731, 785] payload_rows_per_dst=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] dup_rows_per_dst=[3026, 3072, 3070, 3068, 2790, 2986, 3038, 2996] combine_rows_per_src=[9346, 9274, 9316, 9304, 9176, 9312, 9312, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343944 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100616 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531857288 weight_movement_bytes=103809024 total_nvlink_bytes=635666312 weight_to_token_ratio=0.195182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74540 payload_rows_remote=65394 dup_rows=23764 dup_rows_remote=20866 dedup_ratio=0.241740 router_tokens_per_expert=[745, 789, 768, 812, 759, 785, 748, 740, 775, 758, 760, 765, 803, 782, 745, 734, 792, 782, 713, 764, 762, 748, 715, 746, 733, 777, 788, 752, 753, 797, 752, 826, 770, 791, 739, 761, 799, 794, 799, 755, 764, 779, 801, 790, 781, 797, 741, 737, 783, 757, 806, 761, 774, 755, 761, 709, 749, 789, 798, 741, 777, 734, 836, 756] payload_rows_per_dst=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] dup_rows_per_dst=[2952, 2964, 2898, 2962, 2942, 3034, 2974, 3038] combine_rows_per_src=[9340, 9280, 9146, 9394, 9474, 9346, 9238, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=343368 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265231688 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530120008 weight_movement_bytes=34603008 total_nvlink_bytes=564723016 weight_to_token_ratio=0.065274 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343512 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266468824 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594136 weight_movement_bytes=34603008 total_nvlink_bytes=567197144 weight_to_token_ratio=0.064971 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=343704 combine_payload_bytes=266772480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267116184 ingress_bytes=335978496 link_bytes=335978496 token_bytes=533888664 weight_movement_bytes=69206016 total_nvlink_bytes=603094680 weight_to_token_ratio=0.129626 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264260032e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344336 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174160 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003984 weight_movement_bytes=34603008 total_nvlink_bytes=568606992 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266018816 dispatch_meta_bytes=344024 combine_payload_bytes=266018816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266362840 ingress_bytes=335224832 link_bytes=335224832 token_bytes=532381656 weight_movement_bytes=69206016 total_nvlink_bytes=601587672 weight_to_token_ratio=0.129993 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261424055e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267853824 dispatch_meta_bytes=345040 combine_payload_bytes=267853824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268198864 ingress_bytes=302456832 link_bytes=302456832 token_bytes=536052688 weight_movement_bytes=34603008 total_nvlink_bytes=570655696 weight_to_token_ratio=0.064552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138120709e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65080 dup_rows=23956 dup_rows_remote=21050 dedup_ratio=0.243693 router_tokens_per_expert=[772, 775, 742, 770, 781, 786, 792, 761, 790, 771, 848, 760, 738, 736, 776, 729, 730, 770, 720, 755, 754, 762, 836, 760, 737, 799, 747, 817, 742, 778, 750, 753, 767, 786, 765, 770, 773, 732, 792, 801, 755, 775, 717, 745, 742, 768, 784, 749, 751, 819, 775, 744, 800, 799, 792, 799, 717, 780, 771, 792, 778, 769, 735, 773] payload_rows_per_dst=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] dup_rows_per_dst=[3006, 3080, 2906, 3020, 3000, 2832, 3150, 2962] combine_rows_per_src=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=343416 combine_payload_bytes=266035200 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266378616 ingress_bytes=335241216 link_bytes=335241216 token_bytes=532413816 weight_movement_bytes=69206016 total_nvlink_bytes=601619832 weight_to_token_ratio=0.129985 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261485707e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=264888320 dispatch_meta_bytes=343368 combine_payload_bytes=264888320 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265231688 ingress_bytes=299491328 link_bytes=299491328 token_bytes=530120008 weight_movement_bytes=34603008 total_nvlink_bytes=564723016 weight_to_token_ratio=0.065274 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126961756e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343944 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266100616 ingress_bytes=369565696 link_bytes=369565696 token_bytes=531857288 weight_movement_bytes=103809024 total_nvlink_bytes=635666312 weight_to_token_ratio=0.195182 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390645962e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344520 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266912200 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479880 weight_movement_bytes=34603008 total_nvlink_bytes=568082888 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65120 dup_rows=23878 dup_rows_remote=20804 dedup_ratio=0.242900 router_tokens_per_expert=[765, 784, 767, 746, 766, 768, 768, 749, 758, 741, 794, 759, 730, 796, 768, 809, 747, 767, 748, 785, 709, 753, 814, 809, 796, 737, 710, 766, 746, 777, 780, 758, 787, 786, 746, 771, 767, 806, 790, 734, 799, 772, 784, 768, 777, 755, 802, 733, 748, 779, 756, 784, 717, 831, 753, 760, 804, 742, 769, 763, 761, 820, 720, 798] payload_rows_per_dst=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] dup_rows_per_dst=[2966, 2932, 2992, 2932, 3042, 3074, 2918, 3022] combine_rows_per_src=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65098 dup_rows=23958 dup_rows_remote=21000 dedup_ratio=0.243713 router_tokens_per_expert=[732, 774, 809, 766, 766, 778, 765, 756, 762, 761, 752, 786, 720, 810, 766, 738, 762, 809, 769, 746, 768, 772, 725, 755, 757, 795, 772, 757, 741, 770, 732, 779, 756, 727, 764, 742, 790, 801, 822, 740, 795, 765, 769, 767, 831, 826, 796, 774, 767, 760, 745, 780, 817, 769, 770, 774, 760, 765, 741, 743, 742, 785, 754, 765] payload_rows_per_dst=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] dup_rows_per_dst=[3044, 2830, 2942, 2958, 2962, 3142, 3088, 2992] combine_rows_per_src=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74646 payload_rows_remote=65332 dup_rows=23658 dup_rows_remote=20664 dedup_ratio=0.240662 router_tokens_per_expert=[736, 754, 806, 786, 750, 804, 791, 760, 726, 765, 745, 789, 739, 795, 783, 800, 787, 770, 783, 752, 746, 768, 778, 746, 745, 777, 758, 788, 722, 720, 790, 781, 709, 758, 741, 836, 741, 760, 737, 792, 769, 756, 737, 765, 782, 772, 761, 779, 794, 788, 762, 732, 775, 786, 788, 729, 819, 764, 770, 795, 775, 810, 765, 765] payload_rows_per_dst=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] dup_rows_per_dst=[2972, 2928, 3020, 2856, 2936, 2978, 2994, 2974] combine_rows_per_src=[9402, 9356, 9240, 9306, 9212, 9264, 9314, 9552] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64922 dup_rows=24032 dup_rows_remote=20896 dedup_ratio=0.244466 router_tokens_per_expert=[725, 752, 734, 749, 817, 767, 766, 777, 783, 794, 760, 805, 824, 780, 738, 759, 767, 794, 798, 762, 742, 737, 775, 795, 784, 766, 766, 793, 842, 786, 781, 730, 798, 744, 764, 798, 765, 749, 767, 755, 726, 790, 809, 746, 747, 747, 712, 742, 785, 745, 743, 742, 720, 772, 736, 771, 793, 790, 794, 743, 736, 754, 777, 844] payload_rows_per_dst=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] dup_rows_per_dst=[2866, 3136, 3096, 3156, 3036, 2894, 2862, 2986] combine_rows_per_src=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64834 dup_rows=24074 dup_rows_remote=20912 dedup_ratio=0.244893 router_tokens_per_expert=[732, 741, 785, 796, 776, 774, 778, 788, 772, 761, 783, 762, 779, 750, 773, 767, 768, 792, 755, 741, 781, 769, 792, 713, 715, 760, 731, 777, 732, 769, 754, 801, 764, 796, 726, 740, 788, 789, 804, 778, 795, 799, 776, 793, 776, 773, 756, 745, 792, 779, 763, 785, 743, 751, 728, 850, 754, 787, 725, 763, 753, 732, 810, 772] payload_rows_per_dst=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] dup_rows_per_dst=[3034, 3014, 2936, 2958, 2816, 3150, 3004, 3162] combine_rows_per_src=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65120 dup_rows=23992 dup_rows_remote=20880 dedup_ratio=0.244059 router_tokens_per_expert=[793, 751, 788, 752, 793, 772, 777, 765, 750, 782, 755, 808, 751, 757, 737, 800, 734, 737, 752, 763, 769, 788, 756, 792, 768, 758, 810, 708, 749, 765, 766, 739, 776, 730, 745, 796, 792, 731, 770, 812, 749, 759, 787, 786, 816, 784, 792, 799, 751, 758, 768, 781, 765, 731, 805, 784, 751, 765, 767, 731, 787, 751, 760, 788] payload_rows_per_dst=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] dup_rows_per_dst=[2998, 2938, 2936, 2976, 3112, 3090, 2964, 2978] combine_rows_per_src=[9384, 9342, 9246, 9150, 9192, 9454, 9322, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65028 dup_rows=23974 dup_rows_remote=21058 dedup_ratio=0.243876 router_tokens_per_expert=[731, 725, 732, 772, 767, 729, 752, 748, 740, 792, 719, 773, 748, 787, 767, 768, 756, 759, 797, 809, 777, 807, 777, 827, 757, 783, 736, 753, 808, 748, 775, 784, 757, 755, 772, 754, 775, 758, 754, 760, 730, 791, 781, 784, 782, 800, 731, 806, 756, 750, 781, 789, 712, 773, 835, 813, 791, 748, 792, 757, 756, 753, 792, 761] payload_rows_per_dst=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] dup_rows_per_dst=[2916, 2936, 3156, 3170, 2940, 2832, 3026, 2998] combine_rows_per_src=[9302, 9062, 9462, 9514, 9278, 9066, 9344, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74230 payload_rows_remote=64834 dup_rows=24074 dup_rows_remote=20912 dedup_ratio=0.244893 router_tokens_per_expert=[732, 741, 785, 796, 776, 774, 778, 788, 772, 761, 783, 762, 779, 750, 773, 767, 768, 792, 755, 741, 781, 769, 792, 713, 715, 760, 731, 777, 732, 769, 754, 801, 764, 796, 726, 740, 788, 789, 804, 778, 795, 799, 776, 793, 776, 773, 756, 745, 792, 779, 763, 785, 743, 751, 728, 850, 754, 787, 725, 763, 753, 732, 810, 772] payload_rows_per_dst=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] dup_rows_per_dst=[3034, 3014, 2936, 2958, 2816, 3150, 3004, 3162] combine_rows_per_src=[9306, 9280, 9270, 9120, 9188, 9276, 9394, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74272 payload_rows_remote=64922 dup_rows=24032 dup_rows_remote=20896 dedup_ratio=0.244466 router_tokens_per_expert=[725, 752, 734, 749, 817, 767, 766, 777, 783, 794, 760, 805, 824, 780, 738, 759, 767, 794, 798, 762, 742, 737, 775, 795, 784, 766, 766, 793, 842, 786, 781, 730, 798, 744, 764, 798, 765, 749, 767, 755, 726, 790, 809, 746, 747, 747, 712, 742, 785, 745, 743, 742, 720, 772, 736, 771, 793, 790, 794, 743, 736, 754, 777, 844] payload_rows_per_dst=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] dup_rows_per_dst=[2866, 3136, 3096, 3156, 3036, 2894, 2862, 2986] combine_rows_per_src=[9308, 9350, 9244, 9340, 9244, 9144, 9166, 9476] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65098 dup_rows=23958 dup_rows_remote=21000 dedup_ratio=0.243713 router_tokens_per_expert=[732, 774, 809, 766, 766, 778, 765, 756, 762, 761, 752, 786, 720, 810, 766, 738, 762, 809, 769, 746, 768, 772, 725, 755, 757, 795, 772, 757, 741, 770, 732, 779, 756, 727, 764, 742, 790, 801, 822, 740, 795, 765, 769, 767, 831, 826, 796, 774, 767, 760, 745, 780, 817, 769, 770, 774, 760, 765, 741, 743, 742, 785, 754, 765] payload_rows_per_dst=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] dup_rows_per_dst=[3044, 2830, 2942, 2958, 2962, 3142, 3088, 2992] combine_rows_per_src=[9248, 9360, 9270, 9248, 9322, 9504, 9276, 9118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344000 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075520 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807040 weight_movement_bytes=69206016 total_nvlink_bytes=603013056 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344392 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985800 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627208 weight_movement_bytes=34603008 total_nvlink_bytes=568230216 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267599872 dispatch_meta_bytes=343984 combine_payload_bytes=267599872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267943856 ingress_bytes=336805888 link_bytes=336805888 token_bytes=535543728 weight_movement_bytes=69206016 total_nvlink_bytes=604749744 weight_to_token_ratio=0.129226 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.267373441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65230 dup_rows=23832 dup_rows_remote=21030 dedup_ratio=0.242432 router_tokens_per_expert=[730, 785, 769, 761, 734, 803, 796, 779, 798, 732, 743, 769, 785, 830, 791, 777, 768, 753, 821, 765, 763, 769, 749, 766, 785, 752, 776, 770, 767, 760, 757, 797, 812, 745, 789, 732, 800, 788, 769, 799, 756, 794, 746, 740, 761, 746, 794, 814, 820, 803, 722, 746, 728, 717, 749, 737, 738, 739, 720, 780, 748, 746, 772, 802] payload_rows_per_dst=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] dup_rows_per_dst=[2910, 3100, 2994, 3050, 3098, 2948, 2802, 2930] combine_rows_per_src=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344344 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699032 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053720 weight_movement_bytes=34603008 total_nvlink_bytes=567656728 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64908 dup_rows=24050 dup_rows_remote=20910 dedup_ratio=0.244649 router_tokens_per_expert=[778, 798, 719, 790, 815, 754, 759, 721, 741, 811, 752, 696, 787, 739, 775, 744, 766, 765, 792, 780, 736, 732, 769, 762, 751, 786, 807, 745, 832, 810, 810, 729, 755, 790, 771, 749, 789, 795, 790, 775, 776, 776, 787, 797, 793, 761, 766, 787, 832, 795, 749, 718, 756, 774, 754, 732, 787, 725, 755, 739, 780, 730, 751, 767] payload_rows_per_dst=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] dup_rows_per_dst=[2952, 2938, 2970, 3068, 3130, 3140, 2930, 2922] combine_rows_per_src=[9316, 9152, 9234, 9472, 9298, 9346, 9290, 9146] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265920512 dispatch_meta_bytes=343272 combine_payload_bytes=265920512 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266263784 ingress_bytes=335126528 link_bytes=335126528 token_bytes=532184296 weight_movement_bytes=69206016 total_nvlink_bytes=601390312 weight_to_token_ratio=0.130041 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261054145e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64754 dup_rows=24184 dup_rows_remote=21076 dedup_ratio=0.246012 router_tokens_per_expert=[778, 758, 721, 786, 721, 775, 755, 790, 768, 788, 768, 787, 751, 773, 754, 770, 751, 802, 827, 792, 765, 755, 763, 782, 806, 752, 703, 776, 796, 762, 775, 798, 785, 738, 835, 812, 767, 750, 726, 771, 731, 782, 797, 765, 714, 830, 768, 794, 769, 735, 768, 724, 756, 758, 768, 725, 748, 775, 742, 778, 770, 763, 792, 768] payload_rows_per_dst=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] dup_rows_per_dst=[2940, 3094, 3108, 3036, 3024, 3052, 2836, 3094] combine_rows_per_src=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=344000 combine_payload_bytes=266731520 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267075520 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533807040 weight_movement_bytes=69206016 total_nvlink_bytes=603013056 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266354688 dispatch_meta_bytes=344344 combine_payload_bytes=266354688 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266699032 ingress_bytes=300957696 link_bytes=300957696 token_bytes=533053720 weight_movement_bytes=34603008 total_nvlink_bytes=567656728 weight_to_token_ratio=0.064915 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132479581e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265560064 dispatch_meta_bytes=342984 combine_payload_bytes=265560064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265903048 ingress_bytes=300163072 link_bytes=300163072 token_bytes=531463112 weight_movement_bytes=34603008 total_nvlink_bytes=566066120 weight_to_token_ratio=0.065109 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129489475e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344392 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266985800 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627208 weight_movement_bytes=34603008 total_nvlink_bytes=568230216 weight_to_token_ratio=0.064845 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74098 payload_rows_remote=64836 dup_rows=24206 dup_rows_remote=21112 dedup_ratio=0.246236 router_tokens_per_expert=[752, 754, 772, 762, 798, 789, 739, 799, 792, 800, 766, 791, 741, 720, 771, 772, 731, 818, 732, 771, 799, 761, 772, 794, 760, 806, 759, 756, 712, 757, 787, 769, 839, 753, 754, 752, 739, 733, 795, 768, 777, 782, 743, 764, 781, 795, 744, 772, 763, 772, 758, 755, 738, 746, 781, 760, 773, 723, 775, 770, 787, 795, 778, 785] payload_rows_per_dst=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] dup_rows_per_dst=[3010, 2988, 3094, 3000, 3034, 3040, 2938, 3102] combine_rows_per_src=[9320, 9318, 9262, 9212, 9232, 9276, 9208, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=345040 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267527120 ingress_bytes=336388096 link_bytes=336388096 token_bytes=534709200 weight_movement_bytes=69206016 total_nvlink_bytes=603915216 weight_to_token_ratio=0.129427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265801324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343272 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206440 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069608 weight_movement_bytes=69206016 total_nvlink_bytes=601275624 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65088 dup_rows=24050 dup_rows_remote=21118 dedup_ratio=0.244649 router_tokens_per_expert=[796, 811, 742, 810, 805, 743, 762, 777, 776, 724, 768, 772, 726, 827, 754, 852, 754, 763, 809, 806, 806, 727, 687, 739, 762, 758, 771, 746, 803, 763, 791, 774, 750, 758, 730, 777, 765, 749, 765, 755, 749, 760, 854, 761, 808, 810, 758, 785, 780, 752, 773, 758, 770, 723, 744, 793, 748, 767, 735, 747, 727, 775, 782, 740] payload_rows_per_dst=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] dup_rows_per_dst=[3118, 3076, 2914, 3026, 2932, 3090, 3016, 2878] combine_rows_per_src=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=343792 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912048 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480304 weight_movement_bytes=69206016 total_nvlink_bytes=600686320 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64852 dup_rows=24114 dup_rows_remote=21112 dedup_ratio=0.245300 router_tokens_per_expert=[765, 807, 769, 763, 750, 727, 830, 755, 725, 745, 803, 758, 745, 735, 761, 725, 811, 756, 779, 747, 821, 749, 792, 752, 783, 745, 762, 784, 759, 783, 735, 762, 764, 742, 815, 770, 770, 750, 804, 744, 733, 771, 730, 750, 821, 748, 768, 731, 792, 827, 791, 761, 760, 806, 750, 825, 782, 751, 731, 777, 780, 780, 760, 785] payload_rows_per_dst=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] dup_rows_per_dst=[3002, 2982, 2990, 3252, 3000, 2762, 3106, 3020] combine_rows_per_src=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64820 dup_rows=24114 dup_rows_remote=21082 dedup_ratio=0.245300 router_tokens_per_expert=[821, 749, 749, 778, 751, 803, 770, 745, 756, 780, 800, 748, 789, 759, 793, 776, 758, 768, 788, 759, 786, 728, 760, 793, 762, 759, 802, 744, 780, 739, 762, 779, 711, 767, 780, 742, 763, 753, 788, 787, 840, 784, 780, 760, 747, 780, 754, 782, 733, 759, 733, 782, 769, 741, 811, 765, 775, 748, 788, 745, 768, 745, 773, 765] payload_rows_per_dst=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] dup_rows_per_dst=[2958, 3032, 2988, 3050, 2988, 3168, 2910, 3020] combine_rows_per_src=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=64968 dup_rows=23874 dup_rows_remote=20604 dedup_ratio=0.242859 router_tokens_per_expert=[777, 796, 754, 757, 744, 742, 722, 777, 775, 779, 789, 806, 787, 780, 757, 750, 750, 813, 803, 760, 830, 736, 758, 793, 738, 779, 790, 773, 715, 796, 741, 761, 790, 774, 766, 727, 735, 816, 770, 730, 788, 752, 751, 768, 769, 778, 764, 758, 800, 734, 768, 795, 742, 780, 801, 809, 740, 765, 732, 733, 789, 775, 761, 764] payload_rows_per_dst=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] dup_rows_per_dst=[2818, 2900, 3102, 2652, 3186, 2980, 2966, 3270] combine_rows_per_src=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=343320 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265575704 ingress_bytes=334438400 link_bytes=334438400 token_bytes=530808088 weight_movement_bytes=69206016 total_nvlink_bytes=600014104 weight_to_token_ratio=0.130379 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258464775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65186 dup_rows=23896 dup_rows_remote=20936 dedup_ratio=0.243083 router_tokens_per_expert=[756, 755, 786, 739, 787, 779, 762, 744, 758, 766, 768, 801, 773, 784, 743, 772, 766, 766, 725, 730, 806, 809, 816, 811, 778, 752, 755, 795, 788, 786, 796, 740, 807, 784, 791, 769, 770, 721, 759, 751, 754, 755, 738, 784, 749, 777, 769, 765, 739, 783, 808, 769, 794, 749, 776, 734, 745, 783, 743, 787, 749, 755, 752, 751] payload_rows_per_dst=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] dup_rows_per_dst=[2974, 2970, 3054, 3044, 2940, 2960, 3054, 2900] combine_rows_per_src=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65060 dup_rows=24050 dup_rows_remote=20986 dedup_ratio=0.244649 router_tokens_per_expert=[787, 766, 778, 770, 798, 743, 768, 767, 718, 761, 770, 805, 762, 762, 759, 809, 754, 740, 768, 741, 763, 783, 779, 712, 772, 725, 774, 786, 732, 768, 790, 782, 771, 765, 745, 763, 747, 795, 712, 784, 781, 739, 766, 752, 777, 728, 813, 809, 787, 780, 755, 778, 753, 804, 795, 725, 794, 805, 778, 767, 749, 775, 784, 784] payload_rows_per_dst=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] dup_rows_per_dst=[2948, 3018, 2884, 3064, 2950, 3072, 3076, 3038] combine_rows_per_src=[9406, 9274, 9196, 9194, 9214, 9258, 9278, 9434] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=343856 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265977648 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611440 weight_movement_bytes=103809024 total_nvlink_bytes=635420464 weight_to_token_ratio=0.195272 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65044 dup_rows=23948 dup_rows_remote=21066 dedup_ratio=0.243612 router_tokens_per_expert=[814, 739, 795, 760, 792, 754, 783, 758, 802, 741, 796, 779, 760, 745, 794, 768, 770, 762, 759, 771, 755, 728, 735, 754, 808, 783, 785, 778, 772, 797, 804, 783, 744, 739, 786, 755, 785, 773, 772, 762, 774, 775, 806, 747, 779, 754, 745, 760, 743, 790, 778, 770, 790, 738, 731, 757, 752, 795, 784, 690, 764, 721, 782, 787] payload_rows_per_dst=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] dup_rows_per_dst=[2972, 3062, 2866, 3200, 3012, 2996, 2882, 2958] combine_rows_per_src=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=65144 dup_rows=23866 dup_rows_remote=20940 dedup_ratio=0.242778 router_tokens_per_expert=[813, 745, 742, 764, 801, 788, 776, 757, 742, 789, 796, 750, 782, 798, 830, 759, 723, 766, 759, 747, 782, 747, 766, 742, 753, 783, 771, 780, 785, 778, 776, 759, 776, 798, 787, 741, 755, 777, 748, 731, 829, 743, 798, 746, 725, 735, 772, 752, 789, 763, 757, 747, 754, 763, 751, 786, 795, 745, 753, 784, 755, 795, 766, 787] payload_rows_per_dst=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] dup_rows_per_dst=[3050, 3008, 2894, 3062, 2972, 2954, 2926, 3000] combine_rows_per_src=[9322, 9484, 9170, 9308, 9254, 9246, 9294, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64820 dup_rows=24114 dup_rows_remote=21082 dedup_ratio=0.245300 router_tokens_per_expert=[821, 749, 749, 778, 751, 803, 770, 745, 756, 780, 800, 748, 789, 759, 793, 776, 758, 768, 788, 759, 786, 728, 760, 793, 762, 759, 802, 744, 780, 739, 762, 779, 711, 767, 780, 742, 763, 753, 788, 787, 840, 784, 780, 760, 747, 780, 754, 782, 733, 759, 733, 782, 769, 741, 811, 765, 775, 748, 788, 745, 768, 745, 773, 765] payload_rows_per_dst=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] dup_rows_per_dst=[2958, 3032, 2988, 3050, 2988, 3168, 2910, 3020] combine_rows_per_src=[9374, 9370, 9292, 9204, 9194, 9286, 9276, 9194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64852 dup_rows=24114 dup_rows_remote=21112 dedup_ratio=0.245300 router_tokens_per_expert=[765, 807, 769, 763, 750, 727, 830, 755, 725, 745, 803, 758, 745, 735, 761, 725, 811, 756, 779, 747, 821, 749, 792, 752, 783, 745, 762, 784, 759, 783, 735, 762, 764, 742, 815, 770, 770, 750, 804, 744, 733, 771, 730, 750, 821, 748, 768, 731, 792, 827, 791, 761, 760, 806, 750, 825, 782, 751, 731, 777, 780, 780, 760, 785] payload_rows_per_dst=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] dup_rows_per_dst=[3002, 2982, 2990, 3252, 3000, 2762, 3106, 3020] combine_rows_per_src=[9338, 9188, 9248, 9296, 9318, 9020, 9510, 9272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344488 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346344 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348200 weight_movement_bytes=69206016 total_nvlink_bytes=603554216 weight_to_token_ratio=0.129515 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=65088 dup_rows=24050 dup_rows_remote=21118 dedup_ratio=0.244649 router_tokens_per_expert=[796, 811, 742, 810, 805, 743, 762, 777, 776, 724, 768, 772, 726, 827, 754, 852, 754, 763, 809, 806, 806, 727, 687, 739, 762, 758, 771, 746, 803, 763, 791, 774, 750, 758, 730, 777, 765, 749, 765, 755, 749, 760, 854, 761, 808, 810, 758, 785, 780, 752, 773, 758, 770, 723, 744, 793, 748, 767, 735, 747, 727, 775, 782, 740] payload_rows_per_dst=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] dup_rows_per_dst=[3118, 3076, 2914, 3026, 2932, 3090, 3016, 2878] combine_rows_per_src=[9374, 9322, 9268, 9310, 9166, 9480, 9170, 9164] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344184 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266829944 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315704 weight_movement_bytes=69206016 total_nvlink_bytes=602521720 weight_to_token_ratio=0.129766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=64968 dup_rows=23874 dup_rows_remote=20604 dedup_ratio=0.242859 router_tokens_per_expert=[777, 796, 754, 757, 744, 742, 722, 777, 775, 779, 789, 806, 787, 780, 757, 750, 750, 813, 803, 760, 830, 736, 758, 793, 738, 779, 790, 773, 715, 796, 741, 761, 790, 774, 766, 727, 735, 816, 770, 730, 788, 752, 751, 768, 769, 778, 764, 758, 800, 734, 768, 795, 742, 780, 801, 809, 740, 765, 732, 733, 789, 775, 761, 764] payload_rows_per_dst=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] dup_rows_per_dst=[2818, 2900, 3102, 2652, 3186, 2980, 2966, 3270] combine_rows_per_src=[9194, 9124, 9422, 8920, 9452, 9364, 9492, 9462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64998 dup_rows=24096 dup_rows_remote=21130 dedup_ratio=0.245117 router_tokens_per_expert=[711, 775, 771, 727, 763, 837, 760, 780, 753, 779, 744, 754, 746, 811, 757, 767, 743, 732, 693, 776, 770, 836, 777, 761, 746, 774, 743, 776, 734, 734, 739, 749, 781, 749, 804, 789, 770, 753, 744, 728, 767, 788, 795, 793, 785, 758, 789, 779, 794, 769, 757, 747, 774, 804, 809, 780, 773, 761, 803, 794, 753, 757, 793, 794] payload_rows_per_dst=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] dup_rows_per_dst=[2982, 2950, 2966, 2864, 2994, 3188, 3122, 3030] combine_rows_per_src=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344336 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267174160 ingress_bytes=301432832 link_bytes=301432832 token_bytes=534003984 weight_movement_bytes=34603008 total_nvlink_bytes=568606992 weight_to_token_ratio=0.064799 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134267479e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343608 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846328 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349048 weight_movement_bytes=69206016 total_nvlink_bytes=600555064 weight_to_token_ratio=0.130246 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65120 dup_rows=23878 dup_rows_remote=20804 dedup_ratio=0.242900 router_tokens_per_expert=[765, 784, 767, 746, 766, 768, 768, 749, 758, 741, 794, 759, 730, 796, 768, 809, 747, 767, 748, 785, 709, 753, 814, 809, 796, 737, 710, 766, 746, 777, 780, 758, 787, 786, 746, 771, 767, 806, 790, 734, 799, 772, 784, 768, 777, 755, 802, 733, 748, 779, 756, 784, 717, 831, 753, 760, 804, 742, 769, 763, 761, 820, 720, 798] payload_rows_per_dst=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] dup_rows_per_dst=[2966, 2932, 2992, 2932, 3042, 3074, 2918, 3022] combine_rows_per_src=[9260, 9378, 9272, 9208, 9332, 9306, 9338, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=343856 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265977648 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611440 weight_movement_bytes=103809024 total_nvlink_bytes=635420464 weight_to_token_ratio=0.195272 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=344824 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266945272 ingress_bytes=301203456 link_bytes=301203456 token_bytes=533545720 weight_movement_bytes=34603008 total_nvlink_bytes=568148728 weight_to_token_ratio=0.064855 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133404356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342288 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266451216 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532560144 weight_movement_bytes=34603008 total_nvlink_bytes=567163152 weight_to_token_ratio=0.064975 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344184 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266829944 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315704 weight_movement_bytes=69206016 total_nvlink_bytes=602521720 weight_to_token_ratio=0.129766 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65080 dup_rows=23956 dup_rows_remote=21050 dedup_ratio=0.243693 router_tokens_per_expert=[772, 775, 742, 770, 781, 786, 792, 761, 790, 771, 848, 760, 738, 736, 776, 729, 730, 770, 720, 755, 754, 762, 836, 760, 737, 799, 747, 817, 742, 778, 750, 753, 767, 786, 765, 770, 773, 732, 792, 801, 755, 775, 717, 745, 742, 768, 784, 749, 751, 819, 775, 744, 800, 799, 792, 799, 717, 780, 771, 792, 778, 769, 735, 773] payload_rows_per_dst=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] dup_rows_per_dst=[3006, 3080, 2906, 3020, 3000, 2832, 3150, 2962] combine_rows_per_src=[9352, 9216, 9268, 9226, 9372, 9238, 9408, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344440 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764664 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184888 weight_movement_bytes=34603008 total_nvlink_bytes=567787896 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65134 dup_rows=23762 dup_rows_remote=20718 dedup_ratio=0.241720 router_tokens_per_expert=[779, 739, 784, 753, 769, 755, 766, 776, 784, 736, 735, 789, 743, 766, 817, 729, 757, 800, 807, 774, 736, 767, 815, 743, 782, 720, 744, 789, 740, 784, 816, 783, 797, 778, 742, 760, 757, 792, 815, 785, 746, 793, 773, 752, 747, 772, 770, 779, 803, 733, 758, 729, 700, 756, 761, 808, 761, 779, 811, 777, 748, 775, 750, 768] payload_rows_per_dst=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] dup_rows_per_dst=[2936, 2964, 3048, 3010, 3044, 2946, 2834, 2980] combine_rows_per_src=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343696 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267075216 ingress_bytes=335937536 link_bytes=335937536 token_bytes=533806736 weight_movement_bytes=69206016 total_nvlink_bytes=603012752 weight_to_token_ratio=0.129646 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264105903e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64586 dup_rows=24138 dup_rows_remote=20908 dedup_ratio=0.245544 router_tokens_per_expert=[751, 793, 756, 796, 774, 723, 791, 758, 778, 782, 763, 780, 784, 784, 756, 750, 744, 785, 752, 773, 799, 743, 785, 770, 738, 771, 777, 734, 767, 754, 746, 739, 774, 766, 747, 739, 778, 819, 738, 753, 780, 816, 702, 757, 777, 762, 712, 792, 797, 811, 801, 797, 769, 790, 781, 741, 760, 767, 775, 802, 749, 744, 785, 775] payload_rows_per_dst=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] dup_rows_per_dst=[3230, 3142, 3148, 2944, 2804, 2954, 3174, 2742] combine_rows_per_src=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266567680 dispatch_meta_bytes=344520 combine_payload_bytes=266567680 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266912200 ingress_bytes=301170688 link_bytes=301170688 token_bytes=533479880 weight_movement_bytes=34603008 total_nvlink_bytes=568082888 weight_to_token_ratio=0.064863 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133281053e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=64950 dup_rows=23890 dup_rows_remote=20754 dedup_ratio=0.243022 router_tokens_per_expert=[758, 749, 784, 725, 741, 842, 756, 763, 779, 784, 802, 743, 822, 781, 793, 796, 794, 746, 799, 761, 773, 799, 766, 695, 816, 769, 760, 784, 780, 747, 773, 754, 789, 775, 747, 785, 734, 769, 801, 774, 780, 752, 729, 774, 736, 711, 766, 748, 708, 785, 808, 757, 748, 779, 748, 787, 755, 814, 759, 761, 766, 780, 763, 730] payload_rows_per_dst=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] dup_rows_per_dst=[3056, 3136, 2936, 2976, 3042, 2850, 2950, 2944] combine_rows_per_src=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343408 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267132272 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921136 weight_movement_bytes=103809024 total_nvlink_bytes=637730160 weight_to_token_ratio=0.194428 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65150 dup_rows=23796 dup_rows_remote=20764 dedup_ratio=0.242065 router_tokens_per_expert=[767, 732, 742, 784, 699, 822, 756, 703, 783, 779, 786, 710, 772, 745, 753, 768, 766, 742, 723, 777, 764, 726, 790, 728, 816, 777, 795, 807, 788, 736, 793, 826, 798, 754, 788, 789, 754, 769, 773, 792, 783, 759, 731, 784, 731, 720, 811, 774, 776, 744, 724, 787, 803, 792, 797, 773, 760, 768, 764, 761, 783, 763, 818, 774] payload_rows_per_dst=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] dup_rows_per_dst=[2850, 2970, 2844, 3178, 3004, 2892, 3026, 3032] combine_rows_per_src=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344512 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576320 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808128 weight_movement_bytes=34603008 total_nvlink_bytes=567411136 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65108 dup_rows=23846 dup_rows_remote=20922 dedup_ratio=0.242574 router_tokens_per_expert=[765, 761, 787, 744, 753, 769, 713, 785, 800, 756, 765, 812, 788, 712, 801, 739, 778, 692, 777, 757, 749, 776, 777, 808, 794, 777, 757, 766, 761, 789, 720, 761, 761, 774, 769, 767, 818, 778, 719, 772, 770, 755, 808, 751, 797, 736, 779, 741, 788, 768, 747, 748, 757, 743, 785, 775, 782, 773, 776, 808, 784, 776, 755, 803] payload_rows_per_dst=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] dup_rows_per_dst=[2948, 2972, 2944, 2980, 3054, 2924, 2900, 3124] combine_rows_per_src=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74140 payload_rows_remote=64866 dup_rows=24164 dup_rows_remote=21080 dedup_ratio=0.245809 router_tokens_per_expert=[735, 760, 754, 786, 748, 716, 794, 761, 769, 771, 790, 774, 735, 738, 727, 753, 802, 771, 777, 762, 751, 747, 786, 766, 763, 777, 736, 856, 727, 784, 779, 757, 784, 782, 761, 725, 763, 776, 819, 780, 772, 771, 781, 748, 766, 741, 739, 780, 796, 786, 760, 768, 762, 738, 777, 766, 763, 844, 762, 739, 772, 814, 765, 800] payload_rows_per_dst=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] dup_rows_per_dst=[2908, 3010, 3144, 3084, 3076, 2896, 2954, 3092] combine_rows_per_src=[9200, 9104, 9180, 9274, 9304, 9300, 9352, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=341976 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886232 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430488 weight_movement_bytes=34603008 total_nvlink_bytes=564033496 weight_to_token_ratio=0.065359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=65034 dup_rows=24170 dup_rows_remote=21436 dedup_ratio=0.245870 router_tokens_per_expert=[790, 757, 756, 780, 753, 827, 816, 769, 781, 756, 769, 786, 731, 722, 736, 813, 774, 803, 747, 741, 798, 747, 764, 774, 784, 754, 748, 749, 733, 800, 776, 772, 750, 798, 744, 746, 751, 791, 768, 784, 761, 811, 789, 781, 782, 807, 778, 808, 737, 731, 717, 733, 761, 749, 725, 764, 760, 739, 746, 800, 794, 780, 768, 793] payload_rows_per_dst=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] dup_rows_per_dst=[3114, 2964, 3046, 3032, 2974, 3234, 2734, 3072] combine_rows_per_src=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344120 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026488 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708856 weight_movement_bytes=34603008 total_nvlink_bytes=568311864 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74414 payload_rows_remote=64950 dup_rows=23890 dup_rows_remote=20754 dedup_ratio=0.243022 router_tokens_per_expert=[758, 749, 784, 725, 741, 842, 756, 763, 779, 784, 802, 743, 822, 781, 793, 796, 794, 746, 799, 761, 773, 799, 766, 695, 816, 769, 760, 784, 780, 747, 773, 754, 789, 775, 747, 785, 734, 769, 801, 774, 780, 752, 729, 774, 736, 711, 766, 748, 708, 785, 808, 757, 748, 779, 748, 787, 755, 814, 759, 761, 766, 780, 763, 730] payload_rows_per_dst=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] dup_rows_per_dst=[3056, 3136, 2936, 2976, 3042, 2850, 2950, 2944] combine_rows_per_src=[9180, 9464, 9330, 9390, 9306, 9142, 9290, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64586 dup_rows=24138 dup_rows_remote=20908 dedup_ratio=0.245544 router_tokens_per_expert=[751, 793, 756, 796, 774, 723, 791, 758, 778, 782, 763, 780, 784, 784, 756, 750, 744, 785, 752, 773, 799, 743, 785, 770, 738, 771, 777, 734, 767, 754, 746, 739, 774, 766, 747, 739, 778, 819, 738, 753, 780, 816, 702, 757, 777, 762, 712, 792, 797, 811, 801, 797, 769, 790, 781, 741, 760, 767, 775, 802, 749, 744, 785, 775] payload_rows_per_dst=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] dup_rows_per_dst=[3230, 3142, 3148, 2944, 2804, 2954, 3174, 2742] combine_rows_per_src=[9580, 9302, 9436, 9172, 9078, 9296, 9572, 8730] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65230 dup_rows=23832 dup_rows_remote=21030 dedup_ratio=0.242432 router_tokens_per_expert=[730, 785, 769, 761, 734, 803, 796, 779, 798, 732, 743, 769, 785, 830, 791, 777, 768, 753, 821, 765, 763, 769, 749, 766, 785, 752, 776, 770, 767, 760, 757, 797, 812, 745, 789, 732, 800, 788, 769, 799, 756, 794, 746, 740, 761, 746, 794, 814, 820, 803, 722, 746, 728, 717, 749, 737, 738, 739, 720, 780, 748, 746, 772, 802] payload_rows_per_dst=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] dup_rows_per_dst=[2910, 3100, 2994, 3050, 3098, 2948, 2802, 2930] combine_rows_per_src=[9404, 9350, 9314, 9278, 9370, 9354, 9242, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343784 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034920 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531726056 weight_movement_bytes=34603008 total_nvlink_bytes=566329064 weight_to_token_ratio=0.065077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74542 payload_rows_remote=65134 dup_rows=23762 dup_rows_remote=20718 dedup_ratio=0.241720 router_tokens_per_expert=[779, 739, 784, 753, 769, 755, 766, 776, 784, 736, 735, 789, 743, 766, 817, 729, 757, 800, 807, 774, 736, 767, 815, 743, 782, 720, 744, 789, 740, 784, 816, 783, 797, 778, 742, 760, 757, 792, 815, 785, 746, 793, 773, 752, 747, 772, 770, 779, 803, 733, 758, 729, 700, 756, 761, 808, 761, 779, 811, 777, 748, 775, 750, 768] payload_rows_per_dst=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] dup_rows_per_dst=[2936, 2964, 3048, 3010, 3044, 2946, 2834, 2980] combine_rows_per_src=[9306, 9234, 9350, 9306, 9408, 9318, 9262, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65150 dup_rows=23796 dup_rows_remote=20764 dedup_ratio=0.242065 router_tokens_per_expert=[767, 732, 742, 784, 699, 822, 756, 703, 783, 779, 786, 710, 772, 745, 753, 768, 766, 742, 723, 777, 764, 726, 790, 728, 816, 777, 795, 807, 788, 736, 793, 826, 798, 754, 788, 789, 754, 769, 773, 792, 783, 759, 731, 784, 731, 720, 811, 774, 776, 744, 724, 787, 803, 792, 797, 773, 760, 768, 764, 761, 783, 763, 818, 774] payload_rows_per_dst=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] dup_rows_per_dst=[2850, 2970, 2844, 3178, 3004, 2892, 3026, 3032] combine_rows_per_src=[9160, 9222, 9188, 9498, 9422, 9294, 9366, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65262 dup_rows=23758 dup_rows_remote=20714 dedup_ratio=0.241679 router_tokens_per_expert=[778, 749, 738, 756, 766, 770, 743, 857, 796, 783, 763, 825, 804, 747, 733, 730, 731, 773, 797, 798, 755, 781, 800, 729, 745, 774, 755, 784, 758, 762, 778, 779, 809, 765, 792, 793, 768, 770, 755, 765, 759, 755, 775, 731, 779, 739, 777, 758, 730, 748, 721, 733, 739, 824, 771, 812, 784, 758, 828, 761, 755, 770, 739, 752] payload_rows_per_dst=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] dup_rows_per_dst=[2976, 3016, 3044, 2944, 2998, 2906, 2826, 3048] combine_rows_per_src=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=342816 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266378016 ingress_bytes=300638208 link_bytes=300638208 token_bytes=532413216 weight_movement_bytes=34603008 total_nvlink_bytes=567016224 weight_to_token_ratio=0.064993 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131277373e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65186 dup_rows=23896 dup_rows_remote=20936 dedup_ratio=0.243083 router_tokens_per_expert=[756, 755, 786, 739, 787, 779, 762, 744, 758, 766, 768, 801, 773, 784, 743, 772, 766, 766, 725, 730, 806, 809, 816, 811, 778, 752, 755, 795, 788, 786, 796, 740, 807, 784, 791, 769, 770, 721, 759, 751, 754, 755, 738, 784, 749, 777, 769, 765, 739, 783, 808, 769, 794, 749, 776, 734, 745, 783, 743, 787, 749, 755, 752, 751] payload_rows_per_dst=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] dup_rows_per_dst=[2974, 2970, 3054, 3044, 2940, 2960, 3054, 2900] combine_rows_per_src=[9242, 9360, 9404, 9336, 9364, 9222, 9250, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=264544256 dispatch_meta_bytes=341976 combine_payload_bytes=264544256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264886232 ingress_bytes=299147264 link_bytes=299147264 token_bytes=529430488 weight_movement_bytes=34603008 total_nvlink_bytes=564033496 weight_to_token_ratio=0.065359 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125667071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267182080 dispatch_meta_bytes=345040 combine_payload_bytes=267182080 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267527120 ingress_bytes=336388096 link_bytes=336388096 token_bytes=534709200 weight_movement_bytes=69206016 total_nvlink_bytes=603915216 weight_to_token_ratio=0.129427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265801324e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343408 combine_payload_bytes=266788864 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267132272 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533921136 weight_movement_bytes=103809024 total_nvlink_bytes=637730160 weight_to_token_ratio=0.194428 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=343904 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267657056 ingress_bytes=371122176 link_bytes=371122176 token_bytes=534970208 weight_movement_bytes=103809024 total_nvlink_bytes=638779232 weight_to_token_ratio=0.194046 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396502871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266854400 dispatch_meta_bytes=343656 combine_payload_bytes=266854400 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=267198056 ingress_bytes=439869440 link_bytes=439869440 token_bytes=534052456 weight_movement_bytes=173015040 total_nvlink_bytes=707067496 weight_to_token_ratio=0.323966 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.655193290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343784 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034920 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531726056 weight_movement_bytes=34603008 total_nvlink_bytes=566329064 weight_to_token_ratio=0.065077 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=345880 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266725144 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533104408 weight_movement_bytes=34603008 total_nvlink_bytes=567707416 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64754 dup_rows=24184 dup_rows_remote=21076 dedup_ratio=0.246012 router_tokens_per_expert=[778, 758, 721, 786, 721, 775, 755, 790, 768, 788, 768, 787, 751, 773, 754, 770, 751, 802, 827, 792, 765, 755, 763, 782, 806, 752, 703, 776, 796, 762, 775, 798, 785, 738, 835, 812, 767, 750, 726, 771, 731, 782, 797, 765, 714, 830, 768, 794, 769, 735, 768, 724, 756, 758, 768, 725, 748, 775, 742, 778, 770, 763, 792, 768] payload_rows_per_dst=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] dup_rows_per_dst=[2940, 3094, 3108, 3036, 3024, 3052, 2836, 3094] combine_rows_per_src=[9228, 9224, 9366, 9300, 9344, 9310, 9170, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267001856 dispatch_meta_bytes=344488 combine_payload_bytes=267001856 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267346344 ingress_bytes=336207872 link_bytes=336207872 token_bytes=534348200 weight_movement_bytes=69206016 total_nvlink_bytes=603554216 weight_to_token_ratio=0.129515 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265123155e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21002 dedup_ratio=0.244873 router_tokens_per_expert=[770, 758, 761, 732, 770, 794, 746, 704, 799, 769, 696, 753, 738, 769, 775, 770, 748, 788, 793, 808, 734, 797, 788, 702, 789, 776, 781, 760, 770, 744, 758, 761, 764, 790, 813, 777, 740, 814, 745, 749, 697, 763, 779, 736, 788, 743, 781, 804, 772, 776, 818, 784, 780, 752, 762, 765, 769, 791, 823, 741, 762, 817, 764, 792] payload_rows_per_dst=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] dup_rows_per_dst=[2970, 2884, 2998, 3010, 3070, 2918, 3078, 3144] combine_rows_per_src=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65134 dup_rows=23876 dup_rows_remote=20894 dedup_ratio=0.242879 router_tokens_per_expert=[752, 746, 800, 785, 787, 831, 783, 792, 786, 793, 801, 727, 772, 789, 771, 737, 770, 774, 805, 751, 748, 766, 778, 744, 749, 776, 748, 754, 769, 790, 776, 748, 770, 731, 747, 741, 789, 778, 782, 804, 749, 758, 759, 790, 773, 755, 775, 779, 753, 761, 781, 703, 720, 741, 814, 760, 794, 755, 746, 760, 752, 772, 800, 762] payload_rows_per_dst=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] dup_rows_per_dst=[3038, 3050, 2942, 3018, 2988, 2982, 2838, 3020] combine_rows_per_src=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65014 dup_rows=23834 dup_rows_remote=20738 dedup_ratio=0.242452 router_tokens_per_expert=[751, 729, 750, 777, 759, 732, 796, 758, 786, 787, 765, 743, 798, 784, 754, 749, 755, 750, 764, 764, 796, 808, 764, 757, 787, 789, 807, 769, 790, 770, 784, 780, 792, 773, 761, 792, 795, 798, 760, 743, 739, 760, 766, 783, 753, 750, 779, 783, 754, 790, 787, 719, 739, 725, 762, 768, 785, 756, 767, 728, 783, 762, 755, 793] payload_rows_per_dst=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] dup_rows_per_dst=[2898, 3098, 3018, 3096, 3046, 2896, 2838, 2944] combine_rows_per_src=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344112 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132976 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921840 weight_movement_bytes=34603008 total_nvlink_bytes=568524848 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65002 dup_rows=24016 dup_rows_remote=21004 dedup_ratio=0.244303 router_tokens_per_expert=[768, 763, 764, 760, 737, 792, 713, 756, 769, 854, 761, 728, 781, 749, 741, 766, 772, 811, 764, 759, 789, 800, 733, 754, 747, 751, 764, 756, 831, 780, 730, 782, 810, 749, 759, 710, 779, 723, 808, 767, 778, 775, 793, 769, 823, 728, 780, 747, 785, 743, 776, 803, 753, 752, 750, 765, 744, 765, 791, 751, 818, 777, 783, 773] payload_rows_per_dst=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] dup_rows_per_dst=[2864, 3012, 3006, 2990, 2988, 3128, 3004, 3024] combine_rows_per_src=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65082 dup_rows=23786 dup_rows_remote=20492 dedup_ratio=0.241964 router_tokens_per_expert=[759, 779, 760, 802, 808, 804, 769, 792, 766, 784, 811, 796, 734, 801, 806, 763, 743, 777, 749, 755, 764, 780, 802, 759, 738, 739, 771, 798, 752, 804, 746, 804, 697, 769, 772, 782, 772, 769, 682, 765, 774, 717, 782, 779, 758, 771, 763, 741, 731, 802, 796, 764, 806, 717, 745, 784, 765, 768, 785, 758, 760, 801, 725, 737] payload_rows_per_dst=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] dup_rows_per_dst=[3294, 3078, 2842, 2876, 2770, 3326, 3008, 2592] combine_rows_per_src=[9436, 9570, 9234, 9426, 9120, 9556, 9282, 8894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=343320 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265575704 ingress_bytes=334438400 link_bytes=334438400 token_bytes=530808088 weight_movement_bytes=69206016 total_nvlink_bytes=600014104 weight_to_token_ratio=0.130379 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258464775e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64918 dup_rows=23930 dup_rows_remote=20874 dedup_ratio=0.243429 router_tokens_per_expert=[750, 773, 790, 810, 782, 760, 776, 789, 778, 817, 775, 738, 732, 756, 736, 716, 735, 837, 764, 781, 803, 783, 795, 758, 776, 747, 787, 761, 766, 739, 791, 793, 772, 730, 766, 735, 760, 695, 747, 742, 769, 798, 694, 724, 802, 761, 789, 770, 801, 783, 735, 724, 749, 786, 740, 770, 776, 757, 816, 835, 833, 769, 744, 786] payload_rows_per_dst=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] dup_rows_per_dst=[3112, 2836, 3056, 3018, 2778, 2998, 2896, 3236] combine_rows_per_src=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74232 payload_rows_remote=64918 dup_rows=24072 dup_rows_remote=21002 dedup_ratio=0.244873 router_tokens_per_expert=[770, 758, 761, 732, 770, 794, 746, 704, 799, 769, 696, 753, 738, 769, 775, 770, 748, 788, 793, 808, 734, 797, 788, 702, 789, 776, 781, 760, 770, 744, 758, 761, 764, 790, 813, 777, 740, 814, 745, 749, 697, 763, 779, 736, 788, 743, 781, 804, 772, 776, 818, 784, 780, 752, 762, 765, 769, 791, 823, 741, 762, 817, 764, 792] payload_rows_per_dst=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] dup_rows_per_dst=[2970, 2884, 2998, 3010, 3070, 2918, 3078, 3144] combine_rows_per_src=[9100, 9254, 9318, 9268, 9314, 9264, 9340, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343008 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266640352 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532937696 weight_movement_bytes=69206016 total_nvlink_bytes=602143712 weight_to_token_ratio=0.129858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65002 dup_rows=24016 dup_rows_remote=21004 dedup_ratio=0.244303 router_tokens_per_expert=[768, 763, 764, 760, 737, 792, 713, 756, 769, 854, 761, 728, 781, 749, 741, 766, 772, 811, 764, 759, 789, 800, 733, 754, 747, 751, 764, 756, 831, 780, 730, 782, 810, 749, 759, 710, 779, 723, 808, 767, 778, 775, 793, 769, 823, 728, 780, 747, 785, 743, 776, 803, 753, 752, 750, 765, 744, 765, 791, 751, 818, 777, 783, 773] payload_rows_per_dst=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] dup_rows_per_dst=[2864, 3012, 3006, 2990, 2988, 3128, 3004, 3024] combine_rows_per_src=[9242, 9286, 9358, 9292, 9222, 9258, 9250, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343680 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247808 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151936 weight_movement_bytes=69206016 total_nvlink_bytes=601357952 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=64958 dup_rows=23952 dup_rows_remote=20864 dedup_ratio=0.243652 router_tokens_per_expert=[735, 772, 781, 776, 791, 801, 783, 766, 774, 763, 761, 815, 751, 758, 732, 765, 750, 799, 752, 725, 763, 802, 783, 773, 751, 815, 805, 736, 816, 785, 804, 730, 761, 746, 808, 779, 737, 800, 719, 727, 789, 772, 708, 735, 771, 747, 775, 749, 721, 806, 770, 784, 719, 776, 739, 841, 770, 744, 802, 776, 781, 759, 758, 770] payload_rows_per_dst=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] dup_rows_per_dst=[2920, 3062, 2948, 2978, 2922, 2878, 3156, 3088] combine_rows_per_src=[9070, 9176, 9292, 9344, 9286, 9214, 9576, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65014 dup_rows=23834 dup_rows_remote=20738 dedup_ratio=0.242452 router_tokens_per_expert=[751, 729, 750, 777, 759, 732, 796, 758, 786, 787, 765, 743, 798, 784, 754, 749, 755, 750, 764, 764, 796, 808, 764, 757, 787, 789, 807, 769, 790, 770, 784, 780, 792, 773, 761, 792, 795, 798, 760, 743, 739, 760, 766, 783, 753, 750, 779, 783, 754, 790, 787, 719, 739, 725, 762, 768, 785, 756, 767, 728, 783, 762, 755, 793] payload_rows_per_dst=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] dup_rows_per_dst=[2898, 3098, 3018, 3096, 3046, 2896, 2838, 2944] combine_rows_per_src=[9206, 9234, 9298, 9456, 9382, 9330, 9250, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65044 dup_rows=23948 dup_rows_remote=21066 dedup_ratio=0.243612 router_tokens_per_expert=[814, 739, 795, 760, 792, 754, 783, 758, 802, 741, 796, 779, 760, 745, 794, 768, 770, 762, 759, 771, 755, 728, 735, 754, 808, 783, 785, 778, 772, 797, 804, 783, 744, 739, 786, 755, 785, 773, 772, 762, 774, 775, 806, 747, 779, 754, 745, 760, 743, 790, 778, 770, 790, 738, 731, 757, 752, 795, 784, 690, 764, 721, 782, 787] payload_rows_per_dst=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] dup_rows_per_dst=[2972, 3062, 2866, 3200, 3012, 2996, 2882, 2958] combine_rows_per_src=[9418, 9308, 9202, 9420, 9220, 9284, 9312, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344024 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592216 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840408 weight_movement_bytes=34603008 total_nvlink_bytes=567443416 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=342296 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266918168 ingress_bytes=335781888 link_bytes=335781888 token_bytes=533494040 weight_movement_bytes=69206016 total_nvlink_bytes=602700056 weight_to_token_ratio=0.129722 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263520212e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21184 dedup_ratio=0.246195 router_tokens_per_expert=[811, 767, 770, 814, 765, 778, 766, 748, 756, 728, 759, 758, 738, 720, 765, 744, 778, 789, 740, 769, 791, 749, 803, 777, 777, 721, 794, 782, 778, 719, 752, 716, 788, 778, 760, 757, 750, 788, 816, 796, 745, 751, 740, 746, 788, 729, 793, 765, 739, 777, 811, 798, 748, 751, 771, 755, 750, 775, 763, 742, 851, 826, 808, 775] payload_rows_per_dst=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] dup_rows_per_dst=[3152, 2858, 3122, 2938, 2990, 2976, 3018, 3148] combine_rows_per_src=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344024 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266592216 ingress_bytes=300851200 link_bytes=300851200 token_bytes=532840408 weight_movement_bytes=34603008 total_nvlink_bytes=567443416 weight_to_token_ratio=0.064941 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132078845e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65108 dup_rows=23846 dup_rows_remote=20922 dedup_ratio=0.242574 router_tokens_per_expert=[765, 761, 787, 744, 753, 769, 713, 785, 800, 756, 765, 812, 788, 712, 801, 739, 778, 692, 777, 757, 749, 776, 777, 808, 794, 777, 757, 766, 761, 789, 720, 761, 761, 774, 769, 767, 818, 778, 719, 772, 770, 755, 808, 751, 797, 736, 779, 741, 788, 768, 747, 748, 757, 743, 785, 775, 782, 773, 776, 808, 784, 776, 755, 803] payload_rows_per_dst=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] dup_rows_per_dst=[2948, 2972, 2944, 2980, 3054, 2924, 2900, 3124] combine_rows_per_src=[9206, 9374, 9284, 9270, 9262, 9350, 9322, 9390] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343168 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247296 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151424 weight_movement_bytes=34603008 total_nvlink_bytes=566754432 weight_to_token_ratio=0.065025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65282 dup_rows=23910 dup_rows_remote=20946 dedup_ratio=0.243225 router_tokens_per_expert=[759, 759, 780, 782, 773, 766, 777, 783, 758, 767, 778, 784, 774, 724, 771, 725, 760, 752, 802, 769, 769, 787, 713, 808, 756, 791, 728, 801, 760, 780, 812, 766, 751, 725, 761, 763, 775, 746, 778, 784, 727, 772, 794, 759, 744, 777, 730, 735, 787, 756, 798, 761, 740, 779, 786, 745, 770, 850, 787, 723, 796, 777, 804, 758] payload_rows_per_dst=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] dup_rows_per_dst=[3006, 2912, 2976, 3012, 2998, 2964, 2982, 3060] combine_rows_per_src=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343680 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266247808 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532151936 weight_movement_bytes=69206016 total_nvlink_bytes=601357952 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343288 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266411256 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532479224 weight_movement_bytes=69206016 total_nvlink_bytes=601685240 weight_to_token_ratio=0.129969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343008 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266640352 ingress_bytes=335503360 link_bytes=335503360 token_bytes=532937696 weight_movement_bytes=69206016 total_nvlink_bytes=602143712 weight_to_token_ratio=0.129858 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262472133e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344016 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846736 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349456 weight_movement_bytes=34603008 total_nvlink_bytes=565952464 weight_to_token_ratio=0.065123 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64998 dup_rows=24096 dup_rows_remote=21130 dedup_ratio=0.245117 router_tokens_per_expert=[711, 775, 771, 727, 763, 837, 760, 780, 753, 779, 744, 754, 746, 811, 757, 767, 743, 732, 693, 776, 770, 836, 777, 761, 746, 774, 743, 776, 734, 734, 739, 749, 781, 749, 804, 789, 770, 753, 744, 728, 767, 788, 795, 793, 785, 758, 789, 779, 794, 769, 757, 747, 774, 804, 809, 780, 773, 761, 803, 794, 753, 757, 793, 794] payload_rows_per_dst=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] dup_rows_per_dst=[2982, 2950, 2966, 2864, 2994, 3188, 3122, 3030] combine_rows_per_src=[9266, 9272, 9210, 9126, 9242, 9320, 9346, 9426] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344120 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026488 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708856 weight_movement_bytes=34603008 total_nvlink_bytes=568311864 weight_to_token_ratio=0.064835 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65146 dup_rows=23786 dup_rows_remote=20788 dedup_ratio=0.241964 router_tokens_per_expert=[792, 766, 796, 785, 753, 756, 794, 719, 796, 750, 692, 790, 747, 779, 780, 746, 758, 753, 790, 780, 780, 765, 770, 763, 750, 791, 753, 772, 745, 725, 790, 786, 772, 761, 820, 752, 769, 784, 764, 763, 751, 736, 826, 801, 791, 804, 761, 739, 762, 755, 741, 747, 742, 805, 779, 808, 785, 728, 804, 745, 767, 791, 741, 746] payload_rows_per_dst=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] dup_rows_per_dst=[2948, 2816, 2990, 3010, 2998, 3076, 3016, 2932] combine_rows_per_src=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344440 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764664 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184888 weight_movement_bytes=34603008 total_nvlink_bytes=567787896 weight_to_token_ratio=0.064899 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64820 dup_rows=24200 dup_rows_remote=21192 dedup_ratio=0.246175 router_tokens_per_expert=[765, 794, 752, 795, 719, 795, 748, 733, 768, 772, 726, 751, 805, 769, 718, 765, 767, 804, 727, 760, 754, 763, 758, 787, 761, 799, 743, 740, 786, 730, 750, 837, 782, 770, 768, 775, 824, 781, 770, 745, 720, 741, 781, 804, 765, 733, 789, 741, 785, 762, 774, 757, 776, 746, 798, 761, 771, 775, 841, 746, 774, 805, 804, 747] payload_rows_per_dst=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] dup_rows_per_dst=[2978, 3004, 2998, 3008, 3104, 3014, 3038, 3056] combine_rows_per_src=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344912 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739984 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535135056 weight_movement_bytes=34603008 total_nvlink_bytes=569738064 weight_to_token_ratio=0.064662 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65174 dup_rows=23830 dup_rows_remote=20808 dedup_ratio=0.242411 router_tokens_per_expert=[806, 779, 737, 765, 784, 761, 785, 773, 763, 782, 752, 805, 763, 723, 788, 785, 765, 722, 748, 775, 791, 733, 737, 753, 734, 759, 863, 763, 802, 813, 786, 746, 714, 767, 791, 784, 776, 736, 737, 721, 790, 739, 757, 797, 776, 786, 767, 772, 789, 736, 786, 753, 792, 782, 758, 792, 783, 716, 792, 775, 741, 789, 759, 758] payload_rows_per_dst=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] dup_rows_per_dst=[3014, 3022, 2868, 3118, 2874, 2964, 3058, 2912] combine_rows_per_src=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64978 dup_rows=24196 dup_rows_remote=21148 dedup_ratio=0.246134 router_tokens_per_expert=[778, 787, 741, 784, 750, 766, 758, 725, 774, 797, 790, 751, 753, 738, 803, 810, 771, 776, 766, 773, 771, 727, 753, 737, 778, 788, 757, 740, 751, 749, 776, 783, 758, 769, 741, 804, 741, 759, 803, 812, 798, 746, 801, 773, 777, 790, 776, 768, 711, 773, 791, 720, 814, 747, 799, 807, 734, 781, 745, 754, 766, 755, 764, 774] payload_rows_per_dst=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] dup_rows_per_dst=[3048, 3364, 3078, 2882, 3108, 3126, 2878, 2712] combine_rows_per_src=[9130, 9582, 9378, 9124, 9266, 9616, 9266, 8746] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65102 dup_rows=24026 dup_rows_remote=21116 dedup_ratio=0.244405 router_tokens_per_expert=[742, 773, 778, 782, 803, 771, 779, 783, 764, 786, 780, 733, 764, 818, 772, 787, 774, 757, 794, 745, 732, 753, 741, 747, 733, 812, 763, 769, 785, 748, 777, 735, 779, 746, 786, 799, 723, 828, 784, 752, 795, 743, 783, 779, 788, 746, 787, 787, 714, 723, 783, 772, 806, 706, 763, 813, 758, 769, 746, 763, 801, 736, 727, 787] payload_rows_per_dst=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] dup_rows_per_dst=[3086, 3132, 2910, 3006, 3050, 3026, 2884, 2932] combine_rows_per_src=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344048 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846768 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349488 weight_movement_bytes=103809024 total_nvlink_bytes=635158512 weight_to_token_ratio=0.195369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74518 payload_rows_remote=65146 dup_rows=23786 dup_rows_remote=20788 dedup_ratio=0.241964 router_tokens_per_expert=[792, 766, 796, 785, 753, 756, 794, 719, 796, 750, 692, 790, 747, 779, 780, 746, 758, 753, 790, 780, 780, 765, 770, 763, 750, 791, 753, 772, 745, 725, 790, 786, 772, 761, 820, 752, 769, 784, 764, 763, 751, 736, 826, 801, 791, 804, 761, 739, 762, 755, 741, 747, 742, 805, 779, 808, 785, 728, 804, 745, 767, 791, 741, 746] payload_rows_per_dst=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] dup_rows_per_dst=[2948, 2816, 2990, 3010, 2998, 3076, 3016, 2932] combine_rows_per_src=[9374, 9344, 9328, 9214, 9372, 9342, 9262, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266231808 dispatch_meta_bytes=344512 combine_payload_bytes=266231808 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266576320 ingress_bytes=300834816 link_bytes=300834816 token_bytes=532808128 weight_movement_bytes=34603008 total_nvlink_bytes=567411136 weight_to_token_ratio=0.064945 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132017193e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65174 dup_rows=23830 dup_rows_remote=20808 dedup_ratio=0.242411 router_tokens_per_expert=[806, 779, 737, 765, 784, 761, 785, 773, 763, 782, 752, 805, 763, 723, 788, 785, 765, 722, 748, 775, 791, 733, 737, 753, 734, 759, 863, 763, 802, 813, 786, 746, 714, 767, 791, 784, 776, 736, 737, 721, 790, 739, 757, 797, 776, 786, 767, 772, 789, 736, 786, 753, 792, 782, 758, 792, 783, 716, 792, 775, 741, 789, 759, 758] payload_rows_per_dst=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] dup_rows_per_dst=[3014, 3022, 2868, 3118, 2874, 2964, 3058, 2912] combine_rows_per_src=[9366, 9300, 9180, 9414, 9178, 9404, 9318, 9314] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343736 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181752 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019768 weight_movement_bytes=34603008 total_nvlink_bytes=568622776 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74468 payload_rows_remote=65168 dup_rows=23836 dup_rows_remote=20794 dedup_ratio=0.242472 router_tokens_per_expert=[760, 776, 739, 795, 772, 752, 766, 794, 832, 751, 732, 780, 732, 725, 765, 807, 748, 779, 751, 749, 745, 796, 747, 797, 779, 824, 773, 766, 810, 775, 785, 749, 789, 791, 762, 754, 762, 735, 785, 749, 692, 772, 766, 717, 774, 751, 751, 772, 790, 743, 746, 742, 786, 781, 799, 829, 771, 742, 764, 770, 794, 768, 772, 782] payload_rows_per_dst=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] dup_rows_per_dst=[3024, 2932, 2930, 2862, 3122, 2902, 3022, 3042] combine_rows_per_src=[9268, 9316, 9294, 9290, 9502, 9088, 9410, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343928 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296632 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249336 weight_movement_bytes=34603008 total_nvlink_bytes=568852344 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=344504 combine_payload_bytes=266149888 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266494392 ingress_bytes=300752896 link_bytes=300752896 token_bytes=532644280 weight_movement_bytes=34603008 total_nvlink_bytes=567247288 weight_to_token_ratio=0.064965 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131708935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74104 payload_rows_remote=64820 dup_rows=24200 dup_rows_remote=21192 dedup_ratio=0.246175 router_tokens_per_expert=[765, 794, 752, 795, 719, 795, 748, 733, 768, 772, 726, 751, 805, 769, 718, 765, 767, 804, 727, 760, 754, 763, 758, 787, 761, 799, 743, 740, 786, 730, 750, 837, 782, 770, 768, 775, 824, 781, 770, 745, 720, 741, 781, 804, 765, 733, 789, 741, 785, 762, 774, 757, 776, 746, 798, 761, 771, 775, 841, 746, 774, 805, 804, 747] payload_rows_per_dst=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] dup_rows_per_dst=[2978, 3004, 2998, 3008, 3104, 3014, 3038, 3056] combine_rows_per_src=[9224, 9144, 9242, 9284, 9326, 9134, 9280, 9470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65128 dup_rows=23872 dup_rows_remote=20900 dedup_ratio=0.242839 router_tokens_per_expert=[759, 754, 778, 793, 781, 749, 720, 722, 756, 758, 745, 770, 781, 738, 774, 789, 755, 766, 807, 748, 805, 714, 745, 753, 796, 756, 744, 757, 785, 799, 773, 785, 802, 771, 760, 759, 801, 788, 746, 770, 754, 763, 791, 801, 759, 772, 788, 746, 765, 790, 753, 816, 726, 761, 766, 761, 760, 729, 774, 820, 758, 801, 771, 775] payload_rows_per_dst=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] dup_rows_per_dst=[2886, 2982, 2994, 3048, 2984, 3018, 2972, 2988] combine_rows_per_src=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=65034 dup_rows=24170 dup_rows_remote=21436 dedup_ratio=0.245870 router_tokens_per_expert=[790, 757, 756, 780, 753, 827, 816, 769, 781, 756, 769, 786, 731, 722, 736, 813, 774, 803, 747, 741, 798, 747, 764, 774, 784, 754, 748, 749, 733, 800, 776, 772, 750, 798, 744, 746, 751, 791, 768, 784, 761, 811, 789, 781, 782, 807, 778, 808, 737, 731, 717, 733, 761, 749, 725, 764, 760, 739, 746, 800, 794, 780, 768, 793] payload_rows_per_dst=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] dup_rows_per_dst=[3114, 2964, 3046, 3032, 2974, 3234, 2734, 3072] combine_rows_per_src=[9382, 9224, 9250, 9200, 9290, 9400, 9100, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=343928 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296632 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249336 weight_movement_bytes=34603008 total_nvlink_bytes=568852344 weight_to_token_ratio=0.064769 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74428 payload_rows_remote=65134 dup_rows=23876 dup_rows_remote=20894 dedup_ratio=0.242879 router_tokens_per_expert=[752, 746, 800, 785, 787, 831, 783, 792, 786, 793, 801, 727, 772, 789, 771, 737, 770, 774, 805, 751, 748, 766, 778, 744, 749, 776, 748, 754, 769, 790, 776, 748, 770, 731, 747, 741, 789, 778, 782, 804, 749, 758, 759, 790, 773, 755, 775, 779, 753, 761, 781, 703, 720, 741, 814, 760, 794, 755, 746, 760, 752, 772, 800, 762] payload_rows_per_dst=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] dup_rows_per_dst=[3038, 3050, 2942, 3018, 2988, 2982, 2838, 3020] combine_rows_per_src=[9514, 9302, 9330, 9202, 9296, 9294, 9228, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344872 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002664 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660456 weight_movement_bytes=34603008 total_nvlink_bytes=568263464 weight_to_token_ratio=0.064841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=343736 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267181752 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534019768 weight_movement_bytes=34603008 total_nvlink_bytes=568622776 weight_to_token_ratio=0.064797 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65188 dup_rows=23802 dup_rows_remote=20842 dedup_ratio=0.242126 router_tokens_per_expert=[771, 759, 744, 791, 771, 783, 754, 784, 771, 772, 766, 778, 757, 783, 769, 800, 764, 730, 791, 741, 749, 780, 808, 778, 757, 765, 750, 791, 750, 746, 772, 742, 794, 764, 741, 730, 766, 750, 706, 760, 771, 753, 778, 806, 781, 778, 765, 705, 755, 750, 836, 783, 749, 751, 790, 765, 804, 782, 788, 788, 747, 775, 789, 785] payload_rows_per_dst=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] dup_rows_per_dst=[2980, 3060, 2966, 2876, 2926, 2960, 3054, 2980] combine_rows_per_src=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266928128 dispatch_meta_bytes=343848 combine_payload_bytes=266928128 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267271976 ingress_bytes=405340160 link_bytes=405340160 token_bytes=534200104 weight_movement_bytes=138412032 total_nvlink_bytes=672612136 weight_to_token_ratio=0.259101 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525262390e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344048 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846768 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349488 weight_movement_bytes=103809024 total_nvlink_bytes=635158512 weight_to_token_ratio=0.195369 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344112 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108400 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872688 weight_movement_bytes=69206016 total_nvlink_bytes=603078704 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74546 payload_rows_remote=65262 dup_rows=23758 dup_rows_remote=20714 dedup_ratio=0.241679 router_tokens_per_expert=[778, 749, 738, 756, 766, 770, 743, 857, 796, 783, 763, 825, 804, 747, 733, 730, 731, 773, 797, 798, 755, 781, 800, 729, 745, 774, 755, 784, 758, 762, 778, 779, 809, 765, 792, 793, 768, 770, 755, 765, 759, 755, 775, 731, 779, 739, 777, 758, 730, 748, 721, 733, 739, 824, 771, 812, 784, 758, 828, 761, 755, 770, 739, 752] payload_rows_per_dst=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] dup_rows_per_dst=[2976, 3016, 3044, 2944, 2998, 2906, 2826, 3048] combine_rows_per_src=[9338, 9346, 9284, 9326, 9436, 9240, 9330, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344112 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267132976 ingress_bytes=301391872 link_bytes=301391872 token_bytes=533921840 weight_movement_bytes=34603008 total_nvlink_bytes=568524848 weight_to_token_ratio=0.064809 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134113350e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64894 dup_rows=24050 dup_rows_remote=21040 dedup_ratio=0.244649 router_tokens_per_expert=[771, 758, 772, 757, 790, 745, 761, 768, 734, 814, 761, 776, 769, 793, 752, 763, 820, 771, 760, 780, 765, 788, 759, 756, 819, 729, 794, 825, 814, 730, 754, 731, 746, 746, 812, 755, 789, 804, 735, 798, 771, 746, 819, 760, 750, 774, 796, 814, 764, 738, 757, 769, 718, 749, 781, 805, 748, 765, 738, 753, 776, 743, 734, 720] payload_rows_per_dst=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] dup_rows_per_dst=[3008, 3066, 3088, 3030, 3010, 3084, 2904, 2860] combine_rows_per_src=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=345880 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266725144 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533104408 weight_movement_bytes=34603008 total_nvlink_bytes=567707416 weight_to_token_ratio=0.064909 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344120 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354168 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364216 weight_movement_bytes=34603008 total_nvlink_bytes=568967224 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65060 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[750, 747, 834, 738, 758, 827, 788, 727, 787, 757, 778, 746, 732, 800, 779, 741, 812, 799, 767, 738, 760, 718, 782, 792, 783, 738, 741, 724, 841, 765, 762, 747, 745, 782, 753, 731, 772, 781, 771, 775, 778, 784, 789, 734, 808, 783, 728, 767, 777, 739, 775, 771, 724, 822, 777, 762, 744, 766, 794, 801, 812, 772, 751, 726] payload_rows_per_dst=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] dup_rows_per_dst=[2888, 3026, 3072, 2968, 2954, 3098, 2994, 3010] combine_rows_per_src=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64956 dup_rows=24044 dup_rows_remote=20970 dedup_ratio=0.244588 router_tokens_per_expert=[752, 712, 747, 789, 763, 735, 804, 722, 738, 760, 812, 749, 799, 788, 784, 759, 731, 743, 759, 737, 749, 736, 749, 762, 757, 771, 775, 765, 780, 775, 820, 753, 778, 737, 775, 830, 775, 784, 793, 794, 750, 749, 749, 744, 781, 788, 829, 816, 782, 790, 776, 730, 760, 854, 760, 761, 752, 789, 767, 748, 701, 779, 794, 762] payload_rows_per_dst=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] dup_rows_per_dst=[2806, 3074, 2842, 3118, 3028, 3096, 3092, 2988] combine_rows_per_src=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73988 payload_rows_remote=64814 dup_rows=24316 dup_rows_remote=21098 dedup_ratio=0.247355 router_tokens_per_expert=[805, 764, 771, 732, 762, 750, 767, 770, 762, 756, 724, 754, 734, 772, 733, 795, 794, 776, 788, 786, 752, 782, 725, 769, 798, 797, 773, 763, 795, 754, 774, 787, 777, 851, 722, 768, 763, 778, 766, 812, 780, 755, 745, 713, 715, 748, 741, 739, 771, 798, 787, 749, 767, 776, 802, 791, 790, 754, 782, 761, 756, 764, 794, 773] payload_rows_per_dst=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] dup_rows_per_dst=[3218, 2848, 3224, 3096, 2908, 2876, 3032, 3114] combine_rows_per_src=[9174, 9078, 9450, 9386, 9224, 9078, 9364, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64944 dup_rows=24088 dup_rows_remote=21124 dedup_ratio=0.245036 router_tokens_per_expert=[742, 742, 723, 786, 763, 830, 779, 787, 807, 740, 765, 773, 764, 817, 766, 758, 757, 786, 721, 799, 765, 774, 765, 751, 724, 770, 719, 734, 775, 789, 758, 773, 786, 796, 760, 770, 792, 775, 773, 779, 745, 765, 757, 783, 769, 790, 741, 767, 737, 832, 786, 754, 780, 710, 748, 771, 796, 741, 771, 797, 732, 785, 774, 788] payload_rows_per_dst=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] dup_rows_per_dst=[2964, 3052, 2964, 2954, 3152, 2940, 3038, 3024] combine_rows_per_src=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344408 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266830168 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315928 weight_movement_bytes=69206016 total_nvlink_bytes=602521944 weight_to_token_ratio=0.129766 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64894 dup_rows=24050 dup_rows_remote=21040 dedup_ratio=0.244649 router_tokens_per_expert=[771, 758, 772, 757, 790, 745, 761, 768, 734, 814, 761, 776, 769, 793, 752, 763, 820, 771, 760, 780, 765, 788, 759, 756, 819, 729, 794, 825, 814, 730, 754, 731, 746, 746, 812, 755, 789, 804, 735, 798, 771, 746, 819, 760, 750, 774, 796, 814, 764, 738, 757, 769, 718, 749, 781, 805, 748, 765, 738, 753, 776, 743, 734, 720] payload_rows_per_dst=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] dup_rows_per_dst=[3008, 3066, 3088, 3030, 3010, 3084, 2904, 2860] combine_rows_per_src=[9236, 9258, 9310, 9362, 9360, 9376, 9258, 9094] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=343904 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267657056 ingress_bytes=371122176 link_bytes=371122176 token_bytes=534970208 weight_movement_bytes=103809024 total_nvlink_bytes=638779232 weight_to_token_ratio=0.194046 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396502871e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64956 dup_rows=24044 dup_rows_remote=20970 dedup_ratio=0.244588 router_tokens_per_expert=[752, 712, 747, 789, 763, 735, 804, 722, 738, 760, 812, 749, 799, 788, 784, 759, 731, 743, 759, 737, 749, 736, 749, 762, 757, 771, 775, 765, 780, 775, 820, 753, 778, 737, 775, 830, 775, 784, 793, 794, 750, 749, 749, 744, 781, 788, 829, 816, 782, 790, 776, 730, 760, 854, 760, 761, 752, 789, 767, 748, 701, 779, 794, 762] payload_rows_per_dst=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] dup_rows_per_dst=[2806, 3074, 2842, 3118, 3028, 3096, 3092, 2988] combine_rows_per_src=[9242, 9304, 9090, 9274, 9504, 9316, 9334, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343736 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149560 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955384 weight_movement_bytes=103809024 total_nvlink_bytes=635764408 weight_to_token_ratio=0.195146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343648 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265821792 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531299936 weight_movement_bytes=34603008 total_nvlink_bytes=565902944 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64754 dup_rows=24190 dup_rows_remote=20928 dedup_ratio=0.246073 router_tokens_per_expert=[775, 780, 784, 806, 759, 750, 769, 783, 816, 837, 787, 786, 720, 769, 757, 769, 745, 777, 831, 744, 742, 791, 775, 747, 770, 821, 721, 757, 756, 749, 781, 784, 816, 803, 798, 791, 761, 712, 707, 812, 735, 771, 764, 789, 725, 799, 732, 795, 743, 718, 796, 776, 707, 755, 761, 776, 772, 733, 804, 724, 766, 780, 728, 765] payload_rows_per_dst=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] dup_rows_per_dst=[3052, 3030, 3028, 2858, 3062, 2958, 2940, 3262] combine_rows_per_src=[9360, 9406, 9276, 8988, 9338, 9262, 9124, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65060 dup_rows=24010 dup_rows_remote=21042 dedup_ratio=0.244242 router_tokens_per_expert=[750, 747, 834, 738, 758, 827, 788, 727, 787, 757, 778, 746, 732, 800, 779, 741, 812, 799, 767, 738, 760, 718, 782, 792, 783, 738, 741, 724, 841, 765, 762, 747, 745, 782, 753, 731, 772, 781, 771, 775, 778, 784, 789, 734, 808, 783, 728, 767, 777, 739, 775, 771, 724, 822, 777, 762, 744, 766, 794, 801, 812, 772, 751, 726] payload_rows_per_dst=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] dup_rows_per_dst=[2888, 3026, 3072, 2968, 2954, 3098, 2994, 3010] combine_rows_per_src=[9450, 9214, 9264, 9234, 9266, 9244, 9300, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64742 dup_rows=24168 dup_rows_remote=21118 dedup_ratio=0.245850 router_tokens_per_expert=[745, 741, 756, 794, 767, 756, 773, 735, 794, 751, 760, 780, 769, 718, 759, 769, 779, 805, 754, 801, 722, 765, 744, 776, 733, 808, 784, 805, 807, 763, 814, 786, 722, 760, 738, 805, 789, 753, 750, 800, 734, 778, 795, 781, 748, 752, 762, 763, 805, 797, 752, 753, 828, 724, 780, 783, 774, 802, 725, 753, 799, 737, 742, 755] payload_rows_per_dst=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] dup_rows_per_dst=[2946, 2964, 3040, 3216, 3006, 3000, 3050, 2946] combine_rows_per_src=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343704 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266403480 ingress_bytes=369868800 link_bytes=369868800 token_bytes=532463256 weight_movement_bytes=103809024 total_nvlink_bytes=636272280 weight_to_token_ratio=0.194960 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391786518e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=343648 combine_payload_bytes=265478144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265821792 ingress_bytes=300081152 link_bytes=300081152 token_bytes=531299936 weight_movement_bytes=34603008 total_nvlink_bytes=565902944 weight_to_token_ratio=0.065129 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129181216e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21184 dedup_ratio=0.246195 router_tokens_per_expert=[811, 767, 770, 814, 765, 778, 766, 748, 756, 728, 759, 758, 738, 720, 765, 744, 778, 789, 740, 769, 791, 749, 803, 777, 777, 721, 794, 782, 778, 719, 752, 716, 788, 778, 760, 757, 750, 788, 816, 796, 745, 751, 740, 746, 788, 729, 793, 765, 739, 777, 811, 798, 748, 751, 771, 755, 750, 775, 763, 742, 851, 826, 808, 775] payload_rows_per_dst=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] dup_rows_per_dst=[3152, 2858, 3122, 2938, 2990, 2976, 3018, 3148] combine_rows_per_src=[9286, 9078, 9270, 9140, 9476, 9138, 9282, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344272 combine_payload_bytes=266010624 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266354896 ingress_bytes=369819648 link_bytes=369819648 token_bytes=532365520 weight_movement_bytes=103809024 total_nvlink_bytes=636174544 weight_to_token_ratio=0.194996 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391601562e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65296 dup_rows=23700 dup_rows_remote=20716 dedup_ratio=0.241089 router_tokens_per_expert=[733, 743, 759, 768, 781, 811, 848, 736, 818, 745, 807, 790, 765, 739, 779, 721, 779, 772, 766, 766, 743, 731, 799, 735, 733, 786, 726, 757, 793, 704, 762, 801, 789, 775, 748, 774, 750, 773, 754, 761, 804, 772, 782, 785, 762, 732, 761, 748, 802, 787, 778, 798, 777, 783, 775, 820, 769, 791, 706, 721, 764, 746, 795, 774] payload_rows_per_dst=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] dup_rows_per_dst=[2988, 2990, 2918, 2926, 2928, 2984, 3082, 2884] combine_rows_per_src=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265805824 dispatch_meta_bytes=343736 combine_payload_bytes=265805824 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266149560 ingress_bytes=369614848 link_bytes=369614848 token_bytes=531955384 weight_movement_bytes=103809024 total_nvlink_bytes=635764408 weight_to_token_ratio=0.195146 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390830917e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265232384 dispatch_meta_bytes=342728 combine_payload_bytes=265232384 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265575112 ingress_bytes=299835392 link_bytes=299835392 token_bytes=530807496 weight_movement_bytes=34603008 total_nvlink_bytes=565410504 weight_to_token_ratio=0.065189 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128256441e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65282 dup_rows=23910 dup_rows_remote=20946 dedup_ratio=0.243225 router_tokens_per_expert=[759, 759, 780, 782, 773, 766, 777, 783, 758, 767, 778, 784, 774, 724, 771, 725, 760, 752, 802, 769, 769, 787, 713, 808, 756, 791, 728, 801, 760, 780, 812, 766, 751, 725, 761, 763, 775, 746, 778, 784, 727, 772, 794, 759, 744, 777, 730, 735, 787, 756, 798, 761, 740, 779, 786, 745, 770, 850, 787, 723, 796, 777, 804, 758] payload_rows_per_dst=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] dup_rows_per_dst=[3006, 2912, 2976, 3012, 2998, 2964, 2982, 3060] combine_rows_per_src=[9352, 9250, 9344, 9376, 9168, 9112, 9322, 9470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266485760 dispatch_meta_bytes=344408 combine_payload_bytes=266485760 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266830168 ingress_bytes=335691776 link_bytes=335691776 token_bytes=533315928 weight_movement_bytes=69206016 total_nvlink_bytes=602521944 weight_to_token_ratio=0.129766 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263181128e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=343440 combine_payload_bytes=265183232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265526672 ingress_bytes=334389248 link_bytes=334389248 token_bytes=530709904 weight_movement_bytes=69206016 total_nvlink_bytes=599915920 weight_to_token_ratio=0.130403 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258279820e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64896 dup_rows=24158 dup_rows_remote=21074 dedup_ratio=0.245748 router_tokens_per_expert=[779, 778, 763, 783, 763, 757, 723, 756, 731, 720, 769, 730, 766, 742, 781, 734, 789, 800, 748, 779, 755, 711, 800, 777, 794, 793, 778, 742, 816, 751, 784, 730, 753, 738, 765, 768, 808, 788, 783, 764, 822, 780, 813, 751, 752, 808, 773, 755, 791, 791, 751, 760, 771, 763, 777, 808, 750, 757, 798, 760, 763, 789, 735, 745] payload_rows_per_dst=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] dup_rows_per_dst=[2968, 2876, 3028, 3110, 3084, 3048, 3102, 2942] combine_rows_per_src=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344016 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265846736 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531349456 weight_movement_bytes=34603008 total_nvlink_bytes=565952464 weight_to_token_ratio=0.065123 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344048 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267796464 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535248880 weight_movement_bytes=34603008 total_nvlink_bytes=569851888 weight_to_token_ratio=0.064648 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21088 dedup_ratio=0.243958 router_tokens_per_expert=[774, 745, 817, 743, 783, 720, 774, 810, 832, 769, 787, 788, 800, 791, 765, 740, 736, 800, 772, 750, 742, 777, 742, 783, 777, 756, 775, 801, 710, 766, 743, 758, 745, 735, 720, 777, 778, 746, 754, 802, 760, 771, 790, 825, 810, 715, 730, 799, 788, 732, 758, 751, 844, 762, 791, 789, 766, 715, 751, 747, 747, 772, 791, 765] payload_rows_per_dst=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] dup_rows_per_dst=[3098, 3126, 2970, 2894, 2908, 3038, 3050, 2898] combine_rows_per_src=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65018 dup_rows=23998 dup_rows_remote=21022 dedup_ratio=0.244120 router_tokens_per_expert=[794, 735, 798, 778, 773, 776, 760, 771, 757, 746, 766, 811, 753, 793, 711, 795, 782, 794, 785, 747, 795, 823, 776, 757, 795, 727, 759, 727, 739, 775, 736, 741, 800, 747, 778, 738, 806, 802, 814, 711, 787, 772, 796, 683, 732, 718, 766, 796, 789, 796, 798, 685, 755, 823, 757, 747, 749, 776, 754, 750, 754, 781, 874, 743] payload_rows_per_dst=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] dup_rows_per_dst=[3038, 2976, 3130, 2906, 3116, 2834, 2956, 3042] combine_rows_per_src=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64718 dup_rows=24076 dup_rows_remote=20848 dedup_ratio=0.244914 router_tokens_per_expert=[817, 760, 763, 797, 751, 858, 787, 784, 784, 778, 784, 755, 803, 745, 771, 757, 805, 753, 807, 758, 772, 795, 810, 794, 750, 755, 797, 763, 778, 740, 786, 747, 767, 775, 776, 751, 713, 734, 769, 764, 726, 758, 758, 788, 793, 762, 803, 737, 772, 743, 729, 749, 756, 809, 702, 766, 753, 767, 805, 704, 757, 733, 746, 783] payload_rows_per_dst=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] dup_rows_per_dst=[3228, 3050, 3096, 2912, 3006, 3010, 2900, 2874] combine_rows_per_src=[9510, 9306, 9490, 9320, 9092, 9240, 9048, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=64918 dup_rows=23930 dup_rows_remote=20874 dedup_ratio=0.243429 router_tokens_per_expert=[750, 773, 790, 810, 782, 760, 776, 789, 778, 817, 775, 738, 732, 756, 736, 716, 735, 837, 764, 781, 803, 783, 795, 758, 776, 747, 787, 761, 766, 739, 791, 793, 772, 730, 766, 735, 760, 695, 747, 742, 769, 798, 694, 724, 802, 761, 789, 770, 801, 783, 735, 724, 749, 786, 740, 770, 776, 757, 816, 835, 833, 769, 744, 786] payload_rows_per_dst=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] dup_rows_per_dst=[3112, 2836, 3056, 3018, 2778, 2998, 2896, 3236] combine_rows_per_src=[9348, 9260, 9456, 9302, 9116, 9216, 9280, 9396] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267395072 dispatch_meta_bytes=344912 combine_payload_bytes=267395072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267739984 ingress_bytes=301998080 link_bytes=301998080 token_bytes=535135056 weight_movement_bytes=34603008 total_nvlink_bytes=569738064 weight_to_token_ratio=0.064662 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136394462e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65064 dup_rows=23842 dup_rows_remote=20880 dedup_ratio=0.242533 router_tokens_per_expert=[727, 741, 784, 765, 753, 782, 773, 728, 775, 783, 771, 807, 753, 753, 729, 806, 795, 779, 771, 774, 767, 783, 737, 774, 792, 728, 777, 731, 756, 762, 769, 793, 796, 774, 761, 769, 766, 802, 758, 771, 739, 761, 753, 798, 809, 754, 722, 788, 743, 799, 807, 721, 727, 776, 797, 792, 775, 760, 764, 778, 777, 751, 789, 757] payload_rows_per_dst=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] dup_rows_per_dst=[2886, 3030, 2962, 2958, 3064, 2970, 2960, 3012] combine_rows_per_src=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64896 dup_rows=24158 dup_rows_remote=21074 dedup_ratio=0.245748 router_tokens_per_expert=[779, 778, 763, 783, 763, 757, 723, 756, 731, 720, 769, 730, 766, 742, 781, 734, 789, 800, 748, 779, 755, 711, 800, 777, 794, 793, 778, 742, 816, 751, 784, 730, 753, 738, 765, 768, 808, 788, 783, 764, 822, 780, 813, 751, 752, 808, 773, 755, 791, 791, 751, 760, 771, 763, 777, 808, 750, 757, 798, 760, 763, 789, 735, 745] payload_rows_per_dst=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] dup_rows_per_dst=[2968, 2876, 3028, 3110, 3084, 3048, 3102, 2942] combine_rows_per_src=[9236, 9070, 9290, 9266, 9250, 9460, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344528 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764752 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184976 weight_movement_bytes=34603008 total_nvlink_bytes=567787984 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65018 dup_rows=23998 dup_rows_remote=21022 dedup_ratio=0.244120 router_tokens_per_expert=[794, 735, 798, 778, 773, 776, 760, 771, 757, 746, 766, 811, 753, 793, 711, 795, 782, 794, 785, 747, 795, 823, 776, 757, 795, 727, 759, 727, 739, 775, 736, 741, 800, 747, 778, 738, 806, 802, 814, 711, 787, 772, 796, 683, 732, 718, 766, 796, 789, 796, 798, 685, 755, 823, 757, 747, 749, 776, 754, 750, 754, 781, 874, 743] payload_rows_per_dst=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] dup_rows_per_dst=[3038, 2976, 3130, 2906, 3116, 2834, 2956, 3042] combine_rows_per_src=[9332, 9288, 9388, 9092, 9276, 9266, 9344, 9320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74196 payload_rows_remote=64594 dup_rows=24108 dup_rows_remote=20782 dedup_ratio=0.245239 router_tokens_per_expert=[807, 817, 775, 796, 743, 766, 775, 771, 772, 771, 776, 749, 800, 740, 765, 716, 819, 774, 755, 749, 774, 740, 748, 763, 743, 783, 744, 807, 763, 781, 765, 758, 754, 744, 796, 771, 733, 766, 790, 734, 811, 768, 787, 725, 740, 753, 775, 703, 743, 784, 748, 773, 739, 776, 768, 787, 763, 807, 810, 781, 776, 768, 811, 763] payload_rows_per_dst=[8936, 9160, 9170, 9716, 9352, 9256, 9004, 9602] dup_rows_per_dst=[2666, 2780, 2864, 3470, 2990, 3106, 2906, 3326] combine_rows_per_src=[8936, 9160, 9170, 9716, 9352, 9256, 9004, 9602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343880 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157896 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971912 weight_movement_bytes=34603008 total_nvlink_bytes=566574920 weight_to_token_ratio=0.065047 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344160 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657888 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532971616 weight_movement_bytes=69206016 total_nvlink_bytes=602177632 weight_to_token_ratio=0.129849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265084928 dispatch_meta_bytes=342264 combine_payload_bytes=265084928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265427192 ingress_bytes=334290944 link_bytes=334290944 token_bytes=530512120 weight_movement_bytes=69206016 total_nvlink_bytes=599718136 weight_to_token_ratio=0.130451 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257909910e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=343168 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266247296 ingress_bytes=300507136 link_bytes=300507136 token_bytes=532151424 weight_movement_bytes=34603008 total_nvlink_bytes=566754432 weight_to_token_ratio=0.065025 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130784160e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64868 dup_rows=24114 dup_rows_remote=21020 dedup_ratio=0.245300 router_tokens_per_expert=[754, 789, 790, 763, 764, 782, 782, 774, 768, 808, 788, 790, 812, 722, 802, 783, 740, 806, 823, 772, 718, 762, 738, 754, 719, 725, 717, 753, 773, 792, 816, 771, 758, 746, 769, 795, 757, 797, 772, 763, 748, 816, 777, 724, 727, 749, 792, 746, 758, 772, 807, 789, 783, 789, 790, 720, 805, 750, 776, 737, 754, 698, 744, 794] payload_rows_per_dst=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] dup_rows_per_dst=[3052, 3108, 2904, 3000, 3086, 2926, 3094, 2944] combine_rows_per_src=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65044 dup_rows=23982 dup_rows_remote=21088 dedup_ratio=0.243958 router_tokens_per_expert=[774, 745, 817, 743, 783, 720, 774, 810, 832, 769, 787, 788, 800, 791, 765, 740, 736, 800, 772, 750, 742, 777, 742, 783, 777, 756, 775, 801, 710, 766, 743, 758, 745, 735, 720, 777, 778, 746, 754, 802, 760, 771, 790, 825, 810, 715, 730, 799, 788, 732, 758, 751, 844, 762, 791, 789, 766, 715, 751, 747, 747, 772, 791, 765] payload_rows_per_dst=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] dup_rows_per_dst=[3098, 3126, 2970, 2894, 2908, 3038, 3050, 2898] combine_rows_per_src=[9234, 9418, 9234, 9278, 9206, 9362, 9380, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344160 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266657888 ingress_bytes=335519744 link_bytes=335519744 token_bytes=532971616 weight_movement_bytes=69206016 total_nvlink_bytes=602177632 weight_to_token_ratio=0.129849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262533785e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343776 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266845920 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348064 weight_movement_bytes=34603008 total_nvlink_bytes=567951072 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64908 dup_rows=24010 dup_rows_remote=20990 dedup_ratio=0.244242 router_tokens_per_expert=[782, 765, 768, 741, 755, 764, 771, 768, 762, 797, 783, 810, 792, 742, 733, 763, 723, 746, 775, 709, 776, 789, 782, 722, 776, 775, 841, 764, 782, 763, 755, 792, 769, 784, 769, 758, 819, 749, 813, 736, 804, 763, 775, 828, 744, 754, 781, 754, 742, 777, 774, 726, 737, 835, 771, 758, 758, 725, 735, 784, 768, 768, 781, 747] payload_rows_per_dst=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] dup_rows_per_dst=[2990, 3094, 2926, 3098, 2976, 3020, 3018, 2888] combine_rows_per_src=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65128 dup_rows=23872 dup_rows_remote=20900 dedup_ratio=0.242839 router_tokens_per_expert=[759, 754, 778, 793, 781, 749, 720, 722, 756, 758, 745, 770, 781, 738, 774, 789, 755, 766, 807, 748, 805, 714, 745, 753, 796, 756, 744, 757, 785, 799, 773, 785, 802, 771, 760, 759, 801, 788, 746, 770, 754, 763, 791, 801, 759, 772, 788, 746, 765, 790, 753, 816, 726, 761, 766, 761, 760, 729, 774, 820, 758, 801, 771, 775] payload_rows_per_dst=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] dup_rows_per_dst=[2886, 2982, 2994, 3048, 2984, 3018, 2972, 2988] combine_rows_per_src=[9226, 9240, 9192, 9342, 9410, 9330, 9304, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265814016 dispatch_meta_bytes=343880 combine_payload_bytes=265814016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266157896 ingress_bytes=300417024 link_bytes=300417024 token_bytes=531971912 weight_movement_bytes=34603008 total_nvlink_bytes=566574920 weight_to_token_ratio=0.065047 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130445076e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264577024 dispatch_meta_bytes=341504 combine_payload_bytes=264577024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264918528 ingress_bytes=299180032 link_bytes=299180032 token_bytes=529495552 weight_movement_bytes=34603008 total_nvlink_bytes=564098560 weight_to_token_ratio=0.065351 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125790374e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74502 payload_rows_remote=65188 dup_rows=23802 dup_rows_remote=20842 dedup_ratio=0.242126 router_tokens_per_expert=[771, 759, 744, 791, 771, 783, 754, 784, 771, 772, 766, 778, 757, 783, 769, 800, 764, 730, 791, 741, 749, 780, 808, 778, 757, 765, 750, 791, 750, 746, 772, 742, 794, 764, 741, 730, 766, 750, 706, 760, 771, 753, 778, 806, 781, 778, 765, 705, 755, 750, 836, 783, 749, 751, 790, 765, 804, 782, 788, 788, 747, 775, 789, 785] payload_rows_per_dst=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] dup_rows_per_dst=[2980, 3060, 2966, 2876, 2926, 2960, 3054, 2980] combine_rows_per_src=[9334, 9332, 9316, 9270, 9096, 9314, 9304, 9536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343552 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042880 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742208 weight_movement_bytes=103809024 total_nvlink_bytes=635551232 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=344528 combine_payload_bytes=266420224 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266764752 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184976 weight_movement_bytes=34603008 total_nvlink_bytes=567787984 weight_to_token_ratio=0.064899 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64886 dup_rows=23988 dup_rows_remote=20960 dedup_ratio=0.244019 router_tokens_per_expert=[749, 754, 788, 741, 758, 799, 765, 803, 761, 750, 769, 722, 752, 793, 820, 816, 773, 769, 719, 772, 747, 733, 729, 737, 761, 779, 763, 788, 813, 750, 760, 753, 773, 809, 745, 810, 791, 750, 748, 803, 764, 736, 718, 729, 762, 771, 782, 795, 790, 796, 777, 790, 779, 767, 787, 737, 752, 763, 786, 774, 770, 785, 760, 767] payload_rows_per_dst=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] dup_rows_per_dst=[2998, 3054, 2864, 3034, 3028, 2910, 3166, 2934] combine_rows_per_src=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344120 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354168 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364216 weight_movement_bytes=34603008 total_nvlink_bytes=568967224 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343592 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206760 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069928 weight_movement_bytes=34603008 total_nvlink_bytes=566672936 weight_to_token_ratio=0.065035 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266764288 dispatch_meta_bytes=344112 combine_payload_bytes=266764288 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267108400 ingress_bytes=335970304 link_bytes=335970304 token_bytes=533872688 weight_movement_bytes=69206016 total_nvlink_bytes=603078704 weight_to_token_ratio=0.129630 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264229206e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64790 dup_rows=24194 dup_rows_remote=21194 dedup_ratio=0.246114 router_tokens_per_expert=[721, 755, 743, 804, 762, 759, 766, 733, 754, 805, 795, 730, 748, 743, 798, 808, 783, 746, 763, 834, 768, 804, 807, 797, 769, 762, 774, 792, 784, 792, 737, 750, 722, 750, 791, 778, 728, 771, 782, 771, 780, 742, 766, 742, 766, 718, 718, 756, 775, 717, 737, 799, 775, 756, 786, 783, 803, 763, 765, 762, 787, 767, 812, 798] payload_rows_per_dst=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] dup_rows_per_dst=[2902, 3012, 3158, 3000, 2996, 2882, 2998, 3246] combine_rows_per_src=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64788 dup_rows=24238 dup_rows_remote=21346 dedup_ratio=0.246562 router_tokens_per_expert=[787, 769, 797, 766, 747, 809, 819, 726, 771, 714, 774, 783, 733, 783, 742, 785, 749, 752, 782, 706, 743, 761, 774, 747, 772, 765, 779, 760, 770, 733, 760, 764, 739, 784, 846, 760, 792, 786, 814, 744, 739, 789, 789, 764, 773, 753, 764, 747, 781, 786, 758, 748, 772, 761, 728, 750, 758, 778, 793, 732, 805, 785, 825, 787] payload_rows_per_dst=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] dup_rows_per_dst=[3040, 2892, 2956, 3038, 3192, 3032, 2976, 3112] combine_rows_per_src=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65022 dup_rows=23812 dup_rows_remote=20782 dedup_ratio=0.242228 router_tokens_per_expert=[779, 778, 787, 747, 789, 724, 795, 781, 774, 771, 772, 796, 772, 765, 766, 781, 779, 762, 768, 775, 806, 756, 778, 765, 760, 787, 772, 746, 742, 800, 715, 773, 804, 783, 756, 771, 779, 760, 740, 738, 780, 768, 769, 768, 799, 760, 761, 759, 756, 768, 747, 734, 750, 759, 763, 764, 782, 751, 753, 793, 790, 765, 754, 767] payload_rows_per_dst=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] dup_rows_per_dst=[3030, 2914, 3266, 3020, 2952, 2948, 2940, 2742] combine_rows_per_src=[9470, 9490, 9502, 9254, 9188, 9380, 9142, 9066] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65102 dup_rows=24026 dup_rows_remote=21116 dedup_ratio=0.244405 router_tokens_per_expert=[742, 773, 778, 782, 803, 771, 779, 783, 764, 786, 780, 733, 764, 818, 772, 787, 774, 757, 794, 745, 732, 753, 741, 747, 733, 812, 763, 769, 785, 748, 777, 735, 779, 746, 786, 799, 723, 828, 784, 752, 795, 743, 783, 779, 788, 746, 787, 787, 714, 723, 783, 772, 806, 706, 763, 813, 758, 769, 746, 763, 801, 736, 727, 787] payload_rows_per_dst=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] dup_rows_per_dst=[3086, 3132, 2910, 3006, 3050, 3026, 2884, 2932] combine_rows_per_src=[9336, 9276, 9176, 9238, 9344, 9390, 9276, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64992 dup_rows=24028 dup_rows_remote=21012 dedup_ratio=0.244425 router_tokens_per_expert=[765, 724, 777, 809, 740, 780, 801, 714, 740, 774, 758, 735, 756, 766, 771, 797, 816, 749, 751, 788, 770, 746, 752, 778, 774, 810, 762, 793, 769, 803, 784, 785, 742, 742, 777, 774, 696, 766, 779, 796, 792, 757, 755, 742, 772, 804, 739, 736, 805, 752, 690, 742, 751, 758, 751, 789, 785, 793, 787, 780, 816, 821, 757, 769] payload_rows_per_dst=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] dup_rows_per_dst=[2978, 2934, 3016, 3116, 2884, 2996, 2962, 3142] combine_rows_per_src=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74316 payload_rows_remote=64886 dup_rows=23988 dup_rows_remote=20960 dedup_ratio=0.244019 router_tokens_per_expert=[749, 754, 788, 741, 758, 799, 765, 803, 761, 750, 769, 722, 752, 793, 820, 816, 773, 769, 719, 772, 747, 733, 729, 737, 761, 779, 763, 788, 813, 750, 760, 753, 773, 809, 745, 810, 791, 750, 748, 803, 764, 736, 718, 729, 762, 771, 782, 795, 790, 796, 777, 790, 779, 767, 787, 737, 752, 763, 786, 774, 770, 785, 760, 767] payload_rows_per_dst=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] dup_rows_per_dst=[2998, 3054, 2864, 3034, 3028, 2910, 3166, 2934] combine_rows_per_src=[9316, 9312, 9094, 9300, 9430, 9204, 9280, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343936 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723776 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103616 weight_movement_bytes=69206016 total_nvlink_bytes=600309632 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65174 dup_rows=23886 dup_rows_remote=20854 dedup_ratio=0.242981 router_tokens_per_expert=[785, 767, 800, 762, 789, 798, 791, 745, 778, 809, 790, 803, 734, 750, 749, 753, 771, 720, 759, 743, 814, 755, 802, 801, 788, 773, 761, 741, 776, 816, 723, 749, 746, 768, 755, 760, 750, 714, 799, 776, 800, 799, 720, 773, 767, 760, 797, 736, 764, 740, 795, 769, 755, 758, 752, 766, 783, 761, 739, 768, 818, 786, 708, 775] payload_rows_per_dst=[9468, 9274, 9440, 9128, 9028, 9464, 9372, 9244] dup_rows_per_dst=[3006, 3058, 2890, 2832, 2780, 3134, 3154, 3032] combine_rows_per_src=[9468, 9274, 9440, 9128, 9028, 9464, 9372, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343384 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116440 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531889496 weight_movement_bytes=103809024 total_nvlink_bytes=635698520 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74066 payload_rows_remote=64788 dup_rows=24238 dup_rows_remote=21346 dedup_ratio=0.246562 router_tokens_per_expert=[787, 769, 797, 766, 747, 809, 819, 726, 771, 714, 774, 783, 733, 783, 742, 785, 749, 752, 782, 706, 743, 761, 774, 747, 772, 765, 779, 760, 770, 733, 760, 764, 739, 784, 846, 760, 792, 786, 814, 744, 739, 789, 789, 764, 773, 753, 764, 747, 781, 786, 758, 748, 772, 761, 728, 750, 758, 778, 793, 732, 805, 785, 825, 787] payload_rows_per_dst=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] dup_rows_per_dst=[3040, 2892, 2956, 3038, 3192, 3032, 2976, 3112] combine_rows_per_src=[9400, 9278, 9072, 9168, 9338, 9204, 9192, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65220 dup_rows=23798 dup_rows_remote=20822 dedup_ratio=0.242086 router_tokens_per_expert=[799, 735, 769, 807, 745, 707, 733, 783, 726, 741, 732, 755, 726, 771, 787, 738, 729, 754, 757, 820, 785, 755, 735, 771, 734, 819, 758, 751, 810, 777, 768, 832, 776, 750, 780, 783, 772, 756, 783, 797, 810, 761, 799, 733, 741, 822, 784, 749, 750, 731, 843, 778, 766, 756, 745, 762, 766, 739, 733, 767, 795, 818, 806, 792] payload_rows_per_dst=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] dup_rows_per_dst=[2960, 2758, 2982, 3108, 2968, 3046, 2976, 3000] combine_rows_per_src=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344536 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265716184 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087832 weight_movement_bytes=34603008 total_nvlink_bytes=565690840 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344016 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551248 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758480 weight_movement_bytes=34603008 total_nvlink_bytes=567361488 weight_to_token_ratio=0.064951 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64738 dup_rows=24284 dup_rows_remote=21196 dedup_ratio=0.247030 router_tokens_per_expert=[797, 709, 811, 783, 761, 794, 789, 796, 785, 767, 820, 753, 776, 786, 733, 770, 738, 769, 759, 767, 781, 735, 777, 771, 766, 746, 786, 748, 822, 761, 748, 742, 736, 804, 796, 754, 747, 768, 781, 759, 775, 744, 739, 793, 781, 800, 787, 766, 697, 770, 721, 748, 752, 763, 785, 767, 804, 792, 703, 778, 772, 763, 780, 781] payload_rows_per_dst=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] dup_rows_per_dst=[3108, 3096, 3026, 3028, 3014, 3088, 2886, 3038] combine_rows_per_src=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343216 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266673328 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533003440 weight_movement_bytes=34603008 total_nvlink_bytes=567606448 weight_to_token_ratio=0.064921 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344872 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267002664 ingress_bytes=301260800 link_bytes=301260800 token_bytes=533660456 weight_movement_bytes=34603008 total_nvlink_bytes=568263464 weight_to_token_ratio=0.064841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133620137e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64790 dup_rows=24194 dup_rows_remote=21194 dedup_ratio=0.246114 router_tokens_per_expert=[721, 755, 743, 804, 762, 759, 766, 733, 754, 805, 795, 730, 748, 743, 798, 808, 783, 746, 763, 834, 768, 804, 807, 797, 769, 762, 774, 792, 784, 792, 737, 750, 722, 750, 791, 778, 728, 771, 782, 771, 780, 742, 766, 742, 766, 718, 718, 756, 775, 717, 737, 799, 775, 756, 786, 783, 803, 763, 765, 762, 787, 767, 812, 798] payload_rows_per_dst=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] dup_rows_per_dst=[2902, 3012, 3158, 3000, 2996, 2882, 2998, 3246] combine_rows_per_src=[9184, 9350, 9446, 9320, 9190, 9094, 9258, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343384 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116440 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531889496 weight_movement_bytes=103809024 total_nvlink_bytes=635698520 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64742 dup_rows=24168 dup_rows_remote=21118 dedup_ratio=0.245850 router_tokens_per_expert=[745, 741, 756, 794, 767, 756, 773, 735, 794, 751, 760, 780, 769, 718, 759, 769, 779, 805, 754, 801, 722, 765, 744, 776, 733, 808, 784, 805, 807, 763, 814, 786, 722, 760, 738, 805, 789, 753, 750, 800, 734, 778, 795, 781, 748, 752, 762, 763, 805, 797, 752, 753, 828, 724, 780, 783, 774, 802, 725, 753, 799, 737, 742, 755] payload_rows_per_dst=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] dup_rows_per_dst=[2946, 2964, 3040, 3216, 3006, 3000, 3050, 2946] combine_rows_per_src=[9188, 9236, 9252, 9384, 9228, 9226, 9394, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344112 combine_payload_bytes=266952704 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267296816 ingress_bytes=405364736 link_bytes=405364736 token_bytes=534249520 weight_movement_bytes=138412032 total_nvlink_bytes=672661552 weight_to_token_ratio=0.259078 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525354867e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74604 payload_rows_remote=65296 dup_rows=23700 dup_rows_remote=20716 dedup_ratio=0.241089 router_tokens_per_expert=[733, 743, 759, 768, 781, 811, 848, 736, 818, 745, 807, 790, 765, 739, 779, 721, 779, 772, 766, 766, 743, 731, 799, 735, 733, 786, 726, 757, 793, 704, 762, 801, 789, 775, 748, 774, 750, 773, 754, 761, 804, 772, 782, 785, 762, 732, 761, 748, 802, 787, 778, 798, 777, 783, 775, 820, 769, 791, 706, 721, 764, 746, 795, 774] payload_rows_per_dst=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] dup_rows_per_dst=[2988, 2990, 2918, 2926, 2928, 2984, 3082, 2884] combine_rows_per_src=[9370, 9338, 9264, 9198, 9320, 9308, 9558, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265371648 dispatch_meta_bytes=344536 combine_payload_bytes=265371648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265716184 ingress_bytes=299974656 link_bytes=299974656 token_bytes=531087832 weight_movement_bytes=34603008 total_nvlink_bytes=565690840 weight_to_token_ratio=0.065155 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128780481e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344168 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267485288 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534626408 weight_movement_bytes=34603008 total_nvlink_bytes=569229416 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=343936 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265723776 ingress_bytes=334585856 link_bytes=334585856 token_bytes=531103616 weight_movement_bytes=69206016 total_nvlink_bytes=600309632 weight_to_token_ratio=0.130306 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259019640e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65116 dup_rows=24002 dup_rows_remote=21130 dedup_ratio=0.244161 router_tokens_per_expert=[789, 767, 770, 760, 747, 818, 777, 748, 788, 770, 775, 800, 767, 750, 799, 760, 762, 816, 769, 734, 781, 773, 727, 796, 766, 792, 794, 749, 799, 772, 745, 801, 737, 776, 776, 720, 732, 770, 748, 770, 795, 791, 749, 762, 780, 813, 794, 784, 816, 733, 798, 725, 746, 742, 769, 789, 744, 743, 730, 761, 731, 711, 758, 798] payload_rows_per_dst=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] dup_rows_per_dst=[3046, 3110, 3012, 3104, 2872, 3172, 2880, 2806] combine_rows_per_src=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=267452416 dispatch_meta_bytes=344048 combine_payload_bytes=267452416 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267796464 ingress_bytes=302055424 link_bytes=302055424 token_bytes=535248880 weight_movement_bytes=34603008 total_nvlink_bytes=569851888 weight_to_token_ratio=0.064648 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136610243e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343736 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265510584 ingress_bytes=403578880 link_bytes=403578880 token_bytes=530677432 weight_movement_bytes=138412032 total_nvlink_bytes=669089464 weight_to_token_ratio=0.260821 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518634835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=343440 combine_payload_bytes=265183232 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265526672 ingress_bytes=334389248 link_bytes=334389248 token_bytes=530709904 weight_movement_bytes=69206016 total_nvlink_bytes=599915920 weight_to_token_ratio=0.130403 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258279820e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64874 dup_rows=23970 dup_rows_remote=20846 dedup_ratio=0.243835 router_tokens_per_expert=[765, 774, 829, 782, 722, 751, 763, 742, 778, 734, 756, 749, 785, 736, 795, 817, 756, 743, 744, 734, 774, 765, 773, 744, 794, 806, 752, 818, 738, 818, 764, 802, 766, 743, 741, 784, 809, 801, 805, 777, 771, 707, 729, 775, 791, 757, 788, 779, 764, 787, 761, 732, 765, 767, 788, 745, 749, 765, 738, 798, 773, 736, 802, 756] payload_rows_per_dst=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] dup_rows_per_dst=[2918, 3064, 2868, 3124, 3112, 2960, 2922, 3002] combine_rows_per_src=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65236 dup_rows=23810 dup_rows_remote=20834 dedup_ratio=0.242208 router_tokens_per_expert=[744, 799, 786, 787, 811, 743, 752, 769, 853, 755, 749, 812, 732, 775, 709, 732, 760, 756, 796, 763, 778, 765, 756, 764, 779, 760, 733, 762, 790, 735, 742, 732, 755, 802, 745, 732, 825, 786, 735, 818, 769, 733, 753, 741, 778, 804, 758, 785, 743, 804, 780, 772, 811, 778, 756, 785, 793, 768, 692, 746, 749, 781, 764, 832] payload_rows_per_dst=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] dup_rows_per_dst=[3084, 2976, 2966, 2850, 3024, 2972, 3010, 2928] combine_rows_per_src=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65022 dup_rows=24030 dup_rows_remote=21124 dedup_ratio=0.244446 router_tokens_per_expert=[737, 722, 793, 758, 787, 836, 745, 746, 743, 737, 798, 734, 749, 813, 791, 804, 765, 770, 754, 760, 783, 782, 765, 700, 754, 746, 807, 786, 811, 783, 770, 745, 790, 761, 731, 746, 764, 752, 739, 780, 802, 747, 815, 782, 789, 823, 764, 712, 791, 796, 783, 779, 760, 748, 799, 730, 746, 799, 796, 734, 761, 728, 785, 746] payload_rows_per_dst=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] dup_rows_per_dst=[2986, 3024, 2906, 3046, 2986, 3112, 3098, 2872] combine_rows_per_src=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64816 dup_rows=24224 dup_rows_remote=21210 dedup_ratio=0.246419 router_tokens_per_expert=[749, 759, 753, 767, 808, 750, 713, 757, 744, 760, 801, 801, 756, 724, 755, 732, 839, 776, 756, 753, 745, 784, 800, 802, 765, 793, 751, 819, 824, 755, 708, 730, 733, 775, 755, 789, 737, 798, 761, 793, 759, 805, 771, 780, 784, 771, 781, 801, 745, 779, 781, 781, 697, 798, 764, 739, 815, 762, 788, 757, 741, 755, 762, 766] payload_rows_per_dst=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] dup_rows_per_dst=[3014, 3040, 3112, 2884, 3124, 3122, 2868, 3060] combine_rows_per_src=[9264, 9228, 9564, 9118, 9158, 9216, 9300, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=65116 dup_rows=24002 dup_rows_remote=21130 dedup_ratio=0.244161 router_tokens_per_expert=[789, 767, 770, 760, 747, 818, 777, 748, 788, 770, 775, 800, 767, 750, 799, 760, 762, 816, 769, 734, 781, 773, 727, 796, 766, 792, 794, 749, 799, 772, 745, 801, 737, 776, 776, 720, 732, 770, 748, 770, 795, 791, 749, 762, 780, 813, 794, 784, 816, 733, 798, 725, 746, 742, 769, 789, 744, 743, 730, 761, 731, 711, 758, 798] payload_rows_per_dst=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] dup_rows_per_dst=[3046, 3110, 3012, 3104, 2872, 3172, 2880, 2806] combine_rows_per_src=[9306, 9308, 9304, 9332, 9186, 9364, 9356, 9146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=342880 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266066784 ingress_bytes=404135936 link_bytes=404135936 token_bytes=531790688 weight_movement_bytes=138412032 total_nvlink_bytes=670202720 weight_to_token_ratio=0.260275 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520730992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64944 dup_rows=24088 dup_rows_remote=21124 dedup_ratio=0.245036 router_tokens_per_expert=[742, 742, 723, 786, 763, 830, 779, 787, 807, 740, 765, 773, 764, 817, 766, 758, 757, 786, 721, 799, 765, 774, 765, 751, 724, 770, 719, 734, 775, 789, 758, 773, 786, 796, 760, 770, 792, 775, 773, 779, 745, 765, 757, 783, 769, 790, 741, 767, 737, 832, 786, 754, 780, 710, 748, 771, 796, 741, 771, 797, 732, 785, 774, 788] payload_rows_per_dst=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] dup_rows_per_dst=[2964, 3052, 2964, 2954, 3152, 2940, 3038, 3024] combine_rows_per_src=[9340, 9328, 9272, 9130, 9310, 9294, 9198, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344984 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267060120 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533775256 weight_movement_bytes=34603008 total_nvlink_bytes=568378264 weight_to_token_ratio=0.064827 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344280 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550936 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757592 weight_movement_bytes=34603008 total_nvlink_bytes=569360600 weight_to_token_ratio=0.064708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=64958 dup_rows=23994 dup_rows_remote=20976 dedup_ratio=0.244080 router_tokens_per_expert=[732, 752, 771, 752, 746, 767, 797, 747, 763, 725, 761, 732, 804, 749, 785, 776, 767, 769, 762, 714, 822, 754, 758, 746, 761, 773, 808, 799, 775, 798, 757, 803, 795, 770, 740, 761, 782, 720, 766, 772, 786, 724, 783, 784, 764, 789, 799, 795, 781, 776, 813, 786, 763, 779, 744, 747, 720, 797, 849, 765, 763, 743, 781, 690] payload_rows_per_dst=[9200, 9192, 9298, 9250, 9380, 9114, 9524, 9352] dup_rows_per_dst=[2928, 2998, 2886, 3024, 3102, 2880, 3158, 3018] combine_rows_per_src=[9200, 9192, 9298, 9250, 9380, 9114, 9524, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65058 dup_rows=23946 dup_rows_remote=20948 dedup_ratio=0.243591 router_tokens_per_expert=[739, 773, 764, 763, 764, 730, 772, 757, 772, 733, 768, 749, 777, 760, 769, 776, 760, 754, 775, 774, 782, 803, 744, 802, 785, 732, 767, 769, 767, 786, 724, 763, 819, 764, 773, 734, 787, 756, 768, 725, 773, 798, 740, 730, 759, 781, 762, 806, 764, 809, 769, 808, 805, 787, 759, 775, 748, 754, 820, 788, 743, 764, 742, 789] payload_rows_per_dst=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] dup_rows_per_dst=[2920, 2962, 3060, 2958, 2980, 2998, 3130, 2938] combine_rows_per_src=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344584 combine_payload_bytes=266330112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266674696 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533004808 weight_movement_bytes=69206016 total_nvlink_bytes=602210824 weight_to_token_ratio=0.129841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344104 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265830440 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531316776 weight_movement_bytes=34603008 total_nvlink_bytes=565919784 weight_to_token_ratio=0.065127 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74494 payload_rows_remote=65236 dup_rows=23810 dup_rows_remote=20834 dedup_ratio=0.242208 router_tokens_per_expert=[744, 799, 786, 787, 811, 743, 752, 769, 853, 755, 749, 812, 732, 775, 709, 732, 760, 756, 796, 763, 778, 765, 756, 764, 779, 760, 733, 762, 790, 735, 742, 732, 755, 802, 745, 732, 825, 786, 735, 818, 769, 733, 753, 741, 778, 804, 758, 785, 743, 804, 780, 772, 811, 778, 756, 785, 793, 768, 692, 746, 749, 781, 764, 832] payload_rows_per_dst=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] dup_rows_per_dst=[3084, 2976, 2966, 2850, 3024, 2972, 3010, 2928] combine_rows_per_src=[9298, 9258, 9310, 9216, 9372, 9270, 9448, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74010 payload_rows_remote=64690 dup_rows=24294 dup_rows_remote=21154 dedup_ratio=0.247131 router_tokens_per_expert=[752, 743, 762, 755, 756, 778, 779, 779, 750, 786, 766, 773, 777, 765, 748, 795, 798, 762, 798, 782, 737, 766, 770, 733, 745, 788, 768, 732, 743, 782, 765, 749, 755, 765, 800, 709, 804, 744, 771, 752, 816, 773, 760, 768, 759, 745, 861, 736, 745, 783, 745, 808, 754, 782, 807, 806, 781, 777, 789, 778, 735, 745, 764, 753] payload_rows_per_dst=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] dup_rows_per_dst=[3022, 3016, 3148, 3004, 2954, 3076, 3140, 2934] combine_rows_per_src=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64874 dup_rows=23970 dup_rows_remote=20846 dedup_ratio=0.243835 router_tokens_per_expert=[765, 774, 829, 782, 722, 751, 763, 742, 778, 734, 756, 749, 785, 736, 795, 817, 756, 743, 744, 734, 774, 765, 773, 744, 794, 806, 752, 818, 738, 818, 764, 802, 766, 743, 741, 784, 809, 801, 805, 777, 771, 707, 729, 775, 791, 757, 788, 779, 764, 787, 761, 732, 765, 767, 788, 745, 749, 765, 738, 798, 773, 736, 802, 756] payload_rows_per_dst=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] dup_rows_per_dst=[2918, 3064, 2868, 3124, 3112, 2960, 2922, 3002] combine_rows_per_src=[9338, 9236, 9198, 9460, 9340, 9234, 9296, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=344272 combine_payload_bytes=266010624 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266354896 ingress_bytes=369819648 link_bytes=369819648 token_bytes=532365520 weight_movement_bytes=103809024 total_nvlink_bytes=636174544 weight_to_token_ratio=0.194996 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391601562e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344280 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267550936 ingress_bytes=301809664 link_bytes=301809664 token_bytes=534757592 weight_movement_bytes=34603008 total_nvlink_bytes=569360600 weight_to_token_ratio=0.064708 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135685468e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344984 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267060120 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533775256 weight_movement_bytes=34603008 total_nvlink_bytes=568378264 weight_to_token_ratio=0.064827 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64868 dup_rows=24114 dup_rows_remote=21020 dedup_ratio=0.245300 router_tokens_per_expert=[754, 789, 790, 763, 764, 782, 782, 774, 768, 808, 788, 790, 812, 722, 802, 783, 740, 806, 823, 772, 718, 762, 738, 754, 719, 725, 717, 753, 773, 792, 816, 771, 758, 746, 769, 795, 757, 797, 772, 763, 748, 816, 777, 724, 727, 749, 792, 746, 758, 772, 807, 789, 783, 789, 790, 720, 805, 750, 776, 737, 754, 698, 744, 794] payload_rows_per_dst=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] dup_rows_per_dst=[3052, 3108, 2904, 3000, 3086, 2926, 3094, 2944] combine_rows_per_src=[9344, 9438, 9322, 9132, 9228, 9232, 9322, 9172] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=343736 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266411704 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532479672 weight_movement_bytes=34603008 total_nvlink_bytes=567082680 weight_to_token_ratio=0.064985 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64908 dup_rows=24010 dup_rows_remote=20990 dedup_ratio=0.244242 router_tokens_per_expert=[782, 765, 768, 741, 755, 764, 771, 768, 762, 797, 783, 810, 792, 742, 733, 763, 723, 746, 775, 709, 776, 789, 782, 722, 776, 775, 841, 764, 782, 763, 755, 792, 769, 784, 769, 758, 819, 749, 813, 736, 804, 763, 775, 828, 744, 754, 781, 754, 742, 777, 774, 726, 737, 835, 771, 758, 758, 725, 735, 784, 768, 768, 781, 747] payload_rows_per_dst=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] dup_rows_per_dst=[2990, 3094, 2926, 3098, 2976, 3020, 3018, 2888] combine_rows_per_src=[9238, 9270, 9118, 9398, 9418, 9386, 9222, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=343376 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313616 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283856 weight_movement_bytes=34603008 total_nvlink_bytes=564886864 weight_to_token_ratio=0.065254 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=342880 combine_payload_bytes=265723904 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266066784 ingress_bytes=404135936 link_bytes=404135936 token_bytes=531790688 weight_movement_bytes=138412032 total_nvlink_bytes=670202720 weight_to_token_ratio=0.260275 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.520730992e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65032 dup_rows=23952 dup_rows_remote=20900 dedup_ratio=0.243652 router_tokens_per_expert=[781, 731, 793, 727, 713, 755, 770, 765, 781, 732, 778, 745, 749, 751, 766, 750, 805, 765, 794, 766, 713, 805, 758, 780, 796, 752, 760, 793, 755, 792, 753, 735, 800, 752, 766, 759, 783, 725, 821, 780, 780, 800, 754, 750, 745, 767, 737, 803, 750, 808, 737, 791, 778, 810, 724, 780, 764, 741, 829, 797, 825, 774, 747, 766] payload_rows_per_dst=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] dup_rows_per_dst=[2888, 2904, 2962, 2914, 3052, 2972, 3062, 3198] combine_rows_per_src=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344024 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821592 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299160 weight_movement_bytes=69206016 total_nvlink_bytes=602505176 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64976 dup_rows=24116 dup_rows_remote=21006 dedup_ratio=0.245321 router_tokens_per_expert=[749, 730, 777, 754, 774, 758, 780, 790, 769, 760, 783, 778, 850, 765, 727, 761, 758, 740, 785, 850, 801, 809, 755, 766, 764, 742, 823, 806, 747, 742, 805, 732, 757, 745, 716, 784, 755, 757, 730, 735, 780, 780, 730, 769, 767, 722, 731, 818, 730, 799, 739, 808, 787, 735, 786, 712, 763, 821, 793, 768, 768, 777, 774, 786] payload_rows_per_dst=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] dup_rows_per_dst=[2972, 3090, 3088, 3110, 2842, 2934, 2978, 3102] combine_rows_per_src=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343552 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266042880 ingress_bytes=369508352 link_bytes=369508352 token_bytes=531742208 weight_movement_bytes=103809024 total_nvlink_bytes=635551232 weight_to_token_ratio=0.195224 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390430181e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65276 dup_rows=23808 dup_rows_remote=20802 dedup_ratio=0.242188 router_tokens_per_expert=[732, 820, 806, 772, 775, 787, 787, 747, 771, 758, 769, 748, 786, 791, 748, 742, 759, 785, 736, 776, 774, 801, 710, 785, 799, 767, 738, 783, 754, 787, 801, 741, 758, 724, 771, 780, 764, 739, 782, 780, 762, 725, 776, 801, 730, 781, 749, 804, 776, 763, 785, 794, 721, 802, 725, 740, 766, 761, 772, 776, 759, 758, 787, 806] payload_rows_per_dst=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] dup_rows_per_dst=[2946, 3006, 2956, 2998, 3020, 2894, 2958, 3030] combine_rows_per_src=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64856 dup_rows=24170 dup_rows_remote=21152 dedup_ratio=0.245870 router_tokens_per_expert=[794, 751, 755, 744, 715, 774, 741, 770, 758, 697, 788, 797, 823, 763, 744, 726, 794, 835, 725, 761, 799, 758, 756, 720, 780, 807, 774, 789, 786, 745, 764, 789, 792, 753, 813, 801, 769, 781, 758, 760, 732, 796, 748, 784, 792, 822, 815, 752, 779, 771, 751, 748, 782, 736, 761, 792, 725, 770, 746, 801, 756, 751, 752, 741] payload_rows_per_dst=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] dup_rows_per_dst=[2820, 3014, 3018, 3082, 3148, 3114, 3042, 2932] combine_rows_per_src=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73954 payload_rows_remote=64634 dup_rows=24350 dup_rows_remote=21124 dedup_ratio=0.247701 router_tokens_per_expert=[679, 772, 766, 794, 777, 777, 745, 769, 792, 758, 769, 762, 769, 690, 786, 716, 736, 773, 774, 769, 765, 814, 782, 741, 774, 801, 798, 757, 820, 753, 761, 765, 803, 748, 791, 799, 795, 754, 704, 768, 806, 763, 758, 786, 752, 738, 768, 789, 824, 712, 797, 761, 770, 781, 749, 763, 790, 743, 783, 766, 780, 776, 763, 768] payload_rows_per_dst=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] dup_rows_per_dst=[3226, 3098, 3054, 2990, 3260, 3090, 2916, 2716] combine_rows_per_src=[9320, 9312, 9254, 9244, 9470, 9230, 9234, 8890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343592 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266206760 ingress_bytes=300466176 link_bytes=300466176 token_bytes=532069928 weight_movement_bytes=34603008 total_nvlink_bytes=566672936 weight_to_token_ratio=0.065035 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130630031e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74352 payload_rows_remote=65032 dup_rows=23952 dup_rows_remote=20900 dedup_ratio=0.243652 router_tokens_per_expert=[781, 731, 793, 727, 713, 755, 770, 765, 781, 732, 778, 745, 749, 751, 766, 750, 805, 765, 794, 766, 713, 805, 758, 780, 796, 752, 760, 793, 755, 792, 753, 735, 800, 752, 766, 759, 783, 725, 821, 780, 780, 800, 754, 750, 745, 767, 737, 803, 750, 808, 737, 791, 778, 810, 724, 780, 764, 741, 829, 797, 825, 774, 747, 766] payload_rows_per_dst=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] dup_rows_per_dst=[2888, 2904, 2962, 2914, 3052, 2972, 3062, 3198] combine_rows_per_src=[9182, 9200, 9410, 9358, 9320, 9300, 9294, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343928 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266485624 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627320 weight_movement_bytes=69206016 total_nvlink_bytes=601833336 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343728 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266714800 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533085872 weight_movement_bytes=34603008 total_nvlink_bytes=567688880 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=343032 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265083896 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529824760 weight_movement_bytes=34603008 total_nvlink_bytes=564427768 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64968 dup_rows=23946 dup_rows_remote=20648 dedup_ratio=0.243591 router_tokens_per_expert=[766, 717, 771, 784, 779, 757, 750, 798, 821, 731, 791, 757, 751, 765, 718, 787, 729, 711, 769, 758, 743, 736, 797, 788, 787, 774, 776, 808, 778, 772, 714, 784, 835, 765, 734, 730, 767, 775, 772, 786, 740, 749, 760, 765, 756, 771, 783, 815, 751, 751, 758, 789, 729, 802, 731, 783, 796, 802, 788, 736, 811, 797, 758, 800] payload_rows_per_dst=[9306, 9350, 9088, 9338, 9298, 9304, 9284, 9390] dup_rows_per_dst=[2938, 2892, 2916, 3048, 3030, 2888, 2936, 3298] combine_rows_per_src=[9306, 9350, 9088, 9338, 9298, 9304, 9284, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65276 dup_rows=23808 dup_rows_remote=20802 dedup_ratio=0.242188 router_tokens_per_expert=[732, 820, 806, 772, 775, 787, 787, 747, 771, 758, 769, 748, 786, 791, 748, 742, 759, 785, 736, 776, 774, 801, 710, 785, 799, 767, 738, 783, 754, 787, 801, 741, 758, 724, 771, 780, 764, 739, 782, 780, 762, 725, 776, 801, 730, 781, 749, 804, 776, 763, 785, 794, 721, 802, 725, 740, 766, 761, 772, 776, 759, 758, 787, 806] payload_rows_per_dst=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] dup_rows_per_dst=[2946, 3006, 2956, 2998, 3020, 2894, 2958, 3030] combine_rows_per_src=[9506, 9220, 9296, 9342, 9176, 9362, 9254, 9340] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65038 dup_rows=23962 dup_rows_remote=21016 dedup_ratio=0.243754 router_tokens_per_expert=[751, 763, 832, 785, 775, 759, 744, 789, 822, 764, 736, 777, 769, 742, 797, 788, 751, 754, 757, 760, 804, 746, 762, 813, 769, 749, 765, 788, 718, 789, 738, 764, 724, 797, 775, 749, 821, 750, 771, 784, 804, 765, 769, 731, 775, 761, 776, 771, 764, 726, 682, 798, 786, 819, 777, 773, 747, 727, 751, 829, 740, 745, 795, 750] payload_rows_per_dst=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] dup_rows_per_dst=[3028, 3074, 2944, 2982, 3018, 3032, 2946, 2938] combine_rows_per_src=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74462 payload_rows_remote=65064 dup_rows=23842 dup_rows_remote=20880 dedup_ratio=0.242533 router_tokens_per_expert=[727, 741, 784, 765, 753, 782, 773, 728, 775, 783, 771, 807, 753, 753, 729, 806, 795, 779, 771, 774, 767, 783, 737, 774, 792, 728, 777, 731, 756, 762, 769, 793, 796, 774, 761, 769, 766, 802, 758, 771, 739, 761, 753, 798, 809, 754, 722, 788, 743, 799, 807, 721, 727, 776, 797, 792, 775, 760, 764, 778, 777, 751, 789, 757] payload_rows_per_dst=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] dup_rows_per_dst=[2886, 3030, 2962, 2958, 3064, 2970, 2960, 3012] combine_rows_per_src=[9220, 9324, 9398, 9258, 9330, 9278, 9364, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74188 payload_rows_remote=64976 dup_rows=24116 dup_rows_remote=21006 dedup_ratio=0.245321 router_tokens_per_expert=[749, 730, 777, 754, 774, 758, 780, 790, 769, 760, 783, 778, 850, 765, 727, 761, 758, 740, 785, 850, 801, 809, 755, 766, 764, 742, 823, 806, 747, 742, 805, 732, 757, 745, 716, 784, 755, 757, 730, 735, 780, 780, 730, 769, 767, 722, 731, 818, 730, 799, 739, 808, 787, 735, 786, 712, 763, 821, 793, 768, 768, 777, 774, 786] payload_rows_per_dst=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] dup_rows_per_dst=[2972, 3090, 3088, 3110, 2842, 2934, 2978, 3102] combine_rows_per_src=[9252, 9296, 9440, 9212, 9116, 9260, 9214, 9398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344312 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267714808 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085304 weight_movement_bytes=34603008 total_nvlink_bytes=569688312 weight_to_token_ratio=0.064668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=344032 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265994208 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531644384 weight_movement_bytes=69206016 total_nvlink_bytes=600850400 weight_to_token_ratio=0.130174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64908 dup_rows=24012 dup_rows_remote=21004 dedup_ratio=0.244263 router_tokens_per_expert=[716, 747, 742, 760, 752, 805, 743, 768, 738, 736, 785, 711, 803, 811, 758, 754, 787, 789, 805, 753, 764, 744, 757, 745, 741, 769, 757, 823, 782, 768, 814, 790, 781, 786, 750, 800, 765, 760, 778, 804, 771, 778, 810, 746, 777, 824, 762, 728, 763, 755, 752, 761, 759, 704, 789, 818, 798, 765, 733, 787, 755, 813, 732, 731] payload_rows_per_dst=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] dup_rows_per_dst=[2936, 2872, 2994, 3082, 3138, 3008, 2956, 3026] combine_rows_per_src=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264740864 dispatch_meta_bytes=343032 combine_payload_bytes=264740864 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265083896 ingress_bytes=299343872 link_bytes=299343872 token_bytes=529824760 weight_movement_bytes=34603008 total_nvlink_bytes=564427768 weight_to_token_ratio=0.065310 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126406891e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74506 payload_rows_remote=65220 dup_rows=23798 dup_rows_remote=20822 dedup_ratio=0.242086 router_tokens_per_expert=[799, 735, 769, 807, 745, 707, 733, 783, 726, 741, 732, 755, 726, 771, 787, 738, 729, 754, 757, 820, 785, 755, 735, 771, 734, 819, 758, 751, 810, 777, 768, 832, 776, 750, 780, 783, 772, 756, 783, 797, 810, 761, 799, 733, 741, 822, 784, 749, 750, 731, 843, 778, 766, 756, 745, 762, 766, 739, 733, 767, 795, 818, 806, 792] payload_rows_per_dst=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] dup_rows_per_dst=[2960, 2758, 2982, 3108, 2968, 3046, 2976, 3000] combine_rows_per_src=[9196, 9194, 9230, 9390, 9426, 9352, 9286, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343728 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266714800 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533085872 weight_movement_bytes=34603008 total_nvlink_bytes=567688880 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342464 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266451392 ingress_bytes=369917952 link_bytes=369917952 token_bytes=532560320 weight_movement_bytes=103809024 total_nvlink_bytes=636369344 weight_to_token_ratio=0.194924 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391971473e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267370496 dispatch_meta_bytes=344312 combine_payload_bytes=267370496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267714808 ingress_bytes=301973504 link_bytes=301973504 token_bytes=535085304 weight_movement_bytes=34603008 total_nvlink_bytes=569688312 weight_to_token_ratio=0.064668 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136301985e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74020 payload_rows_remote=64738 dup_rows=24284 dup_rows_remote=21196 dedup_ratio=0.247030 router_tokens_per_expert=[797, 709, 811, 783, 761, 794, 789, 796, 785, 767, 820, 753, 776, 786, 733, 770, 738, 769, 759, 767, 781, 735, 777, 771, 766, 746, 786, 748, 822, 761, 748, 742, 736, 804, 796, 754, 747, 768, 781, 759, 775, 744, 739, 793, 781, 800, 787, 766, 697, 770, 721, 748, 752, 763, 785, 767, 804, 792, 703, 778, 772, 763, 780, 781] payload_rows_per_dst=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] dup_rows_per_dst=[3108, 3096, 3026, 3028, 3014, 3088, 2886, 3038] combine_rows_per_src=[9372, 9284, 9168, 9210, 9276, 9282, 9120, 9308] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344216 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739864 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135512 weight_movement_bytes=69206016 total_nvlink_bytes=602341528 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343776 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266845920 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348064 weight_movement_bytes=34603008 total_nvlink_bytes=567951072 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64794 dup_rows=24190 dup_rows_remote=20966 dedup_ratio=0.246073 router_tokens_per_expert=[749, 768, 736, 744, 753, 775, 760, 803, 765, 773, 726, 715, 771, 802, 745, 740, 758, 746, 758, 756, 807, 760, 824, 752, 784, 772, 749, 780, 737, 735, 838, 765, 771, 781, 803, 810, 785, 752, 785, 785, 761, 760, 756, 757, 793, 742, 821, 813, 786, 809, 805, 751, 813, 706, 798, 722, 754, 746, 787, 736, 772, 765, 705, 776] payload_rows_per_dst=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] dup_rows_per_dst=[2912, 2948, 3040, 3018, 3224, 3132, 3030, 2886] combine_rows_per_src=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343928 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266485624 ingress_bytes=335347712 link_bytes=335347712 token_bytes=532627320 weight_movement_bytes=69206016 total_nvlink_bytes=601833336 weight_to_token_ratio=0.129933 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261886443e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206816 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069984 weight_movement_bytes=69206016 total_nvlink_bytes=601276000 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=344168 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267485288 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534626408 weight_movement_bytes=34603008 total_nvlink_bytes=569229416 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64980 dup_rows=24028 dup_rows_remote=20794 dedup_ratio=0.244425 router_tokens_per_expert=[756, 793, 745, 731, 764, 788, 742, 762, 771, 772, 766, 777, 732, 722, 799, 812, 730, 795, 803, 828, 764, 735, 779, 770, 812, 798, 768, 746, 781, 775, 784, 801, 766, 778, 730, 800, 776, 800, 739, 792, 752, 753, 738, 836, 777, 746, 808, 756, 749, 752, 730, 766, 731, 761, 796, 780, 752, 749, 752, 772, 722, 752, 764, 776] payload_rows_per_dst=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] dup_rows_per_dst=[2968, 3018, 3028, 3234, 3072, 2916, 2966, 2826] combine_rows_per_src=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64996 dup_rows=23968 dup_rows_remote=21058 dedup_ratio=0.243815 router_tokens_per_expert=[814, 741, 769, 791, 783, 742, 781, 722, 771, 755, 746, 779, 782, 739, 798, 755, 771, 779, 790, 778, 770, 794, 765, 723, 737, 728, 797, 796, 765, 749, 732, 747, 740, 756, 787, 763, 699, 747, 800, 788, 787, 804, 813, 781, 780, 794, 773, 740, 775, 763, 828, 742, 786, 752, 733, 778, 785, 726, 774, 756, 787, 764, 778, 784] payload_rows_per_dst=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] dup_rows_per_dst=[3008, 2910, 3028, 2940, 2894, 3178, 3014, 2996] combine_rows_per_src=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64708 dup_rows=24214 dup_rows_remote=21216 dedup_ratio=0.246318 router_tokens_per_expert=[738, 772, 728, 779, 803, 785, 740, 763, 768, 797, 769, 755, 786, 754, 812, 778, 763, 783, 765, 739, 816, 762, 770, 792, 774, 749, 744, 770, 746, 808, 743, 761, 752, 785, 763, 769, 776, 812, 731, 766, 761, 756, 799, 744, 724, 775, 764, 742, 777, 734, 714, 771, 753, 796, 764, 764, 739, 822, 794, 752, 795, 747, 824, 775] payload_rows_per_dst=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] dup_rows_per_dst=[2970, 3074, 2998, 3052, 2996, 2992, 3028, 3104] combine_rows_per_src=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74076 payload_rows_remote=64464 dup_rows=24228 dup_rows_remote=20952 dedup_ratio=0.246460 router_tokens_per_expert=[785, 776, 737, 788, 812, 794, 737, 791, 779, 745, 747, 774, 769, 777, 775, 766, 741, 776, 809, 851, 761, 768, 761, 811, 731, 788, 771, 781, 802, 741, 782, 783, 768, 763, 709, 770, 759, 724, 738, 774, 810, 763, 731, 794, 710, 772, 731, 759, 787, 809, 748, 754, 790, 749, 782, 735, 773, 752, 774, 755, 741, 767, 787, 765] payload_rows_per_dst=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] dup_rows_per_dst=[3276, 3332, 3186, 2820, 2924, 2668, 3010, 3012] combine_rows_per_src=[9612, 9684, 9420, 9040, 9086, 8720, 9298, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343736 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265510584 ingress_bytes=403578880 link_bytes=403578880 token_bytes=530677432 weight_movement_bytes=138412032 total_nvlink_bytes=669089464 weight_to_token_ratio=0.260821 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518634835e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343040 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265739264 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135488 weight_movement_bytes=69206016 total_nvlink_bytes=600341504 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=341664 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264386208 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528430752 weight_movement_bytes=34603008 total_nvlink_bytes=563033760 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343096 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501176 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659256 weight_movement_bytes=69206016 total_nvlink_bytes=601865272 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64794 dup_rows=24190 dup_rows_remote=20966 dedup_ratio=0.246073 router_tokens_per_expert=[749, 768, 736, 744, 753, 775, 760, 803, 765, 773, 726, 715, 771, 802, 745, 740, 758, 746, 758, 756, 807, 760, 824, 752, 784, 772, 749, 780, 737, 735, 838, 765, 771, 781, 803, 810, 785, 752, 785, 785, 761, 760, 756, 757, 793, 742, 821, 813, 786, 809, 805, 751, 813, 706, 798, 722, 754, 746, 787, 736, 772, 765, 705, 776] payload_rows_per_dst=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] dup_rows_per_dst=[2912, 2948, 3040, 3018, 3224, 3132, 3030, 2886] combine_rows_per_src=[9264, 9126, 9282, 9302, 9320, 9274, 9350, 9196] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=64792 dup_rows=23956 dup_rows_remote=20750 dedup_ratio=0.243693 router_tokens_per_expert=[739, 749, 839, 772, 757, 780, 779, 776, 774, 752, 788, 733, 700, 775, 780, 697, 781, 730, 730, 771, 758, 776, 782, 774, 776, 774, 800, 753, 795, 776, 745, 779, 774, 784, 763, 799, 756, 763, 720, 783, 825, 779, 772, 730, 728, 789, 787, 770, 746, 761, 767, 725, 777, 782, 782, 754, 797, 770, 795, 795, 774, 790, 766, 759] payload_rows_per_dst=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] dup_rows_per_dst=[2996, 2822, 2888, 3100, 2774, 3032, 3138, 3206] combine_rows_per_src=[9386, 9176, 9094, 9296, 9164, 9404, 9272, 9556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=64996 dup_rows=23968 dup_rows_remote=21058 dedup_ratio=0.243815 router_tokens_per_expert=[814, 741, 769, 791, 783, 742, 781, 722, 771, 755, 746, 779, 782, 739, 798, 755, 771, 779, 790, 778, 770, 794, 765, 723, 737, 728, 797, 796, 765, 749, 732, 747, 740, 756, 787, 763, 699, 747, 800, 788, 787, 804, 813, 781, 780, 794, 773, 740, 775, 763, 828, 742, 786, 752, 733, 778, 785, 726, 774, 756, 787, 764, 778, 784] payload_rows_per_dst=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] dup_rows_per_dst=[3008, 2910, 3028, 2940, 2894, 3178, 3014, 2996] combine_rows_per_src=[9278, 9340, 9312, 9162, 9266, 9366, 9300, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64942 dup_rows=24122 dup_rows_remote=21042 dedup_ratio=0.245382 router_tokens_per_expert=[792, 786, 803, 760, 736, 744, 748, 759, 763, 766, 747, 754, 766, 787, 739, 818, 735, 753, 756, 790, 743, 737, 742, 775, 752, 769, 780, 801, 751, 711, 766, 740, 760, 781, 802, 797, 739, 777, 769, 769, 778, 766, 811, 762, 774, 745, 784, 742, 770, 767, 812, 754, 756, 761, 798, 742, 807, 805, 815, 762, 768, 754, 794, 762] payload_rows_per_dst=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] dup_rows_per_dst=[3024, 2964, 2872, 2884, 3076, 3062, 3080, 3160] combine_rows_per_src=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64992 dup_rows=24028 dup_rows_remote=21012 dedup_ratio=0.244425 router_tokens_per_expert=[765, 724, 777, 809, 740, 780, 801, 714, 740, 774, 758, 735, 756, 766, 771, 797, 816, 749, 751, 788, 770, 746, 752, 778, 774, 810, 762, 793, 769, 803, 784, 785, 742, 742, 777, 774, 696, 766, 779, 796, 792, 757, 755, 742, 772, 804, 739, 736, 805, 752, 690, 742, 751, 758, 751, 789, 785, 793, 787, 780, 816, 821, 757, 769] payload_rows_per_dst=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] dup_rows_per_dst=[2978, 2934, 3016, 3116, 2884, 2996, 2962, 3142] combine_rows_per_src=[9242, 9260, 9284, 9444, 9260, 9198, 9114, 9474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64980 dup_rows=24028 dup_rows_remote=20794 dedup_ratio=0.244425 router_tokens_per_expert=[756, 793, 745, 731, 764, 788, 742, 762, 771, 772, 766, 777, 732, 722, 799, 812, 730, 795, 803, 828, 764, 735, 779, 770, 812, 798, 768, 746, 781, 775, 784, 801, 766, 778, 730, 800, 776, 800, 739, 792, 752, 753, 738, 836, 777, 746, 808, 756, 749, 752, 730, 766, 731, 761, 796, 780, 752, 749, 752, 772, 722, 752, 764, 776] payload_rows_per_dst=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] dup_rows_per_dst=[2968, 3018, 3028, 3234, 3072, 2916, 2966, 2826] combine_rows_per_src=[9194, 9284, 9380, 9296, 9290, 9416, 9164, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344216 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567832 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791448 weight_movement_bytes=34603008 total_nvlink_bytes=567394456 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343696 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265387664 ingress_bytes=368852992 link_bytes=368852992 token_bytes=530431632 weight_movement_bytes=103809024 total_nvlink_bytes=634240656 weight_to_token_ratio=0.195707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387964114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65082 dup_rows=23986 dup_rows_remote=21034 dedup_ratio=0.243998 router_tokens_per_expert=[715, 798, 784, 768, 778, 736, 729, 729, 771, 784, 768, 802, 774, 787, 823, 751, 769, 762, 771, 690, 752, 762, 779, 778, 787, 783, 742, 752, 738, 799, 731, 761, 821, 709, 755, 801, 765, 811, 729, 806, 765, 776, 754, 731, 794, 756, 774, 744, 723, 799, 733, 796, 752, 795, 808, 752, 747, 762, 802, 811, 820, 789, 756, 763] payload_rows_per_dst=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] dup_rows_per_dst=[2888, 3152, 2908, 2976, 3026, 2952, 2962, 3122] combine_rows_per_src=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=264044544 dispatch_meta_bytes=341664 combine_payload_bytes=264044544 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264386208 ingress_bytes=298647552 link_bytes=298647552 token_bytes=528430752 weight_movement_bytes=34603008 total_nvlink_bytes=563033760 weight_to_token_ratio=0.065483 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123786695e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265388032 dispatch_meta_bytes=342168 combine_payload_bytes=265388032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265730200 ingress_bytes=299991040 link_bytes=299991040 token_bytes=531118232 weight_movement_bytes=34603008 total_nvlink_bytes=565721240 weight_to_token_ratio=0.065151 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128842132e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344216 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567832 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791448 weight_movement_bytes=34603008 total_nvlink_bytes=567394456 weight_to_token_ratio=0.064947 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64976 dup_rows=23998 dup_rows_remote=20946 dedup_ratio=0.244120 router_tokens_per_expert=[795, 741, 787, 737, 753, 724, 768, 732, 789, 791, 813, 764, 781, 743, 793, 800, 759, 814, 728, 760, 802, 741, 751, 747, 763, 726, 785, 744, 749, 745, 749, 775, 742, 790, 745, 808, 777, 766, 786, 777, 775, 783, 768, 780, 765, 790, 754, 769, 798, 696, 795, 741, 745, 783, 822, 795, 714, 723, 751, 772, 788, 828, 807, 770] payload_rows_per_dst=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] dup_rows_per_dst=[2890, 3146, 2974, 2864, 3052, 3014, 3068, 2990] combine_rows_per_src=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65058 dup_rows=23946 dup_rows_remote=20948 dedup_ratio=0.243591 router_tokens_per_expert=[739, 773, 764, 763, 764, 730, 772, 757, 772, 733, 768, 749, 777, 760, 769, 776, 760, 754, 775, 774, 782, 803, 744, 802, 785, 732, 767, 769, 767, 786, 724, 763, 819, 764, 773, 734, 787, 756, 768, 725, 773, 798, 740, 730, 759, 781, 762, 806, 764, 809, 769, 808, 805, 787, 759, 775, 748, 754, 820, 788, 743, 764, 742, 789] payload_rows_per_dst=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] dup_rows_per_dst=[2920, 2962, 3060, 2958, 2980, 2998, 3130, 2938] combine_rows_per_src=[9204, 9246, 9328, 9228, 9272, 9300, 9422, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343936 combine_payload_bytes=266002432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266346368 ingress_bytes=300605440 link_bytes=300605440 token_bytes=532348800 weight_movement_bytes=34603008 total_nvlink_bytes=566951808 weight_to_token_ratio=0.065001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131154070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=344016 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266551248 ingress_bytes=300810240 link_bytes=300810240 token_bytes=532758480 weight_movement_bytes=34603008 total_nvlink_bytes=567361488 weight_to_token_ratio=0.064951 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131924716e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266158080 dispatch_meta_bytes=343096 combine_payload_bytes=266158080 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266501176 ingress_bytes=335364096 link_bytes=335364096 token_bytes=532659256 weight_movement_bytes=69206016 total_nvlink_bytes=601865272 weight_to_token_ratio=0.129925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261948094e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74010 payload_rows_remote=64690 dup_rows=24294 dup_rows_remote=21154 dedup_ratio=0.247131 router_tokens_per_expert=[752, 743, 762, 755, 756, 778, 779, 779, 750, 786, 766, 773, 777, 765, 748, 795, 798, 762, 798, 782, 737, 766, 770, 733, 745, 788, 768, 732, 743, 782, 765, 749, 755, 765, 800, 709, 804, 744, 771, 752, 816, 773, 760, 768, 759, 745, 861, 736, 745, 783, 745, 808, 754, 782, 807, 806, 781, 777, 789, 778, 735, 745, 764, 753] payload_rows_per_dst=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] dup_rows_per_dst=[3022, 3016, 3148, 3004, 2954, 3076, 3140, 2934] combine_rows_per_src=[9186, 9304, 9144, 9140, 9246, 9360, 9320, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265396224 dispatch_meta_bytes=343040 combine_payload_bytes=265396224 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265739264 ingress_bytes=334602240 link_bytes=334602240 token_bytes=531135488 weight_movement_bytes=69206016 total_nvlink_bytes=600341504 weight_to_token_ratio=0.130298 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259081291e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258788984 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6133352056 ingress_bytes=7544160256 link_bytes=7544160256 total_nvlink_bytes=13677512312 weight_to_token_ratio=0.115731 prefetch_share_of_ingress=0.087148 dedup_ratio=0.244121 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234263813e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344464 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920336 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496208 weight_movement_bytes=34603008 total_nvlink_bytes=568099216 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64968 dup_rows=23888 dup_rows_remote=20904 dedup_ratio=0.243001 router_tokens_per_expert=[762, 767, 769, 763, 793, 767, 769, 743, 776, 761, 773, 808, 755, 828, 742, 745, 748, 776, 808, 751, 734, 774, 788, 767, 788, 796, 815, 745, 796, 821, 758, 697, 758, 721, 759, 770, 729, 727, 778, 790, 805, 827, 762, 783, 748, 754, 769, 778, 747, 755, 748, 752, 771, 756, 791, 786, 778, 741, 743, 771, 765, 764, 803, 740] payload_rows_per_dst=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] dup_rows_per_dst=[3006, 3018, 3098, 2984, 2826, 3038, 3026, 2892] combine_rows_per_src=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264970240 dispatch_meta_bytes=343376 combine_payload_bytes=264970240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265313616 ingress_bytes=299573248 link_bytes=299573248 token_bytes=530283856 weight_movement_bytes=34603008 total_nvlink_bytes=564886864 weight_to_token_ratio=0.065254 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127270015e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65016 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[750, 780, 781, 803, 777, 761, 802, 810, 754, 724, 764, 790, 816, 778, 753, 759, 740, 786, 771, 764, 749, 768, 740, 702, 767, 762, 762, 767, 736, 723, 735, 773, 832, 733, 759, 795, 743, 763, 781, 763, 783, 788, 736, 762, 729, 802, 811, 762, 789, 787, 755, 805, 802, 750, 795, 783, 732, 776, 749, 779, 760, 755, 785, 761] payload_rows_per_dst=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] dup_rows_per_dst=[3162, 3028, 2928, 2890, 3050, 2982, 3068, 2932] combine_rows_per_src=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12272712288 prefetch_bytes=657457152 reduce_bytes=726663168 weight_movement_bytes=1384120320 egress_bytes=6140312160 ingress_bytes=7516520448 link_bytes=7516520448 total_nvlink_bytes=13656832608 weight_to_token_ratio=0.112780 prefetch_share_of_ingress=0.087468 dedup_ratio=0.243181 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=21 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229741796e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343488 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452416 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561344 weight_movement_bytes=34603008 total_nvlink_bytes=567164352 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64904 dup_rows=23910 dup_rows_remote=20906 dedup_ratio=0.243225 router_tokens_per_expert=[800, 767, 784, 752, 746, 743, 790, 807, 824, 782, 716, 775, 749, 739, 769, 692, 800, 785, 753, 797, 780, 792, 751, 800, 759, 809, 744, 777, 764, 746, 722, 774, 740, 744, 771, 731, 755, 797, 773, 766, 777, 790, 801, 748, 774, 770, 758, 755, 791, 778, 755, 767, 735, 748, 786, 753, 765, 811, 778, 767, 777, 760, 795, 748] payload_rows_per_dst=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] dup_rows_per_dst=[3004, 2942, 3112, 2898, 3114, 3078, 2948, 2814] combine_rows_per_src=[9490, 9040, 9400, 9292, 9480, 9342, 9278, 9072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64822 dup_rows=24168 dup_rows_remote=21116 dedup_ratio=0.245850 router_tokens_per_expert=[756, 828, 801, 773, 743, 781, 749, 732, 750, 783, 754, 762, 775, 777, 747, 791, 782, 790, 782, 783, 771, 840, 781, 747, 750, 784, 731, 805, 798, 781, 746, 766, 724, 736, 750, 786, 745, 748, 831, 769, 720, 796, 708, 782, 768, 790, 776, 757, 713, 753, 752, 772, 757, 738, 787, 772, 770, 782, 810, 781, 752, 779, 746, 763] payload_rows_per_dst=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] dup_rows_per_dst=[3170, 2768, 3094, 3252, 2946, 2976, 2910, 3052] combine_rows_per_src=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=64976 dup_rows=23998 dup_rows_remote=20946 dedup_ratio=0.244120 router_tokens_per_expert=[795, 741, 787, 737, 753, 724, 768, 732, 789, 791, 813, 764, 781, 743, 793, 800, 759, 814, 728, 760, 802, 741, 751, 747, 763, 726, 785, 744, 749, 745, 749, 775, 742, 790, 745, 808, 777, 766, 786, 777, 775, 783, 768, 780, 765, 790, 754, 769, 798, 696, 795, 741, 745, 783, 822, 795, 714, 723, 751, 772, 788, 828, 807, 770] payload_rows_per_dst=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] dup_rows_per_dst=[2890, 3146, 2974, 2864, 3052, 3014, 3068, 2990] combine_rows_per_src=[9184, 9402, 9230, 9208, 9330, 9354, 9282, 9316] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343688 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485384 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627080 weight_movement_bytes=34603008 total_nvlink_bytes=567230088 weight_to_token_ratio=0.064967 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344024 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821592 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299160 weight_movement_bytes=69206016 total_nvlink_bytes=602505176 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344112 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266649648 ingress_bytes=370114560 link_bytes=370114560 token_bytes=532955184 weight_movement_bytes=103809024 total_nvlink_bytes=636764208 weight_to_token_ratio=0.194780 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392711293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258456552 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6133183464 ingress_bytes=7543996416 link_bytes=7543996416 total_nvlink_bytes=13677179880 weight_to_token_ratio=0.115734 prefetch_share_of_ingress=0.091737 dedup_ratio=0.244023 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234237007e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=343240 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266190024 ingress_bytes=300449792 link_bytes=300449792 token_bytes=532036808 weight_movement_bytes=34603008 total_nvlink_bytes=566639816 weight_to_token_ratio=0.065039 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130568379e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74136 payload_rows_remote=64822 dup_rows=24168 dup_rows_remote=21116 dedup_ratio=0.245850 router_tokens_per_expert=[756, 828, 801, 773, 743, 781, 749, 732, 750, 783, 754, 762, 775, 777, 747, 791, 782, 790, 782, 783, 771, 840, 781, 747, 750, 784, 731, 805, 798, 781, 746, 766, 724, 736, 750, 786, 745, 748, 831, 769, 720, 796, 708, 782, 768, 790, 776, 757, 713, 753, 752, 772, 757, 738, 787, 772, 770, 782, 810, 781, 752, 779, 746, 763] payload_rows_per_dst=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] dup_rows_per_dst=[3170, 2768, 3094, 3252, 2946, 2976, 2910, 3052] combine_rows_per_src=[9156, 9004, 9376, 9576, 9232, 9218, 9260, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343752 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854664 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365576 weight_movement_bytes=103809024 total_nvlink_bytes=635174600 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65038 dup_rows=23962 dup_rows_remote=21016 dedup_ratio=0.243754 router_tokens_per_expert=[751, 763, 832, 785, 775, 759, 744, 789, 822, 764, 736, 777, 769, 742, 797, 788, 751, 754, 757, 760, 804, 746, 762, 813, 769, 749, 765, 788, 718, 789, 738, 764, 724, 797, 775, 749, 821, 750, 771, 784, 804, 765, 769, 731, 775, 761, 776, 771, 764, 726, 682, 798, 786, 819, 777, 773, 747, 727, 751, 829, 740, 745, 795, 750] payload_rows_per_dst=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] dup_rows_per_dst=[3028, 3074, 2944, 2982, 3018, 3032, 2946, 2938] combine_rows_per_src=[9368, 9316, 9350, 9178, 9324, 9272, 9304, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=343688 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266485384 ingress_bytes=300744704 link_bytes=300744704 token_bytes=532627080 weight_movement_bytes=34603008 total_nvlink_bytes=567230088 weight_to_token_ratio=0.064967 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131678109e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262175432 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6135042760 ingress_bytes=7407443968 link_bytes=7407443968 total_nvlink_bytes=13542486728 weight_to_token_ratio=0.104411 prefetch_share_of_ingress=0.102770 dedup_ratio=0.243764 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211896318e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233767928 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120831992 ingress_bytes=7427850240 link_bytes=7427850240 total_nvlink_bytes=13548682232 weight_to_token_ratio=0.107482 prefetch_share_of_ingress=0.093171 dedup_ratio=0.244459 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215234890e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12238422760 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6123160296 ingress_bytes=7741603840 link_bytes=7741603840 total_nvlink_bytes=13864764136 weight_to_token_ratio=0.132888 prefetch_share_of_ingress=0.075986 dedup_ratio=0.244481 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266566609e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264226080 prefetch_bytes=692060160 reduce_bytes=519045120 weight_movement_bytes=1211105280 egress_bytes=6136069408 ingress_bytes=7339261952 link_bytes=7339261952 total_nvlink_bytes=13475331360 weight_to_token_ratio=0.098751 prefetch_share_of_ingress=0.094296 dedup_ratio=0.244002 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=15 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200741385e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74264 payload_rows_remote=65016 dup_rows=24040 dup_rows_remote=21012 dedup_ratio=0.244548 router_tokens_per_expert=[750, 780, 781, 803, 777, 761, 802, 810, 754, 724, 764, 790, 816, 778, 753, 759, 740, 786, 771, 764, 749, 768, 740, 702, 767, 762, 762, 767, 736, 723, 735, 773, 832, 733, 759, 795, 743, 763, 781, 763, 783, 788, 736, 762, 729, 802, 811, 762, 789, 787, 755, 805, 802, 750, 795, 783, 732, 776, 749, 779, 760, 755, 785, 761] payload_rows_per_dst=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] dup_rows_per_dst=[3162, 3028, 2928, 2890, 3050, 2982, 3068, 2932] combine_rows_per_src=[9366, 9248, 9112, 9160, 9288, 9364, 9464, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344216 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266739864 ingress_bytes=335601664 link_bytes=335601664 token_bytes=533135512 weight_movement_bytes=69206016 total_nvlink_bytes=602341528 weight_to_token_ratio=0.129809 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262842043e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64942 dup_rows=24122 dup_rows_remote=21042 dedup_ratio=0.245382 router_tokens_per_expert=[792, 786, 803, 760, 736, 744, 748, 759, 763, 766, 747, 754, 766, 787, 739, 818, 735, 753, 756, 790, 743, 737, 742, 775, 752, 769, 780, 801, 751, 711, 766, 740, 760, 781, 802, 797, 739, 777, 769, 769, 778, 766, 811, 762, 774, 745, 784, 742, 770, 767, 812, 754, 756, 761, 798, 742, 807, 805, 815, 762, 768, 754, 794, 762] payload_rows_per_dst=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] dup_rows_per_dst=[3024, 2964, 2872, 2884, 3076, 3062, 3080, 3160] combine_rows_per_src=[9232, 9316, 9190, 9256, 9312, 9262, 9240, 9374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266002432 dispatch_meta_bytes=343936 combine_payload_bytes=266002432 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266346368 ingress_bytes=300605440 link_bytes=300605440 token_bytes=532348800 weight_movement_bytes=34603008 total_nvlink_bytes=566951808 weight_to_token_ratio=0.065001 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131154070e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64970 dup_rows=24090 dup_rows_remote=20982 dedup_ratio=0.245056 router_tokens_per_expert=[755, 806, 790, 755, 760, 762, 756, 779, 745, 788, 788, 760, 779, 829, 738, 769, 755, 772, 777, 729, 792, 751, 793, 771, 764, 764, 795, 771, 756, 792, 742, 794, 734, 779, 713, 825, 754, 735, 732, 751, 674, 763, 741, 792, 760, 794, 792, 787, 776, 789, 731, 828, 780, 736, 744, 792, 827, 753, 746, 777, 745, 784, 742, 799] payload_rows_per_dst=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] dup_rows_per_dst=[3020, 2980, 3000, 3020, 2940, 2972, 3108, 3050] combine_rows_per_src=[9306, 9412, 9280, 9336, 9106, 9234, 9244, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12233767928 prefetch_bytes=692060160 reduce_bytes=622854144 weight_movement_bytes=1314914304 egress_bytes=6120831992 ingress_bytes=7427850240 link_bytes=7427850240 total_nvlink_bytes=13548682232 weight_to_token_ratio=0.107482 prefetch_share_of_ingress=0.093171 dedup_ratio=0.244459 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=18 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.215234890e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64968 dup_rows=23888 dup_rows_remote=20904 dedup_ratio=0.243001 router_tokens_per_expert=[762, 767, 769, 763, 793, 767, 769, 743, 776, 761, 773, 808, 755, 828, 742, 745, 748, 776, 808, 751, 734, 774, 788, 767, 788, 796, 815, 745, 796, 821, 758, 697, 758, 721, 759, 770, 729, 727, 778, 790, 805, 827, 762, 783, 748, 754, 769, 778, 747, 755, 748, 752, 771, 756, 791, 786, 778, 741, 743, 771, 765, 764, 803, 740] payload_rows_per_dst=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] dup_rows_per_dst=[3006, 3018, 3098, 2984, 2826, 3038, 3026, 2892] combine_rows_per_src=[9260, 9358, 9194, 9448, 9238, 9414, 9186, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74274 payload_rows_remote=65022 dup_rows=24030 dup_rows_remote=21124 dedup_ratio=0.244446 router_tokens_per_expert=[737, 722, 793, 758, 787, 836, 745, 746, 743, 737, 798, 734, 749, 813, 791, 804, 765, 770, 754, 760, 783, 782, 765, 700, 754, 746, 807, 786, 811, 783, 770, 745, 790, 761, 731, 746, 764, 752, 739, 780, 802, 747, 815, 782, 789, 823, 764, 712, 791, 796, 783, 779, 760, 748, 799, 730, 746, 799, 796, 734, 761, 728, 785, 746] payload_rows_per_dst=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] dup_rows_per_dst=[2986, 3024, 2906, 3046, 2986, 3112, 3098, 2872] combine_rows_per_src=[9262, 9314, 9252, 9358, 9140, 9356, 9274, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=343752 combine_payload_bytes=265510912 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=265854664 ingress_bytes=369319936 link_bytes=369319936 token_bytes=531365576 weight_movement_bytes=103809024 total_nvlink_bytes=635174600 weight_to_token_ratio=0.195363 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389721186e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266305536 dispatch_meta_bytes=344112 combine_payload_bytes=266305536 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266649648 ingress_bytes=370114560 link_bytes=370114560 token_bytes=532955184 weight_movement_bytes=103809024 total_nvlink_bytes=636764208 weight_to_token_ratio=0.194780 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392711293e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343808 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266460928 ingress_bytes=300720128 link_bytes=300720128 token_bytes=532578048 weight_movement_bytes=34603008 total_nvlink_bytes=567181056 weight_to_token_ratio=0.064973 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131585632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=64908 dup_rows=24012 dup_rows_remote=21004 dedup_ratio=0.244263 router_tokens_per_expert=[716, 747, 742, 760, 752, 805, 743, 768, 738, 736, 785, 711, 803, 811, 758, 754, 787, 789, 805, 753, 764, 744, 757, 745, 741, 769, 757, 823, 782, 768, 814, 790, 781, 786, 750, 800, 765, 760, 778, 804, 771, 778, 810, 746, 777, 824, 762, 728, 763, 755, 752, 761, 759, 704, 789, 818, 798, 765, 733, 787, 755, 813, 732, 731] payload_rows_per_dst=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] dup_rows_per_dst=[2936, 2872, 2994, 3082, 3138, 3008, 2956, 3026] combine_rows_per_src=[9130, 9320, 9294, 9406, 9310, 9384, 9246, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343488 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266452416 ingress_bytes=300711936 link_bytes=300711936 token_bytes=532561344 weight_movement_bytes=34603008 total_nvlink_bytes=567164352 weight_to_token_ratio=0.064975 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131554806e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=344584 combine_payload_bytes=266330112 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266674696 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533004808 weight_movement_bytes=69206016 total_nvlink_bytes=602210824 weight_to_token_ratio=0.129841 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12238422760 prefetch_bytes=588251136 reduce_bytes=1038090240 weight_movement_bytes=1626341376 egress_bytes=6123160296 ingress_bytes=7741603840 link_bytes=7741603840 total_nvlink_bytes=13864764136 weight_to_token_ratio=0.132888 prefetch_share_of_ingress=0.075986 dedup_ratio=0.244481 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=30 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.266566609e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262175432 prefetch_bytes=761266176 reduce_bytes=519045120 weight_movement_bytes=1280311296 egress_bytes=6135042760 ingress_bytes=7407443968 link_bytes=7407443968 total_nvlink_bytes=13542486728 weight_to_token_ratio=0.104411 prefetch_share_of_ingress=0.102770 dedup_ratio=0.243764 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=15 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028696 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211896318e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12253039416 prefetch_bytes=726663168 reduce_bytes=553648128 weight_movement_bytes=1280311296 egress_bytes=6130477880 ingress_bytes=7402872832 link_bytes=7402872832 total_nvlink_bytes=13533350712 weight_to_token_ratio=0.104489 prefetch_share_of_ingress=0.098160 dedup_ratio=0.244448 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=16 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028587 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211148456e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258456552 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6133183464 ingress_bytes=7543996416 link_bytes=7543996416 total_nvlink_bytes=13677179880 weight_to_token_ratio=0.115734 prefetch_share_of_ingress=0.091737 dedup_ratio=0.244023 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234237007e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=343648 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266206816 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532069984 weight_movement_bytes=69206016 total_nvlink_bytes=601276000 weight_to_token_ratio=0.130069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64856 dup_rows=24170 dup_rows_remote=21152 dedup_ratio=0.245870 router_tokens_per_expert=[794, 751, 755, 744, 715, 774, 741, 770, 758, 697, 788, 797, 823, 763, 744, 726, 794, 835, 725, 761, 799, 758, 756, 720, 780, 807, 774, 789, 786, 745, 764, 789, 792, 753, 813, 801, 769, 781, 758, 760, 732, 796, 748, 784, 792, 822, 815, 752, 779, 771, 751, 748, 782, 736, 761, 792, 725, 770, 746, 801, 756, 751, 752, 741] payload_rows_per_dst=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] dup_rows_per_dst=[2820, 3014, 3018, 3082, 3148, 3114, 3042, 2932] combine_rows_per_src=[9268, 9178, 9278, 9386, 9306, 9368, 9198, 9152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=344032 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265994208 ingress_bytes=334856192 link_bytes=334856192 token_bytes=531644384 weight_movement_bytes=69206016 total_nvlink_bytes=600850400 weight_to_token_ratio=0.130174 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260036892e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=65082 dup_rows=23986 dup_rows_remote=21034 dedup_ratio=0.243998 router_tokens_per_expert=[715, 798, 784, 768, 778, 736, 729, 729, 771, 784, 768, 802, 774, 787, 823, 751, 769, 762, 771, 690, 752, 762, 779, 778, 787, 783, 742, 752, 738, 799, 731, 761, 821, 709, 755, 801, 765, 811, 729, 806, 765, 776, 754, 731, 794, 756, 774, 744, 723, 799, 733, 796, 752, 795, 808, 752, 747, 762, 802, 811, 820, 789, 756, 763] payload_rows_per_dst=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] dup_rows_per_dst=[2888, 3152, 2908, 2976, 3026, 2952, 2962, 3122] combine_rows_per_src=[9186, 9368, 9218, 9210, 9368, 9236, 9354, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=344464 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266920336 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533496208 weight_movement_bytes=34603008 total_nvlink_bytes=568099216 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12272712288 prefetch_bytes=657457152 reduce_bytes=726663168 weight_movement_bytes=1384120320 egress_bytes=6140312160 ingress_bytes=7516520448 link_bytes=7516520448 total_nvlink_bytes=13656832608 weight_to_token_ratio=0.112780 prefetch_share_of_ingress=0.087468 dedup_ratio=0.243181 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=21 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229741796e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64708 dup_rows=24214 dup_rows_remote=21216 dedup_ratio=0.246318 router_tokens_per_expert=[738, 772, 728, 779, 803, 785, 740, 763, 768, 797, 769, 755, 786, 754, 812, 778, 763, 783, 765, 739, 816, 762, 770, 792, 774, 749, 744, 770, 746, 808, 743, 761, 752, 785, 763, 769, 776, 812, 731, 766, 761, 756, 799, 744, 724, 775, 764, 742, 777, 734, 714, 771, 753, 796, 764, 764, 739, 822, 794, 752, 795, 747, 824, 775] payload_rows_per_dst=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] dup_rows_per_dst=[2970, 3074, 2998, 3052, 2996, 2992, 3028, 3104] combine_rows_per_src=[9246, 9364, 9382, 9138, 9312, 9138, 9118, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265043968 dispatch_meta_bytes=343696 combine_payload_bytes=265043968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265387664 ingress_bytes=368852992 link_bytes=368852992 token_bytes=530431632 weight_movement_bytes=103809024 total_nvlink_bytes=634240656 weight_to_token_ratio=0.195707 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387964114e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258788984 prefetch_bytes=657457152 reduce_bytes=761266176 weight_movement_bytes=1418723328 egress_bytes=6133352056 ingress_bytes=7544160256 link_bytes=7544160256 total_nvlink_bytes=13677512312 weight_to_token_ratio=0.115731 prefetch_share_of_ingress=0.087148 dedup_ratio=0.244121 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=22 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029348 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.234263813e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:56:40] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 796.5 | throughput per GPU (TFLOP/s/GPU): 160.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.156840E+00 | loss scale: 1.0 | grad norm: 1060.668 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64784 dup_rows=24124 dup_rows_remote=21072 dedup_ratio=0.245402 router_tokens_per_expert=[741, 782, 785, 733, 789, 704, 796, 749, 766, 759, 776, 745, 789, 764, 799, 800, 775, 801, 762, 704, 770, 765, 735, 734, 740, 778, 757, 762, 748, 802, 779, 803, 823, 749, 738, 734, 811, 769, 741, 803, 793, 747, 781, 807, 781, 723, 756, 750, 770, 791, 711, 810, 767, 806, 767, 746, 804, 801, 765, 744, 750, 761, 781, 780] payload_rows_per_dst=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] dup_rows_per_dst=[3052, 3100, 3152, 3030, 3038, 2890, 3004, 2858] combine_rows_per_src=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65166 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[760, 740, 801, 766, 803, 777, 749, 724, 782, 793, 763, 736, 764, 776, 724, 766, 737, 790, 744, 775, 722, 800, 761, 746, 772, 812, 773, 698, 776, 819, 723, 781, 763, 766, 769, 787, 778, 725, 753, 753, 761, 800, 799, 820, 757, 746, 761, 738, 771, 752, 738, 814, 822, 800, 771, 771, 776, 753, 812, 752, 777, 802, 752, 760] payload_rows_per_dst=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] dup_rows_per_dst=[2920, 3068, 2924, 2960, 2928, 3060, 3076, 3062] combine_rows_per_src=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] + [2026-08-26 18:08:46] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 793.3 | throughput per GPU (TFLOP/s/GPU): 160.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.156840E+00 | loss scale: 1.0 | grad norm: 1060.668 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65238 dup_rows=23834 dup_rows_remote=20844 dedup_ratio=0.242452 router_tokens_per_expert=[784, 757, 753, 777, 768, 769, 790, 811, 773, 778, 764, 746, 761, 751, 769, 779, 734, 801, 803, 737, 797, 768, 734, 737, 805, 766, 744, 712, 800, 742, 781, 822, 771, 762, 766, 830, 755, 761, 734, 745, 744, 754, 749, 778, 840, 771, 786, 707, 780, 799, 747, 741, 756, 753, 740, 779, 754, 732, 758, 818, 787, 804, 785, 753] payload_rows_per_dst=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] dup_rows_per_dst=[3044, 2936, 2990, 2952, 2928, 2916, 3008, 3060] combine_rows_per_src=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74018 payload_rows_remote=64758 dup_rows=24286 dup_rows_remote=21208 dedup_ratio=0.247050 router_tokens_per_expert=[773, 774, 726, 797, 741, 784, 766, 796, 808, 802, 754, 769, 756, 784, 777, 768, 763, 751, 799, 793, 735, 762, 742, 786, 797, 761, 793, 781, 742, 803, 756, 736, 823, 784, 747, 764, 754, 709, 763, 734, 775, 775, 760, 775, 797, 804, 745, 721, 797, 749, 768, 764, 757, 758, 757, 762, 799, 711, 805, 770, 715, 766, 819, 750] payload_rows_per_dst=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] dup_rows_per_dst=[3046, 3070, 3032, 3078, 2944, 3072, 3008, 3036] combine_rows_per_src=[9268, 9366, 9230, 9260, 9212, 9232, 9216, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64932 dup_rows=24156 dup_rows_remote=21136 dedup_ratio=0.245728 router_tokens_per_expert=[773, 701, 738, 745, 795, 748, 783, 820, 809, 729, 727, 772, 811, 782, 796, 766, 775, 774, 717, 724, 762, 772, 770, 734, 790, 777, 751, 832, 719, 792, 791, 766, 788, 759, 773, 736, 774, 771, 761, 756, 776, 805, 760, 756, 754, 801, 780, 766, 800, 809, 756, 758, 740, 771, 821, 743, 803, 776, 745, 787, 771, 741, 768, 706] payload_rows_per_dst=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] dup_rows_per_dst=[2984, 3038, 2910, 3064, 3020, 3134, 3054, 2952] combine_rows_per_src=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65304 dup_rows=23816 dup_rows_remote=20818 dedup_ratio=0.242269 router_tokens_per_expert=[787, 814, 771, 737, 763, 749, 755, 772, 706, 783, 793, 725, 780, 743, 786, 742, 779, 740, 811, 808, 731, 813, 770, 770, 743, 783, 794, 779, 741, 753, 762, 759, 808, 742, 749, 744, 807, 794, 729, 758, 808, 736, 797, 753, 739, 763, 755, 792, 766, 829, 815, 724, 746, 693, 789, 729, 768, 762, 761, 794, 813, 798, 744, 805] payload_rows_per_dst=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] dup_rows_per_dst=[3054, 2894, 2986, 2928, 2932, 2962, 2998, 3062] combine_rows_per_src=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65238 dup_rows=23846 dup_rows_remote=20814 dedup_ratio=0.242574 router_tokens_per_expert=[722, 745, 759, 742, 789, 724, 820, 787, 778, 800, 737, 778, 762, 794, 770, 748, 757, 799, 770, 753, 786, 754, 792, 759, 726, 762, 792, 803, 801, 768, 777, 779, 739, 747, 751, 774, 808, 733, 752, 753, 799, 809, 757, 778, 786, 780, 705, 712, 773, 772, 768, 749, 787, 785, 763, 776, 736, 769, 772, 745, 814, 751, 782, 794] payload_rows_per_dst=[9258, 9372, 9330, 9478, 9168, 9220, 9284, 9348] dup_rows_per_dst=[2918, 2962, 3010, 2938, 2946, 3032, 3062, 2978] combine_rows_per_src=[9258, 9372, 9330, 9478, 9168, 9220, 9284, 9348] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65202 dup_rows=23816 dup_rows_remote=20730 dedup_ratio=0.242269 router_tokens_per_expert=[783, 740, 755, 838, 722, 731, 780, 754, 802, 720, 766, 737, 801, 769, 790, 709, 804, 768, 782, 797, 709, 791, 761, 767, 794, 771, 764, 792, 780, 751, 759, 832, 805, 765, 802, 727, 775, 758, 754, 741, 799, 791, 750, 782, 771, 746, 737, 768, 748, 763, 788, 739, 814, 760, 739, 741, 774, 776, 777, 755, 738, 788, 787, 775] payload_rows_per_dst=[9266, 9346, 9310, 9046, 9530, 9456, 9248, 9286] dup_rows_per_dst=[2940, 2842, 3048, 2760, 3234, 3002, 2904, 3086] combine_rows_per_src=[9266, 9346, 9310, 9046, 9530, 9456, 9248, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64784 dup_rows=24124 dup_rows_remote=21072 dedup_ratio=0.245402 router_tokens_per_expert=[741, 782, 785, 733, 789, 704, 796, 749, 766, 759, 776, 745, 789, 764, 799, 800, 775, 801, 762, 704, 770, 765, 735, 734, 740, 778, 757, 762, 748, 802, 779, 803, 823, 749, 738, 734, 811, 769, 741, 803, 793, 747, 781, 807, 781, 723, 756, 750, 770, 791, 711, 810, 767, 806, 767, 746, 804, 801, 765, 744, 750, 761, 781, 780] payload_rows_per_dst=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] dup_rows_per_dst=[3052, 3100, 3152, 3030, 3038, 2890, 3004, 2858] combine_rows_per_src=[9396, 9296, 9320, 9398, 9298, 9096, 9332, 9044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65166 dup_rows=23998 dup_rows_remote=20930 dedup_ratio=0.244120 router_tokens_per_expert=[760, 740, 801, 766, 803, 777, 749, 724, 782, 793, 763, 736, 764, 776, 724, 766, 737, 790, 744, 775, 722, 800, 761, 746, 772, 812, 773, 698, 776, 819, 723, 781, 763, 766, 769, 787, 778, 725, 753, 753, 761, 800, 799, 820, 757, 746, 761, 738, 771, 752, 738, 814, 822, 800, 771, 771, 776, 753, 812, 752, 777, 802, 752, 760] payload_rows_per_dst=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] dup_rows_per_dst=[2920, 3068, 2924, 2960, 2928, 3060, 3076, 3062] combine_rows_per_src=[9320, 9140, 9226, 9348, 9260, 9304, 9402, 9306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64932 dup_rows=24156 dup_rows_remote=21136 dedup_ratio=0.245728 router_tokens_per_expert=[773, 701, 738, 745, 795, 748, 783, 820, 809, 729, 727, 772, 811, 782, 796, 766, 775, 774, 717, 724, 762, 772, 770, 734, 790, 777, 751, 832, 719, 792, 791, 766, 788, 759, 773, 736, 774, 771, 761, 756, 776, 805, 760, 756, 754, 801, 780, 766, 800, 809, 756, 758, 740, 771, 821, 743, 803, 776, 745, 787, 771, 741, 768, 706] payload_rows_per_dst=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] dup_rows_per_dst=[2984, 3038, 2910, 3064, 3020, 3134, 3054, 2952] combine_rows_per_src=[9222, 9346, 9146, 9372, 9216, 9262, 9342, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344328 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559176 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774024 weight_movement_bytes=34603008 total_nvlink_bytes=569377032 weight_to_token_ratio=0.064706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=343864 combine_payload_bytes=265248768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265592632 ingress_bytes=369057792 link_bytes=369057792 token_bytes=530841400 weight_movement_bytes=103809024 total_nvlink_bytes=634650424 weight_to_token_ratio=0.195556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388734760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344208 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267559056 ingress_bytes=336420864 link_bytes=336420864 token_bytes=534773904 weight_movement_bytes=69206016 total_nvlink_bytes=603979920 weight_to_token_ratio=0.129412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343728 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411120 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478512 weight_movement_bytes=103809024 total_nvlink_bytes=638287536 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=343424 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265698688 ingress_bytes=299958272 link_bytes=299958272 token_bytes=531053952 weight_movement_bytes=34603008 total_nvlink_bytes=565656960 weight_to_token_ratio=0.065159 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128718829e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344384 combine_payload_bytes=266919936 prefetch_bytes=0 reduce_bytes=0 egress_bytes=267264320 ingress_bytes=266919936 link_bytes=267264320 token_bytes=534184256 weight_movement_bytes=0 total_nvlink_bytes=534184256 weight_to_token_ratio=0.000000 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.005694120e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=343864 combine_payload_bytes=265248768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265592632 ingress_bytes=369057792 link_bytes=369057792 token_bytes=530841400 weight_movement_bytes=103809024 total_nvlink_bytes=634650424 weight_to_token_ratio=0.195556 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388734760e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74488 payload_rows_remote=65304 dup_rows=23816 dup_rows_remote=20818 dedup_ratio=0.242269 router_tokens_per_expert=[787, 814, 771, 737, 763, 749, 755, 772, 706, 783, 793, 725, 780, 743, 786, 742, 779, 740, 811, 808, 731, 813, 770, 770, 743, 783, 794, 779, 741, 753, 762, 759, 808, 742, 749, 744, 807, 794, 729, 758, 808, 736, 797, 753, 739, 763, 755, 792, 766, 829, 815, 724, 746, 693, 789, 729, 768, 762, 761, 794, 813, 798, 744, 805] payload_rows_per_dst=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] dup_rows_per_dst=[3054, 2894, 2986, 2928, 2932, 2962, 2998, 3062] combine_rows_per_src=[9242, 9222, 9458, 9300, 9330, 9324, 9184, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265961472 dispatch_meta_bytes=344272 combine_payload_bytes=265961472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266305744 ingress_bytes=300564480 link_bytes=300564480 token_bytes=532267216 weight_movement_bytes=34603008 total_nvlink_bytes=566870224 weight_to_token_ratio=0.065011 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130999941e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65182 dup_rows=23968 dup_rows_remote=20976 dedup_ratio=0.243815 router_tokens_per_expert=[769, 789, 719, 766, 771, 810, 808, 751, 793, 778, 698, 760, 766, 751, 770, 781, 741, 764, 746, 769, 770, 804, 709, 770, 772, 824, 766, 776, 788, 763, 709, 770, 763, 742, 752, 796, 773, 724, 788, 774, 765, 779, 776, 799, 766, 769, 759, 751, 777, 737, 761, 776, 773, 774, 805, 775, 830, 788, 783, 763, 737, 743, 748, 785] payload_rows_per_dst=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] dup_rows_per_dst=[3002, 2924, 2992, 2990, 2908, 3078, 3004, 3070] combine_rows_per_src=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65132 dup_rows=23966 dup_rows_remote=20930 dedup_ratio=0.243795 router_tokens_per_expert=[796, 758, 801, 762, 735, 754, 790, 759, 765, 784, 763, 759, 790, 771, 793, 758, 788, 820, 791, 764, 755, 763, 786, 820, 797, 752, 788, 766, 758, 736, 758, 766, 774, 812, 777, 750, 709, 781, 782, 808, 754, 705, 723, 751, 801, 767, 795, 745, 760, 802, 766, 732, 757, 773, 745, 759, 798, 767, 725, 751, 750, 752, 769, 766] payload_rows_per_dst=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] dup_rows_per_dst=[3042, 3132, 3078, 3036, 2952, 2902, 2912, 2912] combine_rows_per_src=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65070 dup_rows=23978 dup_rows_remote=21018 dedup_ratio=0.243917 router_tokens_per_expert=[755, 780, 718, 767, 822, 795, 753, 692, 777, 722, 804, 801, 785, 741, 788, 773, 768, 801, 756, 770, 785, 733, 757, 731, 810, 765, 785, 783, 797, 777, 828, 802, 766, 757, 760, 756, 742, 754, 742, 753, 706, 758, 787, 781, 798, 738, 765, 775, 779, 772, 757, 797, 759, 730, 777, 754, 763, 747, 810, 762, 782, 766, 763, 775] payload_rows_per_dst=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] dup_rows_per_dst=[2862, 3010, 3004, 3216, 2906, 2960, 2962, 3058] combine_rows_per_src=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73982 payload_rows_remote=64688 dup_rows=24322 dup_rows_remote=21280 dedup_ratio=0.247416 router_tokens_per_expert=[807, 716, 789, 817, 782, 752, 721, 751, 774, 788, 779, 769, 764, 756, 765, 759, 767, 761, 776, 738, 802, 725, 791, 748, 777, 775, 747, 754, 750, 756, 806, 757, 758, 819, 736, 783, 762, 748, 778, 808, 791, 830, 830, 782, 768, 763, 767, 714, 746, 735, 781, 750, 806, 767, 757, 800, 777, 791, 757, 735, 782, 740, 747, 725] payload_rows_per_dst=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] dup_rows_per_dst=[2948, 2856, 2898, 2854, 3140, 3276, 3308, 3042] combine_rows_per_src=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267485184 dispatch_meta_bytes=344488 combine_payload_bytes=267485184 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267829672 ingress_bytes=302088192 link_bytes=302088192 token_bytes=535314856 weight_movement_bytes=34603008 total_nvlink_bytes=569917864 weight_to_token_ratio=0.064640 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136733546e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344208 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267559056 ingress_bytes=336420864 link_bytes=336420864 token_bytes=534773904 weight_movement_bytes=69206016 total_nvlink_bytes=603979920 weight_to_token_ratio=0.129412 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265924627e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267067392 dispatch_meta_bytes=343728 combine_payload_bytes=267067392 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267411120 ingress_bytes=370876416 link_bytes=370876416 token_bytes=534478512 weight_movement_bytes=103809024 total_nvlink_bytes=638287536 weight_to_token_ratio=0.194225 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395578095e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=65006 dup_rows=24178 dup_rows_remote=21130 dedup_ratio=0.245951 router_tokens_per_expert=[704, 773, 778, 781, 737, 780, 761, 763, 798, 752, 746, 784, 749, 785, 771, 753, 825, 792, 728, 750, 765, 793, 803, 747, 740, 754, 819, 752, 737, 767, 748, 772, 753, 759, 775, 741, 779, 756, 745, 798, 752, 770, 748, 741, 732, 784, 770, 756, 732, 768, 774, 770, 778, 780, 806, 810, 794, 772, 823, 779, 812, 733, 794, 761] payload_rows_per_dst=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] dup_rows_per_dst=[3048, 2984, 3032, 3070, 2932, 2928, 3048, 3136] combine_rows_per_src=[9120, 9292, 9360, 9182, 9206, 9178, 9388, 9400] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65066 dup_rows=23830 dup_rows_remote=20782 dedup_ratio=0.242411 router_tokens_per_expert=[771, 743, 772, 765, 743, 828, 731, 797, 781, 781, 750, 738, 829, 785, 788, 776, 787, 772, 767, 744, 789, 726, 784, 822, 803, 800, 763, 731, 719, 742, 722, 793, 786, 769, 790, 828, 748, 772, 767, 781, 792, 768, 789, 753, 787, 749, 751, 763, 760, 740, 749, 767, 796, 782, 704, 764, 766, 761, 775, 724, 763, 747, 814, 705] payload_rows_per_dst=[9266, 9408, 9340, 9238, 9410, 9338, 9186, 9288] dup_rows_per_dst=[3034, 3048, 3042, 2908, 3072, 2966, 2938, 2822] combine_rows_per_src=[9266, 9408, 9340, 9238, 9410, 9338, 9186, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65132 dup_rows=23966 dup_rows_remote=20930 dedup_ratio=0.243795 router_tokens_per_expert=[796, 758, 801, 762, 735, 754, 790, 759, 765, 784, 763, 759, 790, 771, 793, 758, 788, 820, 791, 764, 755, 763, 786, 820, 797, 752, 788, 766, 758, 736, 758, 766, 774, 812, 777, 750, 709, 781, 782, 808, 754, 705, 723, 751, 801, 767, 795, 745, 760, 802, 766, 732, 757, 773, 745, 759, 798, 767, 725, 751, 750, 752, 769, 766] payload_rows_per_dst=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] dup_rows_per_dst=[3042, 3132, 3078, 3036, 2952, 2902, 2912, 2912] combine_rows_per_src=[9268, 9234, 9496, 9206, 9434, 9180, 9276, 9244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65238 dup_rows=23834 dup_rows_remote=20844 dedup_ratio=0.242452 router_tokens_per_expert=[784, 757, 753, 777, 768, 769, 790, 811, 773, 778, 764, 746, 761, 751, 769, 779, 734, 801, 803, 737, 797, 768, 734, 737, 805, 766, 744, 712, 800, 742, 781, 822, 771, 762, 766, 830, 755, 761, 734, 745, 744, 754, 749, 778, 840, 771, 786, 707, 780, 799, 747, 741, 756, 753, 740, 779, 754, 732, 758, 818, 787, 804, 785, 753] payload_rows_per_dst=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] dup_rows_per_dst=[3044, 2936, 2990, 2952, 2928, 2916, 3008, 3060] combine_rows_per_src=[9374, 9306, 9232, 9392, 9320, 9342, 9182, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344632 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267330104 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315576 weight_movement_bytes=34603008 total_nvlink_bytes=568918584 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344248 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267124920 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905592 weight_movement_bytes=69206016 total_nvlink_bytes=603111608 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344352 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871072 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533397792 weight_movement_bytes=34603008 total_nvlink_bytes=568000800 weight_to_token_ratio=0.064873 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65062 dup_rows=23926 dup_rows_remote=21072 dedup_ratio=0.243388 router_tokens_per_expert=[808, 745, 774, 785, 795, 755, 788, 762, 813, 722, 799, 775, 782, 749, 786, 765, 750, 752, 751, 755, 749, 741, 761, 748, 779, 810, 765, 777, 730, 812, 779, 779, 787, 800, 739, 740, 779, 756, 768, 803, 758, 807, 788, 756, 766, 777, 749, 720, 749, 774, 780, 719, 749, 796, 765, 753, 732, 798, 750, 796, 788, 736, 732, 801] payload_rows_per_dst=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] dup_rows_per_dst=[3066, 3074, 2854, 3126, 3012, 3024, 2854, 2916] combine_rows_per_src=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64904 dup_rows=24088 dup_rows_remote=21146 dedup_ratio=0.245036 router_tokens_per_expert=[763, 824, 746, 810, 800, 767, 799, 772, 811, 802, 753, 736, 759, 772, 734, 743, 759, 764, 796, 807, 784, 739, 762, 778, 789, 762, 736, 746, 806, 760, 766, 781, 756, 743, 782, 773, 761, 758, 775, 779, 778, 754, 790, 792, 771, 712, 776, 755, 749, 724, 783, 763, 765, 748, 750, 784, 756, 745, 717, 775, 782, 770, 778, 782] payload_rows_per_dst=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] dup_rows_per_dst=[3276, 3014, 3012, 2990, 2942, 3012, 2928, 2914] combine_rows_per_src=[9286, 9206, 9366, 9302, 9312, 9244, 9204, 9296] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65070 dup_rows=23978 dup_rows_remote=21018 dedup_ratio=0.243917 router_tokens_per_expert=[755, 780, 718, 767, 822, 795, 753, 692, 777, 722, 804, 801, 785, 741, 788, 773, 768, 801, 756, 770, 785, 733, 757, 731, 810, 765, 785, 783, 797, 777, 828, 802, 766, 757, 760, 756, 742, 754, 742, 753, 706, 758, 787, 781, 798, 738, 765, 775, 779, 772, 757, 797, 759, 730, 777, 754, 763, 747, 810, 762, 782, 766, 763, 775] payload_rows_per_dst=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] dup_rows_per_dst=[2862, 3010, 3004, 3216, 2906, 2960, 2962, 3058] combine_rows_per_src=[9302, 9372, 9198, 9478, 9154, 9256, 9288, 9278] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73982 payload_rows_remote=64688 dup_rows=24322 dup_rows_remote=21280 dedup_ratio=0.247416 router_tokens_per_expert=[807, 716, 789, 817, 782, 752, 721, 751, 774, 788, 779, 769, 764, 756, 765, 759, 767, 761, 776, 738, 802, 725, 791, 748, 777, 775, 747, 754, 750, 756, 806, 757, 758, 819, 736, 783, 762, 748, 778, 808, 791, 830, 830, 782, 768, 763, 767, 714, 746, 735, 781, 750, 806, 767, 757, 800, 777, 791, 757, 735, 782, 740, 747, 725] payload_rows_per_dst=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] dup_rows_per_dst=[2948, 2856, 2898, 2854, 3140, 3276, 3308, 3042] combine_rows_per_src=[9062, 9172, 9318, 9026, 9276, 9494, 9340, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264962048 dispatch_meta_bytes=343872 combine_payload_bytes=264962048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265305920 ingress_bytes=299565056 link_bytes=299565056 token_bytes=530267968 weight_movement_bytes=34603008 total_nvlink_bytes=564870976 weight_to_token_ratio=0.065256 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127239189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=344544 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266609120 ingress_bytes=370073600 link_bytes=370073600 token_bytes=532873696 weight_movement_bytes=103809024 total_nvlink_bytes=636682720 weight_to_token_ratio=0.194810 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392557163e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266510336 dispatch_meta_bytes=343392 combine_payload_bytes=266510336 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266853728 ingress_bytes=301113344 link_bytes=301113344 token_bytes=533364064 weight_movement_bytes=34603008 total_nvlink_bytes=567967072 weight_to_token_ratio=0.064877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133065272e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344248 combine_payload_bytes=266780672 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267124920 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905592 weight_movement_bytes=69206016 total_nvlink_bytes=603111608 weight_to_token_ratio=0.129622 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=344536 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266838488 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533332440 weight_movement_bytes=34603008 total_nvlink_bytes=567935448 weight_to_token_ratio=0.064881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344200 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190984 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037768 weight_movement_bytes=69206016 total_nvlink_bytes=601243784 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266526720 dispatch_meta_bytes=344352 combine_payload_bytes=266526720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266871072 ingress_bytes=301129728 link_bytes=301129728 token_bytes=533397792 weight_movement_bytes=34603008 total_nvlink_bytes=568000800 weight_to_token_ratio=0.064873 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133126924e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=264962048 dispatch_meta_bytes=343872 combine_payload_bytes=264962048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265305920 ingress_bytes=299565056 link_bytes=299565056 token_bytes=530267968 weight_movement_bytes=34603008 total_nvlink_bytes=564870976 weight_to_token_ratio=0.065256 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127239189e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=344328 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267559176 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534774024 weight_movement_bytes=34603008 total_nvlink_bytes=569377032 weight_to_token_ratio=0.064706 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64614 dup_rows=24270 dup_rows_remote=21244 dedup_ratio=0.246887 router_tokens_per_expert=[753, 804, 757, 748, 775, 856, 730, 798, 776, 738, 704, 796, 703, 774, 733, 810, 739, 788, 785, 752, 778, 754, 794, 775, 765, 784, 798, 808, 742, 774, 761, 753, 725, 764, 774, 763, 754, 776, 733, 753, 806, 804, 735, 741, 767, 796, 774, 797, 784, 793, 749, 792, 775, 785, 817, 807, 752, 704, 814, 763, 746, 707, 760, 737] payload_rows_per_dst=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] dup_rows_per_dst=[3026, 3028, 3086, 3060, 3446, 3120, 2660, 2844] combine_rows_per_src=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65222 dup_rows=23920 dup_rows_remote=20980 dedup_ratio=0.243327 router_tokens_per_expert=[766, 750, 781, 760, 747, 768, 772, 739, 758, 782, 794, 744, 758, 750, 767, 799, 767, 754, 748, 775, 772, 747, 741, 747, 867, 772, 803, 751, 775, 759, 756, 803, 769, 753, 752, 824, 756, 783, 789, 757, 756, 759, 781, 756, 721, 833, 753, 716, 758, 774, 770, 759, 793, 808, 781, 726, 778, 781, 789, 751, 785, 749, 742, 778] payload_rows_per_dst=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] dup_rows_per_dst=[2962, 2928, 2940, 3060, 3086, 2996, 3014, 2934] combine_rows_per_src=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64920 dup_rows=24058 dup_rows_remote=21056 dedup_ratio=0.244731 router_tokens_per_expert=[791, 737, 770, 795, 761, 767, 771, 808, 789, 790, 766, 748, 783, 742, 769, 818, 797, 755, 769, 802, 766, 762, 806, 743, 735, 748, 724, 750, 750, 738, 748, 790, 744, 757, 731, 727, 756, 763, 743, 777, 751, 777, 768, 733, 816, 757, 754, 808, 824, 754, 769, 760, 760, 764, 747, 762, 778, 798, 783, 781, 769, 789, 777, 787] payload_rows_per_dst=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] dup_rows_per_dst=[3088, 3050, 3108, 2862, 2890, 3002, 2940, 3118] combine_rows_per_src=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=64750 dup_rows=24158 dup_rows_remote=21154 dedup_ratio=0.245748 router_tokens_per_expert=[784, 761, 730, 767, 799, 693, 791, 745, 761, 778, 754, 759, 780, 742, 765, 756, 852, 733, 792, 801, 721, 732, 766, 791, 763, 786, 757, 746, 793, 787, 778, 790, 771, 775, 728, 786, 738, 759, 758, 768, 796, 758, 741, 755, 757, 714, 778, 759, 812, 754, 744, 762, 775, 784, 773, 788, 743, 815, 745, 781, 808, 837, 768, 769] payload_rows_per_dst=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] dup_rows_per_dst=[3006, 2950, 3080, 3004, 2972, 2900, 3136, 3110] combine_rows_per_src=[9134, 9240, 9296, 9396, 9194, 9216, 9248, 9422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=65062 dup_rows=23926 dup_rows_remote=21072 dedup_ratio=0.243388 router_tokens_per_expert=[808, 745, 774, 785, 795, 755, 788, 762, 813, 722, 799, 775, 782, 749, 786, 765, 750, 752, 751, 755, 749, 741, 761, 748, 779, 810, 765, 777, 730, 812, 779, 779, 787, 800, 739, 740, 779, 756, 768, 803, 758, 807, 788, 756, 766, 777, 749, 720, 749, 774, 780, 719, 749, 796, 765, 753, 732, 798, 750, 796, 788, 736, 732, 801] payload_rows_per_dst=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] dup_rows_per_dst=[3066, 3074, 2854, 3126, 3012, 3024, 2854, 2916] combine_rows_per_src=[9358, 9308, 9160, 9336, 9332, 9218, 9316, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64904 dup_rows=24192 dup_rows_remote=21128 dedup_ratio=0.246094 router_tokens_per_expert=[758, 774, 769, 765, 764, 757, 788, 792, 742, 768, 771, 778, 778, 785, 782, 755, 786, 753, 798, 766, 736, 857, 691, 775, 767, 817, 779, 753, 782, 786, 722, 785, 744, 772, 782, 778, 729, 769, 732, 778, 804, 783, 784, 762, 722, 757, 774, 729, 769, 767, 743, 779, 760, 748, 807, 771, 744, 794, 773, 784, 754, 779, 766, 736] payload_rows_per_dst=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] dup_rows_per_dst=[3018, 3006, 2944, 3094, 2996, 2780, 3290, 3064] combine_rows_per_src=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64614 dup_rows=24270 dup_rows_remote=21244 dedup_ratio=0.246887 router_tokens_per_expert=[753, 804, 757, 748, 775, 856, 730, 798, 776, 738, 704, 796, 703, 774, 733, 810, 739, 788, 785, 752, 778, 754, 794, 775, 765, 784, 798, 808, 742, 774, 761, 753, 725, 764, 774, 763, 754, 776, 733, 753, 806, 804, 735, 741, 767, 796, 774, 797, 784, 793, 749, 792, 775, 785, 817, 807, 752, 704, 814, 763, 746, 707, 760, 737] payload_rows_per_dst=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] dup_rows_per_dst=[3026, 3028, 3086, 3060, 3446, 3120, 2660, 2844] combine_rows_per_src=[9420, 9136, 9244, 9306, 9694, 9320, 8888, 9026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344808 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267494120 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534643432 weight_movement_bytes=69206016 total_nvlink_bytes=603849448 weight_to_token_ratio=0.129443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343904 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256224 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168544 weight_movement_bytes=69206016 total_nvlink_bytes=601374560 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64966 dup_rows=23950 dup_rows_remote=20988 dedup_ratio=0.243632 router_tokens_per_expert=[786, 752, 760, 747, 830, 749, 728, 767, 746, 821, 811, 803, 776, 757, 732, 729, 804, 714, 779, 748, 804, 772, 798, 784, 782, 761, 775, 751, 786, 788, 777, 789, 791, 754, 776, 745, 731, 793, 797, 737, 809, 751, 755, 827, 784, 745, 817, 718, 743, 755, 775, 741, 736, 738, 703, 826, 768, 734, 780, 730, 780, 747, 791, 769] payload_rows_per_dst=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] dup_rows_per_dst=[2978, 2962, 3086, 3058, 2942, 3056, 2852, 3016] combine_rows_per_src=[9260, 9388, 9320, 9360, 9306, 9356, 9182, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343616 combine_payload_bytes=265216000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265559616 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530775616 weight_movement_bytes=34603008 total_nvlink_bytes=565378624 weight_to_token_ratio=0.065193 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266493952 dispatch_meta_bytes=344536 combine_payload_bytes=266493952 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266838488 ingress_bytes=301096960 link_bytes=301096960 token_bytes=533332440 weight_movement_bytes=34603008 total_nvlink_bytes=567935448 weight_to_token_ratio=0.064881 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133003620e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74014 payload_rows_remote=64658 dup_rows=24290 dup_rows_remote=21308 dedup_ratio=0.247091 router_tokens_per_expert=[795, 748, 770, 727, 787, 741, 825, 774, 782, 805, 718, 764, 816, 787, 775, 756, 721, 781, 808, 767, 753, 755, 777, 727, 745, 740, 769, 758, 726, 772, 798, 749, 738, 754, 741, 796, 756, 785, 748, 760, 774, 829, 821, 759, 783, 747, 773, 796, 785, 719, 759, 796, 743, 765, 791, 811, 740, 786, 742, 776, 769, 792, 751, 751] payload_rows_per_dst=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] dup_rows_per_dst=[3054, 3084, 2916, 3068, 2976, 3146, 2982, 3064] combine_rows_per_src=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=64920 dup_rows=24058 dup_rows_remote=21056 dedup_ratio=0.244731 router_tokens_per_expert=[791, 737, 770, 795, 761, 767, 771, 808, 789, 790, 766, 748, 783, 742, 769, 818, 797, 755, 769, 802, 766, 762, 806, 743, 735, 748, 724, 750, 750, 738, 748, 790, 744, 757, 731, 727, 756, 763, 743, 777, 751, 777, 768, 733, 816, 757, 754, 808, 824, 754, 769, 760, 760, 764, 747, 762, 778, 798, 783, 781, 769, 789, 777, 787] payload_rows_per_dst=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] dup_rows_per_dst=[3088, 3050, 3108, 2862, 2890, 3002, 2940, 3118] combine_rows_per_src=[9312, 9360, 9292, 9104, 9106, 9326, 9340, 9406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64904 dup_rows=24192 dup_rows_remote=21128 dedup_ratio=0.246094 router_tokens_per_expert=[758, 774, 769, 765, 764, 757, 788, 792, 742, 768, 771, 778, 778, 785, 782, 755, 786, 753, 798, 766, 736, 857, 691, 775, 767, 817, 779, 753, 782, 786, 722, 785, 744, 772, 782, 778, 729, 769, 732, 778, 804, 783, 784, 762, 722, 757, 774, 729, 769, 767, 743, 779, 760, 748, 807, 771, 744, 794, 773, 784, 754, 779, 766, 736] payload_rows_per_dst=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] dup_rows_per_dst=[3018, 3006, 2944, 3094, 2996, 2780, 3290, 3064] combine_rows_per_src=[9304, 9112, 9224, 9288, 9172, 9148, 9656, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74422 payload_rows_remote=65244 dup_rows=23882 dup_rows_remote=20996 dedup_ratio=0.242940 router_tokens_per_expert=[782, 744, 750, 831, 773, 754, 742, 753, 775, 739, 774, 788, 735, 797, 800, 773, 756, 799, 734, 791, 804, 745, 763, 780, 762, 746, 760, 710, 784, 795, 731, 755, 784, 783, 738, 744, 776, 715, 731, 761, 767, 802, 813, 787, 743, 765, 776, 730, 802, 774, 758, 745, 808, 770, 783, 778, 734, 786, 790, 775, 817, 726, 794, 772] payload_rows_per_dst=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] dup_rows_per_dst=[2956, 3010, 2956, 2878, 2886, 2984, 3142, 3070] combine_rows_per_src=[9302, 9352, 9388, 9208, 9178, 9382, 9294, 9318] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344128 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190912 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037696 weight_movement_bytes=69206016 total_nvlink_bytes=601243712 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264658944 dispatch_meta_bytes=343432 combine_payload_bytes=264658944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265002376 ingress_bytes=333864960 link_bytes=333864960 token_bytes=529661320 weight_movement_bytes=69206016 total_nvlink_bytes=598867336 weight_to_token_ratio=0.130661 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256306966e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65182 dup_rows=23968 dup_rows_remote=20976 dedup_ratio=0.243815 router_tokens_per_expert=[769, 789, 719, 766, 771, 810, 808, 751, 793, 778, 698, 760, 766, 751, 770, 781, 741, 764, 746, 769, 770, 804, 709, 770, 772, 824, 766, 776, 788, 763, 709, 770, 763, 742, 752, 796, 773, 724, 788, 774, 765, 779, 776, 799, 766, 769, 759, 751, 777, 737, 761, 776, 773, 774, 805, 775, 830, 788, 783, 763, 737, 743, 748, 785] payload_rows_per_dst=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] dup_rows_per_dst=[3002, 2924, 2992, 2990, 2908, 3078, 3004, 3070] combine_rows_per_src=[9364, 9270, 9154, 9346, 9316, 9250, 9352, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265216000 dispatch_meta_bytes=343616 combine_payload_bytes=265216000 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=265559616 ingress_bytes=299819008 link_bytes=299819008 token_bytes=530775616 weight_movement_bytes=34603008 total_nvlink_bytes=565378624 weight_to_token_ratio=0.065193 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128194790e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=343816 combine_payload_bytes=266100736 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266444552 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545288 weight_movement_bytes=69206016 total_nvlink_bytes=601751304 weight_to_token_ratio=0.129953 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=343864 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265183032 ingress_bytes=334045184 link_bytes=334045184 token_bytes=530022200 weight_movement_bytes=69206016 total_nvlink_bytes=599228216 weight_to_token_ratio=0.130572 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256985135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265912320 dispatch_meta_bytes=343904 combine_payload_bytes=265912320 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266256224 ingress_bytes=335118336 link_bytes=335118336 token_bytes=532168544 weight_movement_bytes=69206016 total_nvlink_bytes=601374560 weight_to_token_ratio=0.130045 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261023319e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265846784 dispatch_meta_bytes=344128 combine_payload_bytes=265846784 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266190912 ingress_bytes=335052800 link_bytes=335052800 token_bytes=532037696 weight_movement_bytes=69206016 total_nvlink_bytes=601243712 weight_to_token_ratio=0.130077 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260776712e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267239424 dispatch_meta_bytes=344960 combine_payload_bytes=267239424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267584384 ingress_bytes=301842432 link_bytes=301842432 token_bytes=534823808 weight_movement_bytes=34603008 total_nvlink_bytes=569426816 weight_to_token_ratio=0.064700 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135808771e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344632 combine_payload_bytes=266985472 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=267330104 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315576 weight_movement_bytes=34603008 total_nvlink_bytes=568918584 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65026 dup_rows=23844 dup_rows_remote=20834 dedup_ratio=0.242554 router_tokens_per_expert=[750, 784, 774, 733, 787, 746, 820, 800, 745, 746, 769, 750, 713, 820, 775, 764, 774, 799, 766, 693, 781, 780, 761, 804, 749, 802, 725, 771, 796, 780, 808, 734, 810, 733, 777, 794, 769, 811, 780, 769, 789, 751, 742, 741, 730, 745, 778, 785, 770, 772, 763, 717, 773, 738, 798, 783, 762, 750, 756, 749, 783, 783, 763, 789] payload_rows_per_dst=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] dup_rows_per_dst=[3010, 3212, 3026, 2788, 2946, 2830, 3054, 2978] combine_rows_per_src=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65074 dup_rows=23918 dup_rows_remote=20782 dedup_ratio=0.243306 router_tokens_per_expert=[743, 768, 821, 747, 762, 744, 734, 737, 798, 770, 769, 785, 770, 760, 788, 797, 745, 799, 786, 760, 787, 757, 757, 833, 744, 787, 774, 735, 820, 805, 811, 819, 738, 779, 739, 744, 735, 771, 761, 763, 773, 736, 683, 767, 822, 773, 827, 716, 709, 788, 794, 742, 804, 788, 774, 736, 769, 722, 753, 743, 738, 761, 762, 830] payload_rows_per_dst=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] dup_rows_per_dst=[2906, 3040, 3136, 3116, 2922, 2938, 2946, 2914] combine_rows_per_src=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65032 dup_rows=24010 dup_rows_remote=20968 dedup_ratio=0.244242 router_tokens_per_expert=[776, 718, 729, 761, 769, 760, 758, 733, 735, 792, 750, 763, 794, 752, 744, 739, 778, 799, 771, 773, 747, 797, 776, 796, 783, 719, 876, 791, 749, 752, 752, 765, 777, 782, 740, 775, 835, 787, 734, 769, 749, 775, 782, 754, 738, 754, 792, 808, 780, 793, 742, 768, 747, 763, 810, 736, 744, 757, 762, 751, 761, 820, 776, 794] payload_rows_per_dst=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] dup_rows_per_dst=[2906, 2926, 2988, 2960, 3068, 3042, 3082, 3038] combine_rows_per_src=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74072 payload_rows_remote=64610 dup_rows=24232 dup_rows_remote=21172 dedup_ratio=0.246501 router_tokens_per_expert=[790, 734, 789, 776, 783, 742, 729, 749, 765, 769, 747, 751, 765, 774, 805, 768, 757, 800, 748, 782, 805, 784, 728, 738, 829, 798, 794, 793, 748, 816, 739, 744, 768, 781, 789, 792, 748, 774, 795, 761, 757, 759, 728, 759, 729, 817, 750, 817, 778, 775, 744, 765, 819, 680, 764, 787, 760, 768, 781, 730, 746, 725, 765, 802] payload_rows_per_dst=[9178, 9224, 9320, 9462, 9284, 9246, 9220, 9138] dup_rows_per_dst=[3006, 3064, 2964, 3060, 3132, 2986, 3004, 3016] combine_rows_per_src=[9178, 9224, 9320, 9462, 9284, 9246, 9220, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65038 dup_rows=23892 dup_rows_remote=20926 dedup_ratio=0.243042 router_tokens_per_expert=[771, 817, 766, 788, 800, 779, 763, 745, 794, 825, 741, 737, 720, 798, 760, 795, 794, 729, 785, 714, 781, 760, 726, 753, 769, 712, 799, 758, 790, 727, 813, 772, 754, 802, 818, 776, 696, 760, 758, 786, 746, 779, 802, 798, 730, 760, 772, 745, 738, 755, 751, 788, 809, 770, 758, 753, 782, 723, 764, 769, 795, 761, 774, 799] payload_rows_per_dst=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] dup_rows_per_dst=[3140, 2966, 2946, 2920, 2954, 3020, 2932, 3014] combine_rows_per_src=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65024 dup_rows=23956 dup_rows_remote=20920 dedup_ratio=0.243693 router_tokens_per_expert=[771, 771, 767, 729, 770, 797, 757, 783, 807, 720, 778, 761, 755, 739, 736, 837, 773, 799, 734, 743, 751, 735, 784, 753, 756, 777, 770, 742, 778, 790, 759, 816, 749, 768, 759, 746, 808, 789, 764, 759, 715, 752, 770, 792, 849, 797, 756, 737, 736, 726, 783, 795, 746, 799, 828, 767, 798, 783, 770, 768, 754, 794, 709, 748] payload_rows_per_dst=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] dup_rows_per_dst=[3022, 2994, 2928, 3068, 2976, 2952, 3036, 2980] combine_rows_per_src=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64874 dup_rows=24052 dup_rows_remote=20898 dedup_ratio=0.244670 router_tokens_per_expert=[744, 796, 761, 793, 751, 754, 807, 766, 708, 744, 750, 736, 810, 755, 763, 769, 772, 757, 808, 775, 726, 746, 777, 773, 784, 763, 826, 788, 736, 768, 769, 774, 755, 736, 772, 798, 798, 714, 726, 828, 785, 748, 803, 739, 788, 749, 720, 760, 768, 803, 789, 768, 775, 776, 746, 788, 773, 754, 780, 746, 802, 790, 759, 767] payload_rows_per_dst=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] dup_rows_per_dst=[3020, 2884, 2994, 2794, 2970, 3150, 3086, 3154] combine_rows_per_src=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74014 payload_rows_remote=64658 dup_rows=24290 dup_rows_remote=21308 dedup_ratio=0.247091 router_tokens_per_expert=[795, 748, 770, 727, 787, 741, 825, 774, 782, 805, 718, 764, 816, 787, 775, 756, 721, 781, 808, 767, 753, 755, 777, 727, 745, 740, 769, 758, 726, 772, 798, 749, 738, 754, 741, 796, 756, 785, 748, 760, 774, 829, 821, 759, 783, 747, 773, 796, 785, 719, 759, 796, 743, 765, 791, 811, 740, 786, 742, 776, 769, 792, 751, 751] payload_rows_per_dst=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] dup_rows_per_dst=[3054, 3084, 2916, 3068, 2976, 3146, 2982, 3064] combine_rows_per_src=[9280, 9322, 9262, 9046, 9180, 9418, 9356, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343424 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266886528 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533429632 weight_movement_bytes=34603008 total_nvlink_bytes=568032640 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65032 dup_rows=24010 dup_rows_remote=20968 dedup_ratio=0.244242 router_tokens_per_expert=[776, 718, 729, 761, 769, 760, 758, 733, 735, 792, 750, 763, 794, 752, 744, 739, 778, 799, 771, 773, 747, 797, 776, 796, 783, 719, 876, 791, 749, 752, 752, 765, 777, 782, 740, 775, 835, 787, 734, 769, 749, 775, 782, 754, 738, 754, 792, 808, 780, 793, 742, 768, 747, 763, 810, 736, 744, 757, 762, 751, 761, 820, 776, 794] payload_rows_per_dst=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] dup_rows_per_dst=[2906, 2926, 2988, 2960, 3068, 3042, 3082, 3038] combine_rows_per_src=[9102, 9212, 9486, 9414, 9330, 9262, 9196, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343440 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689936 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036432 weight_movement_bytes=34603008 total_nvlink_bytes=567639440 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64874 dup_rows=24052 dup_rows_remote=20898 dedup_ratio=0.244670 router_tokens_per_expert=[744, 796, 761, 793, 751, 754, 807, 766, 708, 744, 750, 736, 810, 755, 763, 769, 772, 757, 808, 775, 726, 746, 777, 773, 784, 763, 826, 788, 736, 768, 769, 774, 755, 736, 772, 798, 798, 714, 726, 828, 785, 748, 803, 739, 788, 749, 720, 760, 768, 803, 789, 768, 775, 776, 746, 788, 773, 754, 780, 746, 802, 790, 759, 767] payload_rows_per_dst=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] dup_rows_per_dst=[3020, 2884, 2994, 2794, 2970, 3150, 3086, 3154] combine_rows_per_src=[9324, 9186, 9274, 9026, 9284, 9392, 9388, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344000 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715072 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086144 weight_movement_bytes=34603008 total_nvlink_bytes=567689152 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65026 dup_rows=23844 dup_rows_remote=20834 dedup_ratio=0.242554 router_tokens_per_expert=[750, 784, 774, 733, 787, 746, 820, 800, 745, 746, 769, 750, 713, 820, 775, 764, 774, 799, 766, 693, 781, 780, 761, 804, 749, 802, 725, 771, 796, 780, 808, 734, 810, 733, 777, 794, 769, 811, 780, 769, 789, 751, 742, 741, 730, 745, 778, 785, 770, 772, 763, 717, 773, 738, 798, 783, 762, 750, 756, 749, 783, 783, 763, 789] payload_rows_per_dst=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] dup_rows_per_dst=[3010, 3212, 3026, 2788, 2946, 2830, 3054, 2978] combine_rows_per_src=[9434, 9356, 9360, 9150, 9402, 9292, 9174, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264642560 dispatch_meta_bytes=343128 combine_payload_bytes=264642560 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=264985688 ingress_bytes=437657600 link_bytes=437657600 token_bytes=529628248 weight_movement_bytes=173015040 total_nvlink_bytes=702643288 weight_to_token_ratio=0.326673 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.646870314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74412 payload_rows_remote=65038 dup_rows=23892 dup_rows_remote=20926 dedup_ratio=0.243042 router_tokens_per_expert=[771, 817, 766, 788, 800, 779, 763, 745, 794, 825, 741, 737, 720, 798, 760, 795, 794, 729, 785, 714, 781, 760, 726, 753, 769, 712, 799, 758, 790, 727, 813, 772, 754, 802, 818, 776, 696, 760, 758, 786, 746, 779, 802, 798, 730, 760, 772, 745, 738, 755, 751, 788, 809, 770, 758, 753, 782, 723, 764, 769, 795, 761, 774, 799] payload_rows_per_dst=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] dup_rows_per_dst=[3140, 2966, 2946, 2920, 2954, 3020, 2932, 3014] combine_rows_per_src=[9318, 9374, 9138, 9360, 9346, 9244, 9312, 9320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65108 dup_rows=23946 dup_rows_remote=20952 dedup_ratio=0.243591 router_tokens_per_expert=[769, 804, 751, 778, 780, 818, 752, 780, 746, 715, 760, 765, 774, 780, 740, 800, 772, 774, 759, 764, 748, 747, 767, 777, 805, 719, 787, 770, 764, 768, 783, 806, 778, 751, 783, 781, 774, 754, 782, 719, 768, 753, 738, 791, 782, 760, 766, 797, 769, 759, 763, 747, 791, 743, 765, 776, 791, 775, 763, 818, 737, 735, 759, 762] payload_rows_per_dst=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] dup_rows_per_dst=[3130, 3008, 2916, 3026, 2994, 2894, 2978, 3000] combine_rows_per_src=[9334, 9152, 9300, 9378, 9250, 9416, 9248, 9280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=264839168 dispatch_meta_bytes=343864 combine_payload_bytes=264839168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265183032 ingress_bytes=334045184 link_bytes=334045184 token_bytes=530022200 weight_movement_bytes=69206016 total_nvlink_bytes=599228216 weight_to_token_ratio=0.130572 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256985135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343088 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266066992 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531790896 weight_movement_bytes=34603008 total_nvlink_bytes=566393904 weight_to_token_ratio=0.065069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264642560 dispatch_meta_bytes=343128 combine_payload_bytes=264642560 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=264985688 ingress_bytes=437657600 link_bytes=437657600 token_bytes=529628248 weight_movement_bytes=173015040 total_nvlink_bytes=702643288 weight_to_token_ratio=0.326673 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.646870314e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65222 dup_rows=23920 dup_rows_remote=20980 dedup_ratio=0.243327 router_tokens_per_expert=[766, 750, 781, 760, 747, 768, 772, 739, 758, 782, 794, 744, 758, 750, 767, 799, 767, 754, 748, 775, 772, 747, 741, 747, 867, 772, 803, 751, 775, 759, 756, 803, 769, 753, 752, 824, 756, 783, 789, 757, 756, 759, 781, 756, 721, 833, 753, 716, 758, 774, 770, 759, 793, 808, 781, 726, 778, 781, 789, 751, 785, 749, 742, 778] payload_rows_per_dst=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] dup_rows_per_dst=[2962, 2928, 2940, 3060, 3086, 2996, 3014, 2934] combine_rows_per_src=[9204, 9376, 9162, 9512, 9280, 9154, 9324, 9372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343440 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266689936 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036432 weight_movement_bytes=34603008 total_nvlink_bytes=567639440 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=343856 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266739504 ingress_bytes=300998656 link_bytes=300998656 token_bytes=533135152 weight_movement_bytes=34603008 total_nvlink_bytes=567738160 weight_to_token_ratio=0.064905 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132633710e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343776 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682080 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020384 weight_movement_bytes=34603008 total_nvlink_bytes=567623392 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265723904 dispatch_meta_bytes=343088 combine_payload_bytes=265723904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266066992 ingress_bytes=300326912 link_bytes=300326912 token_bytes=531790896 weight_movement_bytes=34603008 total_nvlink_bytes=566393904 weight_to_token_ratio=0.065069 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130105992e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344000 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715072 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086144 weight_movement_bytes=34603008 total_nvlink_bytes=567689152 weight_to_token_ratio=0.064911 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64850 dup_rows=24046 dup_rows_remote=20958 dedup_ratio=0.244609 router_tokens_per_expert=[805, 781, 754, 796, 768, 769, 754, 764, 767, 792, 768, 738, 752, 784, 773, 781, 787, 787, 828, 775, 788, 756, 737, 790, 817, 711, 794, 784, 785, 819, 766, 791, 734, 738, 750, 828, 709, 767, 710, 746, 797, 766, 773, 746, 718, 771, 735, 766, 791, 773, 765, 744, 779, 764, 742, 758, 802, 780, 772, 744, 740, 765, 753, 765] payload_rows_per_dst=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] dup_rows_per_dst=[3104, 2942, 3088, 3110, 2872, 2956, 2974, 3000] combine_rows_per_src=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65030 dup_rows=23984 dup_rows_remote=21092 dedup_ratio=0.243978 router_tokens_per_expert=[725, 763, 750, 768, 794, 759, 791, 735, 778, 733, 840, 777, 749, 740, 762, 768, 716, 796, 798, 758, 783, 750, 753, 760, 789, 758, 798, 781, 799, 763, 801, 757, 762, 740, 781, 778, 752, 771, 800, 781, 746, 746, 751, 737, 774, 733, 788, 816, 800, 781, 748, 812, 761, 772, 748, 761, 730, 791, 781, 753, 762, 744, 808, 752] payload_rows_per_dst=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] dup_rows_per_dst=[3006, 2974, 3036, 3132, 3072, 2892, 2958, 2914] combine_rows_per_src=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=344240 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267026608 ingress_bytes=370491392 link_bytes=370491392 token_bytes=533708976 weight_movement_bytes=103809024 total_nvlink_bytes=637518000 weight_to_token_ratio=0.194505 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394129281e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267149312 dispatch_meta_bytes=344808 combine_payload_bytes=267149312 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267494120 ingress_bytes=336355328 link_bytes=336355328 token_bytes=534643432 weight_movement_bytes=69206016 total_nvlink_bytes=603849448 weight_to_token_ratio=0.129443 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265678020e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64838 dup_rows=24076 dup_rows_remote=20970 dedup_ratio=0.244914 router_tokens_per_expert=[746, 755, 787, 787, 811, 826, 754, 771, 690, 760, 785, 802, 772, 785, 774, 772, 805, 800, 735, 795, 793, 759, 755, 765, 764, 799, 773, 740, 792, 777, 727, 750, 813, 759, 782, 747, 785, 742, 748, 734, 766, 752, 778, 780, 715, 745, 786, 761, 773, 742, 754, 742, 799, 775, 821, 774, 743, 771, 759, 790, 772, 725, 775, 738] payload_rows_per_dst=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] dup_rows_per_dst=[3106, 3048, 3054, 2990, 3166, 2970, 2852, 2890] combine_rows_per_src=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64984 dup_rows=24186 dup_rows_remote=21310 dedup_ratio=0.246033 router_tokens_per_expert=[788, 776, 756, 805, 807, 770, 729, 739, 770, 745, 757, 804, 738, 797, 808, 752, 764, 760, 737, 785, 789, 756, 806, 744, 763, 704, 738, 804, 745, 747, 724, 780, 794, 759, 722, 788, 780, 738, 791, 820, 772, 757, 723, 758, 764, 737, 751, 724, 808, 770, 766, 802, 732, 767, 732, 783, 780, 838, 766, 766, 787, 824, 773, 793] payload_rows_per_dst=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] dup_rows_per_dst=[3130, 2966, 3028, 2876, 3132, 2830, 3030, 3194] combine_rows_per_src=[9210, 9376, 9254, 9134, 9252, 9142, 9290, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343232 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265968832 ingress_bytes=369434624 link_bytes=369434624 token_bytes=531594432 weight_movement_bytes=103809024 total_nvlink_bytes=635403456 weight_to_token_ratio=0.195279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390152748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65262 dup_rows=23736 dup_rows_remote=20790 dedup_ratio=0.241455 router_tokens_per_expert=[774, 809, 759, 751, 755, 773, 724, 729, 789, 708, 773, 773, 827, 742, 769, 791, 757, 791, 752, 778, 769, 771, 803, 776, 769, 734, 721, 786, 777, 791, 780, 790, 775, 770, 744, 741, 751, 779, 769, 771, 777, 792, 759, 745, 776, 811, 752, 768, 726, 787, 745, 739, 807, 761, 762, 799, 724, 773, 753, 778, 797, 768, 741, 821] payload_rows_per_dst=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] dup_rows_per_dst=[2898, 3004, 2942, 2970, 2964, 3050, 2946, 2962] combine_rows_per_src=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64824 dup_rows=24196 dup_rows_remote=21104 dedup_ratio=0.246134 router_tokens_per_expert=[774, 737, 774, 753, 776, 794, 780, 802, 801, 773, 773, 783, 729, 807, 759, 763, 770, 781, 760, 744, 760, 758, 742, 730, 755, 716, 760, 750, 729, 756, 777, 807, 782, 678, 801, 763, 769, 766, 798, 753, 776, 741, 759, 788, 748, 739, 802, 808, 738, 776, 814, 772, 788, 775, 772, 790, 793, 754, 793, 767, 814, 768, 741, 753] payload_rows_per_dst=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] dup_rows_per_dst=[3072, 3092, 2906, 2962, 2902, 3106, 3048, 3108] combine_rows_per_src=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64882 dup_rows=24112 dup_rows_remote=21192 dedup_ratio=0.245280 router_tokens_per_expert=[805, 766, 733, 789, 741, 757, 778, 790, 761, 750, 749, 763, 813, 779, 769, 756, 756, 812, 755, 783, 696, 785, 778, 824, 748, 815, 785, 764, 819, 766, 768, 703, 753, 761, 773, 731, 770, 805, 749, 778, 763, 757, 753, 749, 794, 814, 762, 760, 780, 770, 736, 785, 788, 716, 787, 747, 790, 720, 749, 765, 758, 774, 751, 808] payload_rows_per_dst=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] dup_rows_per_dst=[2930, 2994, 2942, 2858, 3012, 3336, 3120, 2920] combine_rows_per_src=[9158, 9286, 9256, 9144, 9228, 9482, 9328, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=343232 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265919680 ingress_bytes=334782464 link_bytes=334782464 token_bytes=531496128 weight_movement_bytes=69206016 total_nvlink_bytes=600702144 weight_to_token_ratio=0.130210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259759460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65090 dup_rows=23902 dup_rows_remote=20920 dedup_ratio=0.243144 router_tokens_per_expert=[786, 821, 739, 763, 787, 815, 724, 715, 741, 794, 712, 685, 768, 734, 795, 765, 741, 801, 741, 738, 758, 776, 784, 770, 779, 810, 797, 805, 810, 745, 782, 785, 753, 788, 740, 777, 761, 768, 785, 775, 768, 759, 773, 763, 760, 763, 787, 797, 769, 775, 764, 795, 780, 786, 752, 743, 791, 781, 770, 751, 746, 720, 806, 740] payload_rows_per_dst=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] dup_rows_per_dst=[2992, 2814, 2972, 3196, 2982, 3010, 2992, 2944] combine_rows_per_src=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344488 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707368 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070248 weight_movement_bytes=34603008 total_nvlink_bytes=567673256 weight_to_token_ratio=0.064913 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74228 payload_rows_remote=64838 dup_rows=24076 dup_rows_remote=20970 dedup_ratio=0.244914 router_tokens_per_expert=[746, 755, 787, 787, 811, 826, 754, 771, 690, 760, 785, 802, 772, 785, 774, 772, 805, 800, 735, 795, 793, 759, 755, 765, 764, 799, 773, 740, 792, 777, 727, 750, 813, 759, 782, 747, 785, 742, 748, 734, 766, 752, 778, 780, 715, 745, 786, 761, 773, 742, 754, 742, 799, 775, 821, 774, 743, 771, 759, 790, 772, 725, 775, 738] payload_rows_per_dst=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] dup_rows_per_dst=[3106, 3048, 3054, 2990, 3166, 2970, 2852, 2890] combine_rows_per_src=[9390, 9322, 9344, 9254, 9506, 9196, 9056, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74348 payload_rows_remote=65024 dup_rows=23956 dup_rows_remote=20920 dedup_ratio=0.243693 router_tokens_per_expert=[771, 771, 767, 729, 770, 797, 757, 783, 807, 720, 778, 761, 755, 739, 736, 837, 773, 799, 734, 743, 751, 735, 784, 753, 756, 777, 770, 742, 778, 790, 759, 816, 749, 768, 759, 746, 808, 789, 764, 759, 715, 752, 770, 792, 849, 797, 756, 737, 736, 726, 783, 795, 746, 799, 828, 767, 798, 783, 770, 768, 754, 794, 709, 748] payload_rows_per_dst=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] dup_rows_per_dst=[3022, 2994, 2928, 3068, 2976, 2952, 3036, 2980] combine_rows_per_src=[9268, 9272, 9216, 9308, 9308, 9384, 9324, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=345176 combine_payload_bytes=266174464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266519640 ingress_bytes=335380480 link_bytes=335380480 token_bytes=532694104 weight_movement_bytes=69206016 total_nvlink_bytes=601900120 weight_to_token_ratio=0.129917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262009746e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74320 payload_rows_remote=65030 dup_rows=23984 dup_rows_remote=21092 dedup_ratio=0.243978 router_tokens_per_expert=[725, 763, 750, 768, 794, 759, 791, 735, 778, 733, 840, 777, 749, 740, 762, 768, 716, 796, 798, 758, 783, 750, 753, 760, 789, 758, 798, 781, 799, 763, 801, 757, 762, 740, 781, 778, 752, 771, 800, 781, 746, 746, 751, 737, 774, 733, 788, 816, 800, 781, 748, 812, 761, 772, 748, 761, 730, 791, 781, 753, 762, 744, 808, 752] payload_rows_per_dst=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] dup_rows_per_dst=[3006, 2974, 3036, 3132, 3072, 2892, 2958, 2914] combine_rows_per_src=[9164, 9320, 9192, 9360, 9258, 9290, 9408, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65074 dup_rows=23918 dup_rows_remote=20782 dedup_ratio=0.243306 router_tokens_per_expert=[743, 768, 821, 747, 762, 744, 734, 737, 798, 770, 769, 785, 770, 760, 788, 797, 745, 799, 786, 760, 787, 757, 757, 833, 744, 787, 774, 735, 820, 805, 811, 819, 738, 779, 739, 744, 735, 771, 761, 763, 773, 736, 683, 767, 822, 773, 827, 716, 709, 788, 794, 742, 804, 788, 774, 736, 769, 722, 753, 743, 738, 761, 762, 830] payload_rows_per_dst=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] dup_rows_per_dst=[2906, 3040, 3136, 3116, 2922, 2938, 2946, 2914] combine_rows_per_src=[9206, 9434, 9312, 9474, 9138, 9256, 9324, 9242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74108 payload_rows_remote=64824 dup_rows=24196 dup_rows_remote=21104 dedup_ratio=0.246134 router_tokens_per_expert=[774, 737, 774, 753, 776, 794, 780, 802, 801, 773, 773, 783, 729, 807, 759, 763, 770, 781, 760, 744, 760, 758, 742, 730, 755, 716, 760, 750, 729, 756, 777, 807, 782, 678, 801, 763, 769, 766, 798, 753, 776, 741, 759, 788, 748, 739, 802, 808, 738, 776, 814, 772, 788, 775, 772, 790, 793, 754, 793, 767, 814, 768, 741, 753] payload_rows_per_dst=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] dup_rows_per_dst=[3072, 3092, 2906, 2962, 2902, 3106, 3048, 3108] combine_rows_per_src=[9308, 9284, 9184, 9138, 9318, 9216, 9402, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344296 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266100968 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857640 weight_movement_bytes=69206016 total_nvlink_bytes=601063656 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65090 dup_rows=23902 dup_rows_remote=20920 dedup_ratio=0.243144 router_tokens_per_expert=[786, 821, 739, 763, 787, 815, 724, 715, 741, 794, 712, 685, 768, 734, 795, 765, 741, 801, 741, 738, 758, 776, 784, 770, 779, 810, 797, 805, 810, 745, 782, 785, 753, 788, 740, 777, 761, 768, 785, 775, 768, 759, 773, 763, 760, 763, 787, 797, 769, 775, 764, 795, 780, 786, 752, 743, 791, 781, 770, 751, 746, 720, 806, 740] payload_rows_per_dst=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] dup_rows_per_dst=[2992, 2814, 2972, 3196, 2982, 3010, 2992, 2944] combine_rows_per_src=[9308, 9174, 9246, 9430, 9312, 9330, 9336, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265576448 dispatch_meta_bytes=343232 combine_payload_bytes=265576448 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265919680 ingress_bytes=334782464 link_bytes=334782464 token_bytes=531496128 weight_movement_bytes=69206016 total_nvlink_bytes=600702144 weight_to_token_ratio=0.130210 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259759460e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=343776 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682080 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020384 weight_movement_bytes=34603008 total_nvlink_bytes=567623392 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344208 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267657360 ingress_bytes=336519168 link_bytes=336519168 token_bytes=534970512 weight_movement_bytes=69206016 total_nvlink_bytes=604176528 weight_to_token_ratio=0.129364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266294537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265519104 dispatch_meta_bytes=343712 combine_payload_bytes=265519104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265862816 ingress_bytes=300122112 link_bytes=300122112 token_bytes=531381920 weight_movement_bytes=34603008 total_nvlink_bytes=565984928 weight_to_token_ratio=0.065119 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129335346e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344296 combine_payload_bytes=265756672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266100968 ingress_bytes=334962688 link_bytes=334962688 token_bytes=531857640 weight_movement_bytes=69206016 total_nvlink_bytes=601063656 weight_to_token_ratio=0.130121 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260437628e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65424 dup_rows=23660 dup_rows_remote=20672 dedup_ratio=0.240682 router_tokens_per_expert=[762, 754, 754, 770, 751, 787, 799, 759, 733, 765, 782, 754, 744, 785, 735, 735, 765, 743, 782, 750, 784, 746, 774, 760, 712, 806, 832, 751, 799, 763, 762, 776, 680, 757, 727, 758, 710, 720, 770, 758, 798, 770, 806, 828, 799, 797, 768, 747, 781, 742, 787, 769, 777, 807, 845, 768, 769, 795, 762, 767, 743, 770, 819, 784] payload_rows_per_dst=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] dup_rows_per_dst=[2970, 2886, 2988, 3082, 2668, 3122, 3008, 2936] combine_rows_per_src=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344040 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952680 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561320 weight_movement_bytes=34603008 total_nvlink_bytes=568164328 weight_to_token_ratio=0.064853 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=344488 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266707368 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533070248 weight_movement_bytes=34603008 total_nvlink_bytes=567673256 weight_to_token_ratio=0.064913 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=343424 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266886528 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533429632 weight_movement_bytes=34603008 total_nvlink_bytes=568032640 weight_to_token_ratio=0.064869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=65058 dup_rows=24112 dup_rows_remote=21064 dedup_ratio=0.245280 router_tokens_per_expert=[727, 759, 761, 738, 793, 736, 786, 784, 799, 774, 723, 797, 786, 774, 759, 795, 740, 824, 742, 745, 748, 718, 757, 762, 775, 782, 739, 752, 769, 747, 767, 802, 785, 802, 693, 771, 768, 736, 742, 752, 805, 730, 825, 803, 787, 776, 741, 783, 724, 754, 732, 793, 791, 783, 780, 788, 755, 795, 773, 795, 781, 821, 807, 721] payload_rows_per_dst=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] dup_rows_per_dst=[3048, 3364, 3048, 2908, 3210, 2758, 2730, 3046] combine_rows_per_src=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64950 dup_rows=24070 dup_rows_remote=21062 dedup_ratio=0.244853 router_tokens_per_expert=[812, 770, 814, 807, 766, 761, 745, 747, 765, 749, 725, 785, 736, 795, 708, 734, 757, 767, 722, 752, 758, 734, 772, 773, 777, 761, 737, 741, 773, 781, 756, 781, 807, 802, 746, 801, 749, 805, 766, 794, 757, 770, 735, 757, 754, 779, 798, 796, 753, 771, 829, 778, 751, 770, 809, 813, 737, 795, 772, 796, 766, 710, 769, 756] payload_rows_per_dst=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] dup_rows_per_dst=[3106, 2824, 2880, 2876, 3204, 3008, 3162, 3010] combine_rows_per_src=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74118 payload_rows_remote=64820 dup_rows=24186 dup_rows_remote=21132 dedup_ratio=0.246033 router_tokens_per_expert=[754, 752, 722, 778, 813, 853, 803, 768, 758, 757, 806, 796, 779, 739, 788, 778, 745, 772, 813, 733, 727, 732, 767, 783, 770, 769, 788, 750, 769, 783, 738, 809, 767, 757, 812, 766, 755, 731, 781, 761, 739, 742, 755, 732, 767, 838, 775, 751, 769, 750, 751, 748, 749, 786, 762, 791, 780, 761, 728, 759, 757, 768, 776, 796] payload_rows_per_dst=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] dup_rows_per_dst=[3082, 3120, 2958, 3054, 2994, 2942, 2964, 3072] combine_rows_per_src=[9404, 9282, 9186, 9298, 9266, 9256, 9248, 9178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344488 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822056 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299624 weight_movement_bytes=69206016 total_nvlink_bytes=602505640 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344384 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321088 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536297792 weight_movement_bytes=34603008 total_nvlink_bytes=570900800 weight_to_token_ratio=0.064522 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64748 dup_rows=24090 dup_rows_remote=21064 dedup_ratio=0.245056 router_tokens_per_expert=[768, 777, 773, 820, 741, 757, 752, 738, 780, 771, 769, 741, 739, 763, 752, 744, 757, 755, 783, 740, 762, 751, 740, 806, 699, 767, 760, 798, 811, 808, 799, 813, 748, 762, 761, 781, 724, 772, 768, 797, 796, 767, 744, 797, 779, 748, 749, 779, 754, 784, 759, 787, 778, 759, 782, 749, 801, 772, 759, 770, 779, 773, 748, 792] payload_rows_per_dst=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] dup_rows_per_dst=[2986, 2936, 2954, 3088, 2958, 3056, 3086, 3026] combine_rows_per_src=[9162, 9182, 9234, 9314, 9268, 9262, 9326, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64952 dup_rows=24038 dup_rows_remote=21084 dedup_ratio=0.244527 router_tokens_per_expert=[737, 751, 758, 780, 723, 774, 738, 766, 736, 752, 741, 819, 759, 720, 742, 723, 822, 804, 771, 761, 802, 748, 756, 776, 757, 784, 826, 743, 747, 811, 784, 722, 771, 763, 773, 783, 766, 732, 782, 803, 775, 744, 781, 779, 834, 797, 785, 758, 769, 743, 792, 703, 781, 767, 816, 763, 805, 800, 743, 738, 731, 778, 755, 809] payload_rows_per_dst=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] dup_rows_per_dst=[2968, 2832, 3102, 3056, 3008, 3082, 2954, 3036] combine_rows_per_src=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65252 dup_rows=23660 dup_rows_remote=20668 dedup_ratio=0.240682 router_tokens_per_expert=[768, 801, 789, 749, 749, 803, 706, 786, 783, 753, 817, 764, 788, 730, 794, 763, 778, 759, 809, 797, 761, 790, 748, 784, 818, 739, 758, 756, 757, 715, 770, 798, 731, 725, 793, 750, 751, 731, 822, 797, 825, 803, 744, 789, 780, 789, 724, 750, 761, 778, 744, 778, 750, 744, 765, 749, 793, 784, 785, 763, 736, 752, 739, 747] payload_rows_per_dst=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] dup_rows_per_dst=[2930, 2992, 3070, 2952, 2936, 2980, 2864, 2936] combine_rows_per_src=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=65058 dup_rows=24112 dup_rows_remote=21064 dedup_ratio=0.245280 router_tokens_per_expert=[727, 759, 761, 738, 793, 736, 786, 784, 799, 774, 723, 797, 786, 774, 759, 795, 740, 824, 742, 745, 748, 718, 757, 762, 775, 782, 739, 752, 769, 747, 767, 802, 785, 802, 693, 771, 768, 736, 742, 752, 805, 730, 825, 803, 787, 776, 741, 783, 724, 754, 732, 793, 791, 783, 780, 788, 755, 795, 773, 795, 781, 821, 807, 721] payload_rows_per_dst=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] dup_rows_per_dst=[3048, 3364, 3048, 2908, 3210, 2758, 2730, 3046] combine_rows_per_src=[9134, 9742, 9418, 9344, 9416, 8898, 9032, 9208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=344048 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266379248 ingress_bytes=369844224 link_bytes=369844224 token_bytes=532414448 weight_movement_bytes=103809024 total_nvlink_bytes=636223472 weight_to_token_ratio=0.194978 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391694040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65262 dup_rows=23736 dup_rows_remote=20790 dedup_ratio=0.241455 router_tokens_per_expert=[774, 809, 759, 751, 755, 773, 724, 729, 789, 708, 773, 773, 827, 742, 769, 791, 757, 791, 752, 778, 769, 771, 803, 776, 769, 734, 721, 786, 777, 791, 780, 790, 775, 770, 744, 741, 751, 779, 769, 771, 777, 792, 759, 745, 776, 811, 752, 768, 726, 787, 745, 739, 807, 761, 762, 799, 724, 773, 753, 778, 797, 768, 741, 821] payload_rows_per_dst=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] dup_rows_per_dst=[2898, 3004, 2942, 2970, 2964, 3050, 2946, 2962] combine_rows_per_src=[9250, 9340, 9452, 9326, 9236, 9310, 9306, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343808 combine_payload_bytes=265502720 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265846528 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349248 weight_movement_bytes=69206016 total_nvlink_bytes=600555264 weight_to_token_ratio=0.130246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65252 dup_rows=23660 dup_rows_remote=20668 dedup_ratio=0.240682 router_tokens_per_expert=[768, 801, 789, 749, 749, 803, 706, 786, 783, 753, 817, 764, 788, 730, 794, 763, 778, 759, 809, 797, 761, 790, 748, 784, 818, 739, 758, 756, 757, 715, 770, 798, 731, 725, 793, 750, 751, 731, 822, 797, 825, 803, 744, 789, 780, 789, 724, 750, 761, 778, 744, 778, 750, 744, 765, 749, 793, 784, 785, 763, 736, 752, 739, 747] payload_rows_per_dst=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] dup_rows_per_dst=[2930, 2992, 3070, 2952, 2936, 2980, 2864, 2936] combine_rows_per_src=[9372, 9392, 9382, 9270, 9264, 9428, 9274, 9262] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343248 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265551056 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758864 weight_movement_bytes=69206016 total_nvlink_bytes=599964880 weight_to_token_ratio=0.130391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344488 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266822056 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533299624 weight_movement_bytes=69206016 total_nvlink_bytes=602505640 weight_to_token_ratio=0.129769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65030 dup_rows=23972 dup_rows_remote=20928 dedup_ratio=0.243856 router_tokens_per_expert=[845, 784, 755, 778, 790, 764, 745, 747, 729, 824, 718, 730, 809, 803, 717, 795, 810, 760, 751, 764, 759, 818, 804, 682, 766, 788, 776, 728, 790, 746, 748, 714, 789, 843, 751, 736, 755, 764, 749, 786, 772, 766, 754, 776, 775, 763, 755, 789, 800, 769, 769, 744, 791, 755, 805, 773, 776, 754, 707, 735, 760, 809, 774, 771] payload_rows_per_dst=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] dup_rows_per_dst=[3068, 3016, 3044, 2868, 3044, 3012, 3086, 2834] combine_rows_per_src=[9348, 9234, 9252, 9244, 9302, 9288, 9326, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64950 dup_rows=24070 dup_rows_remote=21062 dedup_ratio=0.244853 router_tokens_per_expert=[812, 770, 814, 807, 766, 761, 745, 747, 765, 749, 725, 785, 736, 795, 708, 734, 757, 767, 722, 752, 758, 734, 772, 773, 777, 761, 737, 741, 773, 781, 756, 781, 807, 802, 746, 801, 749, 805, 766, 794, 757, 770, 735, 757, 754, 779, 798, 796, 753, 771, 829, 778, 751, 770, 809, 813, 737, 795, 772, 796, 766, 710, 769, 756] payload_rows_per_dst=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] dup_rows_per_dst=[3106, 2824, 2880, 2876, 3204, 3008, 3162, 3010] combine_rows_per_src=[9338, 9170, 9190, 9338, 9336, 9284, 9386, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64850 dup_rows=24046 dup_rows_remote=20958 dedup_ratio=0.244609 router_tokens_per_expert=[805, 781, 754, 796, 768, 769, 754, 764, 767, 792, 768, 738, 752, 784, 773, 781, 787, 787, 828, 775, 788, 756, 737, 790, 817, 711, 794, 784, 785, 819, 766, 791, 734, 738, 750, 828, 709, 767, 710, 746, 797, 766, 773, 746, 718, 771, 735, 766, 791, 773, 765, 744, 779, 764, 742, 758, 802, 780, 772, 744, 740, 765, 753, 765] payload_rows_per_dst=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] dup_rows_per_dst=[3104, 2942, 3088, 3110, 2872, 2956, 2974, 3000] combine_rows_per_src=[9278, 9368, 9408, 9424, 9092, 9188, 9258, 9242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=267313152 dispatch_meta_bytes=344208 combine_payload_bytes=267313152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267657360 ingress_bytes=336519168 link_bytes=336519168 token_bytes=534970512 weight_movement_bytes=69206016 total_nvlink_bytes=604176528 weight_to_token_ratio=0.129364 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266294537e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265207808 dispatch_meta_bytes=343248 combine_payload_bytes=265207808 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265551056 ingress_bytes=334413824 link_bytes=334413824 token_bytes=530758864 weight_movement_bytes=69206016 total_nvlink_bytes=599964880 weight_to_token_ratio=0.130391 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258372297e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344144 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266387536 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430928 weight_movement_bytes=69206016 total_nvlink_bytes=601636944 weight_to_token_ratio=0.129981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65226 dup_rows=23736 dup_rows_remote=20828 dedup_ratio=0.241455 router_tokens_per_expert=[757, 807, 783, 749, 772, 701, 766, 763, 758, 772, 714, 785, 811, 743, 796, 780, 787, 787, 741, 706, 745, 778, 792, 789, 827, 806, 719, 758, 726, 777, 794, 762, 783, 730, 768, 779, 782, 774, 741, 756, 796, 773, 764, 731, 739, 784, 724, 799, 796, 818, 781, 782, 732, 783, 771, 775, 765, 750, 814, 744, 760, 768, 765, 774] payload_rows_per_dst=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] dup_rows_per_dst=[2876, 3026, 2908, 3022, 2924, 2974, 3064, 2942] combine_rows_per_src=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267272192 dispatch_meta_bytes=343680 combine_payload_bytes=267272192 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267615872 ingress_bytes=301875200 link_bytes=301875200 token_bytes=534888064 weight_movement_bytes=34603008 total_nvlink_bytes=569491072 weight_to_token_ratio=0.064692 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135932075e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266362880 dispatch_meta_bytes=343832 combine_payload_bytes=266362880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266706712 ingress_bytes=300965888 link_bytes=300965888 token_bytes=533069592 weight_movement_bytes=34603008 total_nvlink_bytes=567672600 weight_to_token_ratio=0.064913 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132510407e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266035200 dispatch_meta_bytes=344048 combine_payload_bytes=266035200 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266379248 ingress_bytes=369844224 link_bytes=369844224 token_bytes=532414448 weight_movement_bytes=103809024 total_nvlink_bytes=636223472 weight_to_token_ratio=0.194978 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391694040e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343232 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265968832 ingress_bytes=369434624 link_bytes=369434624 token_bytes=531594432 weight_movement_bytes=103809024 total_nvlink_bytes=635403456 weight_to_token_ratio=0.195279 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390152748e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64800 dup_rows=24208 dup_rows_remote=21146 dedup_ratio=0.246257 router_tokens_per_expert=[722, 779, 759, 794, 762, 748, 774, 759, 726, 747, 748, 821, 731, 782, 706, 794, 786, 768, 758, 762, 765, 710, 781, 766, 732, 789, 742, 727, 766, 792, 724, 752, 742, 836, 753, 825, 785, 772, 791, 771, 785, 816, 805, 801, 764, 733, 778, 777, 786, 769, 802, 781, 799, 716, 789, 774, 741, 780, 731, 796, 788, 790, 764, 740] payload_rows_per_dst=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] dup_rows_per_dst=[3062, 3078, 2984, 3148, 3126, 2902, 3032, 2876] combine_rows_per_src=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64970 dup_rows=23982 dup_rows_remote=20926 dedup_ratio=0.243958 router_tokens_per_expert=[750, 788, 775, 775, 755, 780, 783, 773, 771, 751, 742, 763, 702, 752, 775, 770, 786, 821, 786, 787, 777, 733, 714, 753, 753, 766, 789, 765, 783, 772, 769, 777, 828, 798, 764, 812, 765, 719, 736, 800, 762, 791, 756, 793, 736, 796, 765, 805, 753, 707, 802, 749, 801, 778, 771, 739, 744, 741, 764, 740, 769, 816, 745, 771] payload_rows_per_dst=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] dup_rows_per_dst=[3052, 2920, 3014, 2956, 3122, 3056, 2928, 2934] combine_rows_per_src=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344216 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267509912 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675608 weight_movement_bytes=34603008 total_nvlink_bytes=569278616 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65138 dup_rows=23996 dup_rows_remote=21020 dedup_ratio=0.244100 router_tokens_per_expert=[740, 772, 761, 720, 797, 742, 742, 811, 765, 757, 804, 802, 751, 810, 800, 753, 769, 753, 789, 741, 767, 737, 747, 817, 758, 742, 741, 794, 734, 753, 767, 784, 798, 747, 788, 743, 781, 803, 730, 788, 796, 724, 800, 801, 742, 753, 749, 738, 760, 792, 762, 790, 759, 759, 790, 726, 807, 795, 813, 727, 766, 735, 793, 777] payload_rows_per_dst=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] dup_rows_per_dst=[2880, 3098, 3010, 2976, 2990, 2980, 2990, 3072] combine_rows_per_src=[9290, 9386, 9230, 9170, 9366, 9226, 9286, 9354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343784 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764584 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185384 weight_movement_bytes=34603008 total_nvlink_bytes=565788392 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74404 payload_rows_remote=64962 dup_rows=23900 dup_rows_remote=20740 dedup_ratio=0.243123 router_tokens_per_expert=[798, 774, 767, 768, 732, 784, 756, 773, 792, 744, 809, 758, 730, 714, 787, 749, 791, 753, 778, 805, 724, 792, 776, 764, 744, 808, 775, 761, 722, 766, 783, 796, 722, 761, 715, 802, 781, 748, 775, 734, 752, 805, 773, 753, 766, 795, 780, 780, 742, 787, 721, 738, 805, 761, 724, 774, 743, 786, 771, 788, 780, 830, 758, 829] payload_rows_per_dst=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] dup_rows_per_dst=[2962, 2972, 2994, 3014, 2806, 3006, 2986, 3160] combine_rows_per_src=[9342, 9194, 9372, 9296, 9238, 9278, 9242, 9442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65242 dup_rows=23950 dup_rows_remote=21036 dedup_ratio=0.243632 router_tokens_per_expert=[745, 830, 773, 781, 773, 770, 765, 746, 778, 757, 750, 746, 793, 799, 792, 798, 767, 757, 741, 769, 749, 759, 775, 776, 809, 752, 741, 769, 759, 784, 758, 817, 781, 759, 776, 752, 764, 742, 776, 731, 808, 757, 745, 779, 792, 817, 775, 794, 783, 781, 704, 747, 774, 746, 730, 748, 771, 739, 726, 771, 735, 824, 768, 779] payload_rows_per_dst=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] dup_rows_per_dst=[3040, 3058, 2964, 3034, 2928, 3118, 2914, 2894] combine_rows_per_src=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74096 payload_rows_remote=64800 dup_rows=24208 dup_rows_remote=21146 dedup_ratio=0.246257 router_tokens_per_expert=[722, 779, 759, 794, 762, 748, 774, 759, 726, 747, 748, 821, 731, 782, 706, 794, 786, 768, 758, 762, 765, 710, 781, 766, 732, 789, 742, 727, 766, 792, 724, 752, 742, 836, 753, 825, 785, 772, 791, 771, 785, 816, 805, 801, 764, 733, 778, 777, 786, 769, 802, 781, 799, 716, 789, 774, 741, 780, 731, 796, 788, 790, 764, 740] payload_rows_per_dst=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] dup_rows_per_dst=[3062, 3078, 2984, 3148, 3126, 2902, 3032, 2876] combine_rows_per_src=[9296, 9284, 9320, 9178, 9430, 9080, 9400, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343584 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266460704 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577824 weight_movement_bytes=138412032 total_nvlink_bytes=670989856 weight_to_token_ratio=0.259891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64952 dup_rows=24038 dup_rows_remote=21084 dedup_ratio=0.244527 router_tokens_per_expert=[737, 751, 758, 780, 723, 774, 738, 766, 736, 752, 741, 819, 759, 720, 742, 723, 822, 804, 771, 761, 802, 748, 756, 776, 757, 784, 826, 743, 747, 811, 784, 722, 771, 763, 773, 783, 766, 732, 782, 803, 775, 744, 781, 779, 834, 797, 785, 758, 769, 743, 792, 703, 781, 767, 816, 763, 805, 800, 743, 738, 731, 778, 755, 809] payload_rows_per_dst=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] dup_rows_per_dst=[2968, 2832, 3102, 3056, 3008, 3082, 2954, 3036] combine_rows_per_src=[9086, 9152, 9378, 9292, 9338, 9424, 9314, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344632 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149880 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533955128 weight_movement_bytes=34603008 total_nvlink_bytes=568558136 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=342808 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266427160 ingress_bytes=369893376 link_bytes=369893376 token_bytes=532511512 weight_movement_bytes=103809024 total_nvlink_bytes=636320536 weight_to_token_ratio=0.194942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391878995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74160 payload_rows_remote=64940 dup_rows=24144 dup_rows_remote=21142 dedup_ratio=0.245605 router_tokens_per_expert=[760, 766, 747, 779, 740, 759, 791, 786, 734, 781, 751, 709, 813, 762, 779, 782, 767, 772, 748, 761, 747, 751, 714, 755, 768, 771, 785, 822, 732, 816, 763, 759, 841, 776, 763, 784, 800, 711, 742, 744, 792, 749, 804, 761, 794, 754, 709, 809, 779, 793, 797, 749, 765, 814, 763, 778, 749, 773, 742, 745, 779, 760, 784, 779] payload_rows_per_dst=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] dup_rows_per_dst=[3032, 3002, 2840, 3142, 3058, 2978, 3094, 2998] combine_rows_per_src=[9224, 9220, 9190, 9290, 9264, 9366, 9382, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343784 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764584 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185384 weight_movement_bytes=34603008 total_nvlink_bytes=565788392 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74086 payload_rows_remote=64836 dup_rows=24218 dup_rows_remote=21274 dedup_ratio=0.246358 router_tokens_per_expert=[779, 753, 767, 771, 820, 756, 719, 783, 758, 785, 743, 790, 768, 741, 785, 767, 804, 805, 807, 734, 767, 807, 796, 787, 713, 752, 769, 733, 736, 802, 745, 763, 738, 798, 780, 772, 757, 759, 725, 768, 773, 757, 784, 787, 761, 790, 750, 761, 757, 792, 792, 759, 727, 781, 707, 780, 816, 741, 758, 773, 782, 794, 763, 765] payload_rows_per_dst=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] dup_rows_per_dst=[3054, 2968, 3186, 2880, 2944, 3106, 3016, 3064] combine_rows_per_src=[9242, 9306, 9428, 9146, 9250, 9220, 9174, 9320] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74644 payload_rows_remote=65424 dup_rows=23660 dup_rows_remote=20672 dedup_ratio=0.240682 router_tokens_per_expert=[762, 754, 754, 770, 751, 787, 799, 759, 733, 765, 782, 754, 744, 785, 735, 735, 765, 743, 782, 750, 784, 746, 774, 760, 712, 806, 832, 751, 799, 763, 762, 776, 680, 757, 727, 758, 710, 720, 770, 758, 798, 770, 806, 828, 799, 797, 768, 747, 781, 742, 787, 769, 777, 807, 845, 768, 769, 795, 762, 767, 743, 770, 819, 784] payload_rows_per_dst=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] dup_rows_per_dst=[2970, 2886, 2988, 3082, 2668, 3122, 3008, 2936] combine_rows_per_src=[9302, 9180, 9220, 9320, 9092, 9504, 9544, 9482] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64970 dup_rows=23982 dup_rows_remote=20926 dedup_ratio=0.243958 router_tokens_per_expert=[750, 788, 775, 775, 755, 780, 783, 773, 771, 751, 742, 763, 702, 752, 775, 770, 786, 821, 786, 787, 777, 733, 714, 753, 753, 766, 789, 765, 783, 772, 769, 777, 828, 798, 764, 812, 765, 719, 736, 800, 762, 791, 756, 793, 736, 796, 765, 805, 753, 707, 802, 749, 801, 778, 771, 739, 744, 741, 764, 740, 769, 816, 745, 771] payload_rows_per_dst=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] dup_rows_per_dst=[3052, 2920, 3014, 2956, 3122, 3056, 2928, 2934] combine_rows_per_src=[9306, 9132, 9300, 9392, 9322, 9352, 9272, 9246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=342808 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266427160 ingress_bytes=369893376 link_bytes=369893376 token_bytes=532511512 weight_movement_bytes=103809024 total_nvlink_bytes=636320536 weight_to_token_ratio=0.194942 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391878995e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64974 dup_rows=24098 dup_rows_remote=21116 dedup_ratio=0.245138 router_tokens_per_expert=[783, 764, 748, 753, 719, 804, 768, 793, 772, 831, 755, 751, 744, 717, 775, 769, 719, 748, 770, 773, 770, 748, 789, 790, 837, 761, 762, 838, 771, 702, 750, 740, 774, 749, 768, 762, 730, 772, 828, 742, 795, 750, 790, 787, 803, 748, 751, 722, 796, 779, 796, 780, 787, 771, 707, 778, 768, 742, 810, 787, 771, 793, 717, 785] payload_rows_per_dst=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] dup_rows_per_dst=[3016, 2908, 2982, 3072, 3052, 3044, 3008, 3016] combine_rows_per_src=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344144 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266387536 ingress_bytes=335249408 link_bytes=335249408 token_bytes=532430928 weight_movement_bytes=69206016 total_nvlink_bytes=601636944 weight_to_token_ratio=0.129981 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261516533e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=345112 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267576344 ingress_bytes=301834240 link_bytes=301834240 token_bytes=534807576 weight_movement_bytes=34603008 total_nvlink_bytes=569410584 weight_to_token_ratio=0.064702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135777945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344328 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266338568 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532332808 weight_movement_bytes=34603008 total_nvlink_bytes=566935816 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265568256 dispatch_meta_bytes=344440 combine_payload_bytes=265568256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265912696 ingress_bytes=334774272 link_bytes=334774272 token_bytes=531480952 weight_movement_bytes=69206016 total_nvlink_bytes=600686968 weight_to_token_ratio=0.130214 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259728634e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344384 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268321088 ingress_bytes=302579712 link_bytes=302579712 token_bytes=536297792 weight_movement_bytes=34603008 total_nvlink_bytes=570900800 weight_to_token_ratio=0.064522 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138583097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=343584 combine_payload_bytes=266117120 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266460704 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532577824 weight_movement_bytes=138412032 total_nvlink_bytes=670989856 weight_to_token_ratio=0.259891 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64816 dup_rows=24112 dup_rows_remote=20840 dedup_ratio=0.245280 router_tokens_per_expert=[760, 744, 735, 804, 763, 742, 787, 772, 828, 758, 761, 776, 728, 782, 779, 740, 836, 762, 815, 737, 746, 740, 747, 754, 741, 741, 764, 775, 756, 783, 766, 788, 727, 758, 788, 740, 793, 810, 744, 735, 765, 801, 767, 776, 802, 751, 794, 778, 773, 800, 753, 799, 791, 746, 742, 806, 756, 785, 737, 770, 735, 795, 797, 728] payload_rows_per_dst=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] dup_rows_per_dst=[3272, 3200, 2966, 3112, 2998, 2756, 3036, 2772] combine_rows_per_src=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65074 dup_rows=23970 dup_rows_remote=21054 dedup_ratio=0.243835 router_tokens_per_expert=[753, 710, 767, 801, 787, 781, 787, 758, 816, 734, 778, 760, 742, 756, 736, 784, 783, 726, 784, 821, 743, 796, 733, 750, 771, 757, 793, 777, 781, 785, 804, 763, 800, 772, 766, 767, 811, 758, 707, 741, 754, 779, 736, 739, 758, 750, 789, 783, 787, 785, 762, 781, 756, 751, 763, 764, 778, 775, 793, 783, 800, 753, 757, 737] payload_rows_per_dst=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] dup_rows_per_dst=[2954, 3062, 2990, 3072, 2976, 2916, 2950, 3050] combine_rows_per_src=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344360 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477864 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611368 weight_movement_bytes=34603008 total_nvlink_bytes=567214376 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64986 dup_rows=24078 dup_rows_remote=21136 dedup_ratio=0.244934 router_tokens_per_expert=[783, 759, 725, 741, 775, 775, 793, 735, 775, 739, 772, 749, 771, 781, 735, 788, 754, 768, 751, 744, 775, 808, 803, 797, 777, 749, 735, 778, 749, 786, 736, 781, 767, 824, 743, 770, 773, 773, 780, 739, 789, 707, 762, 778, 792, 805, 725, 807, 737, 735, 750, 781, 733, 773, 759, 771, 788, 782, 786, 807, 767, 794, 755, 813] payload_rows_per_dst=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] dup_rows_per_dst=[2990, 2970, 3096, 2942, 3032, 2992, 2946, 3110] combine_rows_per_src=[9182, 9250, 9304, 9240, 9306, 9338, 9132, 9474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342624 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265828960 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531315296 weight_movement_bytes=34603008 total_nvlink_bytes=565918304 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=64964 dup_rows=23950 dup_rows_remote=21014 dedup_ratio=0.243632 router_tokens_per_expert=[723, 759, 751, 788, 757, 792, 778, 737, 746, 763, 723, 758, 767, 800, 733, 782, 760, 734, 775, 758, 765, 739, 791, 773, 782, 735, 787, 765, 779, 761, 750, 745, 751, 796, 806, 781, 782, 808, 764, 748, 790, 760, 740, 762, 833, 749, 819, 776, 742, 766, 784, 795, 811, 721, 789, 786, 757, 786, 763, 779, 728, 808, 762, 754] payload_rows_per_dst=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] dup_rows_per_dst=[2960, 2914, 2946, 2998, 3038, 3074, 3084, 2936] combine_rows_per_src=[9210, 9230, 9244, 9210, 9382, 9384, 9304, 9390] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344488 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266527144 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709800 weight_movement_bytes=34603008 total_nvlink_bytes=567312808 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65050 dup_rows=23928 dup_rows_remote=20878 dedup_ratio=0.243408 router_tokens_per_expert=[750, 780, 788, 755, 766, 745, 766, 743, 776, 806, 753, 775, 766, 730, 782, 774, 784, 775, 748, 771, 719, 763, 764, 749, 770, 761, 748, 776, 730, 763, 774, 746, 783, 777, 794, 777, 704, 722, 780, 774, 782, 774, 766, 750, 771, 774, 784, 810, 759, 745, 733, 829, 777, 753, 790, 802, 782, 798, 775, 777, 813, 770, 749, 782] payload_rows_per_dst=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] dup_rows_per_dst=[2984, 3000, 2906, 2982, 2966, 2978, 3050, 3062] combine_rows_per_src=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64816 dup_rows=24112 dup_rows_remote=20840 dedup_ratio=0.245280 router_tokens_per_expert=[760, 744, 735, 804, 763, 742, 787, 772, 828, 758, 761, 776, 728, 782, 779, 740, 836, 762, 815, 737, 746, 740, 747, 754, 741, 741, 764, 775, 756, 783, 766, 788, 727, 758, 788, 740, 793, 810, 744, 735, 765, 801, 767, 776, 802, 751, 794, 778, 773, 800, 753, 799, 791, 746, 742, 806, 756, 785, 737, 770, 735, 795, 797, 728] payload_rows_per_dst=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] dup_rows_per_dst=[3272, 3200, 2966, 3112, 2998, 2756, 3036, 2772] combine_rows_per_src=[9376, 9480, 9398, 9518, 9212, 8906, 9360, 8942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344512 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266887616 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430720 weight_movement_bytes=103809024 total_nvlink_bytes=637239744 weight_to_token_ratio=0.194606 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344488 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266527144 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709800 weight_movement_bytes=34603008 total_nvlink_bytes=567312808 weight_to_token_ratio=0.064957 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65242 dup_rows=23950 dup_rows_remote=21036 dedup_ratio=0.243632 router_tokens_per_expert=[745, 830, 773, 781, 773, 770, 765, 746, 778, 757, 750, 746, 793, 799, 792, 798, 767, 757, 741, 769, 749, 759, 775, 776, 809, 752, 741, 769, 759, 784, 758, 817, 781, 759, 776, 752, 764, 742, 776, 731, 808, 757, 745, 779, 792, 817, 775, 794, 783, 781, 704, 747, 774, 746, 730, 748, 771, 739, 726, 771, 735, 824, 768, 779] payload_rows_per_dst=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] dup_rows_per_dst=[3040, 3058, 2964, 3034, 2928, 3118, 2914, 2894] combine_rows_per_src=[9326, 9368, 9222, 9344, 9234, 9416, 9112, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436456 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529000 weight_movement_bytes=103809024 total_nvlink_bytes=636338024 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74252 payload_rows_remote=64954 dup_rows=24052 dup_rows_remote=21078 dedup_ratio=0.244670 router_tokens_per_expert=[767, 790, 752, 743, 810, 775, 779, 749, 714, 794, 817, 769, 753, 797, 807, 685, 758, 812, 755, 776, 754, 794, 805, 703, 780, 776, 723, 761, 763, 768, 803, 757, 758, 776, 705, 786, 782, 771, 763, 767, 770, 767, 785, 806, 790, 828, 744, 816, 783, 723, 767, 794, 713, 786, 744, 748, 750, 764, 812, 776, 725, 786, 726, 752] payload_rows_per_dst=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] dup_rows_per_dst=[3080, 2974, 2938, 3014, 3034, 3096, 2918, 2998] combine_rows_per_src=[9250, 9298, 9376, 9248, 9182, 9516, 9198, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=342624 combine_payload_bytes=265486336 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265828960 ingress_bytes=300089344 link_bytes=300089344 token_bytes=531315296 weight_movement_bytes=34603008 total_nvlink_bytes=565918304 weight_to_token_ratio=0.065127 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129212042e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74248 payload_rows_remote=64864 dup_rows=24056 dup_rows_remote=21036 dedup_ratio=0.244710 router_tokens_per_expert=[762, 805, 740, 789, 762, 724, 761, 780, 792, 730, 788, 803, 774, 771, 746, 708, 776, 754, 807, 752, 735, 851, 775, 752, 794, 790, 779, 715, 760, 725, 730, 758, 765, 775, 827, 767, 735, 764, 739, 830, 735, 797, 733, 780, 775, 820, 725, 811, 774, 748, 755, 768, 783, 712, 805, 787, 731, 770, 789, 759, 768, 766, 797, 774] payload_rows_per_dst=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] dup_rows_per_dst=[2926, 2962, 3186, 2856, 3020, 3012, 3016, 3078] combine_rows_per_src=[9320, 9262, 9218, 9246, 9384, 9340, 9248, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65074 dup_rows=23970 dup_rows_remote=21054 dedup_ratio=0.243835 router_tokens_per_expert=[753, 710, 767, 801, 787, 781, 787, 758, 816, 734, 778, 760, 742, 756, 736, 784, 783, 726, 784, 821, 743, 796, 733, 750, 771, 757, 793, 777, 781, 785, 804, 763, 800, 772, 766, 767, 811, 758, 707, 741, 754, 779, 736, 739, 758, 750, 789, 783, 787, 785, 762, 781, 756, 751, 763, 764, 778, 775, 793, 783, 800, 753, 757, 737] payload_rows_per_dst=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] dup_rows_per_dst=[2954, 3062, 2990, 3072, 2976, 2916, 2950, 3050] combine_rows_per_src=[9334, 9150, 9282, 9390, 9268, 9260, 9348, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74568 payload_rows_remote=65226 dup_rows=23736 dup_rows_remote=20828 dedup_ratio=0.241455 router_tokens_per_expert=[757, 807, 783, 749, 772, 701, 766, 763, 758, 772, 714, 785, 811, 743, 796, 780, 787, 787, 741, 706, 745, 778, 792, 789, 827, 806, 719, 758, 726, 777, 794, 762, 783, 730, 768, 779, 782, 774, 741, 756, 796, 773, 764, 731, 739, 784, 724, 799, 796, 818, 781, 782, 732, 783, 771, 775, 765, 750, 814, 744, 760, 768, 765, 774] payload_rows_per_dst=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] dup_rows_per_dst=[2876, 3026, 2908, 3022, 2924, 2974, 3064, 2942] combine_rows_per_src=[9320, 9292, 9342, 9316, 9302, 9246, 9412, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266436456 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532529000 weight_movement_bytes=103809024 total_nvlink_bytes=636338024 weight_to_token_ratio=0.194936 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64776 dup_rows=24184 dup_rows_remote=21108 dedup_ratio=0.246012 router_tokens_per_expert=[745, 765, 778, 753, 783, 747, 726, 773, 752, 797, 783, 769, 787, 753, 735, 759, 777, 755, 779, 799, 812, 763, 759, 766, 742, 805, 754, 741, 781, 786, 756, 806, 732, 741, 753, 744, 767, 781, 774, 822, 789, 780, 753, 764, 735, 795, 728, 751, 787, 701, 803, 789, 779, 791, 750, 743, 801, 820, 772, 758, 769, 776, 764, 754] payload_rows_per_dst=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] dup_rows_per_dst=[2944, 3052, 3076, 3090, 3022, 2964, 2966, 3070] combine_rows_per_src=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343712 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788512 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233312 weight_movement_bytes=69206016 total_nvlink_bytes=602439328 weight_to_token_ratio=0.129786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267231232 dispatch_meta_bytes=345112 combine_payload_bytes=267231232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267576344 ingress_bytes=301834240 link_bytes=301834240 token_bytes=534807576 weight_movement_bytes=34603008 total_nvlink_bytes=569410584 weight_to_token_ratio=0.064702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135777945e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344128 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266395712 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447296 weight_movement_bytes=34603008 total_nvlink_bytes=567050304 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65082 dup_rows=23832 dup_rows_remote=20870 dedup_ratio=0.242432 router_tokens_per_expert=[766, 783, 747, 768, 758, 747, 808, 791, 743, 755, 771, 778, 738, 778, 732, 828, 761, 768, 742, 759, 732, 788, 795, 741, 797, 764, 746, 745, 762, 775, 712, 775, 762, 787, 792, 810, 750, 771, 740, 776, 773, 832, 786, 735, 775, 760, 779, 736, 875, 773, 732, 764, 739, 745, 748, 814, 791, 826, 793, 710, 782, 723, 775, 745] payload_rows_per_dst=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] dup_rows_per_dst=[2944, 2926, 2964, 2928, 3014, 2962, 3072, 3022] combine_rows_per_src=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343600 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266026544 ingress_bytes=334888960 link_bytes=334888960 token_bytes=531709488 weight_movement_bytes=69206016 total_nvlink_bytes=600915504 weight_to_token_ratio=0.130158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260160196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65136 dup_rows=24140 dup_rows_remote=21272 dedup_ratio=0.245565 router_tokens_per_expert=[789, 698, 710, 752, 746, 745, 736, 749, 776, 816, 761, 768, 759, 735, 811, 771, 751, 745, 788, 762, 755, 774, 773, 741, 780, 780, 762, 791, 770, 766, 786, 790, 804, 830, 723, 744, 794, 794, 741, 790, 750, 749, 760, 760, 766, 772, 745, 772, 834, 768, 787, 767, 802, 820, 771, 784, 756, 771, 769, 781, 727, 755, 754, 776] payload_rows_per_dst=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] dup_rows_per_dst=[2868, 3386, 2936, 3130, 3028, 3260, 2886, 2646] combine_rows_per_src=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344512 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266887616 ingress_bytes=370352128 link_bytes=370352128 token_bytes=533430720 weight_movement_bytes=103809024 total_nvlink_bytes=637239744 weight_to_token_ratio=0.194606 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393605242e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267165696 dispatch_meta_bytes=344216 combine_payload_bytes=267165696 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267509912 ingress_bytes=301768704 link_bytes=301768704 token_bytes=534675608 weight_movement_bytes=34603008 total_nvlink_bytes=569278616 weight_to_token_ratio=0.064718 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135531339e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343536 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265666032 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988528 weight_movement_bytes=69206016 total_nvlink_bytes=600194544 weight_to_token_ratio=0.130334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=65006 dup_rows=23944 dup_rows_remote=20950 dedup_ratio=0.243571 router_tokens_per_expert=[801, 757, 797, 786, 754, 741, 769, 812, 727, 736, 752, 729, 750, 767, 755, 787, 774, 767, 768, 780, 772, 784, 748, 772, 771, 763, 796, 765, 732, 775, 770, 802, 749, 794, 709, 789, 745, 787, 753, 753, 811, 742, 779, 753, 771, 758, 789, 748, 747, 741, 790, 782, 765, 769, 772, 792, 784, 821, 748, 786, 751, 761, 757, 797] payload_rows_per_dst=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] dup_rows_per_dst=[3152, 2868, 3094, 2994, 2914, 2934, 2956, 3032] combine_rows_per_src=[9282, 9138, 9236, 9354, 9244, 9368, 9360, 9378] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=345632 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267142688 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533939744 weight_movement_bytes=34603008 total_nvlink_bytes=568542752 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74482 payload_rows_remote=65124 dup_rows=23822 dup_rows_remote=20726 dedup_ratio=0.242330 router_tokens_per_expert=[771, 776, 746, 792, 740, 805, 767, 787, 732, 813, 754, 742, 773, 732, 760, 732, 789, 739, 779, 787, 756, 733, 733, 795, 749, 768, 785, 764, 746, 779, 767, 829, 789, 780, 750, 813, 726, 758, 765, 790, 761, 737, 771, 749, 791, 732, 730, 776, 783, 758, 808, 784, 801, 777, 779, 817, 802, 744, 742, 769, 808, 753, 745, 744] payload_rows_per_dst=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] dup_rows_per_dst=[3020, 2846, 2908, 3060, 2876, 2928, 3088, 3096] combine_rows_per_src=[9348, 9230, 9284, 9314, 9254, 9166, 9528, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343824 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608400 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532872976 weight_movement_bytes=34603008 total_nvlink_bytes=567475984 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65050 dup_rows=24060 dup_rows_remote=21114 dedup_ratio=0.244751 router_tokens_per_expert=[781, 748, 782, 763, 772, 783, 804, 810, 797, 716, 742, 713, 811, 740, 758, 759, 744, 774, 739, 715, 792, 773, 740, 757, 778, 764, 758, 719, 760, 767, 788, 789, 800, 790, 767, 789, 811, 799, 759, 743, 748, 769, 760, 794, 787, 739, 762, 771, 777, 748, 754, 769, 762, 744, 772, 744, 817, 766, 745, 766, 794, 819, 784, 767] payload_rows_per_dst=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] dup_rows_per_dst=[3146, 2906, 2862, 2942, 3120, 3032, 2946, 3106] combine_rows_per_src=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74164 payload_rows_remote=65136 dup_rows=24140 dup_rows_remote=21272 dedup_ratio=0.245565 router_tokens_per_expert=[789, 698, 710, 752, 746, 745, 736, 749, 776, 816, 761, 768, 759, 735, 811, 771, 751, 745, 788, 762, 755, 774, 773, 741, 780, 780, 762, 791, 770, 766, 786, 790, 804, 830, 723, 744, 794, 794, 741, 790, 750, 749, 760, 760, 766, 772, 745, 772, 834, 768, 787, 767, 802, 820, 771, 784, 756, 771, 769, 781, 727, 755, 754, 776] payload_rows_per_dst=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] dup_rows_per_dst=[2868, 3386, 2936, 3130, 3028, 3260, 2886, 2646] combine_rows_per_src=[9028, 9652, 9242, 9464, 9412, 9562, 8916, 8888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343808 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919680 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495552 weight_movement_bytes=34603008 total_nvlink_bytes=568098560 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266264576 dispatch_meta_bytes=343824 combine_payload_bytes=266264576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266608400 ingress_bytes=300867584 link_bytes=300867584 token_bytes=532872976 weight_movement_bytes=34603008 total_nvlink_bytes=567475984 weight_to_token_ratio=0.064937 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132140497e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343400 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267091304 ingress_bytes=335953920 link_bytes=335953920 token_bytes=533839208 weight_movement_bytes=69206016 total_nvlink_bytes=603045224 weight_to_token_ratio=0.129638 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264167554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65050 dup_rows=23928 dup_rows_remote=20878 dedup_ratio=0.243408 router_tokens_per_expert=[750, 780, 788, 755, 766, 745, 766, 743, 776, 806, 753, 775, 766, 730, 782, 774, 784, 775, 748, 771, 719, 763, 764, 749, 770, 761, 748, 776, 730, 763, 774, 746, 783, 777, 794, 777, 704, 722, 780, 774, 782, 774, 766, 750, 771, 774, 784, 810, 759, 745, 733, 829, 777, 753, 790, 802, 782, 798, 775, 777, 813, 770, 749, 782] payload_rows_per_dst=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] dup_rows_per_dst=[2984, 3000, 2906, 2982, 2966, 2978, 3050, 3062] combine_rows_per_src=[9202, 9324, 9240, 9154, 9256, 9444, 9326, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=345632 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267142688 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533939744 weight_movement_bytes=34603008 total_nvlink_bytes=568542752 weight_to_token_ratio=0.064807 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=65022 dup_rows=23972 dup_rows_remote=20966 dedup_ratio=0.243856 router_tokens_per_expert=[755, 789, 749, 770, 783, 780, 754, 778, 777, 749, 813, 750, 766, 795, 736, 772, 786, 794, 759, 765, 779, 810, 729, 756, 797, 719, 776, 761, 749, 768, 734, 795, 762, 788, 748, 747, 741, 771, 727, 755, 749, 770, 756, 743, 786, 760, 756, 778, 784, 794, 790, 745, 766, 809, 787, 773, 802, 760, 800, 717, 761, 748, 810, 776] payload_rows_per_dst=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] dup_rows_per_dst=[3052, 3006, 3054, 2938, 3014, 2916, 3092, 2900] combine_rows_per_src=[9264, 9310, 9302, 9260, 9064, 9280, 9404, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64940 dup_rows=23894 dup_rows_remote=20806 dedup_ratio=0.243062 router_tokens_per_expert=[760, 737, 739, 778, 775, 791, 785, 812, 793, 801, 746, 735, 780, 734, 739, 750, 812, 772, 767, 781, 762, 809, 801, 775, 781, 724, 752, 739, 790, 726, 798, 735, 760, 813, 766, 730, 819, 754, 764, 777, 774, 697, 763, 801, 784, 774, 806, 736, 738, 751, 803, 757, 751, 783, 739, 781, 724, 794, 803, 729, 826, 733, 778, 765] payload_rows_per_dst=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] dup_rows_per_dst=[3094, 2994, 3088, 2870, 3042, 2966, 2842, 2998] combine_rows_per_src=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74166 payload_rows_remote=64872 dup_rows=24138 dup_rows_remote=21054 dedup_ratio=0.245544 router_tokens_per_expert=[776, 760, 779, 804, 694, 785, 746, 820, 786, 745, 765, 764, 802, 736, 751, 791, 736, 795, 758, 795, 786, 810, 787, 827, 722, 734, 826, 736, 752, 766, 756, 733, 800, 795, 749, 770, 730, 801, 780, 764, 773, 749, 730, 784, 734, 763, 787, 724, 730, 789, 804, 722, 731, 795, 809, 754, 766, 788, 755, 775, 747, 768, 788, 775] payload_rows_per_dst=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] dup_rows_per_dst=[3046, 3042, 3096, 2860, 3084, 2844, 3022, 3144] combine_rows_per_src=[9282, 9238, 9492, 9190, 9294, 9244, 9246, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74472 payload_rows_remote=65082 dup_rows=23832 dup_rows_remote=20870 dedup_ratio=0.242432 router_tokens_per_expert=[766, 783, 747, 768, 758, 747, 808, 791, 743, 755, 771, 778, 738, 778, 732, 828, 761, 768, 742, 759, 732, 788, 795, 741, 797, 764, 746, 745, 762, 775, 712, 775, 762, 787, 792, 810, 750, 771, 740, 776, 773, 832, 786, 735, 775, 760, 779, 736, 875, 773, 732, 764, 739, 745, 748, 814, 791, 826, 793, 710, 782, 723, 775, 745] payload_rows_per_dst=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] dup_rows_per_dst=[2944, 2926, 2964, 2928, 3014, 2962, 3072, 3022] combine_rows_per_src=[9392, 9320, 9208, 9224, 9362, 9390, 9308, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266747904 dispatch_meta_bytes=343400 combine_payload_bytes=266747904 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267091304 ingress_bytes=335953920 link_bytes=335953920 token_bytes=533839208 weight_movement_bytes=69206016 total_nvlink_bytes=603045224 weight_to_token_ratio=0.129638 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264167554e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64974 dup_rows=24098 dup_rows_remote=21116 dedup_ratio=0.245138 router_tokens_per_expert=[783, 764, 748, 753, 719, 804, 768, 793, 772, 831, 755, 751, 744, 717, 775, 769, 719, 748, 770, 773, 770, 748, 789, 790, 837, 761, 762, 838, 771, 702, 750, 740, 774, 749, 768, 762, 730, 772, 828, 742, 795, 750, 790, 787, 803, 748, 751, 722, 796, 779, 796, 780, 787, 771, 707, 778, 768, 742, 810, 787, 771, 793, 717, 785] payload_rows_per_dst=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] dup_rows_per_dst=[3016, 2908, 2982, 3072, 3052, 3044, 3008, 3016] combine_rows_per_src=[9248, 9320, 9232, 9250, 9198, 9248, 9380, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344656 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266789456 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533234256 weight_movement_bytes=138412032 total_nvlink_bytes=671646288 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65076 dup_rows=23768 dup_rows_remote=20656 dedup_ratio=0.241781 router_tokens_per_expert=[807, 811, 759, 743, 750, 793, 773, 753, 771, 763, 761, 774, 794, 787, 756, 782, 786, 769, 708, 750, 757, 709, 793, 788, 729, 747, 749, 800, 780, 714, 760, 808, 726, 762, 755, 774, 774, 743, 821, 773, 740, 797, 750, 811, 757, 758, 727, 782, 780, 772, 794, 747, 778, 761, 722, 809, 780, 759, 771, 782, 774, 803, 745, 801] payload_rows_per_dst=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] dup_rows_per_dst=[3112, 3236, 2830, 2860, 2978, 3146, 2664, 2942] combine_rows_per_src=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=343952 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266674064 ingress_bytes=300933120 link_bytes=300933120 token_bytes=533004176 weight_movement_bytes=34603008 total_nvlink_bytes=567607184 weight_to_token_ratio=0.064921 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132387103e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343712 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788512 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233312 weight_movement_bytes=69206016 total_nvlink_bytes=602439328 weight_to_token_ratio=0.129786 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65364 dup_rows=23818 dup_rows_remote=20956 dedup_ratio=0.242289 router_tokens_per_expert=[815, 777, 766, 765, 790, 766, 770, 784, 807, 783, 744, 747, 767, 805, 781, 761, 768, 764, 771, 763, 767, 761, 761, 731, 768, 791, 741, 761, 797, 780, 762, 792, 815, 731, 770, 761, 734, 805, 775, 805, 723, 734, 790, 764, 700, 775, 763, 743, 734, 780, 761, 743, 754, 761, 776, 777, 785, 786, 810, 777, 784, 717, 771, 742] payload_rows_per_dst=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] dup_rows_per_dst=[3014, 3088, 2880, 2984, 2990, 2862, 2998, 3002] combine_rows_per_src=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=342984 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337224 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331464 weight_movement_bytes=34603008 total_nvlink_bytes=566934472 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343704 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266059416 ingress_bytes=334921728 link_bytes=334921728 token_bytes=531775128 weight_movement_bytes=69206016 total_nvlink_bytes=600981144 weight_to_token_ratio=0.130142 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260283499e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266575872 dispatch_meta_bytes=343808 combine_payload_bytes=266575872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266919680 ingress_bytes=301178880 link_bytes=301178880 token_bytes=533495552 weight_movement_bytes=34603008 total_nvlink_bytes=568098560 weight_to_token_ratio=0.064861 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133311879e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64828 dup_rows=24062 dup_rows_remote=20938 dedup_ratio=0.244771 router_tokens_per_expert=[817, 760, 789, 813, 758, 773, 714, 784, 792, 743, 757, 781, 740, 792, 767, 747, 750, 767, 722, 712, 740, 757, 713, 786, 725, 795, 821, 790, 784, 747, 842, 765, 774, 736, 770, 802, 748, 757, 753, 794, 748, 768, 753, 789, 741, 723, 781, 794, 765, 763, 744, 783, 769, 766, 792, 756, 732, 801, 757, 800, 791, 778, 819, 762] payload_rows_per_dst=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] dup_rows_per_dst=[3124, 3008, 2792, 3124, 2994, 2958, 3026, 3036] combine_rows_per_src=[9292, 9230, 9102, 9414, 9274, 9236, 9250, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344360 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266477864 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611368 weight_movement_bytes=34603008 total_nvlink_bytes=567214376 weight_to_token_ratio=0.064969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=342928 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266894224 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533445520 weight_movement_bytes=69206016 total_nvlink_bytes=602651536 weight_to_token_ratio=0.129734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343064 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265878552 ingress_bytes=369344512 link_bytes=369344512 token_bytes=531414040 weight_movement_bytes=103809024 total_nvlink_bytes=635223064 weight_to_token_ratio=0.195345 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389813664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74312 payload_rows_remote=65086 dup_rows=23992 dup_rows_remote=20908 dedup_ratio=0.244059 router_tokens_per_expert=[782, 742, 758, 727, 734, 797, 766, 758, 787, 774, 746, 753, 728, 777, 783, 772, 762, 752, 802, 728, 792, 734, 761, 811, 768, 811, 815, 750, 775, 741, 738, 791, 759, 760, 790, 771, 785, 748, 750, 750, 718, 794, 762, 753, 760, 794, 791, 781, 728, 785, 808, 807, 780, 766, 752, 808, 763, 776, 754, 750, 750, 786, 824, 734] payload_rows_per_dst=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] dup_rows_per_dst=[2566, 2920, 3334, 3092, 2840, 2940, 3216, 3084] combine_rows_per_src=[8870, 9320, 9642, 9250, 9144, 9366, 9494, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64826 dup_rows=24068 dup_rows_remote=21042 dedup_ratio=0.244832 router_tokens_per_expert=[821, 712, 726, 800, 716, 789, 772, 807, 757, 739, 790, 719, 734, 724, 796, 751, 776, 755, 743, 769, 764, 767, 733, 782, 839, 764, 756, 773, 751, 761, 784, 770, 753, 777, 771, 786, 746, 765, 786, 771, 763, 728, 823, 774, 768, 715, 800, 816, 807, 759, 779, 766, 745, 747, 790, 825, 728, 766, 794, 795, 772, 770, 770, 757] payload_rows_per_dst=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] dup_rows_per_dst=[2974, 2846, 2960, 3034, 3012, 3168, 3026, 3048] combine_rows_per_src=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65076 dup_rows=23768 dup_rows_remote=20656 dedup_ratio=0.241781 router_tokens_per_expert=[807, 811, 759, 743, 750, 793, 773, 753, 771, 763, 761, 774, 794, 787, 756, 782, 786, 769, 708, 750, 757, 709, 793, 788, 729, 747, 749, 800, 780, 714, 760, 808, 726, 762, 755, 774, 774, 743, 821, 773, 740, 797, 750, 811, 757, 758, 727, 782, 780, 772, 794, 747, 778, 761, 722, 809, 780, 759, 771, 782, 774, 803, 745, 801] payload_rows_per_dst=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] dup_rows_per_dst=[3112, 3236, 2830, 2860, 2978, 3146, 2664, 2942] combine_rows_per_src=[9460, 9442, 9290, 9124, 9274, 9610, 9150, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65116 dup_rows=23934 dup_rows_remote=20916 dedup_ratio=0.243469 router_tokens_per_expert=[777, 758, 728, 736, 788, 774, 810, 796, 740, 746, 747, 808, 812, 773, 743, 767, 796, 762, 778, 759, 739, 734, 711, 752, 807, 785, 762, 777, 824, 796, 727, 766, 763, 793, 775, 752, 773, 780, 781, 808, 776, 774, 758, 769, 727, 786, 780, 774, 734, 748, 776, 801, 795, 745, 749, 751, 705, 776, 770, 806, 795, 750, 790, 714] payload_rows_per_dst=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] dup_rows_per_dst=[2962, 3018, 2858, 3128, 3118, 2938, 2970, 2942] combine_rows_per_src=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74486 payload_rows_remote=65364 dup_rows=23818 dup_rows_remote=20956 dedup_ratio=0.242289 router_tokens_per_expert=[815, 777, 766, 765, 790, 766, 770, 784, 807, 783, 744, 747, 767, 805, 781, 761, 768, 764, 771, 763, 767, 761, 761, 731, 768, 791, 741, 761, 797, 780, 762, 792, 815, 731, 770, 761, 734, 805, 775, 805, 723, 734, 790, 764, 700, 775, 763, 743, 734, 780, 761, 743, 754, 761, 776, 777, 785, 786, 810, 777, 784, 717, 771, 742] payload_rows_per_dst=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] dup_rows_per_dst=[3014, 3088, 2880, 2984, 2990, 2862, 2998, 3002] combine_rows_per_src=[9452, 9302, 9292, 9400, 9402, 9122, 9174, 9342] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21056 dedup_ratio=0.245138 router_tokens_per_expert=[755, 785, 784, 784, 732, 811, 770, 799, 772, 782, 776, 766, 740, 774, 770, 775, 769, 794, 752, 780, 772, 786, 741, 778, 742, 763, 768, 693, 771, 792, 763, 749, 740, 786, 797, 767, 768, 755, 750, 778, 754, 718, 772, 770, 774, 764, 735, 779, 746, 791, 781, 766, 810, 810, 792, 750, 762, 783, 740, 793, 772, 721, 724, 816] payload_rows_per_dst=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] dup_rows_per_dst=[3100, 2970, 2970, 2964, 3042, 2938, 3118, 2996] combine_rows_per_src=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=345280 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268076224 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535807168 weight_movement_bytes=34603008 total_nvlink_bytes=570410176 weight_to_token_ratio=0.064581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265535488 dispatch_meta_bytes=343064 combine_payload_bytes=265535488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265878552 ingress_bytes=369344512 link_bytes=369344512 token_bytes=531414040 weight_movement_bytes=103809024 total_nvlink_bytes=635223064 weight_to_token_ratio=0.195345 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389813664e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=343976 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266936232 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533528488 weight_movement_bytes=69206016 total_nvlink_bytes=602734504 weight_to_token_ratio=0.129714 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65116 dup_rows=23934 dup_rows_remote=20916 dedup_ratio=0.243469 router_tokens_per_expert=[777, 758, 728, 736, 788, 774, 810, 796, 740, 746, 747, 808, 812, 773, 743, 767, 796, 762, 778, 759, 739, 734, 711, 752, 807, 785, 762, 777, 824, 796, 727, 766, 763, 793, 775, 752, 773, 780, 781, 808, 776, 774, 758, 769, 727, 786, 780, 774, 734, 748, 776, 801, 795, 745, 749, 751, 705, 776, 770, 806, 795, 750, 790, 714] payload_rows_per_dst=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] dup_rows_per_dst=[2962, 3018, 2858, 3128, 3118, 2938, 2970, 2942] combine_rows_per_src=[9372, 9254, 9204, 9360, 9332, 9350, 9228, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=342928 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266894224 ingress_bytes=335757312 link_bytes=335757312 token_bytes=533445520 weight_movement_bytes=69206016 total_nvlink_bytes=602651536 weight_to_token_ratio=0.129734 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263427734e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65142 dup_rows=23868 dup_rows_remote=20966 dedup_ratio=0.242798 router_tokens_per_expert=[766, 791, 752, 796, 766, 771, 733, 768, 752, 733, 785, 772, 751, 787, 832, 760, 834, 716, 779, 741, 717, 797, 767, 747, 761, 782, 737, 762, 744, 797, 762, 750, 764, 756, 785, 754, 742, 799, 761, 818, 777, 801, 772, 769, 815, 740, 769, 734, 790, 756, 791, 790, 754, 762, 769, 761, 729, 787, 785, 753, 724, 771, 745, 821] payload_rows_per_dst=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] dup_rows_per_dst=[2944, 2998, 2902, 2960, 3058, 3054, 3004, 2948] combine_rows_per_src=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65050 dup_rows=24060 dup_rows_remote=21114 dedup_ratio=0.244751 router_tokens_per_expert=[781, 748, 782, 763, 772, 783, 804, 810, 797, 716, 742, 713, 811, 740, 758, 759, 744, 774, 739, 715, 792, 773, 740, 757, 778, 764, 758, 719, 760, 767, 788, 789, 800, 790, 767, 789, 811, 799, 759, 743, 748, 769, 760, 794, 787, 739, 762, 771, 777, 748, 754, 769, 762, 744, 772, 744, 817, 766, 745, 766, 794, 819, 784, 767] payload_rows_per_dst=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] dup_rows_per_dst=[3146, 2906, 2862, 2942, 3120, 3032, 2946, 3106] combine_rows_per_src=[9340, 9166, 9206, 9304, 9396, 9228, 9194, 9410] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64776 dup_rows=24184 dup_rows_remote=21108 dedup_ratio=0.246012 router_tokens_per_expert=[745, 765, 778, 753, 783, 747, 726, 773, 752, 797, 783, 769, 787, 753, 735, 759, 777, 755, 779, 799, 812, 763, 759, 766, 742, 805, 754, 741, 781, 786, 756, 806, 732, 741, 753, 744, 767, 781, 774, 822, 789, 780, 753, 764, 735, 795, 728, 751, 787, 701, 803, 789, 779, 791, 750, 743, 801, 820, 772, 758, 769, 776, 764, 754] payload_rows_per_dst=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] dup_rows_per_dst=[2944, 3052, 3076, 3090, 3022, 2964, 2966, 3070] combine_rows_per_src=[9196, 9218, 9344, 9252, 9206, 9226, 9320, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64956 dup_rows=23940 dup_rows_remote=20828 dedup_ratio=0.243530 router_tokens_per_expert=[770, 768, 755, 735, 798, 783, 742, 747, 802, 805, 741, 788, 757, 795, 759, 750, 729, 767, 752, 764, 802, 724, 797, 759, 765, 774, 795, 800, 759, 781, 761, 816, 797, 777, 739, 809, 785, 759, 815, 742, 789, 769, 711, 808, 742, 791, 725, 801, 802, 772, 750, 737, 751, 779, 738, 789, 739, 687, 764, 793, 783, 753, 757, 759] payload_rows_per_dst=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] dup_rows_per_dst=[3112, 3114, 2896, 3124, 3070, 2980, 2804, 2840] combine_rows_per_src=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343472 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265870768 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398064 weight_movement_bytes=69206016 total_nvlink_bytes=600604080 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=64966 dup_rows=24098 dup_rows_remote=21056 dedup_ratio=0.245138 router_tokens_per_expert=[755, 785, 784, 784, 732, 811, 770, 799, 772, 782, 776, 766, 740, 774, 770, 775, 769, 794, 752, 780, 772, 786, 741, 778, 742, 763, 768, 693, 771, 792, 763, 749, 740, 786, 797, 767, 768, 755, 750, 778, 754, 718, 772, 770, 774, 764, 735, 779, 746, 791, 781, 766, 810, 810, 792, 750, 762, 783, 740, 793, 772, 721, 724, 816] payload_rows_per_dst=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] dup_rows_per_dst=[3100, 2970, 2970, 2964, 3042, 2938, 3118, 2996] combine_rows_per_src=[9340, 9340, 9374, 9118, 9240, 9194, 9374, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266715136 dispatch_meta_bytes=344128 combine_payload_bytes=266715136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267059264 ingress_bytes=301318144 link_bytes=301318144 token_bytes=533774400 weight_movement_bytes=34603008 total_nvlink_bytes=568377408 weight_to_token_ratio=0.064827 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133835918e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65076 dup_rows=24090 dup_rows_remote=21166 dedup_ratio=0.245056 router_tokens_per_expert=[715, 811, 745, 783, 730, 758, 755, 800, 776, 765, 789, 773, 757, 762, 775, 739, 749, 769, 792, 797, 780, 770, 804, 767, 712, 798, 755, 780, 735, 744, 737, 770, 781, 775, 750, 725, 789, 745, 773, 756, 793, 759, 771, 786, 784, 753, 793, 766, 820, 719, 787, 769, 775, 716, 809, 787, 761, 776, 766, 794, 746, 771, 800, 765] payload_rows_per_dst=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] dup_rows_per_dst=[2996, 2988, 3048, 2924, 3006, 3036, 3022, 3070] combine_rows_per_src=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267730944 dispatch_meta_bytes=345280 combine_payload_bytes=267730944 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268076224 ingress_bytes=302333952 link_bytes=302333952 token_bytes=535807168 weight_movement_bytes=34603008 total_nvlink_bytes=570410176 weight_to_token_ratio=0.064581 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137658321e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344432 combine_payload_bytes=266821632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267166064 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533987696 weight_movement_bytes=69206016 total_nvlink_bytes=603193712 weight_to_token_ratio=0.129602 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65260 dup_rows=23648 dup_rows_remote=20748 dedup_ratio=0.240560 router_tokens_per_expert=[781, 760, 746, 750, 762, 792, 761, 776, 765, 802, 733, 760, 764, 762, 707, 758, 749, 745, 740, 789, 779, 791, 756, 769, 800, 757, 795, 804, 786, 749, 758, 718, 800, 791, 761, 832, 817, 764, 778, 776, 759, 790, 766, 764, 736, 765, 772, 796, 785, 737, 747, 763, 771, 799, 795, 771, 767, 744, 770, 746, 791, 747, 769, 719] payload_rows_per_dst=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] dup_rows_per_dst=[2962, 2908, 3006, 2946, 3124, 2900, 3022, 2780] combine_rows_per_src=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=344656 combine_payload_bytes=266444800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266789456 ingress_bytes=404856832 link_bytes=404856832 token_bytes=533234256 weight_movement_bytes=138412032 total_nvlink_bytes=671646288 weight_to_token_ratio=0.259571 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523443665e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266100736 dispatch_meta_bytes=344088 combine_payload_bytes=266100736 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266444824 ingress_bytes=335306752 link_bytes=335306752 token_bytes=532545560 weight_movement_bytes=69206016 total_nvlink_bytes=601751576 weight_to_token_ratio=0.129953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261732313e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343536 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265666032 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988528 weight_movement_bytes=69206016 total_nvlink_bytes=600194544 weight_to_token_ratio=0.130334 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343136 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402912 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462688 weight_movement_bytes=34603008 total_nvlink_bytes=567065696 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344968 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266896264 ingress_bytes=370360320 link_bytes=370360320 token_bytes=533447560 weight_movement_bytes=103809024 total_nvlink_bytes=637256584 weight_to_token_ratio=0.194600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393636068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=65076 dup_rows=24090 dup_rows_remote=21166 dedup_ratio=0.245056 router_tokens_per_expert=[715, 811, 745, 783, 730, 758, 755, 800, 776, 765, 789, 773, 757, 762, 775, 739, 749, 769, 792, 797, 780, 770, 804, 767, 712, 798, 755, 780, 735, 744, 737, 770, 781, 775, 750, 725, 789, 745, 773, 756, 793, 759, 771, 786, 784, 753, 793, 766, 820, 719, 787, 769, 775, 716, 809, 787, 761, 776, 766, 794, 746, 771, 800, 765] payload_rows_per_dst=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] dup_rows_per_dst=[2996, 2988, 3048, 2924, 3006, 3036, 3022, 3070] combine_rows_per_src=[9198, 9284, 9408, 9138, 9182, 9374, 9342, 9288] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64700 dup_rows=24122 dup_rows_remote=20840 dedup_ratio=0.245382 router_tokens_per_expert=[787, 786, 777, 760, 741, 779, 801, 715, 763, 777, 798, 767, 757, 772, 794, 768, 761, 758, 727, 847, 786, 765, 761, 780, 772, 751, 774, 783, 763, 815, 754, 795, 750, 767, 746, 723, 789, 836, 767, 788, 753, 739, 731, 810, 744, 746, 735, 782, 730, 745, 775, 774, 730, 796, 785, 723, 758, 777, 764, 794, 773, 790, 742, 756] payload_rows_per_dst=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] dup_rows_per_dst=[3004, 3012, 3048, 3060, 2924, 2882, 2910, 3282] combine_rows_per_src=[9288, 9186, 9252, 9338, 9146, 9216, 9274, 9482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=64956 dup_rows=23940 dup_rows_remote=20828 dedup_ratio=0.243530 router_tokens_per_expert=[770, 768, 755, 735, 798, 783, 742, 747, 802, 805, 741, 788, 757, 795, 759, 750, 729, 767, 752, 764, 802, 724, 797, 759, 765, 774, 795, 800, 759, 781, 761, 816, 797, 777, 739, 809, 785, 759, 815, 742, 789, 769, 711, 808, 742, 791, 725, 801, 802, 772, 750, 737, 751, 779, 738, 789, 739, 687, 764, 793, 783, 753, 757, 759] payload_rows_per_dst=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] dup_rows_per_dst=[3112, 3114, 2896, 3124, 3070, 2980, 2804, 2840] combine_rows_per_src=[9408, 9324, 9292, 9334, 9376, 9292, 9108, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65036 dup_rows=23792 dup_rows_remote=20600 dedup_ratio=0.242025 router_tokens_per_expert=[769, 760, 764, 766, 803, 760, 768, 743, 773, 753, 788, 771, 763, 747, 740, 747, 813, 798, 770, 726, 767, 769, 763, 786, 815, 772, 745, 759, 798, 793, 764, 807, 720, 738, 762, 790, 743, 773, 736, 752, 767, 772, 734, 772, 761, 751, 743, 722, 791, 798, 790, 821, 807, 786, 792, 749, 784, 796, 787, 790, 735, 702, 760, 768] payload_rows_per_dst=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] dup_rows_per_dst=[2976, 2830, 2972, 3088, 2902, 2836, 3192, 2996] combine_rows_per_src=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65004 dup_rows=23918 dup_rows_remote=20774 dedup_ratio=0.243306 router_tokens_per_expert=[756, 747, 768, 786, 807, 754, 766, 787, 815, 783, 803, 741, 768, 740, 822, 791, 744, 765, 779, 774, 735, 772, 726, 734, 737, 743, 762, 744, 809, 754, 773, 757, 758, 753, 770, 741, 721, 737, 779, 801, 785, 785, 773, 792, 830, 796, 744, 740, 762, 764, 744, 766, 788, 736, 760, 772, 743, 809, 777, 756, 816, 773, 751, 788] payload_rows_per_dst=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] dup_rows_per_dst=[3070, 3144, 2844, 2932, 2876, 3106, 2952, 2994] combine_rows_per_src=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65260 dup_rows=23648 dup_rows_remote=20748 dedup_ratio=0.240560 router_tokens_per_expert=[781, 760, 746, 750, 762, 792, 761, 776, 765, 802, 733, 760, 764, 762, 707, 758, 749, 745, 740, 789, 779, 791, 756, 769, 800, 757, 795, 804, 786, 749, 758, 718, 800, 791, 761, 832, 817, 764, 778, 776, 759, 790, 766, 764, 736, 765, 772, 796, 785, 737, 747, 763, 771, 799, 795, 771, 767, 744, 770, 746, 791, 747, 769, 719] payload_rows_per_dst=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] dup_rows_per_dst=[2962, 2908, 3006, 2946, 3124, 2900, 3022, 2780] combine_rows_per_src=[9294, 9194, 9230, 9388, 9514, 9396, 9314, 9326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=344032 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648992 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534953952 weight_movement_bytes=34603008 total_nvlink_bytes=569556960 weight_to_token_ratio=0.064684 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266551296 dispatch_meta_bytes=344968 combine_payload_bytes=266551296 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266896264 ingress_bytes=370360320 link_bytes=370360320 token_bytes=533447560 weight_movement_bytes=103809024 total_nvlink_bytes=637256584 weight_to_token_ratio=0.194600 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393636068e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265011200 dispatch_meta_bytes=342160 combine_payload_bytes=265011200 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265353360 ingress_bytes=299614208 link_bytes=299614208 token_bytes=530364560 weight_movement_bytes=34603008 total_nvlink_bytes=564967568 weight_to_token_ratio=0.065244 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127424144e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65004 dup_rows=23918 dup_rows_remote=20774 dedup_ratio=0.243306 router_tokens_per_expert=[756, 747, 768, 786, 807, 754, 766, 787, 815, 783, 803, 741, 768, 740, 822, 791, 744, 765, 779, 774, 735, 772, 726, 734, 737, 743, 762, 744, 809, 754, 773, 757, 758, 753, 770, 741, 721, 737, 779, 801, 785, 785, 773, 792, 830, 796, 744, 740, 762, 764, 744, 766, 788, 736, 760, 772, 743, 809, 777, 756, 816, 773, 751, 788] payload_rows_per_dst=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] dup_rows_per_dst=[3070, 3144, 2844, 2932, 2876, 3106, 2952, 2994] combine_rows_per_src=[9272, 9382, 9214, 9226, 9244, 9384, 9232, 9432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343136 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402912 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462688 weight_movement_bytes=34603008 total_nvlink_bytes=567065696 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65188 dup_rows=23846 dup_rows_remote=20850 dedup_ratio=0.242574 router_tokens_per_expert=[711, 767, 785, 753, 745, 777, 766, 816, 719, 733, 788, 806, 782, 764, 758, 761, 769, 773, 798, 774, 755, 735, 759, 770, 762, 797, 766, 800, 799, 746, 754, 781, 822, 801, 741, 768, 756, 727, 769, 761, 788, 792, 804, 732, 724, 804, 733, 755, 768, 715, 750, 766, 757, 787, 776, 758, 811, 803, 758, 724, 785, 758, 770, 820] payload_rows_per_dst=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] dup_rows_per_dst=[3024, 2932, 2996, 3060, 3024, 2910, 2812, 3088] combine_rows_per_src=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64826 dup_rows=24068 dup_rows_remote=21042 dedup_ratio=0.244832 router_tokens_per_expert=[821, 712, 726, 800, 716, 789, 772, 807, 757, 739, 790, 719, 734, 724, 796, 751, 776, 755, 743, 769, 764, 767, 733, 782, 839, 764, 756, 773, 751, 761, 784, 770, 753, 777, 771, 786, 746, 765, 786, 771, 763, 728, 823, 774, 768, 715, 800, 816, 807, 759, 779, 766, 745, 747, 790, 825, 728, 766, 794, 795, 772, 770, 770, 757] payload_rows_per_dst=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] dup_rows_per_dst=[2974, 2846, 2960, 3034, 3012, 3168, 3026, 3048] combine_rows_per_src=[9312, 9174, 9218, 9362, 9298, 9206, 9410, 9256] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74130 payload_rows_remote=64812 dup_rows=24174 dup_rows_remote=21036 dedup_ratio=0.245911 router_tokens_per_expert=[796, 785, 781, 764, 730, 720, 766, 732, 780, 753, 771, 764, 765, 789, 728, 780, 750, 831, 740, 772, 771, 809, 807, 763, 778, 780, 786, 759, 772, 756, 737, 731, 802, 791, 788, 728, 781, 789, 760, 789, 765, 787, 778, 751, 724, 809, 773, 783, 755, 793, 766, 750, 746, 774, 766, 805, 736, 747, 732, 770, 786, 784, 756, 742] payload_rows_per_dst=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] dup_rows_per_dst=[2964, 3096, 3090, 2980, 3138, 2960, 2978, 2968] combine_rows_per_src=[9184, 9164, 9396, 9218, 9318, 9380, 9332, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266730000 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117456 weight_movement_bytes=69206016 total_nvlink_bytes=602323472 weight_to_token_ratio=0.129814 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64944 dup_rows=23974 dup_rows_remote=20822 dedup_ratio=0.243876 router_tokens_per_expert=[735, 799, 798, 776, 735, 752, 785, 765, 698, 718, 764, 805, 743, 787, 748, 788, 740, 778, 787, 762, 749, 772, 763, 711, 806, 803, 835, 790, 795, 764, 748, 788, 799, 774, 732, 763, 758, 795, 776, 735, 757, 806, 758, 739, 765, 779, 754, 800, 800, 735, 791, 761, 737, 809, 782, 745, 733, 793, 799, 753, 815, 728, 743, 751] payload_rows_per_dst=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] dup_rows_per_dst=[3152, 2906, 3182, 3242, 2836, 2682, 3044, 2930] combine_rows_per_src=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343112 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266599496 ingress_bytes=335462400 link_bytes=335462400 token_bytes=532855880 weight_movement_bytes=69206016 total_nvlink_bytes=602061896 weight_to_token_ratio=0.129878 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262318004e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=64940 dup_rows=23894 dup_rows_remote=20806 dedup_ratio=0.243062 router_tokens_per_expert=[760, 737, 739, 778, 775, 791, 785, 812, 793, 801, 746, 735, 780, 734, 739, 750, 812, 772, 767, 781, 762, 809, 801, 775, 781, 724, 752, 739, 790, 726, 798, 735, 760, 813, 766, 730, 819, 754, 764, 777, 774, 697, 763, 801, 784, 774, 806, 736, 738, 751, 803, 757, 751, 783, 739, 781, 724, 794, 803, 729, 826, 733, 778, 765] payload_rows_per_dst=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] dup_rows_per_dst=[3094, 2994, 3088, 2870, 3042, 2966, 2842, 2998] combine_rows_per_src=[9260, 9162, 9470, 9220, 9324, 9304, 9364, 9306] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64820 dup_rows=24198 dup_rows_remote=21078 dedup_ratio=0.246155 router_tokens_per_expert=[788, 760, 758, 745, 779, 766, 721, 726, 803, 802, 770, 774, 805, 755, 735, 740, 726, 745, 740, 739, 748, 793, 799, 770, 787, 772, 735, 779, 775, 781, 788, 786, 755, 797, 820, 764, 790, 783, 767, 793, 796, 774, 793, 768, 808, 828, 752, 734, 749, 726, 781, 728, 764, 789, 765, 759, 734, 745, 739, 800, 757, 751, 772, 781] payload_rows_per_dst=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] dup_rows_per_dst=[2932, 3032, 2970, 3120, 3242, 3112, 2830, 2960] combine_rows_per_src=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=344032 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267648992 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534953952 weight_movement_bytes=34603008 total_nvlink_bytes=569556960 weight_to_token_ratio=0.064684 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344152 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354200 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364248 weight_movement_bytes=34603008 total_nvlink_bytes=568967256 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64958 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[768, 775, 774, 759, 828, 746, 752, 804, 765, 789, 755, 780, 778, 782, 779, 762, 719, 740, 767, 830, 787, 748, 761, 747, 758, 770, 809, 766, 788, 745, 732, 781, 815, 769, 730, 778, 778, 766, 749, 739, 796, 758, 761, 761, 754, 748, 796, 760, 764, 776, 772, 799, 771, 746, 795, 727, 756, 808, 735, 782, 751, 765, 734, 769] payload_rows_per_dst=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] dup_rows_per_dst=[3042, 3004, 2994, 3132, 2932, 2902, 2990, 2984] combine_rows_per_src=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343472 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265870768 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398064 weight_movement_bytes=69206016 total_nvlink_bytes=600604080 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265469952 dispatch_meta_bytes=343392 combine_payload_bytes=265469952 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265813344 ingress_bytes=334675968 link_bytes=334675968 token_bytes=531283296 weight_movement_bytes=69206016 total_nvlink_bytes=600489312 weight_to_token_ratio=0.130262 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259358724e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343064 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353688 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364312 weight_movement_bytes=69206016 total_nvlink_bytes=601570328 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=342984 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266337224 ingress_bytes=300597248 link_bytes=300597248 token_bytes=532331464 weight_movement_bytes=34603008 total_nvlink_bytes=566934472 weight_to_token_ratio=0.065003 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131123244e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343592 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846312 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349032 weight_movement_bytes=103809024 total_nvlink_bytes=635158056 weight_to_token_ratio=0.195369 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74106 payload_rows_remote=64820 dup_rows=24198 dup_rows_remote=21078 dedup_ratio=0.246155 router_tokens_per_expert=[788, 760, 758, 745, 779, 766, 721, 726, 803, 802, 770, 774, 805, 755, 735, 740, 726, 745, 740, 739, 748, 793, 799, 770, 787, 772, 735, 779, 775, 781, 788, 786, 755, 797, 820, 764, 790, 783, 767, 793, 796, 774, 793, 768, 808, 828, 752, 734, 749, 726, 781, 728, 764, 789, 765, 759, 734, 745, 739, 800, 757, 751, 772, 781] payload_rows_per_dst=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] dup_rows_per_dst=[2932, 3032, 2970, 3120, 3242, 3112, 2830, 2960] combine_rows_per_src=[9154, 9336, 9150, 9286, 9296, 9394, 9292, 9198] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74386 payload_rows_remote=65034 dup_rows=23918 dup_rows_remote=21022 dedup_ratio=0.243306 router_tokens_per_expert=[783, 756, 776, 724, 779, 787, 813, 777, 754, 785, 730, 771, 798, 771, 812, 739, 775, 818, 754, 703, 769, 833, 769, 768, 755, 724, 747, 770, 774, 773, 785, 766, 774, 812, 776, 735, 786, 801, 756, 727, 795, 759, 743, 766, 743, 798, 733, 778, 763, 756, 772, 748, 780, 767, 771, 766, 773, 760, 743, 780, 753, 768, 788, 744] payload_rows_per_dst=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] dup_rows_per_dst=[2888, 3176, 3012, 2616, 3076, 3026, 3228, 2896] combine_rows_per_src=[9150, 9496, 9366, 8730, 9424, 9174, 9694, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64930 dup_rows=24192 dup_rows_remote=21150 dedup_ratio=0.246094 router_tokens_per_expert=[776, 805, 800, 782, 772, 828, 767, 711, 739, 741, 754, 786, 766, 779, 747, 750, 798, 777, 755, 720, 760, 769, 760, 784, 743, 774, 770, 764, 756, 784, 805, 785, 760, 762, 721, 779, 736, 767, 775, 744, 789, 758, 743, 779, 785, 837, 816, 735, 759, 765, 750, 765, 800, 736, 753, 784, 760, 771, 767, 760, 728, 785, 796, 780] payload_rows_per_dst=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] dup_rows_per_dst=[3084, 2962, 3060, 3084, 2876, 3082, 3042, 3002] combine_rows_per_src=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=64944 dup_rows=23974 dup_rows_remote=20822 dedup_ratio=0.243876 router_tokens_per_expert=[735, 799, 798, 776, 735, 752, 785, 765, 698, 718, 764, 805, 743, 787, 748, 788, 740, 778, 787, 762, 749, 772, 763, 711, 806, 803, 835, 790, 795, 764, 748, 788, 799, 774, 732, 763, 758, 795, 776, 735, 757, 806, 758, 739, 765, 779, 754, 800, 800, 735, 791, 761, 737, 809, 782, 745, 733, 793, 799, 753, 815, 728, 743, 751] payload_rows_per_dst=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] dup_rows_per_dst=[3152, 2906, 3182, 3242, 2836, 2682, 3044, 2930] combine_rows_per_src=[9386, 9196, 9594, 9510, 9180, 8982, 9276, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64800 dup_rows=24226 dup_rows_remote=21174 dedup_ratio=0.246440 router_tokens_per_expert=[820, 790, 735, 807, 764, 753, 692, 766, 805, 776, 755, 767, 768, 791, 764, 739, 719, 785, 772, 783, 780, 743, 762, 745, 791, 824, 737, 739, 833, 760, 760, 770, 777, 738, 727, 770, 739, 815, 705, 757, 793, 809, 791, 731, 768, 806, 834, 769, 713, 728, 733, 766, 814, 769, 761, 734, 831, 748, 700, 764, 799, 771, 783, 814] payload_rows_per_dst=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] dup_rows_per_dst=[3004, 3052, 2912, 3090, 2924, 3230, 2886, 3128] combine_rows_per_src=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74324 payload_rows_remote=64958 dup_rows=23980 dup_rows_remote=21078 dedup_ratio=0.243937 router_tokens_per_expert=[768, 775, 774, 759, 828, 746, 752, 804, 765, 789, 755, 780, 778, 782, 779, 762, 719, 740, 767, 830, 787, 748, 761, 747, 758, 770, 809, 766, 788, 745, 732, 781, 815, 769, 730, 778, 778, 766, 749, 739, 796, 758, 761, 761, 754, 748, 796, 760, 764, 776, 772, 799, 771, 746, 795, 727, 756, 808, 735, 782, 751, 765, 734, 769] payload_rows_per_dst=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] dup_rows_per_dst=[3042, 3004, 2994, 3132, 2932, 2902, 2990, 2984] combine_rows_per_src=[9370, 9376, 9204, 9166, 9316, 9366, 9310, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344144 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266412112 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480080 weight_movement_bytes=69206016 total_nvlink_bytes=601686096 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343592 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265846312 ingress_bytes=369311744 link_bytes=369311744 token_bytes=531349032 weight_movement_bytes=103809024 total_nvlink_bytes=635158056 weight_to_token_ratio=0.195369 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.389690361e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344224 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266723488 ingress_bytes=335585280 link_bytes=335585280 token_bytes=533102752 weight_movement_bytes=69206016 total_nvlink_bytes=602308768 weight_to_token_ratio=0.129817 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262780392e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343064 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266353688 ingress_bytes=335216640 link_bytes=335216640 token_bytes=532364312 weight_movement_bytes=69206016 total_nvlink_bytes=601570328 weight_to_token_ratio=0.129997 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261393229e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64800 dup_rows=24226 dup_rows_remote=21174 dedup_ratio=0.246440 router_tokens_per_expert=[820, 790, 735, 807, 764, 753, 692, 766, 805, 776, 755, 767, 768, 791, 764, 739, 719, 785, 772, 783, 780, 743, 762, 745, 791, 824, 737, 739, 833, 760, 760, 770, 777, 738, 727, 770, 739, 815, 705, 757, 793, 809, 791, 731, 768, 806, 834, 769, 713, 728, 733, 766, 814, 769, 761, 734, 831, 748, 700, 764, 799, 771, 783, 814] payload_rows_per_dst=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] dup_rows_per_dst=[3004, 3052, 2912, 3090, 2924, 3230, 2886, 3128] combine_rows_per_src=[9250, 9278, 9266, 9338, 9132, 9372, 9150, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65046 dup_rows=24098 dup_rows_remote=21112 dedup_ratio=0.245138 router_tokens_per_expert=[810, 735, 796, 777, 764, 763, 771, 808, 770, 756, 762, 766, 770, 791, 748, 779, 748, 755, 756, 769, 734, 772, 800, 739, 736, 745, 766, 735, 748, 805, 793, 773, 749, 775, 742, 786, 763, 776, 733, 780, 767, 749, 762, 781, 754, 768, 778, 773, 744, 761, 752, 887, 819, 760, 729, 785, 821, 753, 773, 729, 763, 766, 805, 729] payload_rows_per_dst=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] dup_rows_per_dst=[3074, 2920, 2986, 3020, 2920, 3064, 3056, 3058] combine_rows_per_src=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65036 dup_rows=23792 dup_rows_remote=20600 dedup_ratio=0.242025 router_tokens_per_expert=[769, 760, 764, 766, 803, 760, 768, 743, 773, 753, 788, 771, 763, 747, 740, 747, 813, 798, 770, 726, 767, 769, 763, 786, 815, 772, 745, 759, 798, 793, 764, 807, 720, 738, 762, 790, 743, 773, 736, 752, 767, 772, 734, 772, 761, 751, 743, 722, 791, 798, 790, 821, 807, 786, 792, 749, 784, 796, 787, 790, 735, 702, 760, 768] payload_rows_per_dst=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] dup_rows_per_dst=[2976, 2830, 2972, 3088, 2902, 2836, 3192, 2996] combine_rows_per_src=[9290, 9334, 9412, 9418, 9126, 9208, 9476, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74378 payload_rows_remote=64968 dup_rows=23926 dup_rows_remote=20768 dedup_ratio=0.243388 router_tokens_per_expert=[723, 821, 749, 757, 756, 752, 779, 790, 744, 728, 754, 780, 784, 774, 757, 732, 750, 743, 738, 777, 752, 783, 757, 771, 780, 709, 766, 726, 785, 798, 790, 797, 754, 841, 775, 788, 814, 766, 771, 775, 798, 774, 765, 760, 734, 742, 786, 799, 726, 776, 779, 791, 764, 746, 779, 752, 809, 781, 802, 752, 784, 748, 795, 724] payload_rows_per_dst=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] dup_rows_per_dst=[3016, 2926, 2898, 3020, 3158, 2886, 2998, 3024] combine_rows_per_src=[9238, 9180, 9244, 9282, 9410, 9430, 9228, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=344320 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266297600 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250880 weight_movement_bytes=34603008 total_nvlink_bytes=566853888 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65040 dup_rows=23916 dup_rows_remote=20928 dedup_ratio=0.243286 router_tokens_per_expert=[723, 777, 776, 765, 765, 763, 802, 782, 752, 736, 777, 752, 766, 789, 778, 806, 755, 747, 787, 747, 767, 741, 737, 809, 747, 752, 788, 823, 753, 788, 734, 775, 754, 808, 772, 750, 730, 772, 769, 773, 768, 780, 771, 818, 719, 790, 773, 772, 778, 767, 752, 738, 755, 790, 810, 746, 734, 738, 712, 758, 800, 825, 772, 799] payload_rows_per_dst=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] dup_rows_per_dst=[2988, 3008, 2846, 3084, 3132, 2880, 2960, 3018] combine_rows_per_src=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343896 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764696 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185496 weight_movement_bytes=34603008 total_nvlink_bytes=565788504 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74436 payload_rows_remote=65142 dup_rows=23868 dup_rows_remote=20966 dedup_ratio=0.242798 router_tokens_per_expert=[766, 791, 752, 796, 766, 771, 733, 768, 752, 733, 785, 772, 751, 787, 832, 760, 834, 716, 779, 741, 717, 797, 767, 747, 761, 782, 737, 762, 744, 797, 762, 750, 764, 756, 785, 754, 742, 799, 761, 818, 777, 801, 772, 769, 815, 740, 769, 734, 790, 756, 791, 790, 754, 762, 769, 761, 729, 787, 785, 753, 724, 771, 745, 821] payload_rows_per_dst=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] dup_rows_per_dst=[2944, 2998, 2902, 2960, 3058, 3054, 3004, 2948] combine_rows_per_src=[9342, 9346, 9294, 9230, 9300, 9300, 9342, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344144 combine_payload_bytes=266067968 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266412112 ingress_bytes=335273984 link_bytes=335273984 token_bytes=532480080 weight_movement_bytes=69206016 total_nvlink_bytes=601686096 weight_to_token_ratio=0.129969 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261609010e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=343896 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265764696 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185496 weight_movement_bytes=34603008 total_nvlink_bytes=565788504 weight_to_token_ratio=0.065143 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344632 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266773048 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533201464 weight_movement_bytes=69206016 total_nvlink_bytes=602407480 weight_to_token_ratio=0.129793 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=342544 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266730000 ingress_bytes=335593472 link_bytes=335593472 token_bytes=533117456 weight_movement_bytes=69206016 total_nvlink_bytes=602323472 weight_to_token_ratio=0.129814 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262811218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65078 dup_rows=24016 dup_rows_remote=20998 dedup_ratio=0.244303 router_tokens_per_expert=[811, 720, 756, 768, 743, 778, 779, 791, 736, 822, 795, 795, 743, 802, 787, 798, 798, 745, 810, 734, 765, 758, 783, 795, 747, 728, 764, 744, 751, 773, 732, 799, 771, 811, 767, 777, 746, 745, 799, 763, 760, 774, 760, 759, 775, 776, 773, 737, 717, 765, 816, 793, 764, 741, 790, 715, 729, 763, 744, 749, 799, 779, 778, 767] payload_rows_per_dst=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] dup_rows_per_dst=[3028, 3152, 3020, 2844, 3130, 3018, 2900, 2924] combine_rows_per_src=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64944 dup_rows=24130 dup_rows_remote=21050 dedup_ratio=0.245463 router_tokens_per_expert=[766, 756, 717, 761, 722, 762, 810, 792, 711, 729, 780, 773, 783, 752, 810, 770, 740, 807, 825, 764, 720, 746, 775, 796, 795, 730, 774, 775, 777, 765, 759, 780, 726, 775, 777, 754, 734, 769, 802, 770, 726, 777, 785, 776, 793, 798, 770, 753, 789, 764, 796, 764, 730, 767, 752, 770, 776, 785, 766, 820, 733, 776, 770, 787] payload_rows_per_dst=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] dup_rows_per_dst=[3002, 2962, 3126, 3080, 2922, 3004, 2966, 3068] combine_rows_per_src=[9170, 9254, 9220, 9230, 9292, 9352, 9298, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64850 dup_rows=24220 dup_rows_remote=21064 dedup_ratio=0.246379 router_tokens_per_expert=[775, 800, 797, 767, 719, 782, 747, 781, 771, 804, 752, 754, 771, 693, 771, 741, 757, 788, 771, 747, 762, 828, 765, 752, 789, 772, 715, 781, 737, 771, 758, 777, 781, 756, 769, 790, 748, 780, 785, 759, 794, 801, 745, 796, 770, 736, 792, 770, 761, 789, 770, 788, 754, 789, 790, 755, 743, 750, 737, 728, 830, 799, 723, 779] payload_rows_per_dst=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] dup_rows_per_dst=[3076, 2918, 3018, 2964, 3028, 3096, 2964, 3156] combine_rows_per_src=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=342944 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266451872 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532560800 weight_movement_bytes=69206016 total_nvlink_bytes=601766816 weight_to_token_ratio=0.129950 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74388 payload_rows_remote=65040 dup_rows=23916 dup_rows_remote=20928 dedup_ratio=0.243286 router_tokens_per_expert=[723, 777, 776, 765, 765, 763, 802, 782, 752, 736, 777, 752, 766, 789, 778, 806, 755, 747, 787, 747, 767, 741, 737, 809, 747, 752, 788, 823, 753, 788, 734, 775, 754, 808, 772, 750, 730, 772, 769, 773, 768, 780, 771, 818, 719, 790, 773, 772, 778, 767, 752, 738, 755, 790, 810, 746, 734, 738, 712, 758, 800, 825, 772, 799] payload_rows_per_dst=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] dup_rows_per_dst=[2988, 3008, 2846, 3084, 3132, 2880, 2960, 3018] combine_rows_per_src=[9348, 9388, 9258, 9392, 9446, 9142, 9156, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343872 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747712 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151552 weight_movement_bytes=34603008 total_nvlink_bytes=567754560 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=344432 combine_payload_bytes=266821632 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267166064 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533987696 weight_movement_bytes=69206016 total_nvlink_bytes=603193712 weight_to_token_ratio=0.129602 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64850 dup_rows=24220 dup_rows_remote=21064 dedup_ratio=0.246379 router_tokens_per_expert=[775, 800, 797, 767, 719, 782, 747, 781, 771, 804, 752, 754, 771, 693, 771, 741, 757, 788, 771, 747, 762, 828, 765, 752, 789, 772, 715, 781, 737, 771, 758, 777, 781, 756, 769, 790, 748, 780, 785, 759, 794, 801, 745, 796, 770, 736, 792, 770, 761, 789, 770, 788, 754, 789, 790, 755, 743, 750, 737, 728, 830, 799, 723, 779] payload_rows_per_dst=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] dup_rows_per_dst=[3076, 2918, 3018, 2964, 3028, 3096, 2964, 3156] combine_rows_per_src=[9260, 9196, 9322, 9236, 9308, 9282, 9246, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343976 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266354600 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365224 weight_movement_bytes=34603008 total_nvlink_bytes=566968232 weight_to_token_ratio=0.064999 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64652 dup_rows=24182 dup_rows_remote=20956 dedup_ratio=0.245992 router_tokens_per_expert=[779, 784, 821, 765, 753, 746, 743, 717, 768, 775, 791, 779, 754, 750, 756, 751, 764, 783, 800, 742, 765, 749, 799, 757, 772, 747, 781, 743, 804, 806, 734, 735, 771, 730, 769, 719, 736, 756, 789, 783, 748, 747, 751, 788, 795, 768, 792, 761, 808, 754, 795, 809, 798, 793, 790, 801, 763, 785, 755, 779, 732, 772, 754, 748] payload_rows_per_dst=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] dup_rows_per_dst=[3016, 3004, 2978, 2938, 2928, 3090, 3226, 3002] combine_rows_per_src=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65146 dup_rows=23968 dup_rows_remote=20914 dedup_ratio=0.243815 router_tokens_per_expert=[767, 749, 744, 755, 772, 773, 789, 811, 727, 833, 762, 723, 799, 752, 757, 769, 779, 769, 794, 770, 767, 830, 763, 784, 769, 720, 752, 787, 752, 763, 768, 810, 768, 737, 785, 779, 713, 771, 764, 768, 744, 790, 809, 827, 782, 786, 774, 731, 766, 762, 755, 752, 738, 801, 738, 777, 770, 794, 725, 758, 717, 747, 782, 783] payload_rows_per_dst=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] dup_rows_per_dst=[2986, 3054, 3096, 2948, 2968, 3088, 2944, 2884] combine_rows_per_src=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344304 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266903792 ingress_bytes=404971520 link_bytes=404971520 token_bytes=533463280 weight_movement_bytes=138412032 total_nvlink_bytes=671875312 weight_to_token_ratio=0.259459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523875227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266010624 dispatch_meta_bytes=343976 combine_payload_bytes=266010624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266354600 ingress_bytes=300613632 link_bytes=300613632 token_bytes=532365224 weight_movement_bytes=34603008 total_nvlink_bytes=566968232 weight_to_token_ratio=0.064999 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131184896e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343656 combine_payload_bytes=265625600 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265969256 ingress_bytes=300228608 link_bytes=300228608 token_bytes=531594856 weight_movement_bytes=34603008 total_nvlink_bytes=566197864 weight_to_token_ratio=0.065093 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129736082e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74288 payload_rows_remote=65078 dup_rows=24016 dup_rows_remote=20998 dedup_ratio=0.244303 router_tokens_per_expert=[811, 720, 756, 768, 743, 778, 779, 791, 736, 822, 795, 795, 743, 802, 787, 798, 798, 745, 810, 734, 765, 758, 783, 795, 747, 728, 764, 744, 751, 773, 732, 799, 771, 811, 767, 777, 746, 745, 799, 763, 760, 774, 760, 759, 775, 776, 773, 737, 717, 765, 816, 793, 764, 741, 790, 715, 729, 763, 744, 749, 799, 779, 778, 767] payload_rows_per_dst=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] dup_rows_per_dst=[3028, 3152, 3020, 2844, 3130, 3018, 2900, 2924] combine_rows_per_src=[9264, 9404, 9356, 9232, 9228, 9210, 9302, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266403840 dispatch_meta_bytes=343872 combine_payload_bytes=266403840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266747712 ingress_bytes=301006848 link_bytes=301006848 token_bytes=533151552 weight_movement_bytes=34603008 total_nvlink_bytes=567754560 weight_to_token_ratio=0.064903 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132664536e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65206 dup_rows=23708 dup_rows_remote=20704 dedup_ratio=0.241170 router_tokens_per_expert=[749, 734, 806, 743, 812, 771, 795, 776, 776, 761, 738, 736, 765, 731, 847, 709, 754, 782, 761, 767, 796, 773, 807, 757, 766, 781, 774, 786, 773, 752, 749, 791, 777, 751, 789, 749, 765, 747, 761, 763, 784, 786, 783, 794, 748, 745, 801, 792, 774, 742, 743, 816, 784, 755, 773, 744, 694, 761, 738, 742, 806, 818, 780, 729] payload_rows_per_dst=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] dup_rows_per_dst=[3052, 2856, 3004, 3006, 2868, 3062, 2954, 2906] combine_rows_per_src=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65146 dup_rows=23968 dup_rows_remote=20914 dedup_ratio=0.243815 router_tokens_per_expert=[767, 749, 744, 755, 772, 773, 789, 811, 727, 833, 762, 723, 799, 752, 757, 769, 779, 769, 794, 770, 767, 830, 763, 784, 769, 720, 752, 787, 752, 763, 768, 810, 768, 737, 785, 779, 713, 771, 764, 768, 744, 790, 809, 827, 782, 786, 774, 731, 766, 762, 755, 752, 738, 801, 738, 777, 770, 794, 725, 758, 717, 747, 782, 783] payload_rows_per_dst=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] dup_rows_per_dst=[2986, 3054, 3096, 2948, 2968, 3088, 2944, 2884] combine_rows_per_src=[9334, 9190, 9416, 9294, 9202, 9398, 9234, 9268] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74112 payload_rows_remote=64930 dup_rows=24192 dup_rows_remote=21150 dedup_ratio=0.246094 router_tokens_per_expert=[776, 805, 800, 782, 772, 828, 767, 711, 739, 741, 754, 786, 766, 779, 747, 750, 798, 777, 755, 720, 760, 769, 760, 784, 743, 774, 770, 764, 756, 784, 805, 785, 760, 762, 721, 779, 736, 767, 775, 744, 789, 758, 743, 779, 785, 837, 816, 735, 759, 765, 750, 765, 800, 736, 753, 784, 760, 771, 767, 760, 728, 785, 796, 780] payload_rows_per_dst=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] dup_rows_per_dst=[3084, 2962, 3060, 3084, 2876, 3082, 3042, 3002] combine_rows_per_src=[9398, 9162, 9186, 9278, 9212, 9402, 9182, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64982 dup_rows=24068 dup_rows_remote=20988 dedup_ratio=0.244832 router_tokens_per_expert=[766, 753, 803, 745, 760, 719, 796, 750, 781, 740, 809, 760, 779, 767, 743, 744, 774, 726, 776, 765, 727, 771, 791, 751, 779, 776, 789, 779, 817, 754, 809, 774, 733, 759, 816, 773, 775, 767, 786, 758, 735, 746, 788, 761, 781, 759, 788, 777, 772, 762, 774, 684, 765, 759, 781, 785, 800, 775, 818, 721, 770, 749, 777, 785] payload_rows_per_dst=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] dup_rows_per_dst=[2948, 2926, 2932, 3088, 3080, 2984, 3042, 3068] combine_rows_per_src=[9236, 9320, 9230, 9466, 9254, 9286, 9122, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342432 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265157024 ingress_bytes=368623616 link_bytes=368623616 token_bytes=529971616 weight_movement_bytes=103809024 total_nvlink_bytes=633780640 weight_to_token_ratio=0.195877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387100990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65024 dup_rows=23958 dup_rows_remote=20998 dedup_ratio=0.243713 router_tokens_per_expert=[726, 716, 786, 793, 808, 813, 758, 741, 779, 792, 756, 721, 739, 741, 764, 776, 768, 758, 766, 764, 779, 726, 791, 780, 757, 798, 819, 792, 741, 747, 779, 802, 786, 776, 749, 771, 778, 745, 797, 784, 735, 752, 764, 746, 760, 790, 784, 820, 785, 781, 788, 763, 730, 792, 754, 737, 778, 780, 752, 767, 747, 791, 743, 751] payload_rows_per_dst=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] dup_rows_per_dst=[2960, 3054, 2952, 3400, 3020, 2774, 3028, 2770] combine_rows_per_src=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343640 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427416 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511192 weight_movement_bytes=34603008 total_nvlink_bytes=569114200 weight_to_token_ratio=0.064738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65042 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[795, 723, 748, 769, 792, 723, 803, 809, 812, 736, 797, 776, 799, 757, 754, 749, 791, 720, 764, 791, 758, 777, 736, 745, 756, 769, 792, 804, 797, 746, 749, 785, 751, 788, 785, 749, 781, 764, 728, 785, 753, 781, 794, 784, 753, 805, 731, 797, 781, 740, 728, 785, 748, 791, 792, 692, 743, 809, 734, 752, 788, 792, 752, 774] payload_rows_per_dst=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] dup_rows_per_dst=[3044, 2964, 2986, 3014, 3004, 3134, 2896, 2958] combine_rows_per_src=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344240 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182256 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020272 weight_movement_bytes=34603008 total_nvlink_bytes=568623280 weight_to_token_ratio=0.064797 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74458 payload_rows_remote=65188 dup_rows=23846 dup_rows_remote=20850 dedup_ratio=0.242574 router_tokens_per_expert=[711, 767, 785, 753, 745, 777, 766, 816, 719, 733, 788, 806, 782, 764, 758, 761, 769, 773, 798, 774, 755, 735, 759, 770, 762, 797, 766, 800, 799, 746, 754, 781, 822, 801, 741, 768, 756, 727, 769, 761, 788, 792, 804, 732, 724, 804, 733, 755, 768, 715, 750, 766, 757, 787, 776, 758, 811, 803, 758, 724, 785, 758, 770, 820] payload_rows_per_dst=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] dup_rows_per_dst=[3024, 2932, 2996, 3060, 3024, 2910, 2812, 3088] combine_rows_per_src=[9216, 9290, 9270, 9350, 9266, 9354, 9342, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=344304 combine_payload_bytes=266559488 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266903792 ingress_bytes=404971520 link_bytes=404971520 token_bytes=533463280 weight_movement_bytes=138412032 total_nvlink_bytes=671875312 weight_to_token_ratio=0.259459 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523875227e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74336 payload_rows_remote=65184 dup_rows=23968 dup_rows_remote=21210 dedup_ratio=0.243815 router_tokens_per_expert=[733, 802, 793, 752, 789, 779, 748, 790, 773, 770, 755, 804, 778, 757, 767, 801, 791, 732, 807, 745, 780, 731, 738, 738, 743, 764, 741, 737, 742, 754, 733, 741, 759, 731, 772, 781, 784, 775, 802, 718, 772, 822, 777, 743, 774, 779, 754, 757, 797, 757, 748, 764, 767, 752, 818, 718, 781, 784, 810, 768, 783, 813, 785, 799] payload_rows_per_dst=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] dup_rows_per_dst=[3108, 3104, 2872, 2758, 2938, 3044, 2944, 3200] combine_rows_per_src=[9264, 9306, 9252, 9152, 9306, 9312, 9298, 9446] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265953280 dispatch_meta_bytes=344320 combine_payload_bytes=265953280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266297600 ingress_bytes=300556288 link_bytes=300556288 token_bytes=532250880 weight_movement_bytes=34603008 total_nvlink_bytes=566853888 weight_to_token_ratio=0.065013 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130969115e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64928 dup_rows=23946 dup_rows_remote=20956 dedup_ratio=0.243591 router_tokens_per_expert=[759, 778, 756, 783, 747, 796, 758, 742, 765, 811, 756, 743, 771, 782, 788, 764, 753, 784, 742, 834, 700, 766, 774, 749, 744, 758, 743, 731, 788, 755, 757, 739, 765, 732, 776, 762, 808, 791, 751, 786, 770, 808, 786, 795, 789, 790, 768, 780, 757, 764, 751, 746, 825, 759, 749, 756, 761, 813, 746, 776, 792, 758, 736, 790] payload_rows_per_dst=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] dup_rows_per_dst=[2764, 3026, 3180, 2968, 2948, 2946, 3124, 2990] combine_rows_per_src=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65024 dup_rows=23958 dup_rows_remote=20998 dedup_ratio=0.243713 router_tokens_per_expert=[726, 716, 786, 793, 808, 813, 758, 741, 779, 792, 756, 721, 739, 741, 764, 776, 768, 758, 766, 764, 779, 726, 791, 780, 757, 798, 819, 792, 741, 747, 779, 802, 786, 776, 749, 771, 778, 745, 797, 784, 735, 752, 764, 746, 760, 790, 784, 820, 785, 781, 788, 763, 730, 792, 754, 737, 778, 780, 752, 767, 747, 791, 743, 751] payload_rows_per_dst=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] dup_rows_per_dst=[2960, 3054, 2952, 3400, 3020, 2774, 3028, 2770] combine_rows_per_src=[9322, 9422, 9312, 9652, 9352, 8946, 9232, 9108] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343880 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510152 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676424 weight_movement_bytes=34603008 total_nvlink_bytes=567279432 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344088 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266682392 ingress_bytes=370147328 link_bytes=370147328 token_bytes=533020696 weight_movement_bytes=103809024 total_nvlink_bytes=636829720 weight_to_token_ratio=0.194756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64928 dup_rows=23946 dup_rows_remote=20956 dedup_ratio=0.243591 router_tokens_per_expert=[759, 778, 756, 783, 747, 796, 758, 742, 765, 811, 756, 743, 771, 782, 788, 764, 753, 784, 742, 834, 700, 766, 774, 749, 744, 758, 743, 731, 788, 755, 757, 739, 765, 732, 776, 762, 808, 791, 751, 786, 770, 808, 786, 795, 789, 790, 768, 780, 757, 764, 751, 746, 825, 759, 749, 756, 761, 813, 746, 776, 792, 758, 736, 790] payload_rows_per_dst=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] dup_rows_per_dst=[2764, 3026, 3180, 2968, 2948, 2946, 3124, 2990] combine_rows_per_src=[9042, 9334, 9420, 9062, 9338, 9416, 9316, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344152 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354200 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364248 weight_movement_bytes=34603008 total_nvlink_bytes=568967256 weight_to_token_ratio=0.064755 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=345576 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267339240 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332904 weight_movement_bytes=34603008 total_nvlink_bytes=568935912 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64996 dup_rows=23922 dup_rows_remote=20912 dedup_ratio=0.243347 router_tokens_per_expert=[798, 759, 780, 723, 722, 755, 803, 789, 763, 760, 718, 796, 761, 756, 752, 720, 701, 785, 752, 762, 773, 802, 799, 757, 772, 739, 737, 775, 751, 705, 811, 771, 768, 778, 791, 780, 775, 770, 836, 747, 791, 812, 799, 765, 778, 792, 734, 791, 766, 787, 746, 748, 749, 789, 833, 780, 762, 750, 755, 774, 784, 773, 759, 743] payload_rows_per_dst=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] dup_rows_per_dst=[3056, 2822, 2966, 2902, 3070, 3104, 3010, 2992] combine_rows_per_src=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65042 dup_rows=24060 dup_rows_remote=20974 dedup_ratio=0.244751 router_tokens_per_expert=[775, 740, 766, 786, 798, 748, 827, 797, 769, 800, 751, 770, 744, 819, 715, 776, 712, 795, 816, 755, 782, 809, 731, 765, 779, 774, 748, 750, 774, 775, 790, 785, 749, 766, 764, 771, 790, 753, 721, 742, 740, 782, 729, 741, 760, 760, 817, 747, 739, 720, 791, 774, 797, 749, 780, 793, 777, 768, 744, 794, 751, 735, 771, 816] payload_rows_per_dst=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] dup_rows_per_dst=[3122, 3086, 3088, 3000, 2920, 2796, 3050, 2998] combine_rows_per_src=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343632 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266755664 ingress_bytes=370221056 link_bytes=370221056 token_bytes=533167696 weight_movement_bytes=103809024 total_nvlink_bytes=636976720 weight_to_token_ratio=0.194702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265945088 dispatch_meta_bytes=343536 combine_payload_bytes=265945088 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266288624 ingress_bytes=335151104 link_bytes=335151104 token_bytes=532233712 weight_movement_bytes=69206016 total_nvlink_bytes=601439728 weight_to_token_ratio=0.130029 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261146622e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74304 payload_rows_remote=65042 dup_rows=24000 dup_rows_remote=20866 dedup_ratio=0.244141 router_tokens_per_expert=[795, 723, 748, 769, 792, 723, 803, 809, 812, 736, 797, 776, 799, 757, 754, 749, 791, 720, 764, 791, 758, 777, 736, 745, 756, 769, 792, 804, 797, 746, 749, 785, 751, 788, 785, 749, 781, 764, 728, 785, 753, 781, 794, 784, 753, 805, 731, 797, 781, 740, 728, 785, 748, 791, 792, 692, 743, 809, 734, 752, 788, 792, 752, 774] payload_rows_per_dst=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] dup_rows_per_dst=[3044, 2964, 2986, 3014, 3004, 3134, 2896, 2958] combine_rows_per_src=[9280, 9396, 9178, 9382, 9258, 9262, 9218, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266993664 dispatch_meta_bytes=345576 combine_payload_bytes=266993664 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267339240 ingress_bytes=301596672 link_bytes=301596672 token_bytes=534332904 weight_movement_bytes=34603008 total_nvlink_bytes=568935912 weight_to_token_ratio=0.064759 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134883996e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64774 dup_rows=24296 dup_rows_remote=21188 dedup_ratio=0.247152 router_tokens_per_expert=[790, 800, 743, 811, 753, 815, 723, 775, 760, 753, 764, 774, 701, 683, 775, 731, 738, 762, 720, 724, 815, 833, 786, 793, 712, 809, 807, 779, 789, 762, 799, 778, 738, 796, 767, 721, 762, 775, 794, 801, 778, 821, 727, 753, 768, 807, 730, 761, 765, 762, 783, 806, 761, 739, 803, 779, 800, 714, 784, 770, 701, 762, 825, 742] payload_rows_per_dst=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] dup_rows_per_dst=[3092, 2832, 3108, 3106, 2998, 3010, 3186, 2964] combine_rows_per_src=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344088 combine_payload_bytes=266338304 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266682392 ingress_bytes=370147328 link_bytes=370147328 token_bytes=533020696 weight_movement_bytes=103809024 total_nvlink_bytes=636829720 weight_to_token_ratio=0.194756 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392834596e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=65042 dup_rows=24060 dup_rows_remote=20974 dedup_ratio=0.244751 router_tokens_per_expert=[775, 740, 766, 786, 798, 748, 827, 797, 769, 800, 751, 770, 744, 819, 715, 776, 712, 795, 816, 755, 782, 809, 731, 765, 779, 774, 748, 750, 774, 775, 790, 785, 749, 766, 764, 771, 790, 753, 721, 742, 740, 782, 729, 741, 760, 760, 817, 747, 739, 720, 791, 774, 797, 749, 780, 793, 777, 768, 744, 794, 751, 735, 771, 816] payload_rows_per_dst=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] dup_rows_per_dst=[3122, 3086, 3088, 3000, 2920, 2796, 3050, 2998] combine_rows_per_src=[9352, 9202, 9242, 9350, 9192, 9356, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74122 payload_rows_remote=64652 dup_rows=24182 dup_rows_remote=20956 dedup_ratio=0.245992 router_tokens_per_expert=[779, 784, 821, 765, 753, 746, 743, 717, 768, 775, 791, 779, 754, 750, 756, 751, 764, 783, 800, 742, 765, 749, 799, 757, 772, 747, 781, 743, 804, 806, 734, 735, 771, 730, 769, 719, 736, 756, 789, 783, 748, 747, 751, 788, 795, 768, 792, 761, 808, 754, 795, 809, 798, 793, 790, 801, 763, 785, 755, 779, 732, 772, 754, 748] payload_rows_per_dst=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] dup_rows_per_dst=[3016, 3004, 2978, 2938, 2928, 3090, 3226, 3002] combine_rows_per_src=[9200, 9244, 9340, 9306, 9178, 9210, 9470, 9174] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343848 combine_payload_bytes=265314304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265658152 ingress_bytes=334520320 link_bytes=334520320 token_bytes=530972456 weight_movement_bytes=69206016 total_nvlink_bytes=600178472 weight_to_token_ratio=0.130338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74496 payload_rows_remote=65134 dup_rows=23808 dup_rows_remote=20836 dedup_ratio=0.242188 router_tokens_per_expert=[727, 804, 796, 790, 761, 784, 762, 784, 797, 768, 773, 749, 749, 768, 738, 774, 749, 793, 762, 717, 701, 778, 798, 785, 742, 754, 795, 759, 831, 751, 705, 713, 743, 777, 757, 817, 739, 795, 747, 792, 806, 769, 783, 755, 779, 764, 772, 736, 751, 787, 794, 752, 755, 774, 752, 787, 768, 792, 794, 742, 760, 760, 793, 803] payload_rows_per_dst=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] dup_rows_per_dst=[3036, 2890, 2888, 2892, 2972, 3012, 3052, 3066] combine_rows_per_src=[9380, 9342, 9278, 9208, 9362, 9316, 9252, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343632 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567248 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532790864 weight_movement_bytes=34603008 total_nvlink_bytes=567393872 weight_to_token_ratio=0.064947 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=344064 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266756096 ingress_bytes=301015040 link_bytes=301015040 token_bytes=533168128 weight_movement_bytes=34603008 total_nvlink_bytes=567771136 weight_to_token_ratio=0.064901 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132695362e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21024 dedup_ratio=0.244161 router_tokens_per_expert=[766, 767, 689, 778, 766, 826, 774, 761, 773, 757, 753, 793, 761, 771, 769, 805, 781, 788, 741, 756, 783, 738, 776, 790, 796, 739, 746, 770, 753, 795, 780, 785, 782, 758, 787, 798, 754, 765, 744, 767, 713, 770, 763, 769, 760, 780, 807, 780, 780, 802, 767, 784, 761, 734, 800, 717, 791, 776, 760, 746, 754, 755, 754, 748] payload_rows_per_dst=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] dup_rows_per_dst=[3042, 3124, 2954, 3018, 2942, 2978, 2936, 3008] combine_rows_per_src=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65106 dup_rows=23896 dup_rows_remote=20786 dedup_ratio=0.243083 router_tokens_per_expert=[753, 774, 777, 760, 771, 772, 787, 755, 767, 758, 706, 758, 753, 768, 799, 798, 792, 775, 745, 779, 770, 759, 784, 789, 795, 746, 778, 785, 742, 778, 808, 740, 762, 782, 736, 830, 788, 779, 779, 757, 762, 735, 806, 710, 787, 780, 760, 792, 735, 776, 742, 829, 791, 727, 758, 737, 750, 760, 775, 766, 747, 771, 759, 763] payload_rows_per_dst=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] dup_rows_per_dst=[3110, 3212, 2774, 2876, 2998, 2948, 2998, 2980] combine_rows_per_src=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266412032 dispatch_meta_bytes=343632 combine_payload_bytes=266412032 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266755664 ingress_bytes=370221056 link_bytes=370221056 token_bytes=533167696 weight_movement_bytes=103809024 total_nvlink_bytes=636976720 weight_to_token_ratio=0.194702 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393112028e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74206 payload_rows_remote=65046 dup_rows=24098 dup_rows_remote=21112 dedup_ratio=0.245138 router_tokens_per_expert=[810, 735, 796, 777, 764, 763, 771, 808, 770, 756, 762, 766, 770, 791, 748, 779, 748, 755, 756, 769, 734, 772, 800, 739, 736, 745, 766, 735, 748, 805, 793, 773, 749, 775, 742, 786, 763, 776, 733, 780, 767, 749, 762, 781, 754, 768, 778, 773, 744, 761, 752, 887, 819, 760, 729, 785, 821, 753, 773, 729, 763, 766, 805, 729] payload_rows_per_dst=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] dup_rows_per_dst=[3074, 2920, 2986, 3020, 2920, 3064, 3056, 3058] combine_rows_per_src=[9374, 9364, 9160, 9182, 9288, 9200, 9418, 9220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=264814592 dispatch_meta_bytes=342432 combine_payload_bytes=264814592 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265157024 ingress_bytes=368623616 link_bytes=368623616 token_bytes=529971616 weight_movement_bytes=103809024 total_nvlink_bytes=633780640 weight_to_token_ratio=0.195877 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.387100990e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64956 dup_rows=23888 dup_rows_remote=20834 dedup_ratio=0.243001 router_tokens_per_expert=[777, 767, 764, 828, 733, 772, 789, 743, 762, 777, 757, 773, 748, 792, 793, 748, 739, 720, 733, 793, 734, 769, 774, 781, 722, 781, 802, 758, 735, 775, 763, 814, 774, 763, 800, 772, 755, 756, 768, 809, 738, 774, 775, 771, 760, 767, 798, 780, 801, 742, 757, 779, 816, 718, 733, 781, 764, 786, 771, 776, 747, 744, 761, 800] payload_rows_per_dst=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] dup_rows_per_dst=[2968, 3004, 2818, 2954, 3000, 3052, 3038, 3054] combine_rows_per_src=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74102 payload_rows_remote=64820 dup_rows=24202 dup_rows_remote=21104 dedup_ratio=0.246195 router_tokens_per_expert=[761, 785, 772, 773, 768, 762, 771, 727, 761, 712, 767, 757, 753, 765, 751, 731, 798, 777, 810, 746, 747, 772, 773, 778, 733, 813, 741, 774, 801, 755, 771, 802, 751, 745, 775, 779, 700, 756, 764, 768, 757, 754, 736, 790, 741, 739, 801, 833, 797, 792, 775, 823, 749, 764, 748, 814, 820, 737, 782, 746, 804, 775, 761, 769] payload_rows_per_dst=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] dup_rows_per_dst=[2994, 2860, 3070, 3098, 2948, 2986, 3148, 3098] combine_rows_per_src=[9244, 9134, 9332, 9282, 9128, 9316, 9376, 9290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=343880 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267132744 ingress_bytes=335994880 link_bytes=335994880 token_bytes=533921608 weight_movement_bytes=69206016 total_nvlink_bytes=603127624 weight_to_token_ratio=0.129618 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264321684e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74416 payload_rows_remote=64956 dup_rows=23888 dup_rows_remote=20834 dedup_ratio=0.243001 router_tokens_per_expert=[777, 767, 764, 828, 733, 772, 789, 743, 762, 777, 757, 773, 748, 792, 793, 748, 739, 720, 733, 793, 734, 769, 774, 781, 722, 781, 802, 758, 735, 775, 763, 814, 774, 763, 800, 772, 755, 756, 768, 809, 738, 774, 775, 771, 760, 767, 798, 780, 801, 742, 757, 779, 816, 718, 733, 781, 764, 786, 771, 776, 747, 744, 761, 800] payload_rows_per_dst=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] dup_rows_per_dst=[2968, 3004, 2818, 2954, 3000, 3052, 3038, 3054] combine_rows_per_src=[9378, 9296, 9268, 9346, 9164, 9274, 9230, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343568 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267017744 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533691920 weight_movement_bytes=34603008 total_nvlink_bytes=568294928 weight_to_token_ratio=0.064837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65224 dup_rows=23914 dup_rows_remote=21058 dedup_ratio=0.243266 router_tokens_per_expert=[762, 759, 772, 698, 783, 741, 807, 755, 800, 742, 783, 771, 743, 787, 774, 758, 740, 808, 740, 810, 791, 768, 814, 747, 762, 733, 726, 743, 791, 794, 805, 779, 793, 790, 750, 780, 773, 793, 743, 742, 786, 792, 783, 761, 773, 765, 789, 775, 748, 804, 732, 764, 723, 749, 752, 739, 755, 826, 749, 780, 790, 719, 751, 797] payload_rows_per_dst=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] dup_rows_per_dst=[2874, 3040, 3070, 3004, 3054, 3062, 2856, 2954] combine_rows_per_src=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65106 dup_rows=23896 dup_rows_remote=20786 dedup_ratio=0.243083 router_tokens_per_expert=[753, 774, 777, 760, 771, 772, 787, 755, 767, 758, 706, 758, 753, 768, 799, 798, 792, 775, 745, 779, 770, 759, 784, 789, 795, 746, 778, 785, 742, 778, 808, 740, 762, 782, 736, 830, 788, 779, 779, 757, 762, 735, 806, 710, 787, 780, 760, 792, 735, 776, 742, 829, 791, 727, 758, 737, 750, 760, 775, 766, 747, 771, 759, 763] payload_rows_per_dst=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] dup_rows_per_dst=[3110, 3212, 2774, 2876, 2998, 2948, 2998, 2980] combine_rows_per_src=[9302, 9630, 9146, 9312, 9308, 9364, 9144, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344080 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567696 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791312 weight_movement_bytes=34603008 total_nvlink_bytes=567394320 weight_to_token_ratio=0.064947 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343160 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402936 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462712 weight_movement_bytes=34603008 total_nvlink_bytes=567065720 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=343696 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265846416 ingress_bytes=334708736 link_bytes=334708736 token_bytes=531349136 weight_movement_bytes=69206016 total_nvlink_bytes=600555152 weight_to_token_ratio=0.130246 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259482027e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64902 dup_rows=24136 dup_rows_remote=21196 dedup_ratio=0.245524 router_tokens_per_expert=[765, 774, 768, 744, 756, 779, 771, 780, 756, 764, 764, 796, 780, 809, 762, 742, 789, 762, 730, 771, 729, 773, 778, 771, 718, 784, 762, 799, 784, 724, 801, 815, 786, 830, 791, 757, 748, 779, 742, 793, 748, 773, 729, 757, 797, 790, 735, 729, 752, 792, 741, 769, 770, 785, 735, 710, 754, 767, 776, 768, 769, 781, 802, 797] payload_rows_per_dst=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] dup_rows_per_dst=[3038, 3024, 2940, 3030, 3034, 2970, 3020, 3080] combine_rows_per_src=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266674176 dispatch_meta_bytes=343568 combine_payload_bytes=266674176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267017744 ingress_bytes=301277184 link_bytes=301277184 token_bytes=533691920 weight_movement_bytes=34603008 total_nvlink_bytes=568294928 weight_to_token_ratio=0.064837 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133681789e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=65012 dup_rows=24008 dup_rows_remote=20956 dedup_ratio=0.244222 router_tokens_per_expert=[765, 737, 752, 724, 753, 794, 762, 776, 762, 768, 735, 731, 770, 785, 816, 801, 780, 771, 793, 778, 754, 797, 744, 879, 775, 740, 761, 759, 749, 774, 731, 795, 797, 779, 730, 774, 794, 832, 771, 750, 783, 772, 793, 697, 728, 752, 774, 757, 761, 757, 776, 784, 792, 771, 813, 803, 743, 767, 765, 764, 732, 715, 715, 800] payload_rows_per_dst=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] dup_rows_per_dst=[2872, 3052, 3166, 2906, 3028, 2970, 3088, 2926] combine_rows_per_src=[9254, 9284, 9426, 9262, 9426, 9142, 9426, 9076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266428416 dispatch_meta_bytes=344632 combine_payload_bytes=266428416 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266773048 ingress_bytes=335634432 link_bytes=335634432 token_bytes=533201464 weight_movement_bytes=69206016 total_nvlink_bytes=602407480 weight_to_token_ratio=0.129793 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262965347e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343160 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266402936 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532462712 weight_movement_bytes=34603008 total_nvlink_bytes=567065720 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64996 dup_rows=24002 dup_rows_remote=21024 dedup_ratio=0.244161 router_tokens_per_expert=[766, 767, 689, 778, 766, 826, 774, 761, 773, 757, 753, 793, 761, 771, 769, 805, 781, 788, 741, 756, 783, 738, 776, 790, 796, 739, 746, 770, 753, 795, 780, 785, 782, 758, 787, 798, 754, 765, 744, 767, 713, 770, 763, 769, 760, 780, 807, 780, 780, 802, 767, 784, 761, 734, 800, 717, 791, 776, 760, 746, 754, 755, 754, 748] payload_rows_per_dst=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] dup_rows_per_dst=[3042, 3124, 2954, 3018, 2942, 2978, 2936, 3008] combine_rows_per_src=[9212, 9240, 9352, 9310, 9368, 9306, 9354, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=64996 dup_rows=23922 dup_rows_remote=20912 dedup_ratio=0.243347 router_tokens_per_expert=[798, 759, 780, 723, 722, 755, 803, 789, 763, 760, 718, 796, 761, 756, 752, 720, 701, 785, 752, 762, 773, 802, 799, 757, 772, 739, 737, 775, 751, 705, 811, 771, 768, 778, 791, 780, 775, 770, 836, 747, 791, 812, 799, 765, 778, 792, 734, 791, 766, 787, 746, 748, 749, 789, 833, 780, 762, 750, 755, 774, 784, 773, 759, 743] payload_rows_per_dst=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] dup_rows_per_dst=[3056, 2822, 2966, 2902, 3070, 3104, 3010, 2992] combine_rows_per_src=[9202, 9230, 9296, 9220, 9420, 9420, 9386, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=345128 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267502632 ingress_bytes=336363520 link_bytes=336363520 token_bytes=534660136 weight_movement_bytes=69206016 total_nvlink_bytes=603866152 weight_to_token_ratio=0.129439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265708846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344392 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182984 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532021576 weight_movement_bytes=69206016 total_nvlink_bytes=601227592 weight_to_token_ratio=0.130081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64900 dup_rows=24136 dup_rows_remote=21046 dedup_ratio=0.245524 router_tokens_per_expert=[810, 796, 718, 772, 733, 795, 748, 805, 706, 713, 765, 808, 777, 786, 778, 802, 799, 729, 777, 776, 729, 743, 784, 773, 790, 779, 763, 763, 757, 699, 837, 714, 772, 779, 805, 775, 740, 746, 776, 786, 757, 759, 780, 768, 743, 794, 743, 771, 756, 752, 748, 762, 837, 754, 726, 739, 791, 768, 755, 763, 805, 781, 758, 839] payload_rows_per_dst=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] dup_rows_per_dst=[3094, 3074, 2974, 2964, 3090, 3038, 2878, 3024] combine_rows_per_src=[9260, 9196, 9246, 9240, 9268, 9192, 9270, 9496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343872 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266633024 ingress_bytes=300892160 link_bytes=300892160 token_bytes=532922176 weight_movement_bytes=34603008 total_nvlink_bytes=567525184 weight_to_token_ratio=0.064931 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132232974e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73734 payload_rows_remote=64424 dup_rows=24570 dup_rows_remote=21476 dedup_ratio=0.249939 router_tokens_per_expert=[754, 748, 737, 754, 792, 743, 764, 799, 788, 762, 815, 801, 743, 752, 776, 744, 745, 767, 772, 749, 788, 813, 763, 789, 782, 773, 738, 748, 780, 794, 790, 768, 779, 805, 761, 732, 742, 744, 737, 732, 706, 746, 801, 768, 788, 720, 786, 793, 784, 736, 810, 758, 788, 768, 773, 833, 759, 749, 798, 761, 795, 731, 729, 809] payload_rows_per_dst=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] dup_rows_per_dst=[3094, 3208, 3136, 3140, 2812, 3370, 3122, 2688] combine_rows_per_src=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64972 dup_rows=24050 dup_rows_remote=20994 dedup_ratio=0.244649 router_tokens_per_expert=[735, 771, 789, 803, 734, 734, 773, 762, 758, 767, 733, 756, 776, 785, 761, 774, 765, 800, 765, 759, 775, 813, 763, 778, 769, 763, 780, 790, 749, 738, 780, 775, 764, 765, 786, 754, 716, 790, 726, 754, 793, 794, 756, 716, 766, 767, 785, 792, 750, 777, 764, 789, 780, 763, 764, 769, 776, 749, 680, 826, 771, 796, 813, 788] payload_rows_per_dst=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] dup_rows_per_dst=[2952, 2998, 3028, 2984, 2896, 3056, 3046, 3090] combine_rows_per_src=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344080 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567696 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532791312 weight_movement_bytes=34603008 total_nvlink_bytes=567394320 weight_to_token_ratio=0.064947 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343632 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266567248 ingress_bytes=300826624 link_bytes=300826624 token_bytes=532790864 weight_movement_bytes=34603008 total_nvlink_bytes=567393872 weight_to_token_ratio=0.064947 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131986368e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64768 dup_rows=24038 dup_rows_remote=20898 dedup_ratio=0.244527 router_tokens_per_expert=[731, 755, 815, 774, 769, 715, 789, 806, 758, 748, 757, 762, 756, 822, 763, 732, 763, 795, 757, 782, 782, 745, 753, 794, 809, 764, 777, 810, 749, 799, 768, 756, 751, 800, 779, 749, 756, 772, 778, 760, 730, 779, 747, 743, 741, 767, 736, 747, 805, 790, 727, 822, 738, 730, 795, 730, 763, 779, 728, 815, 794, 764, 804, 778] payload_rows_per_dst=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] dup_rows_per_dst=[2960, 3002, 3058, 2980, 3074, 2918, 2906, 3140] combine_rows_per_src=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343784 combine_payload_bytes=265830400 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266174184 ingress_bytes=404242432 link_bytes=404242432 token_bytes=532004584 weight_movement_bytes=138412032 total_nvlink_bytes=670416616 weight_to_token_ratio=0.260171 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521131727e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65258 dup_rows=23880 dup_rows_remote=20918 dedup_ratio=0.242920 router_tokens_per_expert=[753, 729, 801, 821, 746, 722, 729, 714, 745, 792, 769, 785, 700, 817, 784, 792, 787, 740, 781, 779, 759, 786, 804, 783, 757, 767, 773, 770, 778, 777, 733, 709, 797, 805, 796, 807, 730, 732, 768, 790, 773, 722, 784, 793, 757, 743, 746, 772, 737, 743, 744, 757, 795, 774, 804, 824, 811, 770, 752, 760, 785, 758, 746, 795] payload_rows_per_dst=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] dup_rows_per_dst=[2782, 3000, 3066, 2962, 3032, 2972, 3060, 3006] combine_rows_per_src=[9248, 9368, 9372, 9166, 9418, 9208, 9296, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65206 dup_rows=23708 dup_rows_remote=20704 dedup_ratio=0.241170 router_tokens_per_expert=[749, 734, 806, 743, 812, 771, 795, 776, 776, 761, 738, 736, 765, 731, 847, 709, 754, 782, 761, 767, 796, 773, 807, 757, 766, 781, 774, 786, 773, 752, 749, 791, 777, 751, 789, 749, 765, 747, 761, 763, 784, 786, 783, 794, 748, 745, 801, 792, 774, 742, 743, 816, 784, 755, 773, 744, 694, 761, 738, 742, 806, 818, 780, 729] payload_rows_per_dst=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] dup_rows_per_dst=[3052, 2856, 3004, 3006, 2868, 3062, 2954, 2906] combine_rows_per_src=[9320, 9270, 9390, 9338, 9336, 9404, 9308, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=263880704 dispatch_meta_bytes=343600 combine_payload_bytes=263880704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264224304 ingress_bytes=298483712 link_bytes=298483712 token_bytes=528105008 weight_movement_bytes=34603008 total_nvlink_bytes=562708016 weight_to_token_ratio=0.065523 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123170178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64768 dup_rows=24038 dup_rows_remote=20898 dedup_ratio=0.244527 router_tokens_per_expert=[731, 755, 815, 774, 769, 715, 789, 806, 758, 748, 757, 762, 756, 822, 763, 732, 763, 795, 757, 782, 782, 745, 753, 794, 809, 764, 777, 810, 749, 799, 768, 756, 751, 800, 779, 749, 756, 772, 778, 760, 730, 779, 747, 743, 741, 767, 736, 747, 805, 790, 727, 822, 738, 730, 795, 730, 763, 779, 728, 815, 794, 764, 804, 778] payload_rows_per_dst=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] dup_rows_per_dst=[2960, 3002, 3058, 2980, 3074, 2918, 2906, 3140] combine_rows_per_src=[9348, 9194, 9244, 9296, 9256, 9062, 9368, 9498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=65000 dup_rows=24078 dup_rows_remote=21026 dedup_ratio=0.244934 router_tokens_per_expert=[758, 776, 734, 756, 778, 741, 778, 827, 764, 767, 771, 745, 832, 691, 777, 750, 750, 815, 810, 765, 722, 730, 736, 732, 785, 788, 761, 775, 743, 793, 744, 790, 776, 796, 795, 729, 770, 810, 768, 728, 787, 786, 768, 763, 740, 734, 774, 795, 767, 761, 798, 779, 750, 749, 732, 803, 801, 820, 770, 801, 741, 713, 792, 772] payload_rows_per_dst=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] dup_rows_per_dst=[2958, 3012, 2920, 2980, 3052, 3030, 3052, 3074] combine_rows_per_src=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73734 payload_rows_remote=64424 dup_rows=24570 dup_rows_remote=21476 dedup_ratio=0.249939 router_tokens_per_expert=[754, 748, 737, 754, 792, 743, 764, 799, 788, 762, 815, 801, 743, 752, 776, 744, 745, 767, 772, 749, 788, 813, 763, 789, 782, 773, 738, 748, 780, 794, 790, 768, 779, 805, 761, 732, 742, 744, 737, 732, 706, 746, 801, 768, 788, 720, 786, 793, 784, 736, 810, 758, 788, 768, 773, 833, 759, 749, 798, 761, 795, 731, 729, 809] payload_rows_per_dst=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] dup_rows_per_dst=[3094, 3208, 3136, 3140, 2812, 3370, 3122, 2688] combine_rows_per_src=[9310, 9534, 9238, 9254, 9030, 9426, 9328, 8614] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343864 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469176 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594488 weight_movement_bytes=34603008 total_nvlink_bytes=567197496 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=342664 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632392 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922120 weight_movement_bytes=103809024 total_nvlink_bytes=634731144 weight_to_token_ratio=0.195526 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=64984 dup_rows=23866 dup_rows_remote=20736 dedup_ratio=0.242778 router_tokens_per_expert=[776, 718, 763, 759, 744, 772, 814, 749, 781, 731, 808, 770, 768, 725, 790, 718, 856, 782, 748, 811, 738, 770, 805, 782, 789, 785, 724, 751, 755, 804, 772, 744, 752, 747, 808, 745, 763, 736, 816, 738, 760, 761, 756, 803, 766, 781, 724, 813, 798, 751, 779, 767, 705, 783, 797, 740, 772, 780, 740, 763, 754, 788, 800, 764] payload_rows_per_dst=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] dup_rows_per_dst=[2988, 2966, 3130, 2960, 2956, 2862, 2964, 3040] combine_rows_per_src=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=267296768 dispatch_meta_bytes=344704 combine_payload_bytes=267296768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267641472 ingress_bytes=336502784 link_bytes=336502784 token_bytes=534938240 weight_movement_bytes=69206016 total_nvlink_bytes=604144256 weight_to_token_ratio=0.129372 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266232885e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64844 dup_rows=24278 dup_rows_remote=21154 dedup_ratio=0.246969 router_tokens_per_expert=[810, 805, 781, 783, 769, 751, 758, 742, 788, 728, 746, 770, 789, 770, 791, 771, 788, 808, 767, 777, 751, 762, 771, 754, 778, 800, 730, 741, 737, 777, 739, 776, 761, 765, 688, 797, 759, 767, 818, 781, 784, 780, 769, 766, 793, 798, 769, 750, 807, 795, 769, 730, 725, 753, 787, 762, 756, 771, 765, 743, 782, 754, 744, 756] payload_rows_per_dst=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] dup_rows_per_dst=[3100, 3124, 3102, 2982, 2968, 3064, 2990, 2948] combine_rows_per_src=[9298, 9182, 9254, 9174, 9304, 9354, 9266, 9194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=343640 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267427416 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534511192 weight_movement_bytes=34603008 total_nvlink_bytes=569114200 weight_to_token_ratio=0.064738 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=342664 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265632392 ingress_bytes=369098752 link_bytes=369098752 token_bytes=530922120 weight_movement_bytes=103809024 total_nvlink_bytes=634731144 weight_to_token_ratio=0.195526 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388888889e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74254 payload_rows_remote=64972 dup_rows=24050 dup_rows_remote=20994 dedup_ratio=0.244649 router_tokens_per_expert=[735, 771, 789, 803, 734, 734, 773, 762, 758, 767, 733, 756, 776, 785, 761, 774, 765, 800, 765, 759, 775, 813, 763, 778, 769, 763, 780, 790, 749, 738, 780, 775, 764, 765, 786, 754, 716, 790, 726, 754, 793, 794, 756, 716, 766, 767, 785, 792, 750, 777, 764, 789, 780, 763, 764, 769, 776, 749, 680, 826, 771, 796, 813, 788] payload_rows_per_dst=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] dup_rows_per_dst=[2952, 2998, 3028, 2984, 2896, 3056, 3046, 3090] combine_rows_per_src=[9250, 9222, 9408, 9304, 9214, 9282, 9266, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=263880704 dispatch_meta_bytes=343600 combine_payload_bytes=263880704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264224304 ingress_bytes=298483712 link_bytes=298483712 token_bytes=528105008 weight_movement_bytes=34603008 total_nvlink_bytes=562708016 weight_to_token_ratio=0.065523 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123170178e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65224 dup_rows=23914 dup_rows_remote=21058 dedup_ratio=0.243266 router_tokens_per_expert=[762, 759, 772, 698, 783, 741, 807, 755, 800, 742, 783, 771, 743, 787, 774, 758, 740, 808, 740, 810, 791, 768, 814, 747, 762, 733, 726, 743, 791, 794, 805, 779, 793, 790, 750, 780, 773, 793, 743, 742, 786, 792, 783, 761, 773, 765, 789, 775, 748, 804, 732, 764, 723, 749, 752, 739, 755, 826, 749, 780, 790, 719, 751, 797] payload_rows_per_dst=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] dup_rows_per_dst=[2874, 3040, 3070, 3004, 3054, 3062, 2856, 2954] combine_rows_per_src=[9280, 9276, 9366, 9262, 9274, 9386, 9166, 9380] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344104 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266584104 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532824104 weight_movement_bytes=69206016 total_nvlink_bytes=602030120 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=342880 combine_payload_bytes=266174464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266517344 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532691808 weight_movement_bytes=103809024 total_nvlink_bytes=636500832 weight_to_token_ratio=0.194876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73998 payload_rows_remote=64608 dup_rows=24306 dup_rows_remote=21192 dedup_ratio=0.247253 router_tokens_per_expert=[796, 777, 781, 787, 740, 786, 742, 787, 752, 747, 737, 762, 746, 742, 694, 786, 718, 778, 756, 763, 766, 735, 790, 730, 768, 794, 757, 801, 717, 790, 760, 778, 772, 760, 848, 770, 796, 789, 735, 782, 724, 763, 729, 719, 770, 792, 844, 750, 800, 754, 757, 753, 814, 760, 756, 741, 796, 765, 775, 798, 807, 774, 801, 795] payload_rows_per_dst=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] dup_rows_per_dst=[3048, 2888, 2968, 3054, 3114, 2984, 3022, 3228] combine_rows_per_src=[9344, 9044, 9104, 9276, 9390, 9198, 9248, 9394] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265601024 dispatch_meta_bytes=343992 combine_payload_bytes=265601024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265945016 ingress_bytes=300204032 link_bytes=300204032 token_bytes=531546040 weight_movement_bytes=34603008 total_nvlink_bytes=566149048 weight_to_token_ratio=0.065099 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129643604e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65158 dup_rows=23966 dup_rows_remote=20902 dedup_ratio=0.243795 router_tokens_per_expert=[759, 766, 783, 750, 754, 760, 745, 805, 773, 813, 775, 775, 762, 763, 785, 759, 819, 736, 718, 737, 775, 783, 765, 818, 720, 761, 770, 777, 747, 743, 763, 768, 814, 776, 752, 769, 752, 773, 737, 763, 797, 744, 758, 758, 741, 746, 764, 814, 801, 769, 752, 799, 783, 764, 771, 747, 775, 787, 793, 786, 734, 795, 787, 724] payload_rows_per_dst=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] dup_rows_per_dst=[2938, 3076, 3008, 2840, 2984, 3064, 3080, 2976] combine_rows_per_src=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266125312 dispatch_meta_bytes=343864 combine_payload_bytes=266125312 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266469176 ingress_bytes=300728320 link_bytes=300728320 token_bytes=532594488 weight_movement_bytes=34603008 total_nvlink_bytes=567197496 weight_to_token_ratio=0.064971 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131616458e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64982 dup_rows=24038 dup_rows_remote=21112 dedup_ratio=0.244527 router_tokens_per_expert=[744, 771, 786, 701, 773, 771, 766, 777, 762, 773, 772, 820, 785, 780, 867, 743, 750, 771, 812, 822, 784, 771, 776, 749, 746, 755, 737, 753, 739, 732, 782, 714, 809, 769, 713, 777, 745, 717, 830, 799, 794, 731, 734, 742, 779, 779, 724, 787, 764, 769, 772, 774, 793, 768, 818, 706, 735, 753, 753, 833, 789, 795, 748, 769] payload_rows_per_dst=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] dup_rows_per_dst=[2926, 3356, 3056, 2856, 3176, 2836, 2836, 2996] combine_rows_per_src=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64912 dup_rows=24060 dup_rows_remote=21082 dedup_ratio=0.244751 router_tokens_per_expert=[760, 786, 755, 761, 776, 746, 783, 804, 824, 765, 788, 708, 800, 728, 842, 726, 709, 733, 776, 780, 801, 769, 775, 798, 779, 757, 737, 765, 782, 753, 776, 745, 770, 729, 784, 728, 739, 753, 762, 743, 716, 726, 798, 824, 777, 772, 781, 767, 821, 742, 800, 751, 777, 810, 781, 762, 768, 737, 784, 786, 776, 782, 755, 764] payload_rows_per_dst=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] dup_rows_per_dst=[3090, 2622, 3042, 3444, 2764, 3032, 3088, 2978] combine_rows_per_src=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=345128 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267502632 ingress_bytes=336363520 link_bytes=336363520 token_bytes=534660136 weight_movement_bytes=69206016 total_nvlink_bytes=603866152 weight_to_token_ratio=0.129439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265708846e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=264634368 dispatch_meta_bytes=343200 combine_payload_bytes=264634368 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=264977568 ingress_bytes=333840384 link_bytes=333840384 token_bytes=529611936 weight_movement_bytes=69206016 total_nvlink_bytes=598817952 weight_to_token_ratio=0.130673 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256214489e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65362 dup_rows=23768 dup_rows_remote=20900 dedup_ratio=0.241781 router_tokens_per_expert=[764, 820, 771, 727, 778, 733, 786, 834, 695, 759, 792, 790, 779, 803, 759, 769, 770, 763, 789, 748, 766, 764, 772, 776, 752, 752, 773, 747, 770, 755, 756, 716, 786, 723, 781, 720, 772, 777, 816, 786, 770, 805, 779, 800, 819, 739, 747, 749, 748, 770, 792, 757, 801, 759, 737, 796, 771, 748, 814, 736, 793, 756, 744, 733] payload_rows_per_dst=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] dup_rows_per_dst=[3026, 3000, 2914, 2868, 2984, 3004, 3002, 2970] combine_rows_per_src=[9400, 9292, 9382, 9174, 9338, 9412, 9318, 9220] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64912 dup_rows=24060 dup_rows_remote=21082 dedup_ratio=0.244751 router_tokens_per_expert=[760, 786, 755, 761, 776, 746, 783, 804, 824, 765, 788, 708, 800, 728, 842, 726, 709, 733, 776, 780, 801, 769, 775, 798, 779, 757, 737, 765, 782, 753, 776, 745, 770, 729, 784, 728, 739, 753, 762, 743, 716, 726, 798, 824, 777, 772, 781, 767, 821, 742, 800, 751, 777, 810, 781, 762, 768, 737, 784, 786, 776, 782, 755, 764] payload_rows_per_dst=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] dup_rows_per_dst=[3090, 2622, 3042, 3444, 2764, 3032, 3088, 2978] combine_rows_per_src=[9252, 8836, 9152, 9648, 9252, 9372, 9400, 9332] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74008 payload_rows_remote=64774 dup_rows=24296 dup_rows_remote=21188 dedup_ratio=0.247152 router_tokens_per_expert=[790, 800, 743, 811, 753, 815, 723, 775, 760, 753, 764, 774, 701, 683, 775, 731, 738, 762, 720, 724, 815, 833, 786, 793, 712, 809, 807, 779, 789, 762, 799, 778, 738, 796, 767, 721, 762, 775, 794, 801, 778, 821, 727, 753, 768, 807, 730, 761, 765, 762, 783, 806, 761, 739, 803, 779, 800, 714, 784, 770, 701, 762, 825, 742] payload_rows_per_dst=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] dup_rows_per_dst=[3092, 2832, 3108, 3106, 2998, 3010, 3186, 2964] combine_rows_per_src=[9328, 9050, 9234, 9364, 9310, 9280, 9210, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344376 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510648 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676920 weight_movement_bytes=34603008 total_nvlink_bytes=567279928 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74266 payload_rows_remote=64982 dup_rows=24038 dup_rows_remote=21112 dedup_ratio=0.244527 router_tokens_per_expert=[744, 771, 786, 701, 773, 771, 766, 777, 762, 773, 772, 820, 785, 780, 867, 743, 750, 771, 812, 822, 784, 771, 776, 749, 746, 755, 737, 753, 739, 732, 782, 714, 809, 769, 713, 777, 745, 717, 830, 799, 794, 731, 734, 742, 779, 779, 724, 787, 764, 769, 772, 774, 793, 768, 818, 706, 735, 753, 753, 833, 789, 795, 748, 769] payload_rows_per_dst=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] dup_rows_per_dst=[2926, 3356, 3056, 2856, 3176, 2836, 2836, 2996] combine_rows_per_src=[9284, 9582, 9382, 9060, 9372, 9074, 9188, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344240 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267231408 ingress_bytes=336093184 link_bytes=336093184 token_bytes=534118576 weight_movement_bytes=69206016 total_nvlink_bytes=603324592 weight_to_token_ratio=0.129571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264691594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343976 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223528 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103080 weight_movement_bytes=69206016 total_nvlink_bytes=601309096 weight_to_token_ratio=0.130061 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65136 dup_rows=23902 dup_rows_remote=20932 dedup_ratio=0.243144 router_tokens_per_expert=[767, 768, 809, 753, 786, 721, 765, 776, 753, 806, 758, 806, 798, 764, 744, 766, 766, 745, 752, 730, 781, 704, 796, 800, 740, 731, 774, 817, 811, 791, 760, 747, 757, 739, 825, 731, 799, 772, 783, 763, 759, 808, 755, 782, 790, 783, 750, 786, 724, 753, 832, 778, 725, 732, 792, 782, 742, 792, 790, 700, 749, 726, 770, 798] payload_rows_per_dst=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] dup_rows_per_dst=[2972, 3026, 2958, 2998, 3002, 3090, 2970, 2886] combine_rows_per_src=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65094 dup_rows=23880 dup_rows_remote=21010 dedup_ratio=0.242920 router_tokens_per_expert=[751, 817, 815, 752, 737, 747, 790, 738, 752, 764, 793, 798, 782, 824, 763, 791, 750, 785, 768, 793, 752, 757, 759, 736, 780, 727, 746, 774, 796, 783, 773, 742, 740, 791, 781, 775, 773, 800, 759, 769, 755, 777, 714, 773, 813, 732, 804, 765, 780, 776, 744, 777, 781, 781, 723, 762, 719, 743, 810, 746, 815, 734, 757, 748] payload_rows_per_dst=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] dup_rows_per_dst=[2998, 3094, 2870, 2936, 3034, 2958, 3054, 2936] combine_rows_per_src=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65122 dup_rows=23860 dup_rows_remote=20898 dedup_ratio=0.242716 router_tokens_per_expert=[771, 793, 780, 753, 800, 769, 792, 710, 787, 729, 768, 795, 790, 782, 740, 751, 746, 765, 763, 784, 740, 706, 787, 820, 757, 747, 794, 746, 779, 721, 796, 791, 801, 781, 796, 727, 785, 781, 748, 739, 718, 767, 776, 758, 771, 775, 744, 784, 765, 772, 776, 738, 801, 793, 784, 752, 755, 774, 760, 734, 832, 761, 768, 784] payload_rows_per_dst=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] dup_rows_per_dst=[3010, 2962, 2986, 2964, 2970, 2930, 3034, 3004] combine_rows_per_src=[9326, 9322, 9236, 9298, 9346, 9256, 9328, 9332] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=267722752 dispatch_meta_bytes=345048 combine_payload_bytes=267722752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268067800 ingress_bytes=302325760 link_bytes=302325760 token_bytes=535790552 weight_movement_bytes=34603008 total_nvlink_bytes=570393560 weight_to_token_ratio=0.064583 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137627496e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343976 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266223528 ingress_bytes=335085568 link_bytes=335085568 token_bytes=532103080 weight_movement_bytes=69206016 total_nvlink_bytes=601309096 weight_to_token_ratio=0.130061 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260900016e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=65158 dup_rows=23966 dup_rows_remote=20902 dedup_ratio=0.243795 router_tokens_per_expert=[759, 766, 783, 750, 754, 760, 745, 805, 773, 813, 775, 775, 762, 763, 785, 759, 819, 736, 718, 737, 775, 783, 765, 818, 720, 761, 770, 777, 747, 743, 763, 768, 814, 776, 752, 769, 752, 773, 737, 763, 797, 744, 758, 758, 741, 746, 764, 814, 801, 769, 752, 799, 783, 764, 771, 747, 775, 787, 793, 786, 734, 795, 787, 724] payload_rows_per_dst=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] dup_rows_per_dst=[2938, 3076, 3008, 2840, 2984, 3064, 3080, 2976] combine_rows_per_src=[9306, 9334, 9294, 9258, 9288, 9180, 9292, 9386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265314304 dispatch_meta_bytes=343848 combine_payload_bytes=265314304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265658152 ingress_bytes=334520320 link_bytes=334520320 token_bytes=530972456 weight_movement_bytes=69206016 total_nvlink_bytes=600178472 weight_to_token_ratio=0.130338 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258773033e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344376 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510648 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676920 weight_movement_bytes=34603008 total_nvlink_bytes=567279928 weight_to_token_ratio=0.064961 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=65000 dup_rows=24078 dup_rows_remote=21026 dedup_ratio=0.244934 router_tokens_per_expert=[758, 776, 734, 756, 778, 741, 778, 827, 764, 767, 771, 745, 832, 691, 777, 750, 750, 815, 810, 765, 722, 730, 736, 732, 785, 788, 761, 775, 743, 793, 744, 790, 776, 796, 795, 729, 770, 810, 768, 728, 787, 786, 768, 763, 740, 734, 774, 795, 767, 761, 798, 779, 750, 749, 732, 803, 801, 820, 770, 801, 741, 713, 792, 772] payload_rows_per_dst=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] dup_rows_per_dst=[2958, 3012, 2920, 2980, 3052, 3030, 3052, 3074] combine_rows_per_src=[9338, 9182, 9200, 9378, 9292, 9264, 9226, 9346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344272 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141328 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938384 weight_movement_bytes=69206016 total_nvlink_bytes=603144400 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344416 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969440 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594464 weight_movement_bytes=34603008 total_nvlink_bytes=568197472 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64906 dup_rows=24152 dup_rows_remote=21116 dedup_ratio=0.245687 router_tokens_per_expert=[728, 808, 743, 785, 769, 785, 773, 742, 781, 751, 789, 774, 820, 758, 768, 734, 769, 736, 754, 763, 762, 757, 759, 760, 786, 773, 763, 793, 755, 751, 772, 759, 809, 782, 717, 746, 793, 703, 791, 800, 776, 799, 744, 776, 782, 734, 806, 778, 768, 807, 727, 773, 775, 779, 735, 767, 787, 798, 725, 734, 749, 745, 805, 822] payload_rows_per_dst=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] dup_rows_per_dst=[3018, 3050, 2920, 3014, 3036, 3064, 3044, 3006] combine_rows_per_src=[9248, 9300, 9200, 9290, 9246, 9326, 9218, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=344080 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267083792 ingress_bytes=335945728 link_bytes=335945728 token_bytes=533823504 weight_movement_bytes=69206016 total_nvlink_bytes=603029520 weight_to_token_ratio=0.129642 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264136729e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65050 dup_rows=23908 dup_rows_remote=20884 dedup_ratio=0.243205 router_tokens_per_expert=[794, 746, 750, 728, 770, 760, 750, 748, 769, 768, 803, 759, 830, 730, 779, 708, 792, 780, 766, 747, 739, 807, 778, 808, 805, 749, 769, 764, 796, 741, 767, 758, 802, 722, 786, 786, 784, 734, 762, 756, 781, 725, 834, 793, 752, 771, 754, 775, 741, 768, 799, 751, 843, 771, 774, 781, 740, 776, 742, 735, 799, 781, 727, 749] payload_rows_per_dst=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] dup_rows_per_dst=[2942, 2918, 3082, 2968, 3006, 3024, 3028, 2940] combine_rows_per_src=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344104 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266584104 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532824104 weight_movement_bytes=69206016 total_nvlink_bytes=602030120 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.025000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64830 dup_rows=24092 dup_rows_remote=20944 dedup_ratio=0.245076 router_tokens_per_expert=[756, 750, 752, 757, 810, 722, 756, 767, 772, 793, 775, 741, 778, 742, 767, 763, 781, 744, 747, 799, 744, 744, 788, 781, 760, 714, 799, 790, 746, 796, 759, 795, 750, 771, 787, 759, 764, 757, 765, 762, 817, 750, 814, 780, 736, 788, 786, 724, 789, 794, 767, 782, 797, 752, 786, 747, 785, 782, 778, 762, 811, 714, 768, 740] payload_rows_per_dst=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] dup_rows_per_dst=[2924, 2816, 3016, 3088, 3020, 3054, 3026, 3148] combine_rows_per_src=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64668 dup_rows=24118 dup_rows_remote=20750 dedup_ratio=0.245341 router_tokens_per_expert=[732, 813, 814, 755, 792, 745, 783, 787, 792, 815, 753, 765, 787, 742, 795, 760, 786, 766, 748, 744, 736, 771, 739, 746, 752, 740, 782, 783, 754, 766, 782, 786, 812, 814, 739, 763, 744, 770, 765, 724, 789, 740, 768, 764, 745, 802, 769, 788, 799, 781, 787, 778, 765, 755, 757, 768, 750, 772, 699, 725, 777, 779, 767, 786] payload_rows_per_dst=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] dup_rows_per_dst=[3368, 3104, 2960, 2986, 3018, 3022, 2778, 2882] combine_rows_per_src=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=344240 combine_payload_bytes=266887168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267231408 ingress_bytes=336093184 link_bytes=336093184 token_bytes=534118576 weight_movement_bytes=69206016 total_nvlink_bytes=603324592 weight_to_token_ratio=0.129571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264691594e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265854976 dispatch_meta_bytes=344088 combine_payload_bytes=265854976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266199064 ingress_bytes=300457984 link_bytes=300457984 token_bytes=532054040 weight_movement_bytes=34603008 total_nvlink_bytes=566657048 weight_to_token_ratio=0.065037 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130599205e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65034 dup_rows=23924 dup_rows_remote=20928 dedup_ratio=0.243368 router_tokens_per_expert=[752, 796, 729, 775, 778, 767, 797, 767, 765, 751, 805, 789, 738, 776, 750, 761, 794, 764, 754, 714, 753, 737, 701, 759, 799, 753, 786, 793, 786, 745, 714, 795, 793, 755, 772, 797, 774, 801, 740, 811, 787, 765, 752, 744, 795, 777, 743, 740, 761, 765, 739, 804, 741, 807, 808, 801, 723, 778, 788, 717, 813, 775, 775, 768] payload_rows_per_dst=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] dup_rows_per_dst=[3008, 3006, 2818, 2996, 3066, 2964, 3054, 3012] combine_rows_per_src=[9314, 9264, 9134, 9346, 9420, 9242, 9398, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=341672 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265221800 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530101928 weight_movement_bytes=34603008 total_nvlink_bytes=564704936 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64830 dup_rows=24092 dup_rows_remote=20944 dedup_ratio=0.245076 router_tokens_per_expert=[756, 750, 752, 757, 810, 722, 756, 767, 772, 793, 775, 741, 778, 742, 767, 763, 781, 744, 747, 799, 744, 744, 788, 781, 760, 714, 799, 790, 746, 796, 759, 795, 750, 771, 787, 759, 764, 757, 765, 762, 817, 750, 814, 780, 736, 788, 786, 724, 789, 794, 767, 782, 797, 752, 786, 747, 785, 782, 778, 762, 811, 714, 768, 740] payload_rows_per_dst=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] dup_rows_per_dst=[2924, 2816, 3016, 3088, 3020, 3054, 3026, 3148] combine_rows_per_src=[9216, 8964, 9240, 9336, 9336, 9308, 9430, 9382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64668 dup_rows=24118 dup_rows_remote=20750 dedup_ratio=0.245341 router_tokens_per_expert=[732, 813, 814, 755, 792, 745, 783, 787, 792, 815, 753, 765, 787, 742, 795, 760, 786, 766, 748, 744, 736, 771, 739, 746, 752, 740, 782, 783, 754, 766, 782, 786, 812, 814, 739, 763, 744, 770, 765, 724, 789, 740, 768, 764, 745, 802, 769, 788, 799, 781, 787, 778, 765, 755, 757, 768, 750, 772, 699, 725, 777, 779, 767, 786] payload_rows_per_dst=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] dup_rows_per_dst=[3368, 3104, 2960, 2986, 3018, 3022, 2778, 2882] combine_rows_per_src=[9518, 9326, 9182, 9304, 9244, 9296, 9088, 9228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343096 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886776 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430456 weight_movement_bytes=34603008 total_nvlink_bytes=566033464 weight_to_token_ratio=0.065113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343736 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788536 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233336 weight_movement_bytes=69206016 total_nvlink_bytes=602439352 weight_to_token_ratio=0.129786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64902 dup_rows=24136 dup_rows_remote=21196 dedup_ratio=0.245524 router_tokens_per_expert=[765, 774, 768, 744, 756, 779, 771, 780, 756, 764, 764, 796, 780, 809, 762, 742, 789, 762, 730, 771, 729, 773, 778, 771, 718, 784, 762, 799, 784, 724, 801, 815, 786, 830, 791, 757, 748, 779, 742, 793, 748, 773, 729, 757, 797, 790, 735, 729, 752, 792, 741, 769, 770, 785, 735, 710, 754, 767, 776, 768, 769, 781, 802, 797] payload_rows_per_dst=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] dup_rows_per_dst=[3038, 3024, 2940, 3030, 3034, 2970, 3020, 3080] combine_rows_per_src=[9236, 9322, 9266, 9344, 9418, 9146, 9088, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64890 dup_rows=24114 dup_rows_remote=21090 dedup_ratio=0.245300 router_tokens_per_expert=[787, 749, 771, 748, 765, 734, 730, 777, 780, 746, 736, 785, 728, 794, 748, 752, 802, 810, 765, 789, 771, 771, 779, 773, 756, 758, 732, 732, 783, 781, 771, 799, 806, 778, 738, 734, 767, 760, 752, 775, 798, 755, 762, 771, 764, 794, 751, 752, 772, 781, 746, 769, 775, 791, 733, 795, 807, 805, 804, 820, 739, 746, 766, 744] payload_rows_per_dst=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] dup_rows_per_dst=[2980, 2996, 3052, 2970, 3006, 2954, 3024, 3132] combine_rows_per_src=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343848 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266723112 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533102376 weight_movement_bytes=34603008 total_nvlink_bytes=567705384 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64940 dup_rows=23954 dup_rows_remote=20938 dedup_ratio=0.243673 router_tokens_per_expert=[768, 765, 776, 764, 754, 787, 743, 760, 801, 765, 786, 751, 744, 739, 756, 768, 779, 829, 767, 821, 768, 760, 760, 729, 752, 767, 736, 764, 843, 770, 743, 739, 778, 787, 789, 767, 772, 742, 832, 799, 718, 784, 758, 737, 776, 796, 732, 740, 782, 737, 736, 788, 794, 762, 737, 789, 787, 788, 815, 726, 767, 763, 796, 724] payload_rows_per_dst=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] dup_rows_per_dst=[2960, 2972, 3016, 2906, 3134, 2950, 3034, 2982] combine_rows_per_src=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74538 payload_rows_remote=65088 dup_rows=23766 dup_rows_remote=20596 dedup_ratio=0.241760 router_tokens_per_expert=[785, 781, 807, 756, 755, 726, 749, 785, 805, 788, 737, 812, 805, 768, 829, 766, 770, 793, 747, 767, 789, 760, 795, 800, 749, 821, 749, 746, 768, 779, 772, 748, 762, 771, 784, 777, 746, 780, 724, 733, 758, 755, 759, 747, 745, 733, 797, 759, 787, 720, 756, 720, 785, 723, 776, 783, 749, 754, 799, 773, 805, 762, 763, 760] payload_rows_per_dst=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] dup_rows_per_dst=[2942, 3170, 3146, 2930, 2868, 2882, 2916, 2912] combine_rows_per_src=[9346, 9450, 9296, 9334, 9286, 9224, 9184, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65050 dup_rows=23908 dup_rows_remote=20884 dedup_ratio=0.243205 router_tokens_per_expert=[794, 746, 750, 728, 770, 760, 750, 748, 769, 768, 803, 759, 830, 730, 779, 708, 792, 780, 766, 747, 739, 807, 778, 808, 805, 749, 769, 764, 796, 741, 767, 758, 802, 722, 786, 786, 784, 734, 762, 756, 781, 725, 834, 793, 752, 771, 754, 775, 741, 768, 799, 751, 843, 771, 774, 781, 740, 776, 742, 735, 799, 781, 727, 749] payload_rows_per_dst=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] dup_rows_per_dst=[2942, 2918, 3082, 2968, 3006, 3024, 3028, 2940] combine_rows_per_src=[9150, 9374, 9352, 9330, 9258, 9346, 9428, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265838592 dispatch_meta_bytes=344392 combine_payload_bytes=265838592 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266182984 ingress_bytes=335044608 link_bytes=335044608 token_bytes=532021576 weight_movement_bytes=69206016 total_nvlink_bytes=601227592 weight_to_token_ratio=0.130081 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260745887e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64982 dup_rows=24008 dup_rows_remote=21136 dedup_ratio=0.244222 router_tokens_per_expert=[772, 768, 766, 765, 772, 729, 742, 766, 744, 738, 792, 759, 745, 767, 797, 778, 769, 733, 799, 771, 762, 776, 800, 769, 743, 742, 704, 756, 819, 734, 807, 805, 762, 723, 784, 750, 750, 786, 762, 776, 719, 756, 801, 745, 828, 791, 779, 772, 771, 740, 770, 769, 769, 809, 776, 797, 752, 768, 746, 775, 771, 829, 808, 729] payload_rows_per_dst=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] dup_rows_per_dst=[3008, 3006, 2948, 2952, 2872, 3132, 3058, 3032] combine_rows_per_src=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265543680 dispatch_meta_bytes=343096 combine_payload_bytes=265543680 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265886776 ingress_bytes=300146688 link_bytes=300146688 token_bytes=531430456 weight_movement_bytes=34603008 total_nvlink_bytes=566033464 weight_to_token_ratio=0.065113 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129427823e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=343848 combine_payload_bytes=266379264 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266723112 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533102376 weight_movement_bytes=34603008 total_nvlink_bytes=567705384 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=341672 combine_payload_bytes=264880128 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265221800 ingress_bytes=299483136 link_bytes=299483136 token_bytes=530101928 weight_movement_bytes=34603008 total_nvlink_bytes=564704936 weight_to_token_ratio=0.065276 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126930930e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74402 payload_rows_remote=65136 dup_rows=23902 dup_rows_remote=20932 dedup_ratio=0.243144 router_tokens_per_expert=[767, 768, 809, 753, 786, 721, 765, 776, 753, 806, 758, 806, 798, 764, 744, 766, 766, 745, 752, 730, 781, 704, 796, 800, 740, 731, 774, 817, 811, 791, 760, 747, 757, 739, 825, 731, 799, 772, 783, 763, 759, 808, 755, 782, 790, 783, 750, 786, 724, 753, 832, 778, 725, 732, 792, 782, 742, 792, 790, 700, 749, 726, 770, 798] payload_rows_per_dst=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] dup_rows_per_dst=[2972, 3026, 2958, 2998, 3002, 3090, 2970, 2886] combine_rows_per_src=[9318, 9364, 9190, 9344, 9336, 9336, 9266, 9248] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343920 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266133360 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922800 weight_movement_bytes=103809024 total_nvlink_bytes=635731824 weight_to_token_ratio=0.195158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64776 dup_rows=24178 dup_rows_remote=21014 dedup_ratio=0.245951 router_tokens_per_expert=[789, 749, 781, 813, 801, 777, 770, 777, 816, 780, 776, 739, 780, 760, 773, 773, 762, 717, 721, 760, 749, 818, 770, 774, 766, 805, 717, 734, 739, 782, 722, 768, 754, 797, 804, 750, 791, 772, 774, 790, 766, 767, 747, 778, 764, 730, 749, 788, 778, 758, 804, 777, 748, 786, 726, 825, 768, 754, 749, 768, 793, 748, 777, 714] payload_rows_per_dst=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] dup_rows_per_dst=[3164, 3096, 3044, 2808, 3082, 2912, 3072, 3000] combine_rows_per_src=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343512 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266337752 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532331992 weight_movement_bytes=69206016 total_nvlink_bytes=601538008 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74296 payload_rows_remote=64982 dup_rows=24008 dup_rows_remote=21136 dedup_ratio=0.244222 router_tokens_per_expert=[772, 768, 766, 765, 772, 729, 742, 766, 744, 738, 792, 759, 745, 767, 797, 778, 769, 733, 799, 771, 762, 776, 800, 769, 743, 742, 704, 756, 819, 734, 807, 805, 762, 723, 784, 750, 750, 786, 762, 776, 719, 756, 801, 745, 828, 791, 779, 772, 771, 740, 770, 769, 769, 809, 776, 797, 752, 768, 746, 775, 771, 829, 808, 729] payload_rows_per_dst=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] dup_rows_per_dst=[3008, 3006, 2948, 2952, 2872, 3132, 3058, 3032] combine_rows_per_src=[9152, 9234, 9410, 9268, 9314, 9250, 9344, 9324] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266600448 dispatch_meta_bytes=342736 combine_payload_bytes=266600448 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266943184 ingress_bytes=405012480 link_bytes=405012480 token_bytes=533543632 weight_movement_bytes=138412032 total_nvlink_bytes=671955664 weight_to_token_ratio=0.259420 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524029356e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266444800 dispatch_meta_bytes=343736 combine_payload_bytes=266444800 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266788536 ingress_bytes=335650816 link_bytes=335650816 token_bytes=533233336 weight_movement_bytes=69206016 total_nvlink_bytes=602439352 weight_to_token_ratio=0.129786 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263026999e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344472 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510744 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532677016 weight_movement_bytes=34603008 total_nvlink_bytes=567280024 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64636 dup_rows=24148 dup_rows_remote=20926 dedup_ratio=0.245646 router_tokens_per_expert=[769, 765, 741, 814, 757, 740, 799, 750, 742, 808, 746, 796, 741, 806, 743, 763, 795, 759, 777, 773, 752, 759, 785, 775, 786, 737, 791, 784, 736, 795, 842, 777, 760, 736, 731, 776, 756, 739, 750, 740, 793, 822, 781, 744, 787, 783, 753, 796, 731, 746, 785, 762, 740, 755, 769, 794, 800, 739, 803, 722, 774, 756, 768, 758] payload_rows_per_dst=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] dup_rows_per_dst=[2956, 2808, 2832, 3138, 2700, 3306, 3186, 3222] combine_rows_per_src=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344272 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267141328 ingress_bytes=336003072 link_bytes=336003072 token_bytes=533938384 weight_movement_bytes=69206016 total_nvlink_bytes=603144400 weight_to_token_ratio=0.129614 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264352509e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64786 dup_rows=24270 dup_rows_remote=21362 dedup_ratio=0.246887 router_tokens_per_expert=[763, 748, 849, 764, 790, 754, 810, 766, 781, 737, 776, 781, 726, 814, 783, 760, 782, 723, 744, 779, 793, 766, 782, 734, 793, 724, 726, 798, 763, 766, 742, 773, 741, 779, 760, 764, 786, 798, 754, 784, 743, 740, 757, 737, 786, 779, 760, 776, 780, 811, 787, 809, 733, 767, 737, 764, 796, 745, 789, 734, 772, 765, 784, 745] payload_rows_per_dst=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] dup_rows_per_dst=[3112, 3090, 2982, 3020, 3084, 2908, 3054, 3020] combine_rows_per_src=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64856 dup_rows=24118 dup_rows_remote=21000 dedup_ratio=0.245341 router_tokens_per_expert=[748, 759, 761, 770, 749, 785, 777, 817, 767, 738, 763, 797, 714, 792, 783, 798, 753, 756, 774, 774, 791, 730, 783, 738, 783, 763, 721, 796, 792, 758, 804, 807, 777, 701, 769, 806, 747, 780, 763, 811, 732, 789, 764, 743, 824, 756, 783, 759, 715, 811, 718, 721, 791, 803, 787, 785, 761, 778, 754, 741, 774, 759, 765, 744] payload_rows_per_dst=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] dup_rows_per_dst=[3010, 3056, 2922, 3118, 3072, 3012, 3038, 2890] combine_rows_per_src=[9322, 9248, 9276, 9330, 9236, 9288, 9224, 9262] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343160 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265665656 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988152 weight_movement_bytes=69206016 total_nvlink_bytes=600194168 weight_to_token_ratio=0.130334 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74438 payload_rows_remote=64984 dup_rows=23866 dup_rows_remote=20736 dedup_ratio=0.242778 router_tokens_per_expert=[776, 718, 763, 759, 744, 772, 814, 749, 781, 731, 808, 770, 768, 725, 790, 718, 856, 782, 748, 811, 738, 770, 805, 782, 789, 785, 724, 751, 755, 804, 772, 744, 752, 747, 808, 745, 763, 736, 816, 738, 760, 761, 756, 803, 766, 781, 724, 813, 798, 751, 779, 767, 705, 783, 797, 740, 772, 780, 740, 763, 754, 788, 800, 764] payload_rows_per_dst=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] dup_rows_per_dst=[2988, 2966, 3130, 2960, 2956, 2862, 2964, 3040] combine_rows_per_src=[9202, 9216, 9454, 9288, 9254, 9466, 9276, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=344472 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510744 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532677016 weight_movement_bytes=34603008 total_nvlink_bytes=567280024 weight_to_token_ratio=0.064961 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=342248 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265091304 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529840360 weight_movement_bytes=34603008 total_nvlink_bytes=564443368 weight_to_token_ratio=0.065308 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64776 dup_rows=24178 dup_rows_remote=21014 dedup_ratio=0.245951 router_tokens_per_expert=[789, 749, 781, 813, 801, 777, 770, 777, 816, 780, 776, 739, 780, 760, 773, 773, 762, 717, 721, 760, 749, 818, 770, 774, 766, 805, 717, 734, 739, 782, 722, 768, 754, 797, 804, 750, 791, 772, 774, 790, 766, 767, 747, 778, 764, 730, 749, 788, 778, 758, 804, 777, 748, 786, 726, 825, 768, 754, 749, 768, 793, 748, 777, 714] payload_rows_per_dst=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] dup_rows_per_dst=[3164, 3096, 3044, 2808, 3082, 2912, 3072, 3000] combine_rows_per_src=[9350, 9420, 9194, 9040, 9382, 9266, 9332, 9142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344592 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708048 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071504 weight_movement_bytes=34603008 total_nvlink_bytes=565674512 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64636 dup_rows=24148 dup_rows_remote=20926 dedup_ratio=0.245646 router_tokens_per_expert=[769, 765, 741, 814, 757, 740, 799, 750, 742, 808, 746, 796, 741, 806, 743, 763, 795, 759, 777, 773, 752, 759, 785, 775, 786, 737, 791, 784, 736, 795, 842, 777, 760, 736, 731, 776, 756, 739, 750, 740, 793, 822, 781, 744, 787, 783, 753, 796, 731, 746, 785, 762, 740, 755, 769, 794, 800, 739, 803, 722, 774, 756, 768, 758] payload_rows_per_dst=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] dup_rows_per_dst=[2956, 2808, 2832, 3138, 2700, 3306, 3186, 3222] combine_rows_per_src=[9196, 9004, 9128, 9368, 8882, 9602, 9456, 9520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65172 dup_rows=23922 dup_rows_remote=21070 dedup_ratio=0.243347 router_tokens_per_expert=[798, 750, 738, 797, 805, 805, 804, 752, 761, 783, 790, 784, 765, 770, 806, 759, 803, 755, 749, 778, 781, 785, 794, 790, 778, 746, 763, 769, 778, 801, 757, 782, 810, 741, 792, 744, 795, 790, 762, 806, 711, 715, 729, 781, 738, 762, 781, 783, 738, 734, 813, 772, 729, 759, 728, 758, 754, 702, 730, 770, 775, 733, 805, 736] payload_rows_per_dst=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] dup_rows_per_dst=[3128, 3108, 3044, 3008, 3040, 2912, 2852, 2830] combine_rows_per_src=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65194 dup_rows=23834 dup_rows_remote=20912 dedup_ratio=0.242452 router_tokens_per_expert=[762, 774, 770, 794, 749, 787, 791, 768, 813, 748, 787, 783, 725, 800, 776, 747, 719, 769, 790, 768, 733, 774, 769, 777, 769, 782, 749, 755, 744, 792, 774, 727, 704, 737, 764, 742, 760, 803, 774, 773, 799, 770, 762, 743, 715, 759, 780, 731, 760, 740, 839, 765, 771, 810, 770, 756, 781, 743, 784, 799, 749, 812, 827, 765] payload_rows_per_dst=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] dup_rows_per_dst=[2998, 2932, 2922, 2934, 2924, 2984, 3036, 3104] combine_rows_per_src=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265650176 dispatch_meta_bytes=343424 combine_payload_bytes=265650176 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265993600 ingress_bytes=369459200 link_bytes=369459200 token_bytes=531643776 weight_movement_bytes=103809024 total_nvlink_bytes=635452800 weight_to_token_ratio=0.195260 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390245226e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65058 dup_rows=23924 dup_rows_remote=20882 dedup_ratio=0.243368 router_tokens_per_expert=[691, 778, 777, 726, 759, 748, 737, 761, 781, 806, 759, 833, 761, 773, 741, 728, 732, 747, 756, 813, 755, 744, 780, 795, 798, 779, 769, 799, 771, 780, 763, 814, 780, 732, 787, 799, 746, 729, 793, 774, 798, 777, 805, 801, 768, 772, 722, 784, 741, 732, 766, 749, 826, 770, 705, 774, 758, 761, 753, 800, 743, 802, 797, 754] payload_rows_per_dst=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] dup_rows_per_dst=[2750, 3042, 2974, 3110, 3016, 3114, 2916, 3002] combine_rows_per_src=[9204, 9322, 9270, 9436, 9264, 9340, 9210, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74034 payload_rows_remote=64786 dup_rows=24270 dup_rows_remote=21362 dedup_ratio=0.246887 router_tokens_per_expert=[763, 748, 849, 764, 790, 754, 810, 766, 781, 737, 776, 781, 726, 814, 783, 760, 782, 723, 744, 779, 793, 766, 782, 734, 793, 724, 726, 798, 763, 766, 742, 773, 741, 779, 760, 764, 786, 798, 754, 784, 743, 740, 757, 737, 786, 779, 760, 776, 780, 811, 787, 809, 733, 767, 737, 764, 796, 745, 789, 734, 772, 765, 784, 745] payload_rows_per_dst=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] dup_rows_per_dst=[3112, 3090, 2982, 3020, 3084, 2908, 3054, 3020] combine_rows_per_src=[9376, 9226, 9224, 9150, 9248, 9248, 9322, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266174464 dispatch_meta_bytes=342880 combine_payload_bytes=266174464 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266517344 ingress_bytes=369983488 link_bytes=369983488 token_bytes=532691808 weight_movement_bytes=103809024 total_nvlink_bytes=636500832 weight_to_token_ratio=0.194876 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392218079e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64964 dup_rows=24034 dup_rows_remote=21014 dedup_ratio=0.244486 router_tokens_per_expert=[765, 717, 793, 766, 784, 749, 755, 737, 785, 764, 786, 762, 779, 779, 751, 762, 787, 793, 783, 739, 789, 780, 813, 797, 780, 735, 769, 749, 763, 782, 753, 774, 781, 772, 757, 747, 740, 771, 770, 825, 750, 769, 764, 776, 730, 769, 754, 801, 825, 744, 739, 714, 777, 746, 757, 789, 736, 759, 780, 733, 827, 781, 777, 772] payload_rows_per_dst=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] dup_rows_per_dst=[2940, 2956, 3210, 2926, 3020, 2956, 2974, 3052] combine_rows_per_src=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64890 dup_rows=24114 dup_rows_remote=21090 dedup_ratio=0.245300 router_tokens_per_expert=[787, 749, 771, 748, 765, 734, 730, 777, 780, 746, 736, 785, 728, 794, 748, 752, 802, 810, 765, 789, 771, 771, 779, 773, 756, 758, 732, 732, 783, 781, 771, 799, 806, 778, 738, 734, 767, 760, 752, 775, 798, 755, 762, 771, 764, 794, 751, 752, 772, 781, 746, 769, 775, 791, 733, 795, 807, 805, 804, 820, 739, 746, 766, 744] payload_rows_per_dst=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] dup_rows_per_dst=[2980, 2996, 3052, 2970, 3006, 2954, 3024, 3132] combine_rows_per_src=[9142, 9142, 9468, 9254, 9214, 9340, 9300, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265322496 dispatch_meta_bytes=343160 combine_payload_bytes=265322496 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265665656 ingress_bytes=334528512 link_bytes=334528512 token_bytes=530988152 weight_movement_bytes=69206016 total_nvlink_bytes=600194168 weight_to_token_ratio=0.130334 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258803859e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=264749056 dispatch_meta_bytes=342248 combine_payload_bytes=264749056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265091304 ingress_bytes=299352064 link_bytes=299352064 token_bytes=529840360 weight_movement_bytes=34603008 total_nvlink_bytes=564443368 weight_to_token_ratio=0.065308 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126437717e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344968 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289480 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233992 weight_movement_bytes=34603008 total_nvlink_bytes=568837000 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65064 dup_rows=23898 dup_rows_remote=20930 dedup_ratio=0.243103 router_tokens_per_expert=[776, 751, 752, 782, 741, 771, 774, 782, 769, 781, 815, 721, 783, 742, 795, 770, 732, 792, 708, 816, 742, 740, 728, 809, 773, 731, 776, 790, 727, 744, 765, 731, 779, 780, 765, 761, 752, 785, 844, 735, 754, 759, 759, 795, 753, 735, 785, 792, 803, 779, 770, 745, 764, 795, 774, 729, 762, 794, 757, 814, 781, 786, 785, 772] payload_rows_per_dst=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] dup_rows_per_dst=[2968, 3088, 3118, 2920, 2964, 3220, 2736, 2884] combine_rows_per_src=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344424 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379048 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413672 weight_movement_bytes=34603008 total_nvlink_bytes=569016680 weight_to_token_ratio=0.064749 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64818 dup_rows=24216 dup_rows_remote=21052 dedup_ratio=0.246338 router_tokens_per_expert=[746, 731, 778, 826, 775, 752, 771, 766, 771, 755, 778, 772, 716, 766, 807, 805, 780, 738, 740, 721, 788, 792, 715, 791, 738, 764, 793, 772, 769, 734, 742, 775, 809, 792, 785, 741, 768, 788, 731, 771, 792, 754, 794, 748, 732, 737, 749, 791, 795, 769, 786, 752, 788, 787, 767, 765, 760, 734, 777, 805, 773, 798, 785, 762] payload_rows_per_dst=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] dup_rows_per_dst=[2990, 2998, 2968, 2970, 3084, 2930, 3112, 3164] combine_rows_per_src=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=343760 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266821328 ingress_bytes=335683584 link_bytes=335683584 token_bytes=533298896 weight_movement_bytes=69206016 total_nvlink_bytes=602504912 weight_to_token_ratio=0.129770 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263150302e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265363456 dispatch_meta_bytes=344592 combine_payload_bytes=265363456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265708048 ingress_bytes=299966464 link_bytes=299966464 token_bytes=531071504 weight_movement_bytes=34603008 total_nvlink_bytes=565674512 weight_to_token_ratio=0.065157 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128749655e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436456 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532529000 weight_movement_bytes=69206016 total_nvlink_bytes=601735016 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64964 dup_rows=24034 dup_rows_remote=21014 dedup_ratio=0.244486 router_tokens_per_expert=[765, 717, 793, 766, 784, 749, 755, 737, 785, 764, 786, 762, 779, 779, 751, 762, 787, 793, 783, 739, 789, 780, 813, 797, 780, 735, 769, 749, 763, 782, 753, 774, 781, 772, 757, 747, 740, 771, 770, 825, 750, 769, 764, 776, 730, 769, 754, 801, 825, 744, 739, 714, 777, 746, 757, 789, 736, 759, 780, 733, 827, 781, 777, 772] payload_rows_per_dst=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] dup_rows_per_dst=[2940, 2956, 3210, 2926, 3020, 2956, 2974, 3052] combine_rows_per_src=[9192, 9380, 9352, 9284, 9306, 9270, 9208, 9278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343920 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266133360 ingress_bytes=369598464 link_bytes=369598464 token_bytes=531922800 weight_movement_bytes=103809024 total_nvlink_bytes=635731824 weight_to_token_ratio=0.195158 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390769265e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65086 dup_rows=24006 dup_rows_remote=21032 dedup_ratio=0.244202 router_tokens_per_expert=[775, 762, 782, 777, 807, 782, 788, 726, 782, 745, 797, 742, 791, 783, 747, 792, 762, 760, 754, 738, 780, 800, 751, 724, 766, 786, 751, 801, 764, 787, 735, 768, 779, 784, 742, 772, 788, 748, 767, 755, 781, 777, 730, 791, 747, 764, 771, 732, 819, 732, 721, 767, 753, 800, 768, 809, 784, 829, 724, 793, 794, 727, 745, 754] payload_rows_per_dst=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] dup_rows_per_dst=[3004, 3056, 3022, 3016, 2982, 2974, 3010, 2942] combine_rows_per_src=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343480 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838008 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332536 weight_movement_bytes=69206016 total_nvlink_bytes=600538552 weight_to_token_ratio=0.130250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64848 dup_rows=24268 dup_rows_remote=21246 dedup_ratio=0.246867 router_tokens_per_expert=[797, 779, 799, 784, 784, 774, 770, 764, 743, 751, 780, 794, 769, 749, 778, 752, 774, 789, 798, 792, 751, 766, 736, 775, 783, 798, 731, 749, 763, 770, 779, 732, 737, 778, 790, 762, 726, 781, 767, 800, 791, 752, 716, 785, 749, 803, 772, 785, 781, 749, 815, 729, 789, 768, 803, 715, 711, 791, 774, 725, 770, 739, 762, 784] payload_rows_per_dst=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] dup_rows_per_dst=[3136, 2996, 3112, 3022, 2946, 3090, 2994, 2972] combine_rows_per_src=[9366, 9236, 9250, 9188, 9336, 9216, 9304, 9140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343976 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846120 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348264 weight_movement_bytes=34603008 total_nvlink_bytes=567951272 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74424 payload_rows_remote=65094 dup_rows=23880 dup_rows_remote=21010 dedup_ratio=0.242920 router_tokens_per_expert=[751, 817, 815, 752, 737, 747, 790, 738, 752, 764, 793, 798, 782, 824, 763, 791, 750, 785, 768, 793, 752, 757, 759, 736, 780, 727, 746, 774, 796, 783, 773, 742, 740, 791, 781, 775, 773, 800, 759, 769, 755, 777, 714, 773, 813, 732, 804, 765, 780, 776, 744, 777, 781, 781, 723, 762, 719, 743, 810, 746, 815, 734, 757, 748] payload_rows_per_dst=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] dup_rows_per_dst=[2998, 3094, 2870, 2936, 3034, 2958, 3054, 2936] combine_rows_per_src=[9296, 9440, 9330, 9306, 9342, 9308, 9194, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74088 payload_rows_remote=64818 dup_rows=24216 dup_rows_remote=21052 dedup_ratio=0.246338 router_tokens_per_expert=[746, 731, 778, 826, 775, 752, 771, 766, 771, 755, 778, 772, 716, 766, 807, 805, 780, 738, 740, 721, 788, 792, 715, 791, 738, 764, 793, 772, 769, 734, 742, 775, 809, 792, 785, 741, 768, 788, 731, 771, 792, 754, 794, 748, 732, 737, 749, 791, 795, 769, 786, 752, 788, 787, 767, 765, 760, 734, 777, 805, 773, 798, 785, 762] payload_rows_per_dst=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] dup_rows_per_dst=[2990, 2998, 2968, 2970, 3084, 2930, 3112, 3164] combine_rows_per_src=[9300, 9342, 9162, 9204, 9240, 9264, 9306, 9270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=343912 combine_payload_bytes=266092544 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266436456 ingress_bytes=335298560 link_bytes=335298560 token_bytes=532529000 weight_movement_bytes=69206016 total_nvlink_bytes=601735016 weight_to_token_ratio=0.129957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261701488e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344472 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936728 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528984 weight_movement_bytes=34603008 total_nvlink_bytes=568131992 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74406 payload_rows_remote=65064 dup_rows=23898 dup_rows_remote=20930 dedup_ratio=0.243103 router_tokens_per_expert=[776, 751, 752, 782, 741, 771, 774, 782, 769, 781, 815, 721, 783, 742, 795, 770, 732, 792, 708, 816, 742, 740, 728, 809, 773, 731, 776, 790, 727, 744, 765, 731, 779, 780, 765, 761, 752, 785, 844, 735, 754, 759, 759, 795, 753, 735, 785, 792, 803, 779, 770, 745, 764, 795, 774, 729, 762, 794, 757, 814, 781, 786, 785, 772] payload_rows_per_dst=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] dup_rows_per_dst=[2968, 3088, 3118, 2920, 2964, 3220, 2736, 2884] combine_rows_per_src=[9342, 9520, 9364, 9196, 9364, 9500, 8850, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64964 dup_rows=23954 dup_rows_remote=20746 dedup_ratio=0.243673 router_tokens_per_expert=[768, 733, 786, 741, 756, 773, 791, 768, 773, 712, 723, 774, 727, 765, 738, 776, 772, 762, 766, 742, 751, 793, 770, 789, 772, 802, 753, 787, 711, 809, 746, 749, 740, 838, 755, 767, 741, 768, 784, 791, 793, 732, 775, 792, 773, 771, 775, 742, 767, 786, 781, 782, 795, 809, 785, 792, 804, 720, 714, 777, 799, 778, 761, 787] payload_rows_per_dst=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] dup_rows_per_dst=[2960, 2840, 3022, 2952, 3012, 2968, 3208, 2992] combine_rows_per_src=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64952 dup_rows=24212 dup_rows_remote=21254 dedup_ratio=0.246297 router_tokens_per_expert=[738, 742, 698, 743, 758, 753, 798, 792, 762, 723, 794, 778, 789, 729, 712, 762, 779, 798, 789, 797, 753, 780, 771, 770, 777, 775, 791, 759, 818, 788, 730, 779, 776, 754, 733, 804, 764, 774, 783, 777, 806, 739, 785, 826, 781, 770, 749, 761, 753, 753, 702, 797, 792, 771, 757, 833, 725, 747, 774, 791, 754, 717, 765, 814] payload_rows_per_dst=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] dup_rows_per_dst=[2882, 2958, 3170, 3178, 2970, 3074, 3022, 2958] combine_rows_per_src=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65374 dup_rows=23848 dup_rows_remote=20880 dedup_ratio=0.242594 router_tokens_per_expert=[762, 783, 770, 804, 790, 807, 738, 788, 765, 804, 773, 744, 772, 817, 779, 814, 762, 762, 753, 756, 748, 743, 780, 721, 772, 731, 739, 780, 741, 768, 712, 764, 778, 824, 749, 791, 773, 765, 800, 735, 712, 786, 765, 763, 736, 757, 769, 767, 768, 798, 749, 765, 765, 779, 781, 780, 750, 770, 732, 740, 773, 806, 815, 769] payload_rows_per_dst=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] dup_rows_per_dst=[3096, 3126, 2968, 2884, 2914, 2892, 2988, 2980] combine_rows_per_src=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=344376 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265961784 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531579192 weight_movement_bytes=34603008 total_nvlink_bytes=566182200 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=65086 dup_rows=24006 dup_rows_remote=21032 dedup_ratio=0.244202 router_tokens_per_expert=[775, 762, 782, 777, 807, 782, 788, 726, 782, 745, 797, 742, 791, 783, 747, 792, 762, 760, 754, 738, 780, 800, 751, 724, 766, 786, 751, 801, 764, 787, 735, 768, 779, 784, 742, 772, 788, 748, 767, 755, 781, 777, 730, 791, 747, 764, 771, 732, 819, 732, 721, 767, 753, 800, 768, 809, 784, 829, 724, 793, 794, 727, 745, 754] payload_rows_per_dst=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] dup_rows_per_dst=[3004, 3056, 3022, 3016, 2982, 2974, 3010, 2942] combine_rows_per_src=[9394, 9302, 9116, 9300, 9288, 9212, 9328, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344416 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266969440 ingress_bytes=301228032 link_bytes=301228032 token_bytes=533594464 weight_movement_bytes=34603008 total_nvlink_bytes=568197472 weight_to_token_ratio=0.064849 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133496834e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265494528 dispatch_meta_bytes=343480 combine_payload_bytes=265494528 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265838008 ingress_bytes=334700544 link_bytes=334700544 token_bytes=531332536 weight_movement_bytes=69206016 total_nvlink_bytes=600538552 weight_to_token_ratio=0.130250 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259451201e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64970 dup_rows=24002 dup_rows_remote=21046 dedup_ratio=0.244161 router_tokens_per_expert=[755, 790, 751, 683, 759, 789, 787, 755, 769, 800, 769, 758, 749, 782, 790, 750, 774, 809, 736, 768, 761, 754, 758, 755, 778, 769, 800, 769, 804, 786, 806, 794, 802, 725, 766, 765, 763, 800, 737, 786, 805, 694, 793, 723, 729, 750, 750, 776, 735, 816, 786, 783, 755, 802, 708, 781, 791, 799, 766, 730, 774, 750, 784, 771] payload_rows_per_dst=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] dup_rows_per_dst=[2962, 3066, 2924, 3198, 2956, 2860, 3068, 2968] combine_rows_per_src=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64952 dup_rows=24212 dup_rows_remote=21254 dedup_ratio=0.246297 router_tokens_per_expert=[738, 742, 698, 743, 758, 753, 798, 792, 762, 723, 794, 778, 789, 729, 712, 762, 779, 798, 789, 797, 753, 780, 771, 770, 777, 775, 791, 759, 818, 788, 730, 779, 776, 754, 733, 804, 764, 774, 783, 777, 806, 739, 785, 826, 781, 770, 749, 761, 753, 753, 702, 797, 792, 771, 757, 833, 725, 747, 774, 791, 754, 717, 765, 814] payload_rows_per_dst=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] dup_rows_per_dst=[2882, 2958, 3170, 3178, 2970, 3074, 3022, 2958] combine_rows_per_src=[9162, 9140, 9304, 9256, 9360, 9360, 9294, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65172 dup_rows=23922 dup_rows_remote=21070 dedup_ratio=0.243347 router_tokens_per_expert=[798, 750, 738, 797, 805, 805, 804, 752, 761, 783, 790, 784, 765, 770, 806, 759, 803, 755, 749, 778, 781, 785, 794, 790, 778, 746, 763, 769, 778, 801, 757, 782, 810, 741, 792, 744, 795, 790, 762, 806, 711, 715, 729, 781, 738, 762, 781, 783, 738, 734, 813, 772, 729, 759, 728, 758, 754, 702, 730, 770, 775, 733, 805, 736] payload_rows_per_dst=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] dup_rows_per_dst=[3128, 3108, 3044, 3008, 3040, 2912, 2852, 2830] combine_rows_per_src=[9370, 9328, 9426, 9340, 9440, 9088, 9210, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267771904 dispatch_meta_bytes=345016 combine_payload_bytes=267771904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268116920 ingress_bytes=302374912 link_bytes=302374912 token_bytes=535888824 weight_movement_bytes=34603008 total_nvlink_bytes=570491832 weight_to_token_ratio=0.064571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137812451e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64882 dup_rows=24004 dup_rows_remote=20978 dedup_ratio=0.244181 router_tokens_per_expert=[757, 775, 773, 719, 745, 801, 766, 771, 768, 780, 696, 784, 792, 785, 756, 802, 736, 789, 741, 796, 769, 772, 792, 776, 771, 757, 760, 762, 778, 716, 767, 799, 764, 715, 732, 750, 794, 742, 804, 771, 768, 741, 802, 765, 770, 775, 788, 821, 735, 776, 762, 803, 768, 760, 781, 764, 781, 767, 775, 764, 756, 803, 772, 732] payload_rows_per_dst=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] dup_rows_per_dst=[2968, 2632, 3002, 2882, 3252, 3240, 3002, 3026] combine_rows_per_src=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=343976 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846120 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348264 weight_movement_bytes=34603008 total_nvlink_bytes=567951272 weight_to_token_ratio=0.064879 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342840 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266435384 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532527928 weight_movement_bytes=103809024 total_nvlink_bytes=636336952 weight_to_token_ratio=0.194936 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20974 dedup_ratio=0.243876 router_tokens_per_expert=[761, 773, 736, 787, 754, 787, 779, 781, 768, 784, 760, 829, 762, 772, 804, 776, 785, 804, 807, 746, 787, 739, 763, 737, 734, 787, 776, 779, 786, 767, 757, 740, 774, 722, 746, 786, 744, 771, 743, 815, 781, 758, 792, 795, 733, 759, 712, 808, 759, 832, 790, 722, 768, 752, 755, 803, 757, 739, 739, 748, 754, 775, 747, 766] payload_rows_per_dst=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] dup_rows_per_dst=[3000, 3090, 3258, 3048, 2984, 2958, 2724, 2912] combine_rows_per_src=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64970 dup_rows=24002 dup_rows_remote=21046 dedup_ratio=0.244161 router_tokens_per_expert=[755, 790, 751, 683, 759, 789, 787, 755, 769, 800, 769, 758, 749, 782, 790, 750, 774, 809, 736, 768, 761, 754, 758, 755, 778, 769, 800, 769, 804, 786, 806, 794, 802, 725, 766, 765, 763, 800, 737, 786, 805, 694, 793, 723, 729, 750, 750, 776, 735, 816, 786, 783, 755, 802, 708, 781, 791, 799, 766, 730, 774, 750, 784, 771] payload_rows_per_dst=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] dup_rows_per_dst=[2962, 3066, 2924, 3198, 2956, 2860, 3068, 2968] combine_rows_per_src=[9176, 9268, 9306, 9414, 9332, 9180, 9264, 9362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64806 dup_rows=24194 dup_rows_remote=21074 dedup_ratio=0.246114 router_tokens_per_expert=[776, 787, 812, 726, 733, 759, 739, 793, 743, 769, 817, 762, 773, 784, 752, 799, 734, 810, 765, 768, 781, 753, 768, 791, 778, 784, 776, 772, 826, 757, 754, 770, 783, 693, 769, 774, 759, 781, 753, 756, 762, 710, 793, 757, 791, 810, 805, 784, 748, 779, 775, 714, 756, 747, 756, 786, 738, 814, 752, 741, 787, 775, 727, 766] payload_rows_per_dst=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] dup_rows_per_dst=[3046, 3070, 3014, 3114, 2976, 3120, 2944, 2910] combine_rows_per_src=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266043392 dispatch_meta_bytes=344824 combine_payload_bytes=266043392 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266388216 ingress_bytes=300646400 link_bytes=300646400 token_bytes=532431608 weight_movement_bytes=34603008 total_nvlink_bytes=567034616 weight_to_token_ratio=0.064991 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131308199e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=344472 combine_payload_bytes=266592256 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266936728 ingress_bytes=301195264 link_bytes=301195264 token_bytes=533528984 weight_movement_bytes=34603008 total_nvlink_bytes=568131992 weight_to_token_ratio=0.064857 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133373530e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344064 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266461184 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532578304 weight_movement_bytes=138412032 total_nvlink_bytes=670990336 weight_to_token_ratio=0.259890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343952 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266641296 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938640 weight_movement_bytes=34603008 total_nvlink_bytes=567541648 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266944512 dispatch_meta_bytes=344968 combine_payload_bytes=266944512 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267289480 ingress_bytes=301547520 link_bytes=301547520 token_bytes=534233992 weight_movement_bytes=34603008 total_nvlink_bytes=568837000 weight_to_token_ratio=0.064771 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134699041e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100112 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531856784 weight_movement_bytes=34603008 total_nvlink_bytes=566459792 weight_to_token_ratio=0.065061 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74632 payload_rows_remote=65216 dup_rows=23672 dup_rows_remote=20636 dedup_ratio=0.240804 router_tokens_per_expert=[735, 786, 769, 776, 778, 739, 759, 768, 779, 773, 745, 754, 739, 749, 725, 727, 757, 829, 779, 765, 805, 730, 729, 833, 787, 793, 787, 808, 760, 741, 756, 794, 762, 768, 800, 766, 801, 718, 772, 790, 755, 816, 778, 784, 742, 700, 771, 773, 760, 800, 786, 772, 804, 751, 778, 753, 775, 745, 750, 759, 783, 756, 764, 766] payload_rows_per_dst=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] dup_rows_per_dst=[2930, 2780, 3086, 3036, 2938, 2962, 2992, 2948] combine_rows_per_src=[9290, 9202, 9368, 9416, 9416, 9276, 9416, 9248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64882 dup_rows=24004 dup_rows_remote=20978 dedup_ratio=0.244181 router_tokens_per_expert=[757, 775, 773, 719, 745, 801, 766, 771, 768, 780, 696, 784, 792, 785, 756, 802, 736, 789, 741, 796, 769, 772, 792, 776, 771, 757, 760, 762, 778, 716, 767, 799, 764, 715, 732, 750, 794, 742, 804, 771, 768, 741, 802, 765, 770, 775, 788, 821, 735, 776, 762, 803, 768, 760, 781, 764, 781, 767, 775, 764, 756, 803, 772, 732] payload_rows_per_dst=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] dup_rows_per_dst=[2968, 2632, 3002, 2882, 3252, 3240, 3002, 3026] combine_rows_per_src=[9242, 8808, 9340, 9338, 9398, 9456, 9300, 9418] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64940 dup_rows=23954 dup_rows_remote=20938 dedup_ratio=0.243673 router_tokens_per_expert=[768, 765, 776, 764, 754, 787, 743, 760, 801, 765, 786, 751, 744, 739, 756, 768, 779, 829, 767, 821, 768, 760, 760, 729, 752, 767, 736, 764, 843, 770, 743, 739, 778, 787, 789, 767, 772, 742, 832, 799, 718, 784, 758, 737, 776, 796, 732, 740, 782, 737, 736, 788, 794, 762, 737, 789, 787, 788, 815, 726, 767, 763, 796, 724] payload_rows_per_dst=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] dup_rows_per_dst=[2960, 2972, 3016, 2906, 3134, 2950, 3034, 2982] combine_rows_per_src=[9274, 9248, 9410, 9322, 9398, 9132, 9216, 9350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266117120 dispatch_meta_bytes=344064 combine_payload_bytes=266117120 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266461184 ingress_bytes=404529152 link_bytes=404529152 token_bytes=532578304 weight_movement_bytes=138412032 total_nvlink_bytes=670990336 weight_to_token_ratio=0.259890 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522210632e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343520 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265788896 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531234272 weight_movement_bytes=34603008 total_nvlink_bytes=565837280 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64800 dup_rows=24054 dup_rows_remote=20910 dedup_ratio=0.244690 router_tokens_per_expert=[799, 754, 716, 788, 791, 785, 745, 786, 817, 786, 760, 827, 808, 754, 800, 809, 716, 752, 707, 754, 753, 771, 726, 737, 768, 743, 770, 717, 797, 715, 790, 769, 796, 761, 774, 730, 777, 793, 758, 769, 770, 734, 702, 740, 796, 735, 738, 823, 809, 749, 786, 831, 755, 801, 792, 774, 795, 790, 738, 760, 778, 742, 790, 756] payload_rows_per_dst=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] dup_rows_per_dst=[3054, 3196, 2810, 2958, 3002, 2896, 3144, 2994] combine_rows_per_src=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73976 payload_rows_remote=64756 dup_rows=24328 dup_rows_remote=21280 dedup_ratio=0.247477 router_tokens_per_expert=[797, 841, 776, 780, 777, 799, 756, 808, 749, 734, 766, 786, 829, 725, 774, 786, 761, 728, 742, 747, 771, 792, 780, 813, 759, 789, 753, 788, 745, 755, 781, 798, 733, 770, 768, 788, 747, 772, 809, 741, 802, 807, 765, 733, 758, 810, 691, 756, 727, 734, 757, 761, 749, 786, 794, 764, 714, 778, 773, 772, 708, 744, 780, 776] payload_rows_per_dst=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] dup_rows_per_dst=[3224, 3008, 3048, 3026, 2970, 3050, 3026, 2976] combine_rows_per_src=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267124736 dispatch_meta_bytes=343408 combine_payload_bytes=267124736 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267468144 ingress_bytes=301727744 link_bytes=301727744 token_bytes=534592880 weight_movement_bytes=34603008 total_nvlink_bytes=569195888 weight_to_token_ratio=0.064728 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135377210e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65014 dup_rows=23974 dup_rows_remote=20974 dedup_ratio=0.243876 router_tokens_per_expert=[761, 773, 736, 787, 754, 787, 779, 781, 768, 784, 760, 829, 762, 772, 804, 776, 785, 804, 807, 746, 787, 739, 763, 737, 734, 787, 776, 779, 786, 767, 757, 740, 774, 722, 746, 786, 744, 771, 743, 815, 781, 758, 792, 795, 733, 759, 712, 808, 759, 832, 790, 722, 768, 752, 755, 803, 757, 739, 739, 748, 754, 775, 747, 766] payload_rows_per_dst=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] dup_rows_per_dst=[3000, 3090, 3258, 3048, 2984, 2958, 2724, 2912] combine_rows_per_src=[9316, 9480, 9470, 9380, 9218, 9318, 9010, 9138] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74302 payload_rows_remote=64978 dup_rows=24002 dup_rows_remote=20890 dedup_ratio=0.244161 router_tokens_per_expert=[817, 743, 822, 798, 780, 782, 769, 759, 780, 794, 795, 732, 785, 788, 776, 768, 753, 798, 737, 779, 706, 808, 773, 761, 767, 762, 807, 790, 710, 774, 729, 779, 733, 758, 752, 770, 778, 724, 759, 753, 742, 769, 749, 767, 741, 774, 752, 745, 807, 745, 771, 785, 765, 790, 810, 722, 824, 727, 796, 752, 725, 803, 785, 758] payload_rows_per_dst=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] dup_rows_per_dst=[3028, 3112, 2894, 3042, 2914, 2896, 3062, 3054] combine_rows_per_src=[9512, 9324, 9336, 9194, 9140, 9182, 9328, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343440 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100112 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531856784 weight_movement_bytes=34603008 total_nvlink_bytes=566459792 weight_to_token_ratio=0.065061 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64806 dup_rows=24194 dup_rows_remote=21074 dedup_ratio=0.246114 router_tokens_per_expert=[776, 787, 812, 726, 733, 759, 739, 793, 743, 769, 817, 762, 773, 784, 752, 799, 734, 810, 765, 768, 781, 753, 768, 791, 778, 784, 776, 772, 826, 757, 754, 770, 783, 693, 769, 774, 759, 781, 753, 756, 762, 710, 793, 757, 791, 810, 805, 784, 748, 779, 775, 714, 756, 747, 756, 786, 738, 814, 752, 741, 787, 775, 727, 766] payload_rows_per_dst=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] dup_rows_per_dst=[3046, 3070, 3014, 3114, 2976, 3120, 2944, 2910] combine_rows_per_src=[9204, 9328, 9326, 9320, 9160, 9304, 9178, 9290] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65032 dup_rows=24084 dup_rows_remote=21110 dedup_ratio=0.244995 router_tokens_per_expert=[760, 746, 787, 750, 748, 719, 719, 749, 749, 792, 739, 750, 733, 778, 791, 778, 791, 827, 738, 765, 784, 751, 760, 748, 806, 807, 729, 771, 751, 793, 764, 770, 725, 781, 765, 750, 758, 768, 732, 802, 734, 756, 780, 780, 795, 795, 801, 762, 805, 778, 768, 762, 788, 760, 764, 767, 769, 784, 808, 732, 782, 803, 765, 790] payload_rows_per_dst=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] dup_rows_per_dst=[2798, 3044, 3022, 3082, 2974, 3032, 3054, 3078] combine_rows_per_src=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343512 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266337752 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532331992 weight_movement_bytes=69206016 total_nvlink_bytes=601538008 weight_to_token_ratio=0.130005 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=344144 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584720 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530825296 weight_movement_bytes=34603008 total_nvlink_bytes=565428304 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64764 dup_rows=24242 dup_rows_remote=21128 dedup_ratio=0.246602 router_tokens_per_expert=[794, 758, 836, 757, 771, 742, 838, 771, 762, 745, 772, 782, 755, 814, 735, 779, 804, 790, 790, 770, 779, 744, 752, 760, 781, 785, 785, 784, 790, 729, 784, 733, 750, 739, 803, 775, 766, 734, 787, 742, 772, 718, 732, 806, 738, 787, 756, 773, 761, 764, 774, 734, 767, 742, 830, 741, 751, 754, 772, 752, 810, 726, 791, 734] payload_rows_per_dst=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] dup_rows_per_dst=[2856, 3280, 3028, 2974, 3024, 2930, 3036, 3114] combine_rows_per_src=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=64964 dup_rows=23954 dup_rows_remote=20746 dedup_ratio=0.243673 router_tokens_per_expert=[768, 733, 786, 741, 756, 773, 791, 768, 773, 712, 723, 774, 727, 765, 738, 776, 772, 762, 766, 742, 751, 793, 770, 789, 772, 802, 753, 787, 711, 809, 746, 749, 740, 838, 755, 767, 741, 768, 784, 791, 793, 732, 775, 792, 773, 771, 775, 742, 767, 786, 781, 782, 795, 809, 785, 792, 804, 720, 714, 777, 799, 778, 761, 787] payload_rows_per_dst=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] dup_rows_per_dst=[2960, 2840, 3022, 2952, 3012, 2968, 3208, 2992] combine_rows_per_src=[9272, 9136, 9268, 9306, 9356, 9338, 9386, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266297344 dispatch_meta_bytes=343952 combine_payload_bytes=266297344 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266641296 ingress_bytes=300900352 link_bytes=300900352 token_bytes=532938640 weight_movement_bytes=34603008 total_nvlink_bytes=567541648 weight_to_token_ratio=0.064929 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132263800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64852 dup_rows=24298 dup_rows_remote=21220 dedup_ratio=0.247172 router_tokens_per_expert=[781, 791, 746, 753, 752, 761, 784, 741, 786, 732, 734, 770, 783, 706, 762, 770, 754, 779, 785, 750, 794, 770, 758, 754, 775, 748, 818, 745, 752, 786, 733, 745, 762, 735, 775, 777, 769, 783, 759, 787, 762, 770, 834, 693, 766, 790, 805, 809, 779, 771, 691, 806, 791, 761, 778, 752, 772, 793, 765, 788, 782, 761, 795, 793] payload_rows_per_dst=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] dup_rows_per_dst=[3078, 2882, 3086, 3014, 3002, 3250, 2932, 3054] combine_rows_per_src=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=342840 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265763640 ingress_bytes=403832832 link_bytes=403832832 token_bytes=531184440 weight_movement_bytes=138412032 total_nvlink_bytes=669596472 weight_to_token_ratio=0.260572 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519590436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64936 dup_rows=23970 dup_rows_remote=20914 dedup_ratio=0.243835 router_tokens_per_expert=[750, 772, 746, 759, 842, 746, 763, 789, 785, 769, 775, 775, 725, 770, 804, 745, 772, 763, 762, 778, 720, 792, 761, 729, 744, 771, 731, 775, 776, 754, 744, 770, 780, 764, 743, 791, 730, 762, 809, 724, 774, 754, 811, 773, 793, 777, 774, 771, 784, 773, 774, 755, 772, 775, 753, 803, 783, 769, 783, 743, 746, 758, 786, 808] payload_rows_per_dst=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] dup_rows_per_dst=[3112, 2950, 2954, 2974, 2960, 3056, 2964, 3000] combine_rows_per_src=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74220 payload_rows_remote=65032 dup_rows=24084 dup_rows_remote=21110 dedup_ratio=0.244995 router_tokens_per_expert=[760, 746, 787, 750, 748, 719, 719, 749, 749, 792, 739, 750, 733, 778, 791, 778, 791, 827, 738, 765, 784, 751, 760, 748, 806, 807, 729, 771, 751, 793, 764, 770, 725, 781, 765, 750, 758, 768, 732, 802, 734, 756, 780, 780, 795, 795, 801, 762, 805, 778, 768, 762, 788, 760, 764, 767, 769, 784, 808, 732, 782, 803, 765, 790] payload_rows_per_dst=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] dup_rows_per_dst=[2798, 3044, 3022, 3082, 2974, 3032, 3054, 3078] combine_rows_per_src=[9158, 9176, 9306, 9300, 9188, 9374, 9330, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266149888 dispatch_meta_bytes=343472 combine_payload_bytes=266149888 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266493360 ingress_bytes=404561920 link_bytes=404561920 token_bytes=532643248 weight_movement_bytes=138412032 total_nvlink_bytes=671055280 weight_to_token_ratio=0.259859 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.522333935e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265445376 dispatch_meta_bytes=343520 combine_payload_bytes=265445376 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265788896 ingress_bytes=300048384 link_bytes=300048384 token_bytes=531234272 weight_movement_bytes=34603008 total_nvlink_bytes=565837280 weight_to_token_ratio=0.065137 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129057913e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344568 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715640 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086712 weight_movement_bytes=34603008 total_nvlink_bytes=567689720 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344288 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265978080 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611872 weight_movement_bytes=103809024 total_nvlink_bytes=635420896 weight_to_token_ratio=0.195272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=343568 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265616912 ingress_bytes=299876352 link_bytes=299876352 token_bytes=530890256 weight_movement_bytes=34603008 total_nvlink_bytes=565493264 weight_to_token_ratio=0.065179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128410571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=65090 dup_rows=23908 dup_rows_remote=20916 dedup_ratio=0.243205 router_tokens_per_expert=[775, 766, 759, 762, 791, 751, 760, 750, 766, 739, 773, 784, 744, 734, 745, 777, 778, 766, 753, 758, 714, 743, 781, 797, 776, 737, 778, 773, 758, 795, 759, 773, 748, 774, 786, 789, 771, 795, 811, 848, 758, 800, 789, 742, 789, 769, 714, 763, 830, 754, 761, 781, 720, 710, 781, 771, 755, 788, 739, 757, 775, 816, 781, 772] payload_rows_per_dst=[9370, 9242, 9154, 9306, 9486, 9278, 9198, 9362] dup_rows_per_dst=[2858, 2882, 3026, 2992, 3158, 2970, 3018, 3004] combine_rows_per_src=[9370, 9242, 9154, 9306, 9486, 9278, 9198, 9362] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74062 payload_rows_remote=64764 dup_rows=24242 dup_rows_remote=21128 dedup_ratio=0.246602 router_tokens_per_expert=[794, 758, 836, 757, 771, 742, 838, 771, 762, 745, 772, 782, 755, 814, 735, 779, 804, 790, 790, 770, 779, 744, 752, 760, 781, 785, 785, 784, 790, 729, 784, 733, 750, 739, 803, 775, 766, 734, 787, 742, 772, 718, 732, 806, 738, 787, 756, 773, 761, 764, 774, 734, 767, 742, 830, 741, 751, 754, 772, 752, 810, 726, 791, 734] payload_rows_per_dst=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] dup_rows_per_dst=[2856, 3280, 3028, 2974, 3024, 2930, 3036, 3114] combine_rows_per_src=[8996, 9458, 9350, 9322, 9172, 9234, 9232, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266092544 dispatch_meta_bytes=342840 combine_payload_bytes=266092544 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266435384 ingress_bytes=369901568 link_bytes=369901568 token_bytes=532527928 weight_movement_bytes=103809024 total_nvlink_bytes=636336952 weight_to_token_ratio=0.194936 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391909821e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343400 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321256 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299112 weight_movement_bytes=34603008 total_nvlink_bytes=566902120 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=344568 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266715640 ingress_bytes=300974080 link_bytes=300974080 token_bytes=533086712 weight_movement_bytes=34603008 total_nvlink_bytes=567689720 weight_to_token_ratio=0.064911 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132541233e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64892 dup_rows=23986 dup_rows_remote=20940 dedup_ratio=0.243998 router_tokens_per_expert=[726, 827, 760, 780, 766, 737, 812, 763, 774, 705, 794, 733, 815, 769, 739, 782, 750, 784, 730, 777, 753, 789, 764, 798, 772, 785, 742, 799, 749, 784, 743, 801, 759, 774, 772, 811, 805, 779, 739, 767, 739, 739, 742, 732, 754, 750, 791, 743, 794, 812, 766, 756, 796, 754, 792, 766, 754, 812, 767, 757, 767, 741, 752, 768] payload_rows_per_dst=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] dup_rows_per_dst=[2968, 2964, 3034, 3024, 3132, 2856, 3046, 2962] combine_rows_per_src=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65194 dup_rows=23834 dup_rows_remote=20912 dedup_ratio=0.242452 router_tokens_per_expert=[762, 774, 770, 794, 749, 787, 791, 768, 813, 748, 787, 783, 725, 800, 776, 747, 719, 769, 790, 768, 733, 774, 769, 777, 769, 782, 749, 755, 744, 792, 774, 727, 704, 737, 764, 742, 760, 803, 774, 773, 799, 770, 762, 743, 715, 759, 780, 731, 760, 740, 839, 765, 771, 810, 770, 756, 781, 743, 784, 799, 749, 812, 827, 765] payload_rows_per_dst=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] dup_rows_per_dst=[2998, 2932, 2922, 2934, 2924, 2984, 3036, 3104] combine_rows_per_src=[9392, 9426, 9276, 9250, 9190, 9134, 9386, 9416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64940 dup_rows=24236 dup_rows_remote=21282 dedup_ratio=0.246541 router_tokens_per_expert=[740, 762, 771, 790, 798, 807, 786, 717, 707, 767, 740, 810, 761, 750, 797, 774, 713, 850, 740, 745, 719, 744, 752, 778, 777, 782, 784, 753, 747, 724, 741, 726, 776, 798, 754, 754, 836, 738, 771, 789, 768, 760, 814, 834, 747, 765, 785, 757, 796, 718, 773, 773, 806, 801, 790, 754, 778, 716, 746, 822, 784, 740, 757, 800] payload_rows_per_dst=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] dup_rows_per_dst=[3028, 3020, 2954, 2952, 3062, 3074, 3022, 3124] combine_rows_per_src=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344024 combine_payload_bytes=266608640 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266952664 ingress_bytes=335814656 link_bytes=335814656 token_bytes=533561304 weight_movement_bytes=69206016 total_nvlink_bytes=602767320 weight_to_token_ratio=0.129706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263643515e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265273344 dispatch_meta_bytes=343568 combine_payload_bytes=265273344 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265616912 ingress_bytes=299876352 link_bytes=299876352 token_bytes=530890256 weight_movement_bytes=34603008 total_nvlink_bytes=565493264 weight_to_token_ratio=0.065179 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128410571e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74006 payload_rows_remote=64852 dup_rows=24298 dup_rows_remote=21220 dedup_ratio=0.247172 router_tokens_per_expert=[781, 791, 746, 753, 752, 761, 784, 741, 786, 732, 734, 770, 783, 706, 762, 770, 754, 779, 785, 750, 794, 770, 758, 754, 775, 748, 818, 745, 752, 786, 733, 745, 762, 735, 775, 777, 769, 783, 759, 787, 762, 770, 834, 693, 766, 790, 805, 809, 779, 771, 691, 806, 791, 761, 778, 752, 772, 793, 765, 788, 782, 761, 795, 793] payload_rows_per_dst=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] dup_rows_per_dst=[3078, 2882, 3086, 3014, 3002, 3250, 2932, 3054] combine_rows_per_src=[9154, 9204, 9402, 9190, 9078, 9354, 9180, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344888 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266339128 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532333368 weight_movement_bytes=69206016 total_nvlink_bytes=601539384 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64956 dup_rows=23982 dup_rows_remote=20956 dedup_ratio=0.243958 router_tokens_per_expert=[828, 777, 754, 760, 757, 765, 775, 702, 807, 791, 773, 766, 757, 761, 777, 749, 782, 799, 752, 783, 755, 755, 774, 761, 757, 757, 755, 774, 744, 796, 770, 811, 799, 750, 789, 791, 781, 818, 741, 810, 727, 781, 739, 792, 757, 744, 708, 746, 736, 784, 761, 775, 772, 740, 784, 793, 783, 828, 738, 736, 766, 764, 786, 709] payload_rows_per_dst=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] dup_rows_per_dst=[3012, 3034, 3028, 2982, 3116, 2808, 2976, 3026] combine_rows_per_src=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74478 payload_rows_remote=65164 dup_rows=23826 dup_rows_remote=20798 dedup_ratio=0.242371 router_tokens_per_expert=[763, 771, 745, 745, 762, 807, 752, 758, 748, 782, 790, 790, 827, 735, 723, 776, 781, 755, 748, 793, 755, 728, 761, 794, 756, 771, 785, 802, 793, 745, 763, 757, 784, 807, 747, 746, 698, 773, 750, 771, 738, 789, 792, 779, 772, 789, 774, 803, 769, 764, 743, 801, 762, 779, 755, 798, 773, 776, 784, 765, 763, 763, 741, 743] payload_rows_per_dst=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] dup_rows_per_dst=[2900, 3028, 3028, 2938, 2970, 3050, 2974, 2938] combine_rows_per_src=[9306, 9314, 9202, 9406, 9182, 9422, 9368, 9278] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65170 dup_rows=23778 dup_rows_remote=20776 dedup_ratio=0.241882 router_tokens_per_expert=[801, 800, 806, 750, 763, 761, 800, 740, 771, 761, 728, 754, 753, 813, 753, 766, 755, 756, 788, 799, 781, 770, 748, 761, 796, 788, 751, 741, 809, 798, 742, 748, 751, 816, 775, 788, 722, 779, 763, 785, 767, 710, 789, 718, 816, 765, 719, 785, 810, 790, 742, 773, 789, 791, 745, 746, 784, 734, 726, 808, 748, 817, 689, 761] payload_rows_per_dst=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] dup_rows_per_dst=[3146, 2960, 3012, 2884, 3002, 2894, 2986, 2894] combine_rows_per_src=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=64936 dup_rows=23970 dup_rows_remote=20914 dedup_ratio=0.243835 router_tokens_per_expert=[750, 772, 746, 759, 842, 746, 763, 789, 785, 769, 775, 775, 725, 770, 804, 745, 772, 763, 762, 778, 720, 792, 761, 729, 744, 771, 731, 775, 776, 754, 744, 770, 780, 764, 743, 791, 730, 762, 809, 724, 774, 754, 811, 773, 793, 777, 774, 771, 784, 773, 774, 755, 772, 775, 753, 803, 783, 769, 783, 743, 746, 758, 786, 808] payload_rows_per_dst=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] dup_rows_per_dst=[3112, 2950, 2954, 2974, 2960, 3056, 2964, 3000] combine_rows_per_src=[9222, 9346, 9200, 9156, 9246, 9398, 9414, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64910 dup_rows=24194 dup_rows_remote=21246 dedup_ratio=0.246114 router_tokens_per_expert=[764, 754, 750, 786, 718, 769, 773, 742, 827, 754, 765, 732, 767, 765, 789, 799, 773, 770, 830, 750, 729, 725, 771, 774, 813, 754, 799, 775, 748, 786, 789, 753, 773, 746, 718, 794, 742, 773, 780, 769, 765, 792, 776, 742, 778, 752, 716, 804, 799, 791, 784, 779, 767, 786, 746, 788, 798, 763, 759, 753, 772, 765, 754, 735] payload_rows_per_dst=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] dup_rows_per_dst=[2948, 3296, 3006, 3122, 2800, 2926, 3100, 2996] combine_rows_per_src=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64800 dup_rows=24054 dup_rows_remote=20910 dedup_ratio=0.244690 router_tokens_per_expert=[799, 754, 716, 788, 791, 785, 745, 786, 817, 786, 760, 827, 808, 754, 800, 809, 716, 752, 707, 754, 753, 771, 726, 737, 768, 743, 770, 717, 797, 715, 790, 769, 796, 761, 774, 730, 777, 793, 758, 769, 770, 734, 702, 740, 796, 735, 738, 823, 809, 749, 786, 831, 755, 801, 792, 774, 795, 790, 738, 760, 778, 742, 790, 756] payload_rows_per_dst=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] dup_rows_per_dst=[3054, 3196, 2810, 2958, 3002, 2896, 3144, 2994] combine_rows_per_src=[9274, 9526, 9022, 9180, 9314, 9180, 9450, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65044 dup_rows=23910 dup_rows_remote=20944 dedup_ratio=0.243225 router_tokens_per_expert=[783, 748, 738, 786, 720, 748, 762, 793, 772, 704, 767, 754, 719, 723, 763, 742, 790, 800, 766, 805, 809, 744, 731, 800, 748, 758, 757, 807, 753, 771, 769, 734, 750, 747, 786, 826, 817, 785, 761, 772, 733, 734, 795, 776, 737, 823, 813, 747, 741, 761, 757, 736, 793, 797, 749, 753, 819, 795, 771, 766, 768, 762, 796, 822] payload_rows_per_dst=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] dup_rows_per_dst=[2910, 2814, 3036, 3000, 3074, 2966, 2960, 3150] combine_rows_per_src=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343328 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266140960 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531938592 weight_movement_bytes=69206016 total_nvlink_bytes=601144608 weight_to_token_ratio=0.130102 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344424 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379048 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413672 weight_movement_bytes=34603008 total_nvlink_bytes=569016680 weight_to_token_ratio=0.064749 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74526 payload_rows_remote=65170 dup_rows=23778 dup_rows_remote=20776 dedup_ratio=0.241882 router_tokens_per_expert=[801, 800, 806, 750, 763, 761, 800, 740, 771, 761, 728, 754, 753, 813, 753, 766, 755, 756, 788, 799, 781, 770, 748, 761, 796, 788, 751, 741, 809, 798, 742, 748, 751, 816, 775, 788, 722, 779, 763, 785, 767, 710, 789, 718, 816, 765, 719, 785, 810, 790, 742, 773, 789, 791, 745, 746, 784, 734, 726, 808, 748, 817, 689, 761] payload_rows_per_dst=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] dup_rows_per_dst=[3146, 2960, 3012, 2884, 3002, 2894, 2986, 2894] combine_rows_per_src=[9296, 9238, 9304, 9462, 9356, 9244, 9386, 9240] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265633792 dispatch_meta_bytes=344288 combine_payload_bytes=265633792 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265978080 ingress_bytes=369442816 link_bytes=369442816 token_bytes=531611872 weight_movement_bytes=103809024 total_nvlink_bytes=635420896 weight_to_token_ratio=0.195272 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390183574e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266799912 prefetch_bytes=657457152 reduce_bytes=553648128 weight_movement_bytes=1211105280 egress_bytes=6137357096 ingress_bytes=7340548096 link_bytes=7340548096 total_nvlink_bytes=13477905192 weight_to_token_ratio=0.098730 prefetch_share_of_ingress=0.089565 dedup_ratio=0.243767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=16 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200951805e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343648 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403424 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463200 weight_movement_bytes=34603008 total_nvlink_bytes=567066208 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64862 dup_rows=24062 dup_rows_remote=21132 dedup_ratio=0.244771 router_tokens_per_expert=[756, 805, 736, 788, 798, 745, 758, 745, 742, 754, 781, 742, 781, 781, 810, 759, 787, 782, 762, 733, 753, 791, 780, 827, 753, 756, 741, 836, 756, 826, 744, 743, 747, 778, 777, 765, 790, 758, 757, 773, 816, 736, 743, 745, 817, 745, 805, 726, 771, 755, 754, 740, 760, 796, 768, 766, 788, 757, 760, 809, 779, 729, 749, 742] payload_rows_per_dst=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] dup_rows_per_dst=[2994, 3074, 3104, 2930, 3016, 2980, 3008, 2956] combine_rows_per_src=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343848 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255592 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167336 weight_movement_bytes=34603008 total_nvlink_bytes=568770344 weight_to_token_ratio=0.064779 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343784 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267280104 ingress_bytes=370745344 link_bytes=370745344 token_bytes=534216424 weight_movement_bytes=103809024 total_nvlink_bytes=638025448 weight_to_token_ratio=0.194320 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395084882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343400 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321256 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299112 weight_movement_bytes=34603008 total_nvlink_bytes=566902120 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344624 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266215984 ingress_bytes=335077376 link_bytes=335077376 token_bytes=532087344 weight_movement_bytes=69206016 total_nvlink_bytes=601293360 weight_to_token_ratio=0.130065 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260869190e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74242 payload_rows_remote=64862 dup_rows=24062 dup_rows_remote=21132 dedup_ratio=0.244771 router_tokens_per_expert=[756, 805, 736, 788, 798, 745, 758, 745, 742, 754, 781, 742, 781, 781, 810, 759, 787, 782, 762, 733, 753, 791, 780, 827, 753, 756, 741, 836, 756, 826, 744, 743, 747, 778, 777, 765, 790, 758, 757, 773, 816, 736, 743, 745, 817, 745, 805, 726, 771, 755, 754, 740, 760, 796, 768, 766, 788, 757, 760, 809, 779, 729, 749, 742] payload_rows_per_dst=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] dup_rows_per_dst=[2994, 3074, 3104, 2930, 3016, 2980, 3008, 2956] combine_rows_per_src=[9268, 9226, 9326, 9380, 9274, 9286, 9212, 9270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64956 dup_rows=23982 dup_rows_remote=20956 dedup_ratio=0.243958 router_tokens_per_expert=[828, 777, 754, 760, 757, 765, 775, 702, 807, 791, 773, 766, 757, 761, 777, 749, 782, 799, 752, 783, 755, 755, 774, 761, 757, 757, 755, 774, 744, 796, 770, 811, 799, 750, 789, 791, 781, 818, 741, 810, 727, 781, 739, 792, 757, 744, 708, 746, 736, 784, 761, 775, 772, 740, 784, 793, 783, 828, 738, 736, 766, 764, 786, 709] payload_rows_per_dst=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] dup_rows_per_dst=[3012, 3034, 3028, 2982, 3116, 2808, 2976, 3026] combine_rows_per_src=[9158, 9328, 9294, 9346, 9442, 9180, 9208, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=342840 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=265763640 ingress_bytes=403832832 link_bytes=403832832 token_bytes=531184440 weight_movement_bytes=138412032 total_nvlink_bytes=669596472 weight_to_token_ratio=0.260572 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519590436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343952 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764176 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184400 weight_movement_bytes=34603008 total_nvlink_bytes=567787408 weight_to_token_ratio=0.064899 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12237856392 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=6122880648 ingress_bytes=7360684032 link_bytes=7360684032 total_nvlink_bytes=13483564680 weight_to_token_ratio=0.101791 prefetch_share_of_ingress=0.094021 dedup_ratio=0.244930 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.204246149e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266936320 dispatch_meta_bytes=343784 combine_payload_bytes=266936320 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267280104 ingress_bytes=370745344 link_bytes=370745344 token_bytes=534216424 weight_movement_bytes=103809024 total_nvlink_bytes=638025448 weight_to_token_ratio=0.194320 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395084882e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12234197168 prefetch_bytes=761266176 reduce_bytes=622854144 weight_movement_bytes=1384120320 egress_bytes=6121048240 ingress_bytes=7497269248 link_bytes=7497269248 total_nvlink_bytes=13618317488 weight_to_token_ratio=0.113135 prefetch_share_of_ingress=0.101539 dedup_ratio=0.244638 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=18 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226592200e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265674752 dispatch_meta_bytes=343976 combine_payload_bytes=265674752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266018728 ingress_bytes=300277760 link_bytes=300277760 token_bytes=531693480 weight_movement_bytes=34603008 total_nvlink_bytes=566296488 weight_to_token_ratio=0.065081 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129921037e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64976 dup_rows=24112 dup_rows_remote=21134 dedup_ratio=0.245280 router_tokens_per_expert=[755, 791, 805, 848, 804, 762, 811, 759, 776, 744, 771, 741, 757, 768, 747, 827, 812, 769, 784, 766, 698, 763, 744, 752, 750, 782, 774, 785, 740, 779, 816, 779, 770, 746, 717, 759, 760, 783, 777, 758, 796, 754, 759, 775, 806, 759, 720, 784, 739, 780, 757, 764, 787, 754, 761, 755, 775, 723, 800, 792, 779, 733, 721, 750] payload_rows_per_dst=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] dup_rows_per_dst=[3196, 3014, 2916, 3000, 2992, 3062, 2978, 2954] combine_rows_per_src=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65374 dup_rows=23848 dup_rows_remote=20880 dedup_ratio=0.242594 router_tokens_per_expert=[762, 783, 770, 804, 790, 807, 738, 788, 765, 804, 773, 744, 772, 817, 779, 814, 762, 762, 753, 756, 748, 743, 780, 721, 772, 731, 739, 780, 741, 768, 712, 764, 778, 824, 749, 791, 773, 765, 800, 735, 712, 786, 765, 763, 736, 757, 769, 767, 768, 798, 749, 765, 765, 779, 781, 780, 750, 770, 732, 740, 773, 806, 815, 769] payload_rows_per_dst=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] dup_rows_per_dst=[3096, 3126, 2968, 2884, 2914, 2892, 2988, 2980] combine_rows_per_src=[9388, 9410, 9082, 9130, 9516, 9218, 9382, 9330] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343648 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403424 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463200 weight_movement_bytes=34603008 total_nvlink_bytes=567066208 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262343336 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135128744 ingress_bytes=7511334912 link_bytes=7511334912 total_nvlink_bytes=13646463656 weight_to_token_ratio=0.112876 prefetch_share_of_ingress=0.082922 dedup_ratio=0.243905 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228893416e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74110 payload_rows_remote=64910 dup_rows=24194 dup_rows_remote=21246 dedup_ratio=0.246114 router_tokens_per_expert=[764, 754, 750, 786, 718, 769, 773, 742, 827, 754, 765, 732, 767, 765, 789, 799, 773, 770, 830, 750, 729, 725, 771, 774, 813, 754, 799, 775, 748, 786, 789, 753, 773, 746, 718, 794, 742, 773, 780, 769, 765, 792, 776, 742, 778, 752, 716, 804, 799, 791, 784, 779, 767, 786, 746, 788, 798, 763, 759, 753, 772, 765, 754, 735] payload_rows_per_dst=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] dup_rows_per_dst=[2948, 3296, 3006, 3122, 2800, 2926, 3100, 2996] combine_rows_per_src=[9200, 9610, 9222, 9418, 8870, 9314, 9274, 9202] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249891808 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6128903136 ingress_bytes=7608918016 link_bytes=7608918016 total_nvlink_bytes=13737821152 weight_to_token_ratio=0.121465 prefetch_share_of_ingress=0.090954 dedup_ratio=0.244708 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244858519e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64924 dup_rows=24172 dup_rows_remote=21192 dedup_ratio=0.245890 router_tokens_per_expert=[775, 791, 755, 766, 782, 783, 741, 804, 797, 744, 735, 750, 767, 741, 786, 774, 756, 711, 792, 791, 781, 768, 729, 790, 775, 790, 751, 744, 766, 734, 775, 794, 788, 750, 779, 749, 763, 830, 794, 714, 758, 790, 747, 728, 775, 779, 792, 768, 781, 711, 768, 782, 798, 768, 815, 785, 753, 757, 774, 776, 770, 765, 781, 726] payload_rows_per_dst=[9364, 9208, 9302, 9188, 9304, 9316, 9290, 9160] dup_rows_per_dst=[3030, 2980, 2934, 3070, 3030, 2958, 3126, 3044] combine_rows_per_src=[9364, 9208, 9302, 9188, 9304, 9316, 9290, 9160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64892 dup_rows=23986 dup_rows_remote=20940 dedup_ratio=0.243998 router_tokens_per_expert=[726, 827, 760, 780, 766, 737, 812, 763, 774, 705, 794, 733, 815, 769, 739, 782, 750, 784, 730, 777, 753, 789, 764, 798, 772, 785, 742, 799, 749, 784, 743, 801, 759, 774, 772, 811, 805, 779, 739, 767, 739, 739, 742, 732, 754, 750, 791, 743, 794, 812, 766, 756, 796, 754, 792, 766, 754, 812, 767, 757, 767, 741, 752, 768] payload_rows_per_dst=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] dup_rows_per_dst=[2968, 2964, 3034, 3024, 3132, 2856, 3046, 2962] combine_rows_per_src=[9374, 9258, 9256, 9326, 9280, 9124, 9426, 9274] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74286 payload_rows_remote=65078 dup_rows=24018 dup_rows_remote=20834 dedup_ratio=0.244324 router_tokens_per_expert=[759, 752, 716, 789, 766, 763, 794, 742, 767, 739, 772, 818, 760, 721, 801, 715, 787, 774, 808, 778, 731, 765, 780, 747, 736, 752, 809, 768, 740, 779, 731, 783, 797, 702, 824, 743, 762, 775, 784, 809, 766, 782, 770, 788, 786, 777, 750, 808, 748, 763, 804, 767, 769, 794, 777, 738, 755, 788, 779, 784, 757, 742, 778, 744] payload_rows_per_dst=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] dup_rows_per_dst=[2912, 2850, 3004, 2948, 3184, 3136, 3016, 2968] combine_rows_per_src=[9250, 9336, 9336, 9248, 9208, 9318, 9304, 9286] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249891808 prefetch_bytes=692060160 reduce_bytes=795869184 weight_movement_bytes=1487929344 egress_bytes=6128903136 ingress_bytes=7608918016 link_bytes=7608918016 total_nvlink_bytes=13737821152 weight_to_token_ratio=0.121465 prefetch_share_of_ingress=0.090954 dedup_ratio=0.244708 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=23 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028043 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.244858519e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65044 dup_rows=23910 dup_rows_remote=20944 dedup_ratio=0.243225 router_tokens_per_expert=[783, 748, 738, 786, 720, 748, 762, 793, 772, 704, 767, 754, 719, 723, 763, 742, 790, 800, 766, 805, 809, 744, 731, 800, 748, 758, 757, 807, 753, 771, 769, 734, 750, 747, 786, 826, 817, 785, 761, 772, 733, 734, 795, 776, 737, 823, 813, 747, 741, 761, 757, 736, 793, 797, 749, 753, 819, 795, 771, 766, 768, 762, 796, 822] payload_rows_per_dst=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] dup_rows_per_dst=[2910, 2814, 3036, 3000, 3074, 2966, 2960, 3150] combine_rows_per_src=[9246, 9074, 9454, 9194, 9414, 9350, 9214, 9448] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344440 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266486136 ingress_bytes=369950720 link_bytes=369950720 token_bytes=532627832 weight_movement_bytes=103809024 total_nvlink_bytes=636436856 weight_to_token_ratio=0.194900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392094776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265871360 dispatch_meta_bytes=344624 combine_payload_bytes=265871360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266215984 ingress_bytes=335077376 link_bytes=335077376 token_bytes=532087344 weight_movement_bytes=69206016 total_nvlink_bytes=601293360 weight_to_token_ratio=0.130065 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260869190e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12234197168 prefetch_bytes=761266176 reduce_bytes=622854144 weight_movement_bytes=1384120320 egress_bytes=6121048240 ingress_bytes=7497269248 link_bytes=7497269248 total_nvlink_bytes=13618317488 weight_to_token_ratio=0.113135 prefetch_share_of_ingress=0.101539 dedup_ratio=0.244638 prefetch_slots=44 unused_prefetch_slots=324 slot_utilization=0.119565 reduce_slots=18 remote_demand=22 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=29083 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226592200e-04 prefetch_bytes_per_gemm_flop=1.245471014e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267771904 dispatch_meta_bytes=345016 combine_payload_bytes=267771904 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268116920 ingress_bytes=302374912 link_bytes=302374912 token_bytes=535888824 weight_movement_bytes=34603008 total_nvlink_bytes=570491832 weight_to_token_ratio=0.064571 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.137812451e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=344464 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266273168 ingress_bytes=300531712 link_bytes=300531712 token_bytes=532201872 weight_movement_bytes=34603008 total_nvlink_bytes=566804880 weight_to_token_ratio=0.065019 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130876637e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=343328 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266140960 ingress_bytes=335003648 link_bytes=335003648 token_bytes=531938592 weight_movement_bytes=69206016 total_nvlink_bytes=601144608 weight_to_token_ratio=0.130102 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260591757e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12237856392 prefetch_bytes=692060160 reduce_bytes=553648128 weight_movement_bytes=1245708288 egress_bytes=6122880648 ingress_bytes=7360684032 link_bytes=7360684032 total_nvlink_bytes=13483564680 weight_to_token_ratio=0.101791 prefetch_share_of_ingress=0.094021 dedup_ratio=0.244930 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=16 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.204246149e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343648 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903136 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462624 weight_movement_bytes=103809024 total_nvlink_bytes=637271648 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266420224 dispatch_meta_bytes=343952 combine_payload_bytes=266420224 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266764176 ingress_bytes=301023232 link_bytes=301023232 token_bytes=533184400 weight_movement_bytes=34603008 total_nvlink_bytes=567787408 weight_to_token_ratio=0.064899 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132726188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255753552 prefetch_bytes=657457152 reduce_bytes=968884224 weight_movement_bytes=1626341376 egress_bytes=6131832144 ingress_bytes=7750262784 link_bytes=7750262784 total_nvlink_bytes=13882094928 weight_to_token_ratio=0.132700 prefetch_share_of_ingress=0.084830 dedup_ratio=0.244093 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=28 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267983257e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12258865512 prefetch_bytes=726663168 reduce_bytes=380633088 weight_movement_bytes=1107296256 egress_bytes=6133387624 ingress_bytes=7232774144 link_bytes=7233118528 total_nvlink_bytes=13366161768 weight_to_token_ratio=0.090326 prefetch_share_of_ingress=0.100468 dedup_ratio=0.244073 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=11 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029674 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.183375770e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249642576 prefetch_bytes=692060160 reduce_bytes=865075200 weight_movement_bytes=1557135360 egress_bytes=6128776784 ingress_bytes=7678001152 link_bytes=7678001152 total_nvlink_bytes=13806777936 weight_to_token_ratio=0.127117 prefetch_share_of_ingress=0.090135 dedup_ratio=0.244561 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=25 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.027935 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.256160878e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12262343336 prefetch_bytes=622854144 reduce_bytes=761266176 weight_movement_bytes=1384120320 egress_bytes=6135128744 ingress_bytes=7511334912 link_bytes=7511334912 total_nvlink_bytes=13646463656 weight_to_token_ratio=0.112876 prefetch_share_of_ingress=0.082922 dedup_ratio=0.243905 prefetch_slots=36 unused_prefetch_slots=332 slot_utilization=0.097826 reduce_slots=22 remote_demand=18 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28575 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.228893416e-04 prefetch_bytes_per_gemm_flop=1.019021739e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73976 payload_rows_remote=64756 dup_rows=24328 dup_rows_remote=21280 dedup_ratio=0.247477 router_tokens_per_expert=[797, 841, 776, 780, 777, 799, 756, 808, 749, 734, 766, 786, 829, 725, 774, 786, 761, 728, 742, 747, 771, 792, 780, 813, 759, 789, 753, 788, 745, 755, 781, 798, 733, 770, 768, 788, 747, 772, 809, 741, 802, 807, 765, 733, 758, 810, 691, 756, 727, 734, 757, 761, 749, 786, 794, 764, 714, 778, 773, 772, 708, 744, 780, 776] payload_rows_per_dst=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] dup_rows_per_dst=[3224, 3008, 3048, 3026, 2970, 3050, 3026, 2976] combine_rows_per_src=[9444, 9290, 9220, 9310, 9286, 9194, 9118, 9114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265240576 dispatch_meta_bytes=344144 combine_payload_bytes=265240576 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265584720 ingress_bytes=299843584 link_bytes=299843584 token_bytes=530825296 weight_movement_bytes=34603008 total_nvlink_bytes=565428304 weight_to_token_ratio=0.065187 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128287267e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74068 payload_rows_remote=64940 dup_rows=24236 dup_rows_remote=21282 dedup_ratio=0.246541 router_tokens_per_expert=[740, 762, 771, 790, 798, 807, 786, 717, 707, 767, 740, 810, 761, 750, 797, 774, 713, 850, 740, 745, 719, 744, 752, 778, 777, 782, 784, 753, 747, 724, 741, 726, 776, 798, 754, 754, 836, 738, 771, 789, 768, 760, 814, 834, 747, 765, 785, 757, 796, 718, 773, 773, 806, 801, 790, 754, 778, 716, 746, 822, 784, 740, 757, 800] payload_rows_per_dst=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] dup_rows_per_dst=[3028, 3020, 2954, 2952, 3062, 3074, 3022, 3124] combine_rows_per_src=[9314, 9192, 9128, 9116, 9370, 9386, 9400, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344888 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266339128 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532333368 weight_movement_bytes=69206016 total_nvlink_bytes=601539384 weight_to_token_ratio=0.130005 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266799912 prefetch_bytes=657457152 reduce_bytes=553648128 weight_movement_bytes=1211105280 egress_bytes=6137357096 ingress_bytes=7340548096 link_bytes=7340548096 total_nvlink_bytes=13477905192 weight_to_token_ratio=0.098730 prefetch_share_of_ingress=0.089565 dedup_ratio=0.243767 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=16 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029022 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200951805e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none - [2026-08-26 16:56:41] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 802.6 | throughput per GPU (TFLOP/s/GPU): 159.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.020314E+01 | loss scale: 1.0 | grad norm: 158.229 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64976 dup_rows=24112 dup_rows_remote=21134 dedup_ratio=0.245280 router_tokens_per_expert=[755, 791, 805, 848, 804, 762, 811, 759, 776, 744, 771, 741, 757, 768, 747, 827, 812, 769, 784, 766, 698, 763, 744, 752, 750, 782, 774, 785, 740, 779, 816, 779, 770, 746, 717, 759, 760, 783, 777, 758, 796, 754, 759, 775, 806, 759, 720, 784, 739, 780, 757, 764, 787, 754, 761, 755, 775, 723, 800, 792, 779, 733, 721, 750] payload_rows_per_dst=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] dup_rows_per_dst=[3196, 3014, 2916, 3000, 2992, 3062, 2978, 2954] combine_rows_per_src=[9474, 9248, 9260, 9410, 9148, 9244, 9216, 9192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266141696 dispatch_meta_bytes=344440 combine_payload_bytes=266141696 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266486136 ingress_bytes=369950720 link_bytes=369950720 token_bytes=532627832 weight_movement_bytes=103809024 total_nvlink_bytes=636436856 weight_to_token_ratio=0.194900 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392094776e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255753552 prefetch_bytes=657457152 reduce_bytes=968884224 weight_movement_bytes=1626341376 egress_bytes=6131832144 ingress_bytes=7750262784 link_bytes=7750262784 total_nvlink_bytes=13882094928 weight_to_token_ratio=0.132700 prefetch_share_of_ingress=0.084830 dedup_ratio=0.244093 prefetch_slots=38 unused_prefetch_slots=330 slot_utilization=0.103261 reduce_slots=28 remote_demand=19 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028261 gemm_inner_pad_upper_bound=28702 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.267983257e-04 prefetch_bytes_per_gemm_flop=1.075634058e-05 anomalies=none + [2026-08-26 18:08:47] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 797.2 | throughput per GPU (TFLOP/s/GPU): 160.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 1.020314E+01 | loss scale: 1.0 | grad norm: 158.229 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65026 dup_rows=23914 dup_rows_remote=20866 dedup_ratio=0.243266 router_tokens_per_expert=[835, 772, 793, 789, 752, 752, 731, 781, 737, 809, 781, 775, 727, 792, 749, 786, 776, 789, 747, 788, 782, 726, 728, 766, 764, 757, 764, 769, 774, 783, 802, 793, 790, 735, 735, 793, 778, 755, 789, 725, 739, 784, 730, 762, 746, 775, 755, 723, 811, 794, 789, 730, 773, 771, 803, 766, 765, 802, 771, 730, 751, 759, 816, 738] payload_rows_per_dst=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] dup_rows_per_dst=[2986, 2966, 3002, 3048, 2922, 2952, 3090, 2948] combine_rows_per_src=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65084 dup_rows=24046 dup_rows_remote=21106 dedup_ratio=0.244609 router_tokens_per_expert=[771, 751, 771, 744, 781, 737, 727, 802, 737, 734, 737, 784, 788, 795, 751, 770, 738, 809, 740, 738, 769, 793, 763, 801, 785, 701, 784, 756, 777, 814, 787, 765, 796, 755, 734, 787, 749, 774, 772, 766, 754, 701, 750, 757, 747, 772, 780, 796, 768, 793, 822, 768, 732, 797, 838, 791, 800, 777, 815, 763, 758, 761, 775, 704] payload_rows_per_dst=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] dup_rows_per_dst=[2966, 2932, 3058, 3004, 3026, 2940, 3106, 3014] combine_rows_per_src=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64702 dup_rows=24066 dup_rows_remote=20808 dedup_ratio=0.244812 router_tokens_per_expert=[751, 793, 791, 778, 782, 749, 780, 794, 726, 777, 808, 767, 774, 799, 752, 706, 750, 816, 747, 711, 746, 764, 747, 774, 758, 738, 753, 841, 724, 765, 774, 752, 769, 744, 768, 774, 753, 774, 748, 784, 791, 778, 806, 765, 758, 762, 733, 756, 767, 771, 770, 773, 802, 814, 753, 742, 743, 778, 742, 792, 789, 813, 759, 794] payload_rows_per_dst=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] dup_rows_per_dst=[3062, 2968, 2940, 2946, 2966, 3014, 2912, 3258] combine_rows_per_src=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74156 payload_rows_remote=64832 dup_rows=24148 dup_rows_remote=21102 dedup_ratio=0.245646 router_tokens_per_expert=[773, 795, 777, 772, 737, 759, 738, 780, 736, 777, 829, 744, 744, 775, 777, 803, 758, 758, 784, 753, 804, 781, 768, 817, 776, 711, 756, 755, 786, 752, 782, 760, 784, 773, 747, 776, 807, 764, 747, 788, 756, 740, 754, 757, 764, 791, 737, 800, 763, 766, 720, 772, 762, 763, 726, 784, 769, 766, 809, 752, 783, 803, 747, 765] payload_rows_per_dst=[9324, 9324, 9398, 9086, 9386, 9220, 9098, 9320] dup_rows_per_dst=[2938, 3046, 3048, 3070, 2986, 2978, 3014, 3068] combine_rows_per_src=[9324, 9324, 9398, 9086, 9386, 9220, 9098, 9320] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64876 dup_rows=24150 dup_rows_remote=20984 dedup_ratio=0.245667 router_tokens_per_expert=[747, 774, 758, 802, 774, 825, 727, 747, 706, 796, 791, 807, 742, 743, 761, 804, 765, 766, 797, 775, 754, 724, 799, 743, 774, 776, 706, 760, 800, 763, 740, 749, 774, 754, 738, 759, 790, 849, 770, 788, 734, 770, 773, 719, 790, 766, 735, 695, 729, 756, 779, 809, 774, 804, 785, 826, 766, 784, 762, 788, 759, 763, 780, 789] payload_rows_per_dst=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] dup_rows_per_dst=[3064, 2982, 3048, 2898, 3166, 2742, 3144, 3106] combine_rows_per_src=[9244, 9318, 9198, 9238, 9278, 9222, 9380, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74238 payload_rows_remote=64702 dup_rows=24066 dup_rows_remote=20808 dedup_ratio=0.244812 router_tokens_per_expert=[751, 793, 791, 778, 782, 749, 780, 794, 726, 777, 808, 767, 774, 799, 752, 706, 750, 816, 747, 711, 746, 764, 747, 774, 758, 738, 753, 841, 724, 765, 774, 752, 769, 744, 768, 774, 753, 774, 748, 784, 791, 778, 806, 765, 758, 762, 733, 756, 767, 771, 770, 773, 802, 814, 753, 742, 743, 778, 742, 792, 789, 813, 759, 794] payload_rows_per_dst=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] dup_rows_per_dst=[3062, 2968, 2940, 2946, 2966, 3014, 2912, 3258] combine_rows_per_src=[9374, 9250, 9160, 9264, 9236, 9284, 9134, 9536] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=65084 dup_rows=24046 dup_rows_remote=21106 dedup_ratio=0.244609 router_tokens_per_expert=[771, 751, 771, 744, 781, 737, 727, 802, 737, 734, 737, 784, 788, 795, 751, 770, 738, 809, 740, 738, 769, 793, 763, 801, 785, 701, 784, 756, 777, 814, 787, 765, 796, 755, 734, 787, 749, 774, 772, 766, 754, 701, 750, 757, 747, 772, 780, 796, 768, 793, 822, 768, 732, 797, 838, 791, 800, 777, 815, 763, 758, 761, 775, 704] payload_rows_per_dst=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] dup_rows_per_dst=[2966, 2932, 3058, 3004, 3026, 2940, 3106, 3014] combine_rows_per_src=[9202, 9260, 9244, 9334, 9240, 9174, 9512, 9292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74390 payload_rows_remote=65026 dup_rows=23914 dup_rows_remote=20866 dedup_ratio=0.243266 router_tokens_per_expert=[835, 772, 793, 789, 752, 752, 731, 781, 737, 809, 781, 775, 727, 792, 749, 786, 776, 789, 747, 788, 782, 726, 728, 766, 764, 757, 764, 769, 774, 783, 802, 793, 790, 735, 735, 793, 778, 755, 789, 725, 739, 784, 730, 762, 746, 775, 755, 723, 811, 794, 789, 730, 773, 771, 803, 766, 765, 802, 771, 730, 751, 759, 816, 738] payload_rows_per_dst=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] dup_rows_per_dst=[2986, 2966, 3002, 3048, 2922, 2952, 3090, 2948] combine_rows_per_src=[9424, 9346, 9202, 9364, 9278, 9076, 9384, 9316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65002 dup_rows=24122 dup_rows_remote=21154 dedup_ratio=0.245382 router_tokens_per_expert=[781, 759, 725, 816, 751, 712, 798, 763, 774, 797, 761, 723, 788, 817, 768, 726, 782, 790, 791, 766, 746, 756, 765, 749, 794, 769, 756, 761, 765, 735, 758, 770, 828, 800, 790, 768, 742, 782, 770, 739, 723, 806, 811, 735, 776, 776, 729, 801, 744, 782, 784, 765, 739, 728, 774, 758, 768, 773, 819, 761, 770, 774, 784, 741] payload_rows_per_dst=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] dup_rows_per_dst=[3018, 2992, 3018, 2982, 3112, 2992, 2968, 3040] combine_rows_per_src=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21092 dedup_ratio=0.245117 router_tokens_per_expert=[783, 767, 739, 744, 740, 750, 797, 761, 778, 757, 759, 788, 762, 730, 766, 765, 811, 755, 736, 761, 788, 770, 806, 766, 765, 794, 745, 800, 751, 756, 813, 805, 745, 771, 778, 743, 739, 792, 761, 829, 759, 798, 810, 738, 745, 757, 744, 788, 750, 779, 769, 772, 792, 766, 748, 770, 778, 743, 744, 778, 771, 753, 737, 797] payload_rows_per_dst=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] dup_rows_per_dst=[2974, 2982, 3004, 3086, 2998, 3006, 2998, 3048] combine_rows_per_src=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343568 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690064 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036560 weight_movement_bytes=34603008 total_nvlink_bytes=567639568 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344760 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266928824 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512888 weight_movement_bytes=34603008 total_nvlink_bytes=568115896 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=342040 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265361432 ingress_bytes=334225408 link_bytes=334225408 token_bytes=530380824 weight_movement_bytes=69206016 total_nvlink_bytes=599586840 weight_to_token_ratio=0.130484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257663303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74410 payload_rows_remote=65118 dup_rows=23894 dup_rows_remote=20838 dedup_ratio=0.243062 router_tokens_per_expert=[779, 781, 768, 780, 706, 745, 743, 790, 784, 732, 766, 741, 771, 769, 801, 792, 830, 770, 801, 762, 804, 728, 768, 784, 771, 739, 732, 751, 816, 745, 778, 764, 813, 770, 747, 777, 751, 767, 771, 730, 749, 742, 817, 751, 767, 790, 791, 749, 748, 746, 767, 791, 727, 790, 774, 743, 763, 742, 793, 779, 791, 797, 762, 766] payload_rows_per_dst=[9292, 9330, 9388, 9374, 9254, 9640, 9242, 8890] dup_rows_per_dst=[3056, 3064, 2942, 3004, 2812, 3470, 2848, 2698] combine_rows_per_src=[9292, 9330, 9388, 9374, 9254, 9640, 9242, 8890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343736 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895608 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447480 weight_movement_bytes=34603008 total_nvlink_bytes=566050488 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343440 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266075536 ingress_bytes=369541120 link_bytes=369541120 token_bytes=531807632 weight_movement_bytes=103809024 total_nvlink_bytes=635616656 weight_to_token_ratio=0.195200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390553484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265019392 dispatch_meta_bytes=342040 combine_payload_bytes=265019392 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265361432 ingress_bytes=334225408 link_bytes=334225408 token_bytes=530380824 weight_movement_bytes=69206016 total_nvlink_bytes=599586840 weight_to_token_ratio=0.130484 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257663303e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266584064 dispatch_meta_bytes=344760 combine_payload_bytes=266584064 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266928824 ingress_bytes=301187072 link_bytes=301187072 token_bytes=533512888 weight_movement_bytes=34603008 total_nvlink_bytes=568115896 weight_to_token_ratio=0.064859 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133342704e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=343568 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266690064 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533036560 weight_movement_bytes=34603008 total_nvlink_bytes=567639568 weight_to_token_ratio=0.064917 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=65002 dup_rows=24122 dup_rows_remote=21154 dedup_ratio=0.245382 router_tokens_per_expert=[781, 759, 725, 816, 751, 712, 798, 763, 774, 797, 761, 723, 788, 817, 768, 726, 782, 790, 791, 766, 746, 756, 765, 749, 794, 769, 756, 761, 765, 735, 758, 770, 828, 800, 790, 768, 742, 782, 770, 739, 723, 806, 811, 735, 776, 776, 729, 801, 744, 782, 784, 765, 739, 728, 774, 758, 768, 773, 819, 761, 770, 774, 784, 741] payload_rows_per_dst=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] dup_rows_per_dst=[3018, 2992, 3018, 2982, 3112, 2992, 2968, 3040] combine_rows_per_src=[9192, 9316, 9272, 9234, 9326, 9322, 9180, 9340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343736 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895608 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447480 weight_movement_bytes=34603008 total_nvlink_bytes=566050488 weight_to_token_ratio=0.065111 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265732096 dispatch_meta_bytes=343440 combine_payload_bytes=265732096 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266075536 ingress_bytes=369541120 link_bytes=369541120 token_bytes=531807632 weight_movement_bytes=103809024 total_nvlink_bytes=635616656 weight_to_token_ratio=0.195200 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390553484e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343672 combine_payload_bytes=265527296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265870968 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398264 weight_movement_bytes=69206016 total_nvlink_bytes=600604280 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344624 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592816 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532841008 weight_movement_bytes=103809024 total_nvlink_bytes=636650032 weight_to_token_ratio=0.194822 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64826 dup_rows=24096 dup_rows_remote=21092 dedup_ratio=0.245117 router_tokens_per_expert=[783, 767, 739, 744, 740, 750, 797, 761, 778, 757, 759, 788, 762, 730, 766, 765, 811, 755, 736, 761, 788, 770, 806, 766, 765, 794, 745, 800, 751, 756, 813, 805, 745, 771, 778, 743, 739, 792, 761, 829, 759, 798, 810, 738, 745, 757, 744, 788, 750, 779, 769, 772, 792, 766, 748, 770, 778, 743, 744, 778, 771, 753, 737, 797] payload_rows_per_dst=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] dup_rows_per_dst=[2974, 2982, 3004, 3086, 2998, 3006, 2998, 3048] combine_rows_per_src=[9188, 9228, 9382, 9372, 9318, 9272, 9294, 9154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=343824 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067152 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790480 weight_movement_bytes=34603008 total_nvlink_bytes=568393488 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65008 dup_rows=23940 dup_rows_remote=20944 dedup_ratio=0.243530 router_tokens_per_expert=[757, 763, 777, 753, 758, 776, 714, 815, 774, 798, 735, 756, 700, 775, 811, 759, 760, 746, 774, 745, 757, 796, 735, 734, 761, 807, 787, 725, 802, 719, 821, 754, 726, 795, 793, 763, 795, 743, 722, 806, 794, 754, 786, 804, 781, 766, 763, 801, 748, 774, 825, 804, 734, 782, 745, 785, 774, 797, 754, 748, 742, 748, 776, 780] payload_rows_per_dst=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] dup_rows_per_dst=[2968, 2998, 2896, 2996, 2956, 3040, 3054, 3032] combine_rows_per_src=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65108 dup_rows=24054 dup_rows_remote=21166 dedup_ratio=0.244690 router_tokens_per_expert=[738, 797, 823, 752, 782, 792, 810, 778, 761, 746, 760, 793, 750, 728, 785, 760, 804, 769, 766, 780, 801, 740, 829, 747, 738, 744, 773, 738, 733, 845, 772, 778, 791, 756, 757, 717, 767, 733, 817, 805, 786, 736, 723, 782, 789, 736, 767, 696, 792, 811, 773, 740, 769, 723, 766, 751, 744, 827, 810, 755, 769, 773, 746, 733] payload_rows_per_dst=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] dup_rows_per_dst=[3082, 2982, 3146, 2976, 2984, 2888, 2986, 3010] combine_rows_per_src=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64886 dup_rows=24080 dup_rows_remote=20932 dedup_ratio=0.244954 router_tokens_per_expert=[807, 775, 720, 808, 791, 729, 750, 774, 807, 768, 741, 772, 737, 749, 727, 811, 764, 793, 759, 761, 749, 788, 760, 804, 767, 776, 779, 816, 747, 729, 763, 777, 777, 793, 746, 782, 787, 808, 752, 730, 760, 749, 779, 803, 789, 749, 754, 780, 747, 757, 761, 754, 820, 753, 781, 742, 719, 790, 748, 781, 766, 752, 780, 765] payload_rows_per_dst=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] dup_rows_per_dst=[2994, 2962, 3028, 2870, 2976, 2972, 3130, 3148] combine_rows_per_src=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=344624 combine_payload_bytes=266248192 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266592816 ingress_bytes=370057216 link_bytes=370057216 token_bytes=532841008 weight_movement_bytes=103809024 total_nvlink_bytes=636650032 weight_to_token_ratio=0.194822 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392495512e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343808 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266616576 ingress_bytes=335478784 link_bytes=335478784 token_bytes=532889344 weight_movement_bytes=69206016 total_nvlink_bytes=602095360 weight_to_token_ratio=0.129869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343272 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116328 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531889384 weight_movement_bytes=34603008 total_nvlink_bytes=566492392 weight_to_token_ratio=0.065057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74456 payload_rows_remote=65032 dup_rows=23848 dup_rows_remote=20778 dedup_ratio=0.242594 router_tokens_per_expert=[742, 768, 766, 732, 764, 749, 791, 767, 773, 803, 754, 804, 796, 746, 781, 790, 743, 832, 741, 736, 729, 812, 767, 745, 728, 783, 767, 797, 761, 785, 778, 773, 791, 761, 748, 781, 775, 810, 735, 754, 742, 793, 802, 766, 741, 772, 737, 757, 758, 728, 794, 787, 776, 768, 748, 775, 758, 752, 777, 811, 772, 756, 757, 767] payload_rows_per_dst=[9208, 9424, 9320, 9266, 9366, 9242, 9322, 9308] dup_rows_per_dst=[2950, 3070, 2890, 3078, 2944, 2978, 2946, 2992] combine_rows_per_src=[9208, 9424, 9320, 9266, 9366, 9242, 9322, 9308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=345096 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267027464 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533709832 weight_movement_bytes=69206016 total_nvlink_bytes=602915848 weight_to_token_ratio=0.129670 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64726 dup_rows=24250 dup_rows_remote=21066 dedup_ratio=0.246684 router_tokens_per_expert=[736, 750, 708, 773, 790, 771, 738, 783, 804, 749, 739, 764, 798, 764, 741, 784, 795, 808, 757, 814, 753, 768, 784, 777, 825, 785, 709, 798, 752, 773, 760, 785, 790, 773, 765, 754, 718, 771, 811, 756, 778, 802, 767, 813, 757, 737, 760, 721, 786, 775, 776, 766, 751, 733, 772, 751, 750, 758, 744, 760, 817, 801, 741, 763] payload_rows_per_dst=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] dup_rows_per_dst=[2908, 3012, 3184, 3060, 3008, 3070, 3050, 2958] combine_rows_per_src=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64880 dup_rows=24048 dup_rows_remote=21022 dedup_ratio=0.244629 router_tokens_per_expert=[806, 731, 790, 735, 723, 722, 767, 732, 780, 726, 768, 761, 738, 730, 845, 793, 753, 777, 798, 776, 748, 779, 751, 794, 771, 754, 769, 772, 750, 784, 741, 825, 781, 768, 745, 793, 788, 782, 756, 788, 774, 815, 776, 781, 771, 781, 768, 778, 763, 749, 750, 774, 759, 825, 770, 762, 753, 687, 716, 770, 797, 827, 764, 752] payload_rows_per_dst=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] dup_rows_per_dst=[2858, 2988, 2990, 3092, 3026, 3062, 3042, 2990] combine_rows_per_src=[9154, 9294, 9362, 9240, 9376, 9426, 9262, 9142] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64886 dup_rows=24080 dup_rows_remote=20932 dedup_ratio=0.244954 router_tokens_per_expert=[807, 775, 720, 808, 791, 729, 750, 774, 807, 768, 741, 772, 737, 749, 727, 811, 764, 793, 759, 761, 749, 788, 760, 804, 767, 776, 779, 816, 747, 729, 763, 777, 777, 793, 746, 782, 787, 808, 752, 730, 760, 749, 779, 803, 789, 749, 754, 780, 747, 757, 761, 754, 820, 753, 781, 742, 719, 790, 748, 781, 766, 752, 780, 765] payload_rows_per_dst=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] dup_rows_per_dst=[2994, 2962, 3028, 2870, 2976, 2972, 3130, 3148] combine_rows_per_src=[9314, 9182, 9328, 9098, 9138, 9354, 9472, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64958 dup_rows=24130 dup_rows_remote=21224 dedup_ratio=0.245463 router_tokens_per_expert=[759, 728, 752, 768, 760, 734, 770, 770, 776, 768, 769, 762, 770, 767, 773, 727, 782, 805, 778, 747, 740, 807, 779, 808, 737, 743, 816, 789, 791, 760, 775, 792, 758, 753, 804, 784, 742, 799, 754, 760, 693, 791, 752, 791, 775, 798, 780, 795, 758, 774, 766, 765, 763, 726, 744, 765, 770, 768, 829, 751, 754, 735, 770, 783] payload_rows_per_dst=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] dup_rows_per_dst=[2876, 2950, 3172, 3048, 3106, 3062, 2906, 3010] combine_rows_per_src=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74364 payload_rows_remote=65008 dup_rows=23940 dup_rows_remote=20944 dedup_ratio=0.243530 router_tokens_per_expert=[757, 763, 777, 753, 758, 776, 714, 815, 774, 798, 735, 756, 700, 775, 811, 759, 760, 746, 774, 745, 757, 796, 735, 734, 761, 807, 787, 725, 802, 719, 821, 754, 726, 795, 793, 763, 795, 743, 722, 806, 794, 754, 786, 804, 781, 766, 763, 801, 748, 774, 825, 804, 734, 782, 745, 785, 774, 797, 754, 748, 742, 748, 776, 780] payload_rows_per_dst=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] dup_rows_per_dst=[2968, 2998, 2896, 2996, 2956, 3040, 3054, 3032] combine_rows_per_src=[9258, 9218, 9198, 9356, 9330, 9458, 9340, 9206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343672 combine_payload_bytes=265527296 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265870968 ingress_bytes=334733312 link_bytes=334733312 token_bytes=531398264 weight_movement_bytes=69206016 total_nvlink_bytes=600604280 weight_to_token_ratio=0.130234 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259574505e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=65108 dup_rows=24054 dup_rows_remote=21166 dedup_ratio=0.244690 router_tokens_per_expert=[738, 797, 823, 752, 782, 792, 810, 778, 761, 746, 760, 793, 750, 728, 785, 760, 804, 769, 766, 780, 801, 740, 829, 747, 738, 744, 773, 738, 733, 845, 772, 778, 791, 756, 757, 717, 767, 733, 817, 805, 786, 736, 723, 782, 789, 736, 767, 696, 792, 811, 773, 740, 769, 723, 766, 751, 744, 827, 810, 755, 769, 773, 746, 733] payload_rows_per_dst=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] dup_rows_per_dst=[3082, 2982, 3146, 2976, 2984, 2888, 2986, 3010] combine_rows_per_src=[9462, 9184, 9326, 9266, 9302, 9142, 9264, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266371072 dispatch_meta_bytes=343240 combine_payload_bytes=266371072 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266714312 ingress_bytes=370180096 link_bytes=370180096 token_bytes=533085384 weight_movement_bytes=103809024 total_nvlink_bytes=636894408 weight_to_token_ratio=0.194732 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392957899e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266092088 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840568 weight_movement_bytes=69206016 total_nvlink_bytes=601046584 weight_to_token_ratio=0.130125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343272 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116328 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531889384 weight_movement_bytes=34603008 total_nvlink_bytes=566492392 weight_to_token_ratio=0.065057 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343168 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265460864 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530578560 weight_movement_bytes=69206016 total_nvlink_bytes=599784576 weight_to_token_ratio=0.130435 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64710 dup_rows=24006 dup_rows_remote=20718 dedup_ratio=0.244202 router_tokens_per_expert=[764, 785, 767, 759, 783, 814, 727, 736, 765, 816, 770, 757, 789, 797, 775, 775, 756, 754, 825, 782, 729, 762, 771, 787, 789, 740, 743, 771, 741, 786, 796, 777, 730, 803, 786, 787, 800, 734, 782, 739, 760, 751, 801, 756, 756, 791, 782, 711, 761, 746, 701, 779, 800, 756, 764, 765, 773, 743, 745, 752, 795, 769, 797, 749] payload_rows_per_dst=[9588, 9422, 8984, 9304, 9348, 9194, 9182, 9276] dup_rows_per_dst=[3288, 3070, 2832, 2982, 2974, 2928, 2962, 2970] combine_rows_per_src=[9588, 9422, 8984, 9304, 9348, 9194, 9182, 9276] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344728 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412696 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480664 weight_movement_bytes=34603008 total_nvlink_bytes=567083672 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=345096 combine_payload_bytes=266682368 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267027464 ingress_bytes=335888384 link_bytes=335888384 token_bytes=533709832 weight_movement_bytes=69206016 total_nvlink_bytes=602915848 weight_to_token_ratio=0.129670 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263920948e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=343808 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266616576 ingress_bytes=335478784 link_bytes=335478784 token_bytes=532889344 weight_movement_bytes=69206016 total_nvlink_bytes=602095360 weight_to_token_ratio=0.129869 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262379656e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343608 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266092088 ingress_bytes=334954496 link_bytes=334954496 token_bytes=531840568 weight_movement_bytes=69206016 total_nvlink_bytes=601046584 weight_to_token_ratio=0.130125 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260406802e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64958 dup_rows=24130 dup_rows_remote=21224 dedup_ratio=0.245463 router_tokens_per_expert=[759, 728, 752, 768, 760, 734, 770, 770, 776, 768, 769, 762, 770, 767, 773, 727, 782, 805, 778, 747, 740, 807, 779, 808, 737, 743, 816, 789, 791, 760, 775, 792, 758, 753, 804, 784, 742, 799, 754, 760, 693, 791, 752, 791, 775, 798, 780, 795, 758, 774, 766, 765, 763, 726, 744, 765, 770, 768, 829, 751, 754, 735, 770, 783] payload_rows_per_dst=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] dup_rows_per_dst=[2876, 2950, 3172, 3048, 3106, 3062, 2906, 3010] combine_rows_per_src=[9206, 9274, 9320, 9358, 9202, 9288, 9216, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65196 dup_rows=23730 dup_rows_remote=20730 dedup_ratio=0.241394 router_tokens_per_expert=[747, 768, 745, 798, 739, 748, 758, 790, 738, 771, 694, 764, 747, 797, 751, 756, 761, 775, 780, 728, 801, 766, 758, 788, 821, 731, 800, 788, 736, 757, 748, 808, 840, 782, 812, 819, 701, 717, 762, 760, 745, 773, 747, 746, 796, 779, 777, 749, 781, 765, 699, 777, 754, 781, 838, 765, 775, 794, 779, 754, 751, 761, 841, 775] payload_rows_per_dst=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] dup_rows_per_dst=[2914, 2904, 2894, 3000, 3002, 2996, 2972, 3048] combine_rows_per_src=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65022 dup_rows=23796 dup_rows_remote=20662 dedup_ratio=0.242065 router_tokens_per_expert=[734, 719, 775, 767, 729, 734, 803, 739, 730, 807, 763, 768, 767, 706, 792, 791, 764, 809, 801, 797, 765, 800, 776, 755, 766, 768, 785, 782, 771, 780, 767, 750, 772, 772, 755, 705, 816, 802, 798, 777, 744, 745, 706, 800, 712, 818, 778, 781, 752, 762, 744, 788, 748, 732, 754, 777, 734, 756, 777, 752, 798, 817, 789, 831] payload_rows_per_dst=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] dup_rows_per_dst=[2844, 2956, 3116, 3078, 3016, 2820, 2832, 3134] combine_rows_per_src=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65026 dup_rows=23954 dup_rows_remote=21014 dedup_ratio=0.243673 router_tokens_per_expert=[721, 764, 804, 802, 741, 757, 785, 779, 796, 814, 814, 766, 774, 776, 767, 701, 759, 739, 812, 775, 766, 737, 761, 778, 733, 770, 773, 825, 754, 786, 784, 738, 756, 779, 734, 780, 799, 746, 803, 794, 775, 770, 797, 780, 716, 765, 789, 740, 797, 810, 742, 722, 781, 767, 778, 746, 691, 766, 747, 790, 759, 757, 778, 747] payload_rows_per_dst=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] dup_rows_per_dst=[3006, 3112, 2968, 2984, 3042, 2940, 3018, 2884] combine_rows_per_src=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265052160 dispatch_meta_bytes=341712 combine_payload_bytes=265052160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265393872 ingress_bytes=299655168 link_bytes=299655168 token_bytes=530446032 weight_movement_bytes=34603008 total_nvlink_bytes=565049040 weight_to_token_ratio=0.065234 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127578273e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=266067968 dispatch_meta_bytes=344728 combine_payload_bytes=266067968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266412696 ingress_bytes=300670976 link_bytes=300670976 token_bytes=532480664 weight_movement_bytes=34603008 total_nvlink_bytes=567083672 weight_to_token_ratio=0.064985 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131400677e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=343704 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267386520 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534429336 weight_movement_bytes=69206016 total_nvlink_bytes=603635352 weight_to_token_ratio=0.129495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=342736 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266672848 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533002960 weight_movement_bytes=69206016 total_nvlink_bytes=602208976 weight_to_token_ratio=0.129842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344160 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266690656 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037152 weight_movement_bytes=103809024 total_nvlink_bytes=636846176 weight_to_token_ratio=0.194750 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74606 payload_rows_remote=65210 dup_rows=23698 dup_rows_remote=20722 dedup_ratio=0.241069 router_tokens_per_expert=[802, 775, 763, 768, 719, 748, 783, 795, 782, 751, 763, 768, 818, 778, 779, 747, 783, 716, 785, 761, 739, 747, 763, 753, 742, 746, 793, 789, 741, 784, 836, 777, 765, 788, 811, 769, 749, 772, 787, 724, 749, 804, 746, 738, 759, 755, 753, 786, 768, 773, 808, 760, 754, 795, 733, 768, 751, 769, 760, 807, 753, 741, 794, 769] payload_rows_per_dst=[9370, 9396, 9166, 9376, 9282, 9252, 9344, 9420] dup_rows_per_dst=[2936, 2976, 2928, 3040, 3048, 2928, 2974, 2868] combine_rows_per_src=[9370, 9396, 9166, 9376, 9282, 9252, 9344, 9420] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=65156 dup_rows=23850 dup_rows_remote=20824 dedup_ratio=0.242615 router_tokens_per_expert=[757, 758, 770, 750, 782, 793, 796, 809, 764, 732, 687, 748, 738, 818, 772, 736, 749, 794, 755, 760, 765, 778, 789, 772, 834, 739, 816, 786, 778, 752, 744, 779, 770, 775, 750, 750, 760, 785, 725, 767, 759, 805, 790, 798, 768, 773, 815, 759, 781, 767, 782, 726, 813, 742, 759, 749, 731, 769, 777, 734, 775, 772, 731, 795] payload_rows_per_dst=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] dup_rows_per_dst=[2952, 2876, 3026, 3024, 2922, 3070, 3058, 2922] combine_rows_per_src=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65130 dup_rows=23976 dup_rows_remote=20898 dedup_ratio=0.243896 router_tokens_per_expert=[803, 732, 776, 778, 719, 776, 733, 769, 782, 774, 750, 785, 787, 761, 769, 749, 761, 751, 798, 741, 711, 749, 774, 783, 757, 727, 766, 749, 794, 795, 796, 767, 789, 761, 781, 728, 746, 764, 765, 804, 803, 781, 819, 775, 728, 760, 790, 749, 758, 783, 766, 780, 780, 817, 733, 695, 785, 766, 797, 782, 786, 789, 782, 748] payload_rows_per_dst=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] dup_rows_per_dst=[2972, 3036, 2868, 2898, 3078, 3068, 2998, 3058] combine_rows_per_src=[9200, 9278, 9268, 9404, 9198, 9342, 9226, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74508 payload_rows_remote=65022 dup_rows=23796 dup_rows_remote=20662 dedup_ratio=0.242065 router_tokens_per_expert=[734, 719, 775, 767, 729, 734, 803, 739, 730, 807, 763, 768, 767, 706, 792, 791, 764, 809, 801, 797, 765, 800, 776, 755, 766, 768, 785, 782, 771, 780, 767, 750, 772, 772, 755, 705, 816, 802, 798, 777, 744, 745, 706, 800, 712, 818, 778, 781, 752, 762, 744, 788, 748, 732, 754, 777, 734, 756, 777, 752, 798, 817, 789, 831] payload_rows_per_dst=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] dup_rows_per_dst=[2844, 2956, 3116, 3078, 3016, 2820, 2832, 3134] combine_rows_per_src=[9156, 9292, 9418, 9260, 9374, 9240, 9282, 9486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74054 payload_rows_remote=64726 dup_rows=24250 dup_rows_remote=21066 dedup_ratio=0.246684 router_tokens_per_expert=[736, 750, 708, 773, 790, 771, 738, 783, 804, 749, 739, 764, 798, 764, 741, 784, 795, 808, 757, 814, 753, 768, 784, 777, 825, 785, 709, 798, 752, 773, 760, 785, 790, 773, 765, 754, 718, 771, 811, 756, 778, 802, 767, 813, 757, 737, 760, 721, 786, 775, 776, 766, 751, 733, 772, 751, 750, 758, 744, 760, 817, 801, 741, 763] payload_rows_per_dst=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] dup_rows_per_dst=[2908, 3012, 3184, 3060, 3008, 3070, 3050, 2958] combine_rows_per_src=[9190, 9274, 9328, 9314, 9268, 9200, 9170, 9310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65280 dup_rows=23814 dup_rows_remote=20950 dedup_ratio=0.242249 router_tokens_per_expert=[800, 757, 785, 764, 804, 766, 754, 769, 735, 769, 758, 762, 816, 763, 781, 777, 766, 727, 742, 720, 811, 777, 824, 830, 780, 741, 764, 738, 742, 788, 778, 753, 731, 754, 764, 772, 796, 786, 820, 777, 713, 743, 748, 816, 756, 801, 729, 781, 750, 742, 796, 776, 753, 778, 718, 724, 780, 769, 768, 775, 779, 764, 776, 776] payload_rows_per_dst=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] dup_rows_per_dst=[3028, 3078, 2954, 2928, 3068, 2890, 2864, 3004] combine_rows_per_src=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343920 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267222896 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534101872 weight_movement_bytes=34603008 total_nvlink_bytes=568704880 weight_to_token_ratio=0.064787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=343728 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267443888 ingress_bytes=301703168 link_bytes=301703168 token_bytes=534544048 weight_movement_bytes=34603008 total_nvlink_bytes=569147056 weight_to_token_ratio=0.064734 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135284732e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74574 payload_rows_remote=65196 dup_rows=23730 dup_rows_remote=20730 dedup_ratio=0.241394 router_tokens_per_expert=[747, 768, 745, 798, 739, 748, 758, 790, 738, 771, 694, 764, 747, 797, 751, 756, 761, 775, 780, 728, 801, 766, 758, 788, 821, 731, 800, 788, 736, 757, 748, 808, 840, 782, 812, 819, 701, 717, 762, 760, 745, 773, 747, 746, 796, 779, 777, 749, 781, 765, 699, 777, 754, 781, 838, 765, 775, 794, 779, 754, 751, 761, 841, 775] payload_rows_per_dst=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] dup_rows_per_dst=[2914, 2904, 2894, 3000, 3002, 2996, 2972, 3048] combine_rows_per_src=[9272, 9132, 9420, 9378, 9384, 9228, 9348, 9412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65026 dup_rows=23954 dup_rows_remote=21014 dedup_ratio=0.243673 router_tokens_per_expert=[721, 764, 804, 802, 741, 757, 785, 779, 796, 814, 814, 766, 774, 776, 767, 701, 759, 739, 812, 775, 766, 737, 761, 778, 733, 770, 773, 825, 754, 786, 784, 738, 756, 779, 734, 780, 799, 746, 803, 794, 775, 770, 797, 780, 716, 765, 789, 740, 797, 810, 742, 722, 781, 767, 778, 746, 691, 766, 747, 790, 759, 757, 778, 747] payload_rows_per_dst=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] dup_rows_per_dst=[3006, 3112, 2968, 2984, 3042, 2940, 3018, 2884] combine_rows_per_src=[9300, 9304, 9286, 9342, 9340, 9324, 9268, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266772480 dispatch_meta_bytes=344112 combine_payload_bytes=266772480 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267116592 ingress_bytes=370581504 link_bytes=370581504 token_bytes=533889072 weight_movement_bytes=103809024 total_nvlink_bytes=637698096 weight_to_token_ratio=0.194439 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394468365e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266330112 dispatch_meta_bytes=342736 combine_payload_bytes=266330112 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266672848 ingress_bytes=335536128 link_bytes=335536128 token_bytes=533002960 weight_movement_bytes=69206016 total_nvlink_bytes=602208976 weight_to_token_ratio=0.129842 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262595437e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65138 dup_rows=23906 dup_rows_remote=20860 dedup_ratio=0.243184 router_tokens_per_expert=[746, 762, 781, 753, 743, 742, 774, 800, 769, 795, 785, 796, 762, 737, 747, 771, 813, 759, 764, 819, 765, 735, 755, 814, 768, 800, 757, 739, 791, 709, 823, 766, 687, 787, 764, 759, 793, 760, 784, 785, 788, 779, 789, 752, 756, 775, 748, 713, 730, 762, 773, 787, 737, 781, 788, 775, 739, 765, 767, 812, 788, 756, 762, 771] payload_rows_per_dst=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] dup_rows_per_dst=[2950, 2974, 3028, 3046, 3016, 2912, 3004, 2976] combine_rows_per_src=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74284 payload_rows_remote=64868 dup_rows=24020 dup_rows_remote=20930 dedup_ratio=0.244344 router_tokens_per_expert=[769, 755, 778, 777, 741, 799, 781, 792, 781, 750, 771, 767, 789, 793, 732, 755, 779, 761, 739, 749, 764, 792, 751, 781, 709, 777, 775, 750, 753, 792, 766, 797, 722, 788, 793, 751, 774, 764, 711, 757, 792, 778, 812, 764, 783, 772, 758, 776, 755, 764, 743, 779, 774, 773, 751, 774, 762, 810, 764, 786, 749, 771, 765, 772] payload_rows_per_dst=[9416, 9424, 9366, 9136, 9030, 9368, 9518, 9026] dup_rows_per_dst=[3090, 3182, 2888, 2784, 2946, 3090, 3212, 2828] combine_rows_per_src=[9416, 9424, 9366, 9136, 9030, 9368, 9518, 9026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=344920 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267731800 ingress_bytes=301989888 link_bytes=301989888 token_bytes=535118680 weight_movement_bytes=34603008 total_nvlink_bytes=569721688 weight_to_token_ratio=0.064664 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136363636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=265117696 dispatch_meta_bytes=343168 combine_payload_bytes=265117696 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265460864 ingress_bytes=334323712 link_bytes=334323712 token_bytes=530578560 weight_movement_bytes=69206016 total_nvlink_bytes=599784576 weight_to_token_ratio=0.130435 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258033213e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267042816 dispatch_meta_bytes=343704 combine_payload_bytes=267042816 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267386520 ingress_bytes=336248832 link_bytes=336248832 token_bytes=534429336 weight_movement_bytes=69206016 total_nvlink_bytes=603635352 weight_to_token_ratio=0.129495 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265277285e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64814 dup_rows=24036 dup_rows_remote=20888 dedup_ratio=0.244507 router_tokens_per_expert=[768, 735, 721, 732, 753, 772, 793, 767, 754, 788, 749, 773, 795, 748, 782, 817, 762, 763, 777, 755, 743, 817, 770, 768, 743, 789, 763, 790, 713, 769, 793, 716, 786, 784, 766, 735, 737, 762, 741, 778, 797, 723, 756, 778, 754, 786, 770, 772, 812, 737, 753, 762, 760, 771, 785, 768, 784, 798, 779, 752, 770, 764, 801, 853] payload_rows_per_dst=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] dup_rows_per_dst=[2642, 2950, 2636, 3046, 3444, 3094, 3076, 3148] combine_rows_per_src=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65280 dup_rows=23814 dup_rows_remote=20950 dedup_ratio=0.242249 router_tokens_per_expert=[800, 757, 785, 764, 804, 766, 754, 769, 735, 769, 758, 762, 816, 763, 781, 777, 766, 727, 742, 720, 811, 777, 824, 830, 780, 741, 764, 738, 742, 788, 778, 753, 731, 754, 764, 772, 796, 786, 820, 777, 713, 743, 748, 816, 756, 801, 729, 781, 750, 742, 796, 776, 753, 778, 718, 724, 780, 769, 768, 775, 779, 764, 776, 776] payload_rows_per_dst=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] dup_rows_per_dst=[3028, 3078, 2954, 2928, 3068, 2890, 2864, 3004] combine_rows_per_src=[9370, 9244, 9440, 9240, 9332, 9284, 9210, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64960 dup_rows=24130 dup_rows_remote=21090 dedup_ratio=0.245463 router_tokens_per_expert=[780, 754, 789, 777, 794, 759, 749, 759, 772, 757, 805, 750, 762, 739, 779, 759, 778, 741, 722, 784, 777, 787, 760, 743, 813, 746, 774, 813, 747, 817, 809, 765, 745, 772, 777, 766, 782, 785, 747, 744, 791, 787, 784, 761, 742, 778, 749, 735, 824, 742, 769, 749, 772, 831, 743, 798, 780, 708, 744, 792, 761, 754, 744, 736] payload_rows_per_dst=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] dup_rows_per_dst=[3108, 3050, 2950, 3116, 2962, 3040, 3098, 2806] combine_rows_per_src=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344160 combine_payload_bytes=266346496 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266690656 ingress_bytes=370155520 link_bytes=370155520 token_bytes=533037152 weight_movement_bytes=103809024 total_nvlink_bytes=636846176 weight_to_token_ratio=0.194750 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392865422e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=343992 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149240 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954488 weight_movement_bytes=34603008 total_nvlink_bytes=568557496 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=265699328 dispatch_meta_bytes=343192 combine_payload_bytes=265699328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266042520 ingress_bytes=300302336 link_bytes=300302336 token_bytes=531741848 weight_movement_bytes=34603008 total_nvlink_bytes=566344856 weight_to_token_ratio=0.065075 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130013514e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342808 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265820952 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299096 weight_movement_bytes=69206016 total_nvlink_bytes=600505112 weight_to_token_ratio=0.130258 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=267386880 dispatch_meta_bytes=344920 combine_payload_bytes=267386880 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267731800 ingress_bytes=301989888 link_bytes=301989888 token_bytes=535118680 weight_movement_bytes=34603008 total_nvlink_bytes=569721688 weight_to_token_ratio=0.064664 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136363636e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344200 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420360 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496520 weight_movement_bytes=34603008 total_nvlink_bytes=567099528 weight_to_token_ratio=0.064983 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64618 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[779, 779, 800, 766, 773, 777, 854, 785, 769, 755, 771, 769, 744, 802, 763, 735, 766, 757, 807, 791, 820, 789, 780, 786, 761, 735, 813, 732, 769, 764, 772, 746, 735, 745, 729, 785, 786, 746, 748, 682, 798, 759, 788, 730, 742, 758, 706, 752, 749, 742, 801, 778, 728, 790, 763, 784, 799, 735, 793, 808, 757, 762, 817, 748] payload_rows_per_dst=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] dup_rows_per_dst=[3212, 3050, 3042, 2930, 2760, 2972, 3068, 3102] combine_rows_per_src=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74570 payload_rows_remote=65208 dup_rows=23734 dup_rows_remote=20772 dedup_ratio=0.241435 router_tokens_per_expert=[777, 798, 812, 785, 735, 810, 778, 743, 746, 799, 804, 751, 722, 750, 816, 774, 745, 756, 743, 756, 740, 777, 786, 784, 742, 753, 768, 792, 773, 761, 788, 771, 778, 739, 729, 779, 806, 763, 727, 740, 760, 754, 777, 778, 737, 783, 708, 806, 809, 756, 779, 743, 759, 781, 807, 766, 812, 764, 794, 734, 796, 753, 743, 757] payload_rows_per_dst=[9440, 9362, 9228, 9388, 9296, 9158, 9388, 9310] dup_rows_per_dst=[3036, 2962, 2946, 2908, 2826, 3048, 3012, 2996] combine_rows_per_src=[9440, 9362, 9228, 9388, 9296, 9158, 9388, 9310] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64814 dup_rows=24036 dup_rows_remote=20888 dedup_ratio=0.244507 router_tokens_per_expert=[768, 735, 721, 732, 753, 772, 793, 767, 754, 788, 749, 773, 795, 748, 782, 817, 762, 763, 777, 755, 743, 817, 770, 768, 743, 789, 763, 790, 713, 769, 793, 716, 786, 784, 766, 735, 737, 762, 741, 778, 797, 723, 756, 778, 754, 786, 770, 772, 812, 737, 753, 762, 760, 771, 785, 768, 784, 798, 779, 752, 770, 764, 801, 853] payload_rows_per_dst=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] dup_rows_per_dst=[2642, 2950, 2636, 3046, 3444, 3094, 3076, 3148] combine_rows_per_src=[8858, 9204, 8956, 9364, 9606, 9606, 9220, 9454] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74734 payload_rows_remote=65238 dup_rows=23570 dup_rows_remote=20552 dedup_ratio=0.239766 router_tokens_per_expert=[763, 749, 805, 712, 796, 728, 759, 795, 753, 771, 742, 733, 768, 727, 778, 793, 766, 716, 754, 775, 765, 761, 794, 742, 771, 769, 779, 769, 775, 749, 772, 773, 785, 811, 720, 794, 798, 773, 780, 796, 746, 786, 782, 800, 781, 761, 791, 807, 726, 787, 771, 790, 732, 776, 735, 743, 793, 791, 778, 803, 723, 777, 809, 705] payload_rows_per_dst=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] dup_rows_per_dst=[2896, 2912, 2896, 2906, 3018, 3066, 2884, 2992] combine_rows_per_src=[9318, 9218, 9250, 9408, 9496, 9442, 9236, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64990 dup_rows=24162 dup_rows_remote=21244 dedup_ratio=0.245789 router_tokens_per_expert=[725, 783, 801, 808, 784, 772, 760, 782, 806, 772, 722, 789, 791, 797, 787, 783, 775, 814, 767, 792, 807, 825, 794, 806, 792, 738, 766, 714, 747, 751, 737, 778, 789, 732, 737, 767, 739, 711, 729, 744, 774, 775, 740, 738, 739, 774, 757, 786, 690, 791, 777, 785, 776, 763, 705, 748, 770, 772, 811, 758, 757, 783, 755, 815] payload_rows_per_dst=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] dup_rows_per_dst=[3122, 3100, 3246, 2954, 2860, 2948, 2918, 3014] combine_rows_per_src=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=342848 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018176 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529693504 weight_movement_bytes=69206016 total_nvlink_bytes=598899520 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267091968 dispatch_meta_bytes=343920 combine_payload_bytes=267091968 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267435888 ingress_bytes=370900992 link_bytes=370900992 token_bytes=534527856 weight_movement_bytes=103809024 total_nvlink_bytes=638336880 weight_to_token_ratio=0.194207 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.395670573e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74398 payload_rows_remote=65138 dup_rows=23906 dup_rows_remote=20860 dedup_ratio=0.243184 router_tokens_per_expert=[746, 762, 781, 753, 743, 742, 774, 800, 769, 795, 785, 796, 762, 737, 747, 771, 813, 759, 764, 819, 765, 735, 755, 814, 768, 800, 757, 739, 791, 709, 823, 766, 687, 787, 764, 759, 793, 760, 784, 785, 788, 779, 789, 752, 756, 775, 748, 713, 730, 762, 773, 787, 737, 781, 788, 775, 739, 765, 767, 812, 788, 756, 762, 771] payload_rows_per_dst=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] dup_rows_per_dst=[2950, 2974, 3028, 3046, 3016, 2912, 3004, 2976] combine_rows_per_src=[9252, 9350, 9420, 9260, 9222, 9288, 9262, 9344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64960 dup_rows=24130 dup_rows_remote=21090 dedup_ratio=0.245463 router_tokens_per_expert=[780, 754, 789, 777, 794, 759, 749, 759, 772, 757, 805, 750, 762, 739, 779, 759, 778, 741, 722, 784, 777, 787, 760, 743, 813, 746, 774, 813, 747, 817, 809, 765, 745, 772, 777, 766, 782, 785, 747, 744, 791, 787, 784, 761, 742, 778, 749, 735, 824, 742, 769, 749, 772, 831, 743, 798, 780, 708, 744, 792, 761, 754, 744, 736] payload_rows_per_dst=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] dup_rows_per_dst=[3108, 3050, 2950, 3116, 2962, 3040, 3098, 2806] combine_rows_per_src=[9214, 9196, 9234, 9452, 9274, 9214, 9358, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74454 payload_rows_remote=65156 dup_rows=23850 dup_rows_remote=20824 dedup_ratio=0.242615 router_tokens_per_expert=[757, 758, 770, 750, 782, 793, 796, 809, 764, 732, 687, 748, 738, 818, 772, 736, 749, 794, 755, 760, 765, 778, 789, 772, 834, 739, 816, 786, 778, 752, 744, 779, 770, 775, 750, 750, 760, 785, 725, 767, 759, 805, 790, 798, 768, 773, 815, 759, 781, 767, 782, 726, 813, 742, 759, 749, 731, 769, 777, 734, 775, 772, 731, 795] payload_rows_per_dst=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] dup_rows_per_dst=[2952, 2876, 3026, 3024, 2922, 3070, 3058, 2922] combine_rows_per_src=[9478, 9114, 9298, 9432, 9242, 9464, 9180, 9246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265478144 dispatch_meta_bytes=342808 combine_payload_bytes=265478144 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265820952 ingress_bytes=334684160 link_bytes=334684160 token_bytes=531299096 weight_movement_bytes=69206016 total_nvlink_bytes=600505112 weight_to_token_ratio=0.130258 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259389550e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20986 dedup_ratio=0.243917 router_tokens_per_expert=[726, 728, 779, 795, 701, 774, 798, 776, 745, 826, 792, 828, 763, 792, 757, 762, 803, 735, 790, 747, 754, 749, 729, 724, 756, 730, 793, 799, 790, 756, 761, 783, 792, 770, 763, 773, 766, 752, 757, 762, 801, 740, 756, 833, 784, 760, 751, 780, 761, 764, 758, 787, 776, 799, 766, 754, 767, 763, 770, 769, 751, 749, 790, 747] payload_rows_per_dst=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] dup_rows_per_dst=[2900, 3100, 2938, 2992, 2994, 2994, 3034, 3026] combine_rows_per_src=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=267214848 dispatch_meta_bytes=343160 combine_payload_bytes=267214848 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267558008 ingress_bytes=301817856 link_bytes=301817856 token_bytes=534772856 weight_movement_bytes=34603008 total_nvlink_bytes=569375864 weight_to_token_ratio=0.064706 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135716294e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64954 dup_rows=23966 dup_rows_remote=20810 dedup_ratio=0.243795 router_tokens_per_expert=[759, 794, 770, 765, 762, 780, 735, 811, 788, 749, 776, 740, 722, 778, 756, 857, 739, 774, 795, 778, 770, 747, 764, 750, 759, 769, 811, 776, 747, 724, 722, 805, 735, 761, 740, 808, 756, 820, 803, 815, 753, 713, 747, 733, 705, 789, 718, 777, 815, 780, 836, 772, 726, 769, 779, 773, 747, 806, 748, 794, 805, 736, 743, 778] payload_rows_per_dst=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] dup_rows_per_dst=[2894, 2818, 3056, 2934, 3230, 2880, 2998, 3156] combine_rows_per_src=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74216 payload_rows_remote=64738 dup_rows=24088 dup_rows_remote=21032 dedup_ratio=0.245036 router_tokens_per_expert=[766, 788, 763, 781, 792, 815, 787, 766, 734, 786, 795, 793, 750, 788, 811, 757, 752, 782, 786, 788, 812, 778, 755, 751, 732, 742, 781, 782, 741, 775, 769, 768, 767, 786, 767, 796, 746, 806, 801, 699, 780, 771, 734, 791, 739, 768, 757, 762, 765, 693, 766, 773, 785, 833, 760, 730, 725, 771, 771, 744, 771, 764, 706, 759] payload_rows_per_dst=[9478, 9290, 9348, 9204, 9252, 9240, 9310, 9094] dup_rows_per_dst=[3056, 3126, 3060, 2976, 3084, 2964, 3116, 2706] combine_rows_per_src=[9478, 9290, 9348, 9204, 9252, 9240, 9310, 9094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344936 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543976 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532743016 weight_movement_bytes=34603008 total_nvlink_bytes=567346024 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=343992 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149240 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954488 weight_movement_bytes=34603008 total_nvlink_bytes=568557496 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74142 payload_rows_remote=64990 dup_rows=24162 dup_rows_remote=21244 dedup_ratio=0.245789 router_tokens_per_expert=[725, 783, 801, 808, 784, 772, 760, 782, 806, 772, 722, 789, 791, 797, 787, 783, 775, 814, 767, 792, 807, 825, 794, 806, 792, 738, 766, 714, 747, 751, 737, 778, 789, 732, 737, 767, 739, 711, 729, 744, 774, 775, 740, 738, 739, 774, 757, 786, 690, 791, 777, 785, 776, 763, 705, 748, 770, 772, 811, 758, 757, 783, 755, 815] payload_rows_per_dst=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] dup_rows_per_dst=[3122, 3100, 3246, 2954, 2860, 2948, 2918, 3014] combine_rows_per_src=[9308, 9394, 9514, 9092, 9036, 9218, 9152, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65136 dup_rows=23924 dup_rows_remote=20866 dedup_ratio=0.243368 router_tokens_per_expert=[784, 750, 753, 733, 803, 825, 760, 811, 782, 762, 768, 765, 810, 776, 736, 813, 803, 757, 763, 828, 778, 725, 751, 733, 769, 775, 775, 729, 776, 767, 746, 782, 743, 725, 712, 780, 729, 805, 719, 827, 804, 759, 729, 767, 836, 746, 763, 747, 758, 723, 716, 777, 799, 785, 736, 782, 795, 743, 744, 777, 780, 779, 792, 787] payload_rows_per_dst=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] dup_rows_per_dst=[3024, 3042, 2978, 3038, 2916, 3058, 2916, 2952] combine_rows_per_src=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266076160 dispatch_meta_bytes=344200 combine_payload_bytes=266076160 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266420360 ingress_bytes=300679168 link_bytes=300679168 token_bytes=532496520 weight_movement_bytes=34603008 total_nvlink_bytes=567099528 weight_to_token_ratio=0.064983 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131431503e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=266878976 dispatch_meta_bytes=343920 combine_payload_bytes=266878976 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267222896 ingress_bytes=301481984 link_bytes=301481984 token_bytes=534101872 weight_movement_bytes=34603008 total_nvlink_bytes=568704880 weight_to_token_ratio=0.064787 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134452435e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343872 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510144 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676416 weight_movement_bytes=34603008 total_nvlink_bytes=567279424 weight_to_token_ratio=0.064961 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343056 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266394640 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446224 weight_movement_bytes=34603008 total_nvlink_bytes=567049232 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=265166848 dispatch_meta_bytes=343080 combine_payload_bytes=265166848 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265509928 ingress_bytes=368975872 link_bytes=368975872 token_bytes=530676776 weight_movement_bytes=103809024 total_nvlink_bytes=634485800 weight_to_token_ratio=0.195616 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388426501e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64826 dup_rows=24130 dup_rows_remote=21138 dedup_ratio=0.245463 router_tokens_per_expert=[762, 817, 761, 718, 774, 781, 767, 769, 731, 769, 773, 748, 797, 740, 744, 776, 777, 750, 811, 780, 739, 768, 785, 760, 733, 775, 783, 743, 808, 781, 749, 735, 789, 681, 782, 747, 744, 768, 756, 796, 742, 736, 766, 797, 787, 749, 791, 736, 796, 773, 783, 748, 778, 834, 778, 773, 764, 831, 751, 755, 780, 792, 780, 765] payload_rows_per_dst=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] dup_rows_per_dst=[3086, 2940, 2992, 3040, 2952, 2890, 3068, 3162] combine_rows_per_src=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344936 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266543976 ingress_bytes=300802048 link_bytes=300802048 token_bytes=532743016 weight_movement_bytes=34603008 total_nvlink_bytes=567346024 weight_to_token_ratio=0.064953 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131893890e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344008 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141064 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938120 weight_movement_bytes=34603008 total_nvlink_bytes=568541128 weight_to_token_ratio=0.064807 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74430 payload_rows_remote=65004 dup_rows=23874 dup_rows_remote=20938 dedup_ratio=0.242859 router_tokens_per_expert=[802, 783, 814, 757, 761, 790, 761, 749, 738, 827, 777, 797, 728, 788, 752, 774, 767, 726, 711, 747, 778, 770, 776, 767, 812, 772, 799, 734, 739, 809, 767, 748, 802, 737, 770, 750, 764, 780, 779, 759, 737, 732, 795, 748, 770, 777, 764, 773, 775, 719, 813, 798, 808, 806, 774, 775, 752, 760, 757, 803, 734, 736, 773, 712] payload_rows_per_dst=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] dup_rows_per_dst=[3072, 2936, 2910, 3052, 2954, 2898, 3114, 2938] combine_rows_per_src=[9362, 9426, 9174, 9308, 9328, 9294, 9422, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74338 payload_rows_remote=64954 dup_rows=23966 dup_rows_remote=20810 dedup_ratio=0.243795 router_tokens_per_expert=[759, 794, 770, 765, 762, 780, 735, 811, 788, 749, 776, 740, 722, 778, 756, 857, 739, 774, 795, 778, 770, 747, 764, 750, 759, 769, 811, 776, 747, 724, 722, 805, 735, 761, 740, 808, 756, 820, 803, 815, 753, 713, 747, 733, 705, 789, 718, 777, 815, 780, 836, 772, 726, 769, 779, 773, 747, 806, 748, 794, 805, 736, 743, 778] payload_rows_per_dst=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] dup_rows_per_dst=[2894, 2818, 3056, 2934, 3230, 2880, 2998, 3156] combine_rows_per_src=[9228, 9120, 9338, 9292, 9456, 9224, 9296, 9384] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74240 payload_rows_remote=65018 dup_rows=24064 dup_rows_remote=21072 dedup_ratio=0.244792 router_tokens_per_expert=[749, 797, 725, 761, 748, 785, 770, 818, 772, 755, 796, 775, 767, 786, 736, 759, 783, 765, 753, 777, 732, 784, 822, 760, 792, 780, 732, 820, 749, 697, 794, 754, 781, 750, 782, 777, 752, 763, 726, 776, 785, 771, 756, 763, 753, 784, 780, 821, 756, 741, 753, 764, 757, 798, 742, 786, 754, 751, 753, 745, 800, 790, 797, 752] payload_rows_per_dst=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] dup_rows_per_dst=[2952, 3028, 3118, 2906, 2992, 3058, 3026, 2984] combine_rows_per_src=[9354, 9264, 9234, 9330, 9222, 9368, 9168, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343768 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266600152 ingress_bytes=370065408 link_bytes=370065408 token_bytes=532856536 weight_movement_bytes=103809024 total_nvlink_bytes=636665560 weight_to_token_ratio=0.194816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392526338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=64982 dup_rows=23978 dup_rows_remote=20986 dedup_ratio=0.243917 router_tokens_per_expert=[726, 728, 779, 795, 701, 774, 798, 776, 745, 826, 792, 828, 763, 792, 757, 762, 803, 735, 790, 747, 754, 749, 729, 724, 756, 730, 793, 799, 790, 756, 761, 783, 792, 770, 763, 773, 766, 752, 757, 762, 801, 740, 756, 833, 784, 760, 751, 780, 761, 764, 758, 787, 776, 799, 766, 754, 767, 763, 770, 769, 751, 749, 790, 747] payload_rows_per_dst=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] dup_rows_per_dst=[2900, 3100, 2938, 2992, 2994, 2994, 3034, 3026] combine_rows_per_src=[9254, 9430, 9124, 9344, 9276, 9416, 9296, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65118 dup_rows=23864 dup_rows_remote=20904 dedup_ratio=0.242757 router_tokens_per_expert=[789, 770, 771, 756, 750, 756, 752, 798, 744, 784, 749, 830, 783, 790, 777, 780, 725, 811, 759, 805, 716, 810, 784, 740, 755, 802, 706, 723, 793, 765, 755, 772, 743, 746, 771, 795, 753, 798, 793, 757, 728, 779, 796, 759, 748, 787, 780, 764, 794, 763, 758, 773, 775, 782, 746, 750, 807, 757, 761, 738, 804, 819, 717, 711] payload_rows_per_dst=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] dup_rows_per_dst=[3006, 3054, 2938, 2968, 2962, 3000, 2960, 2976] combine_rows_per_src=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74380 payload_rows_remote=65136 dup_rows=23924 dup_rows_remote=20866 dedup_ratio=0.243368 router_tokens_per_expert=[784, 750, 753, 733, 803, 825, 760, 811, 782, 762, 768, 765, 810, 776, 736, 813, 803, 757, 763, 828, 778, 725, 751, 733, 769, 775, 775, 729, 776, 767, 746, 782, 743, 725, 712, 780, 729, 805, 719, 827, 804, 759, 729, 767, 836, 746, 763, 747, 758, 723, 716, 777, 799, 785, 736, 782, 795, 743, 744, 777, 780, 779, 792, 787] payload_rows_per_dst=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] dup_rows_per_dst=[3024, 3042, 2978, 3038, 2916, 3058, 2916, 2952] combine_rows_per_src=[9414, 9382, 9298, 9200, 9164, 9244, 9236, 9442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=343056 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266394640 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532446224 weight_movement_bytes=34603008 total_nvlink_bytes=567049232 weight_to_token_ratio=0.064989 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343856 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871152 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398448 weight_movement_bytes=34603008 total_nvlink_bytes=566001456 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65220 dup_rows=23994 dup_rows_remote=21126 dedup_ratio=0.244080 router_tokens_per_expert=[731, 749, 745, 746, 828, 793, 811, 783, 793, 815, 803, 723, 796, 799, 753, 780, 746, 796, 808, 802, 797, 773, 723, 806, 778, 741, 722, 745, 767, 692, 779, 755, 777, 807, 761, 773, 799, 772, 743, 724, 764, 791, 745, 780, 737, 794, 759, 755, 759, 749, 801, 770, 805, 750, 757, 782, 778, 752, 763, 737, 750, 755, 758, 727] payload_rows_per_dst=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] dup_rows_per_dst=[3016, 3058, 3130, 2868, 2906, 3050, 2986, 2980] combine_rows_per_src=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266256384 dispatch_meta_bytes=343768 combine_payload_bytes=266256384 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266600152 ingress_bytes=370065408 link_bytes=370065408 token_bytes=532856536 weight_movement_bytes=103809024 total_nvlink_bytes=636665560 weight_to_token_ratio=0.194816 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392526338e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266313728 dispatch_meta_bytes=344360 combine_payload_bytes=266313728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266658088 ingress_bytes=300916736 link_bytes=300916736 token_bytes=532971816 weight_movement_bytes=34603008 total_nvlink_bytes=567574824 weight_to_token_ratio=0.064925 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132325452e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=64618 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[779, 779, 800, 766, 773, 777, 854, 785, 769, 755, 771, 769, 744, 802, 763, 735, 766, 757, 807, 791, 820, 789, 780, 786, 761, 735, 813, 732, 769, 764, 772, 746, 735, 745, 729, 785, 786, 746, 748, 682, 798, 759, 788, 730, 742, 758, 706, 752, 749, 742, 801, 778, 728, 790, 763, 784, 799, 735, 793, 808, 757, 762, 817, 748] payload_rows_per_dst=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] dup_rows_per_dst=[3212, 3050, 3042, 2930, 2760, 2972, 3068, 3102] combine_rows_per_src=[9414, 9166, 9550, 9254, 9152, 9094, 9202, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64768 dup_rows=24178 dup_rows_remote=21008 dedup_ratio=0.245951 router_tokens_per_expert=[797, 720, 757, 708, 810, 713, 772, 771, 784, 794, 749, 761, 773, 793, 685, 799, 758, 766, 738, 734, 715, 777, 834, 787, 787, 746, 745, 777, 776, 805, 760, 749, 781, 710, 770, 822, 772, 736, 780, 752, 797, 784, 770, 829, 757, 798, 703, 815, 768, 761, 742, 779, 752, 789, 800, 804, 821, 731, 759, 759, 764, 822, 740, 745] payload_rows_per_dst=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] dup_rows_per_dst=[2438, 2608, 2968, 3520, 3044, 3118, 3312, 3170] combine_rows_per_src=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74084 payload_rows_remote=64784 dup_rows=24220 dup_rows_remote=21252 dedup_ratio=0.246379 router_tokens_per_expert=[796, 753, 779, 736, 777, 733, 795, 765, 748, 733, 762, 734, 772, 780, 738, 771, 767, 778, 782, 801, 807, 798, 786, 729, 731, 744, 779, 721, 820, 723, 749, 787, 768, 741, 775, 781, 744, 774, 820, 764, 772, 765, 756, 807, 778, 802, 764, 746, 756, 779, 748, 801, 810, 793, 767, 789, 755, 754, 776, 810, 724, 757, 737, 765] payload_rows_per_dst=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] dup_rows_per_dst=[2968, 2998, 3126, 3098, 3060, 3074, 2866, 3030] combine_rows_per_src=[9300, 9130, 9370, 9454, 9274, 9306, 9124, 9126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344088 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267067416 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790744 weight_movement_bytes=138412032 total_nvlink_bytes=672202776 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343872 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266510144 ingress_bytes=300769280 link_bytes=300769280 token_bytes=532676416 weight_movement_bytes=34603008 total_nvlink_bytes=567279424 weight_to_token_ratio=0.064961 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131770587e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65146 dup_rows=23928 dup_rows_remote=21078 dedup_ratio=0.243408 router_tokens_per_expert=[783, 763, 765, 819, 752, 810, 808, 779, 750, 755, 815, 773, 784, 795, 763, 779, 723, 744, 798, 727, 747, 749, 762, 780, 770, 747, 775, 769, 761, 780, 770, 785, 741, 750, 773, 722, 762, 778, 775, 778, 755, 790, 714, 752, 745, 757, 781, 746, 833, 783, 785, 759, 780, 776, 781, 757, 790, 796, 743, 785, 747, 724, 737, 777] payload_rows_per_dst=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] dup_rows_per_dst=[3156, 3050, 2880, 2922, 2970, 2850, 3168, 2932] combine_rows_per_src=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74440 payload_rows_remote=65118 dup_rows=23864 dup_rows_remote=20904 dedup_ratio=0.242757 router_tokens_per_expert=[789, 770, 771, 756, 750, 756, 752, 798, 744, 784, 749, 830, 783, 790, 777, 780, 725, 811, 759, 805, 716, 810, 784, 740, 755, 802, 706, 723, 793, 765, 755, 772, 743, 746, 771, 795, 753, 798, 793, 757, 728, 779, 796, 759, 748, 787, 780, 764, 794, 763, 758, 773, 775, 782, 746, 750, 807, 757, 761, 738, 804, 819, 717, 711] payload_rows_per_dst=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] dup_rows_per_dst=[3006, 3054, 2938, 2968, 2962, 3000, 2960, 2976] combine_rows_per_src=[9278, 9420, 9362, 9174, 9350, 9282, 9322, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345384 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486504 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627624 weight_movement_bytes=34603008 total_nvlink_bytes=569230632 weight_to_token_ratio=0.064724 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266797056 dispatch_meta_bytes=344008 combine_payload_bytes=266797056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267141064 ingress_bytes=301400064 link_bytes=301400064 token_bytes=533938120 weight_movement_bytes=34603008 total_nvlink_bytes=568541128 weight_to_token_ratio=0.064807 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134144176e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=264675328 dispatch_meta_bytes=342848 combine_payload_bytes=264675328 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265018176 ingress_bytes=333881344 link_bytes=333881344 token_bytes=529693504 weight_movement_bytes=69206016 total_nvlink_bytes=598899520 weight_to_token_ratio=0.130653 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.256368618e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343104 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265632832 ingress_bytes=299892736 link_bytes=299892736 token_bytes=530922560 weight_movement_bytes=34603008 total_nvlink_bytes=565525568 weight_to_token_ratio=0.065175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128472222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=344144 combine_payload_bytes=265355264 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265699408 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531054672 weight_movement_bytes=69206016 total_nvlink_bytes=600260688 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344896 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182912 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020928 weight_movement_bytes=34603008 total_nvlink_bytes=568623936 weight_to_token_ratio=0.064797 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64968 dup_rows=23944 dup_rows_remote=20934 dedup_ratio=0.243571 router_tokens_per_expert=[773, 753, 756, 826, 800, 742, 801, 743, 741, 744, 770, 758, 769, 753, 826, 775, 757, 765, 767, 753, 860, 809, 741, 749, 732, 703, 731, 808, 761, 766, 765, 796, 763, 738, 768, 759, 762, 786, 787, 768, 776, 773, 763, 723, 774, 770, 762, 702, 788, 778, 763, 750, 786, 770, 734, 725, 776, 819, 738, 790, 785, 773, 793, 817] payload_rows_per_dst=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] dup_rows_per_dst=[2958, 3032, 3010, 2920, 3010, 2962, 2972, 3080] combine_rows_per_src=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344088 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267067416 ingress_bytes=405135360 link_bytes=405135360 token_bytes=533790744 weight_movement_bytes=138412032 total_nvlink_bytes=672202776 weight_to_token_ratio=0.259300 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524491744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74186 payload_rows_remote=64864 dup_rows=24118 dup_rows_remote=21076 dedup_ratio=0.245341 router_tokens_per_expert=[790, 769, 827, 792, 777, 756, 762, 781, 822, 736, 765, 747, 807, 737, 772, 796, 734, 777, 787, 720, 765, 825, 760, 760, 760, 777, 774, 742, 745, 732, 723, 763, 763, 793, 798, 816, 755, 705, 752, 760, 793, 750, 749, 756, 802, 773, 748, 765, 782, 765, 763, 762, 776, 731, 749, 828, 773, 749, 789, 733, 806, 743, 813, 732] payload_rows_per_dst=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] dup_rows_per_dst=[3028, 3042, 2978, 2912, 2996, 3020, 3076, 3066] combine_rows_per_src=[9480, 9322, 9278, 9120, 9288, 9252, 9236, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74126 payload_rows_remote=64768 dup_rows=24178 dup_rows_remote=21008 dedup_ratio=0.245951 router_tokens_per_expert=[797, 720, 757, 708, 810, 713, 772, 771, 784, 794, 749, 761, 773, 793, 685, 799, 758, 766, 738, 734, 715, 777, 834, 787, 787, 746, 745, 777, 776, 805, 760, 749, 781, 710, 770, 822, 772, 736, 780, 752, 797, 784, 770, 829, 757, 798, 703, 815, 768, 761, 742, 779, 752, 789, 800, 804, 821, 731, 759, 759, 764, 822, 740, 745] payload_rows_per_dst=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] dup_rows_per_dst=[2438, 2608, 2968, 3520, 3044, 3118, 3312, 3170] combine_rows_per_src=[8572, 8910, 9312, 9856, 9202, 9444, 9472, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74280 payload_rows_remote=65020 dup_rows=24024 dup_rows_remote=21008 dedup_ratio=0.244385 router_tokens_per_expert=[791, 794, 761, 806, 803, 778, 759, 807, 749, 760, 756, 724, 757, 773, 781, 763, 742, 734, 772, 806, 785, 777, 812, 782, 776, 751, 751, 753, 789, 753, 784, 734, 780, 815, 721, 785, 750, 750, 764, 773, 777, 764, 738, 778, 779, 744, 780, 813, 755, 792, 723, 729, 792, 733, 773, 779, 777, 778, 779, 730, 740, 751, 777, 770] payload_rows_per_dst=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] dup_rows_per_dst=[3162, 2844, 2980, 3002, 3016, 3046, 2950, 3024] combine_rows_per_src=[9436, 9282, 9440, 9180, 9260, 9300, 9202, 9180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65176 dup_rows=23948 dup_rows_remote=21122 dedup_ratio=0.243612 router_tokens_per_expert=[796, 777, 747, 771, 765, 741, 781, 798, 792, 802, 807, 808, 808, 768, 734, 781, 792, 768, 755, 768, 791, 771, 730, 785, 741, 794, 773, 718, 753, 724, 784, 733, 793, 816, 756, 772, 740, 749, 719, 810, 762, 780, 790, 775, 798, 718, 760, 795, 721, 792, 735, 762, 768, 729, 768, 728, 757, 748, 787, 758, 750, 781, 789, 790] payload_rows_per_dst=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] dup_rows_per_dst=[3010, 3186, 2938, 2954, 2994, 3002, 2826, 3038] combine_rows_per_src=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266026704 ingress_bytes=438697984 link_bytes=438697984 token_bytes=531709648 weight_movement_bytes=173015040 total_nvlink_bytes=704724688 weight_to_token_ratio=0.325394 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650785196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74310 payload_rows_remote=65220 dup_rows=23994 dup_rows_remote=21126 dedup_ratio=0.244080 router_tokens_per_expert=[731, 749, 745, 746, 828, 793, 811, 783, 793, 815, 803, 723, 796, 799, 753, 780, 746, 796, 808, 802, 797, 773, 723, 806, 778, 741, 722, 745, 767, 692, 779, 755, 777, 807, 761, 773, 799, 772, 743, 724, 764, 791, 745, 780, 737, 794, 759, 755, 759, 749, 801, 770, 805, 750, 757, 782, 778, 752, 763, 737, 750, 755, 758, 727] payload_rows_per_dst=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] dup_rows_per_dst=[3016, 3058, 3130, 2868, 2906, 3050, 2986, 2980] combine_rows_per_src=[9356, 9466, 9372, 9090, 9406, 9200, 9360, 9060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452536 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561464 weight_movement_bytes=69206016 total_nvlink_bytes=601767480 weight_to_token_ratio=0.129949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65146 dup_rows=23928 dup_rows_remote=21078 dedup_ratio=0.243408 router_tokens_per_expert=[783, 763, 765, 819, 752, 810, 808, 779, 750, 755, 815, 773, 784, 795, 763, 779, 723, 744, 798, 727, 747, 749, 762, 780, 770, 747, 775, 769, 761, 780, 770, 785, 741, 750, 773, 722, 762, 778, 775, 778, 755, 790, 714, 752, 745, 757, 781, 746, 833, 783, 785, 759, 780, 776, 781, 757, 790, 796, 743, 785, 747, 724, 737, 777] payload_rows_per_dst=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] dup_rows_per_dst=[3156, 3050, 2880, 2922, 2970, 2850, 3168, 2932] combine_rows_per_src=[9402, 9378, 9180, 9392, 9188, 9230, 9340, 9266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265289728 dispatch_meta_bytes=343104 combine_payload_bytes=265289728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265632832 ingress_bytes=299892736 link_bytes=299892736 token_bytes=530922560 weight_movement_bytes=34603008 total_nvlink_bytes=565525568 weight_to_token_ratio=0.065175 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128472222e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74174 payload_rows_remote=64826 dup_rows=24130 dup_rows_remote=21138 dedup_ratio=0.245463 router_tokens_per_expert=[762, 817, 761, 718, 774, 781, 767, 769, 731, 769, 773, 748, 797, 740, 744, 776, 777, 750, 811, 780, 739, 768, 785, 760, 733, 775, 783, 743, 808, 781, 749, 735, 789, 681, 782, 747, 744, 768, 756, 796, 742, 736, 766, 797, 787, 749, 791, 736, 796, 773, 783, 748, 778, 834, 778, 773, 764, 831, 751, 755, 780, 792, 780, 765] payload_rows_per_dst=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] dup_rows_per_dst=[3086, 2940, 2992, 3040, 2952, 2890, 3068, 3162] combine_rows_per_src=[9212, 9216, 9348, 9174, 9174, 9318, 9458, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64882 dup_rows=24042 dup_rows_remote=21120 dedup_ratio=0.244568 router_tokens_per_expert=[736, 801, 746, 752, 757, 763, 783, 735, 765, 755, 736, 793, 797, 779, 779, 814, 782, 740, 715, 779, 762, 790, 751, 752, 803, 737, 762, 766, 754, 771, 775, 783, 753, 785, 752, 761, 742, 757, 771, 755, 790, 796, 739, 748, 817, 756, 760, 839, 729, 745, 764, 775, 798, 731, 773, 775, 756, 796, 762, 800, 779, 754, 771, 810] payload_rows_per_dst=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] dup_rows_per_dst=[3004, 3114, 2864, 2922, 2930, 3126, 3004, 3078] combine_rows_per_src=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=265682944 dispatch_meta_bytes=343760 combine_payload_bytes=265682944 prefetch_bytes=34603008 reduce_bytes=138412032 egress_bytes=266026704 ingress_bytes=438697984 link_bytes=438697984 token_bytes=531709648 weight_movement_bytes=173015040 total_nvlink_bytes=704724688 weight_to_token_ratio=0.325394 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.650785196e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65264 dup_rows=23824 dup_rows_remote=20860 dedup_ratio=0.242350 router_tokens_per_expert=[762, 774, 805, 764, 821, 800, 762, 776, 765, 706, 770, 760, 747, 768, 795, 779, 782, 808, 789, 787, 754, 771, 726, 774, 792, 772, 719, 738, 751, 776, 789, 784, 771, 715, 784, 734, 793, 793, 758, 784, 756, 767, 765, 746, 806, 737, 731, 786, 776, 774, 762, 815, 768, 747, 774, 770, 758, 785, 806, 752, 701, 754, 747, 771] payload_rows_per_dst=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] dup_rows_per_dst=[3084, 2872, 3060, 2960, 2960, 2800, 3124, 2964] combine_rows_per_src=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=344112 combine_payload_bytes=266321920 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266666032 ingress_bytes=300924928 link_bytes=300924928 token_bytes=532987952 weight_movement_bytes=34603008 total_nvlink_bytes=567590960 weight_to_token_ratio=0.064923 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132356278e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=345192 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267306088 ingress_bytes=301563904 link_bytes=301563904 token_bytes=534266984 weight_movement_bytes=34603008 total_nvlink_bytes=568869992 weight_to_token_ratio=0.064767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134760693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74298 payload_rows_remote=64974 dup_rows=24006 dup_rows_remote=20812 dedup_ratio=0.244202 router_tokens_per_expert=[786, 727, 718, 771, 819, 784, 779, 766, 738, 785, 772, 777, 775, 807, 798, 782, 769, 778, 761, 742, 780, 717, 809, 785, 749, 788, 782, 745, 775, 772, 768, 760, 800, 845, 738, 812, 731, 799, 737, 768, 746, 767, 794, 773, 770, 814, 772, 740, 768, 755, 768, 754, 749, 742, 737, 731, 744, 743, 793, 706, 833, 703, 777, 779] payload_rows_per_dst=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] dup_rows_per_dst=[3194, 3034, 2946, 2944, 3098, 2992, 2878, 2920] combine_rows_per_src=[9324, 9438, 9118, 9334, 9358, 9360, 9130, 9236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345384 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486504 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627624 weight_movement_bytes=34603008 total_nvlink_bytes=569230632 weight_to_token_ratio=0.064724 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266838016 dispatch_meta_bytes=344896 combine_payload_bytes=266838016 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267182912 ingress_bytes=301441024 link_bytes=301441024 token_bytes=534020928 weight_movement_bytes=34603008 total_nvlink_bytes=568623936 weight_to_token_ratio=0.064797 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134298305e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=265527296 dispatch_meta_bytes=343856 combine_payload_bytes=265527296 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265871152 ingress_bytes=300130304 link_bytes=300130304 token_bytes=531398448 weight_movement_bytes=34603008 total_nvlink_bytes=566001456 weight_to_token_ratio=0.065117 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129366171e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64954 dup_rows=24150 dup_rows_remote=21236 dedup_ratio=0.245667 router_tokens_per_expert=[780, 746, 805, 813, 750, 779, 722, 773, 722, 733, 815, 754, 733, 758, 790, 752, 737, 759, 751, 778, 738, 798, 807, 788, 756, 743, 760, 789, 808, 787, 762, 754, 762, 727, 723, 772, 733, 765, 779, 780, 747, 797, 808, 769, 769, 711, 796, 771, 777, 810, 794, 795, 741, 713, 750, 813, 767, 759, 750, 723, 828, 785, 814, 784] payload_rows_per_dst=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] dup_rows_per_dst=[3080, 2914, 3068, 3016, 2922, 3056, 3062, 3032] combine_rows_per_src=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343144 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266476648 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532610152 weight_movement_bytes=69206016 total_nvlink_bytes=601816168 weight_to_token_ratio=0.129937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65264 dup_rows=23824 dup_rows_remote=20860 dedup_ratio=0.242350 router_tokens_per_expert=[762, 774, 805, 764, 821, 800, 762, 776, 765, 706, 770, 760, 747, 768, 795, 779, 782, 808, 789, 787, 754, 771, 726, 774, 792, 772, 719, 738, 751, 776, 789, 784, 771, 715, 784, 734, 793, 793, 758, 784, 756, 767, 765, 746, 806, 737, 731, 786, 776, 774, 762, 815, 768, 747, 774, 770, 758, 785, 806, 752, 701, 754, 747, 771] payload_rows_per_dst=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] dup_rows_per_dst=[3084, 2872, 3060, 2960, 2960, 2800, 3124, 2964] combine_rows_per_src=[9192, 9308, 9322, 9282, 9304, 9356, 9500, 9216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344760 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266396344 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447928 weight_movement_bytes=34603008 total_nvlink_bytes=567050936 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64918 dup_rows=24128 dup_rows_remote=21230 dedup_ratio=0.245443 router_tokens_per_expert=[781, 772, 766, 758, 798, 771, 766, 769, 795, 741, 759, 724, 764, 751, 801, 744, 815, 766, 761, 751, 781, 749, 761, 794, 751, 805, 753, 746, 796, 768, 769, 763, 760, 779, 755, 745, 755, 760, 782, 783, 759, 712, 797, 783, 803, 731, 720, 803, 732, 781, 765, 768, 755, 736, 774, 767, 769, 779, 790, 783, 812, 769, 781, 775] payload_rows_per_dst=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] dup_rows_per_dst=[3012, 3026, 2990, 2970, 3022, 3064, 2898, 3146] combine_rows_per_src=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65090 dup_rows=23958 dup_rows_remote=20934 dedup_ratio=0.243713 router_tokens_per_expert=[771, 809, 783, 739, 711, 740, 760, 731, 756, 782, 758, 737, 781, 752, 748, 782, 817, 845, 784, 756, 747, 804, 766, 768, 779, 743, 768, 770, 745, 760, 762, 779, 763, 773, 769, 742, 774, 803, 764, 752, 778, 849, 774, 731, 736, 775, 805, 767, 714, 757, 832, 748, 740, 787, 740, 762, 796, 783, 763, 788, 788, 786, 707, 773] payload_rows_per_dst=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] dup_rows_per_dst=[2846, 2970, 3080, 2946, 3024, 3112, 2906, 3074] combine_rows_per_src=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74262 payload_rows_remote=64882 dup_rows=24042 dup_rows_remote=21120 dedup_ratio=0.244568 router_tokens_per_expert=[736, 801, 746, 752, 757, 763, 783, 735, 765, 755, 736, 793, 797, 779, 779, 814, 782, 740, 715, 779, 762, 790, 751, 752, 803, 737, 762, 766, 754, 771, 775, 783, 753, 785, 752, 761, 742, 757, 771, 755, 790, 796, 739, 748, 817, 756, 760, 839, 729, 745, 764, 775, 798, 731, 773, 775, 756, 796, 762, 800, 779, 754, 771, 810] payload_rows_per_dst=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] dup_rows_per_dst=[3004, 3114, 2864, 2922, 2930, 3126, 3004, 3078] combine_rows_per_src=[9142, 9322, 9278, 9380, 9222, 9364, 9176, 9378] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344008 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100680 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857352 weight_movement_bytes=34603008 total_nvlink_bytes=566460360 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=64850 dup_rows=24010 dup_rows_remote=21058 dedup_ratio=0.244242 router_tokens_per_expert=[796, 740, 761, 777, 754, 772, 776, 736, 786, 779, 775, 744, 767, 704, 801, 789, 776, 788, 761, 735, 721, 744, 739, 781, 789, 781, 779, 769, 721, 759, 753, 734, 775, 753, 806, 727, 793, 767, 763, 805, 778, 786, 785, 746, 782, 762, 718, 841, 767, 772, 771, 785, 782, 751, 770, 776, 767, 813, 780, 769, 782, 759, 746, 788] payload_rows_per_dst=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] dup_rows_per_dst=[2972, 3050, 2908, 2950, 3112, 2952, 3024, 3042] combine_rows_per_src=[9252, 9240, 9182, 9220, 9266, 9444, 9324, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65176 dup_rows=23948 dup_rows_remote=21122 dedup_ratio=0.243612 router_tokens_per_expert=[796, 777, 747, 771, 765, 741, 781, 798, 792, 802, 807, 808, 808, 768, 734, 781, 792, 768, 755, 768, 791, 771, 730, 785, 741, 794, 773, 718, 753, 724, 784, 733, 793, 816, 756, 772, 740, 749, 719, 810, 762, 780, 790, 775, 798, 718, 760, 795, 721, 792, 735, 762, 768, 729, 768, 728, 757, 748, 787, 758, 750, 781, 789, 790] payload_rows_per_dst=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] dup_rows_per_dst=[3010, 3186, 2938, 2954, 2994, 3002, 2826, 3038] combine_rows_per_src=[9342, 9414, 9382, 9086, 9316, 9354, 9180, 9282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344496 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665840 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534987184 weight_movement_bytes=103809024 total_nvlink_bytes=638796208 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344592 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248720 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152848 weight_movement_bytes=69206016 total_nvlink_bytes=601358864 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74360 payload_rows_remote=64968 dup_rows=23944 dup_rows_remote=20934 dedup_ratio=0.243571 router_tokens_per_expert=[773, 753, 756, 826, 800, 742, 801, 743, 741, 744, 770, 758, 769, 753, 826, 775, 757, 765, 767, 753, 860, 809, 741, 749, 732, 703, 731, 808, 761, 766, 765, 796, 763, 738, 768, 759, 762, 786, 787, 768, 776, 773, 763, 723, 774, 770, 762, 702, 788, 778, 763, 750, 786, 770, 734, 725, 776, 819, 738, 790, 785, 773, 793, 817] payload_rows_per_dst=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] dup_rows_per_dst=[2958, 3032, 3010, 2920, 3010, 2962, 2972, 3080] combine_rows_per_src=[9430, 9240, 9392, 9204, 9252, 9124, 9216, 9502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344096 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952736 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561376 weight_movement_bytes=34603008 total_nvlink_bytes=568164384 weight_to_token_ratio=0.064853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=344008 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100680 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857352 weight_movement_bytes=34603008 total_nvlink_bytes=566460360 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64784 dup_rows=24082 dup_rows_remote=20954 dedup_ratio=0.244975 router_tokens_per_expert=[796, 784, 761, 768, 798, 790, 742, 774, 780, 787, 764, 763, 771, 738, 796, 751, 775, 779, 777, 720, 791, 728, 747, 782, 734, 764, 763, 770, 744, 779, 736, 751, 757, 743, 770, 791, 739, 773, 793, 706, 772, 783, 753, 737, 770, 766, 766, 801, 725, 788, 800, 766, 752, 750, 763, 822, 813, 786, 788, 828, 786, 734, 746, 782] payload_rows_per_dst=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] dup_rows_per_dst=[3128, 3232, 2948, 3140, 3250, 2930, 2824, 2630] combine_rows_per_src=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=343144 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266476648 ingress_bytes=335339520 link_bytes=335339520 token_bytes=532610152 weight_movement_bytes=69206016 total_nvlink_bytes=601816168 weight_to_token_ratio=0.129937 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261855617e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64734 dup_rows=24212 dup_rows_remote=21218 dedup_ratio=0.246297 router_tokens_per_expert=[737, 742, 802, 796, 796, 729, 789, 762, 738, 763, 761, 742, 812, 798, 753, 733, 834, 759, 794, 745, 771, 812, 730, 731, 788, 754, 818, 769, 759, 761, 702, 761, 755, 710, 755, 824, 787, 783, 718, 726, 729, 742, 752, 772, 743, 779, 786, 708, 776, 784, 735, 778, 823, 818, 789, 773, 788, 787, 780, 766, 777, 811, 827, 730] payload_rows_per_dst=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] dup_rows_per_dst=[3010, 2952, 2994, 3040, 2936, 2914, 3156, 3210] combine_rows_per_src=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265625600 dispatch_meta_bytes=343632 combine_payload_bytes=265625600 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265969232 ingress_bytes=334831616 link_bytes=334831616 token_bytes=531594832 weight_movement_bytes=69206016 total_nvlink_bytes=600800848 weight_to_token_ratio=0.130186 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259944415e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65142 dup_rows=23776 dup_rows_remote=20654 dedup_ratio=0.241862 router_tokens_per_expert=[760, 740, 775, 814, 753, 765, 778, 775, 749, 776, 793, 762, 801, 772, 789, 812, 766, 768, 745, 786, 734, 749, 780, 759, 773, 765, 810, 792, 777, 740, 727, 725, 775, 768, 780, 758, 760, 755, 768, 817, 763, 781, 775, 712, 794, 836, 727, 791, 770, 724, 750, 762, 804, 741, 773, 791, 737, 761, 790, 746, 743, 808, 733, 749] payload_rows_per_dst=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] dup_rows_per_dst=[3000, 3122, 2954, 2956, 2946, 2914, 3010, 2874] combine_rows_per_src=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266108928 dispatch_meta_bytes=343608 combine_payload_bytes=266108928 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266452536 ingress_bytes=335314944 link_bytes=335314944 token_bytes=532561464 weight_movement_bytes=69206016 total_nvlink_bytes=601767480 weight_to_token_ratio=0.129949 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261763139e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=342952 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265698216 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531053480 weight_movement_bytes=69206016 total_nvlink_bytes=600259496 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=64962 dup_rows=23800 dup_rows_remote=20442 dedup_ratio=0.242106 router_tokens_per_expert=[776, 785, 786, 743, 756, 778, 796, 795, 786, 793, 754, 739, 736, 810, 782, 783, 738, 717, 795, 766, 749, 708, 741, 747, 762, 788, 709, 764, 760, 733, 779, 801, 795, 783, 762, 777, 753, 759, 798, 802, 778, 728, 739, 795, 753, 783, 812, 749, 776, 764, 765, 763, 828, 766, 785, 739, 725, 770, 833, 776, 757, 796, 753, 735] payload_rows_per_dst=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] dup_rows_per_dst=[3042, 3034, 2790, 2928, 2992, 2970, 2686, 3358] combine_rows_per_src=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65120 dup_rows=23840 dup_rows_remote=20846 dedup_ratio=0.242513 router_tokens_per_expert=[797, 751, 775, 792, 782, 768, 729, 755, 803, 764, 747, 753, 756, 788, 765, 769, 732, 795, 796, 803, 773, 711, 731, 796, 712, 758, 703, 799, 767, 780, 786, 768, 774, 807, 784, 750, 775, 776, 804, 773, 755, 773, 753, 744, 753, 781, 803, 740, 768, 714, 792, 850, 781, 735, 748, 781, 766, 762, 752, 750, 776, 730, 810, 788] payload_rows_per_dst=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] dup_rows_per_dst=[2990, 2922, 2994, 2940, 3050, 2934, 2994, 3016] combine_rows_per_src=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343184 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267164816 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533986448 weight_movement_bytes=69206016 total_nvlink_bytes=603192464 weight_to_token_ratio=0.129603 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=266960896 dispatch_meta_bytes=345192 combine_payload_bytes=266960896 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267306088 ingress_bytes=301563904 link_bytes=301563904 token_bytes=534266984 weight_movement_bytes=34603008 total_nvlink_bytes=568869992 weight_to_token_ratio=0.064767 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134760693e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64954 dup_rows=24150 dup_rows_remote=21236 dedup_ratio=0.245667 router_tokens_per_expert=[780, 746, 805, 813, 750, 779, 722, 773, 722, 733, 815, 754, 733, 758, 790, 752, 737, 759, 751, 778, 738, 798, 807, 788, 756, 743, 760, 789, 808, 787, 762, 754, 762, 727, 723, 772, 733, 765, 779, 780, 747, 797, 808, 769, 769, 711, 796, 771, 777, 810, 794, 795, 741, 713, 750, 813, 767, 759, 750, 723, 828, 785, 814, 784] payload_rows_per_dst=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] dup_rows_per_dst=[3080, 2914, 3068, 3016, 2922, 3056, 3062, 3032] combine_rows_per_src=[9256, 9200, 9244, 9302, 9160, 9280, 9324, 9388] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74346 payload_rows_remote=65090 dup_rows=23958 dup_rows_remote=20934 dedup_ratio=0.243713 router_tokens_per_expert=[771, 809, 783, 739, 711, 740, 760, 731, 756, 782, 758, 737, 781, 752, 748, 782, 817, 845, 784, 756, 747, 804, 766, 768, 779, 743, 768, 770, 745, 760, 762, 779, 763, 773, 769, 742, 774, 803, 764, 752, 778, 849, 774, 731, 736, 775, 805, 767, 714, 757, 832, 748, 740, 787, 740, 762, 796, 783, 763, 788, 788, 786, 707, 773] payload_rows_per_dst=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] dup_rows_per_dst=[2846, 2970, 3080, 2946, 3024, 3112, 2906, 3074] combine_rows_per_src=[9242, 9222, 9494, 9266, 9256, 9318, 9254, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343808 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265494272 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530644736 weight_movement_bytes=69206016 total_nvlink_bytes=599850752 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74148 payload_rows_remote=64800 dup_rows=24156 dup_rows_remote=21254 dedup_ratio=0.245728 router_tokens_per_expert=[778, 778, 781, 773, 717, 753, 800, 777, 744, 733, 738, 745, 709, 785, 765, 773, 791, 796, 763, 752, 850, 761, 809, 774, 732, 777, 752, 748, 815, 751, 794, 756, 726, 777, 756, 832, 792, 795, 771, 747, 745, 753, 799, 823, 743, 768, 749, 735, 810, 776, 765, 779, 779, 831, 780, 717, 738, 785, 755, 747, 787, 747, 723, 752] payload_rows_per_dst=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] dup_rows_per_dst=[3022, 2824, 3202, 2902, 3120, 3042, 3134, 2910] combine_rows_per_src=[9292, 9160, 9390, 9348, 9272, 9188, 9340, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65094 dup_rows=23970 dup_rows_remote=20912 dedup_ratio=0.243835 router_tokens_per_expert=[734, 716, 793, 769, 737, 736, 776, 761, 784, 791, 785, 772, 766, 726, 712, 792, 726, 797, 789, 777, 720, 785, 793, 791, 730, 797, 767, 766, 794, 769, 779, 853, 789, 751, 735, 771, 756, 820, 762, 765, 708, 821, 797, 768, 744, 766, 771, 794, 810, 738, 772, 732, 821, 790, 776, 759, 741, 718, 795, 794, 767, 787, 739, 712] payload_rows_per_dst=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] dup_rows_per_dst=[2846, 2928, 3058, 3080, 3058, 3074, 3046, 2880] combine_rows_per_src=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=64962 dup_rows=23800 dup_rows_remote=20442 dedup_ratio=0.242106 router_tokens_per_expert=[776, 785, 786, 743, 756, 778, 796, 795, 786, 793, 754, 739, 736, 810, 782, 783, 738, 717, 795, 766, 749, 708, 741, 747, 762, 788, 709, 764, 760, 733, 779, 801, 795, 783, 762, 777, 753, 759, 798, 802, 778, 728, 739, 795, 753, 783, 812, 749, 776, 764, 765, 763, 828, 766, 785, 739, 725, 770, 833, 776, 757, 796, 753, 735] payload_rows_per_dst=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] dup_rows_per_dst=[3042, 3034, 2790, 2928, 2992, 2970, 2686, 3358] combine_rows_per_src=[9388, 9332, 9132, 9264, 9466, 9304, 9076, 9542] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266051584 dispatch_meta_bytes=344760 combine_payload_bytes=266051584 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266396344 ingress_bytes=300654592 link_bytes=300654592 token_bytes=532447928 weight_movement_bytes=34603008 total_nvlink_bytes=567050936 weight_to_token_ratio=0.064989 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131339025e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=266608640 dispatch_meta_bytes=344096 combine_payload_bytes=266608640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266952736 ingress_bytes=301211648 link_bytes=301211648 token_bytes=533561376 weight_movement_bytes=34603008 total_nvlink_bytes=568164384 weight_to_token_ratio=0.064853 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133435182e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74222 payload_rows_remote=64784 dup_rows=24082 dup_rows_remote=20954 dedup_ratio=0.244975 router_tokens_per_expert=[796, 784, 761, 768, 798, 790, 742, 774, 780, 787, 764, 763, 771, 738, 796, 751, 775, 779, 777, 720, 791, 728, 747, 782, 734, 764, 763, 770, 744, 779, 736, 751, 757, 743, 770, 791, 739, 773, 793, 706, 772, 783, 753, 737, 770, 766, 766, 801, 725, 788, 800, 766, 752, 750, 763, 822, 813, 786, 788, 828, 786, 734, 746, 782] payload_rows_per_dst=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] dup_rows_per_dst=[3128, 3232, 2948, 3140, 3250, 2930, 2824, 2630] combine_rows_per_src=[9438, 9502, 9148, 9394, 9544, 9328, 9074, 8794] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74208 payload_rows_remote=64832 dup_rows=24096 dup_rows_remote=20934 dedup_ratio=0.245117 router_tokens_per_expert=[759, 774, 790, 753, 720, 773, 730, 740, 743, 781, 768, 791, 750, 799, 774, 777, 759, 759, 770, 775, 746, 750, 792, 799, 740, 755, 722, 787, 800, 759, 820, 762, 718, 734, 763, 738, 796, 793, 768, 777, 771, 789, 764, 785, 810, 800, 763, 787, 798, 746, 741, 766, 760, 791, 786, 788, 770, 728, 770, 787, 752, 778, 737, 781] payload_rows_per_dst=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] dup_rows_per_dst=[2896, 3032, 2916, 3020, 2948, 3162, 3072, 3050] combine_rows_per_src=[9182, 9334, 9384, 9270, 9226, 9376, 9280, 9156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=341616 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266425968 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532510320 weight_movement_bytes=34603008 total_nvlink_bytes=567113328 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343864 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267075384 ingress_bytes=405143552 link_bytes=405143552 token_bytes=533806904 weight_movement_bytes=138412032 total_nvlink_bytes=672218936 weight_to_token_ratio=0.259292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524522569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64918 dup_rows=24128 dup_rows_remote=21230 dedup_ratio=0.245443 router_tokens_per_expert=[781, 772, 766, 758, 798, 771, 766, 769, 795, 741, 759, 724, 764, 751, 801, 744, 815, 766, 761, 751, 781, 749, 761, 794, 751, 805, 753, 746, 796, 768, 769, 763, 760, 779, 755, 745, 755, 760, 782, 783, 759, 712, 797, 783, 803, 731, 720, 803, 732, 781, 765, 768, 755, 736, 774, 767, 769, 779, 790, 783, 812, 769, 781, 775] payload_rows_per_dst=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] dup_rows_per_dst=[3012, 3026, 2990, 2970, 3022, 3064, 2898, 3146] combine_rows_per_src=[9350, 9132, 9366, 9332, 9216, 9152, 9258, 9370] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265420800 dispatch_meta_bytes=344216 combine_payload_bytes=265420800 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265765016 ingress_bytes=300023808 link_bytes=300023808 token_bytes=531185816 weight_movement_bytes=34603008 total_nvlink_bytes=565788824 weight_to_token_ratio=0.065143 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128965436e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266084352 dispatch_meta_bytes=341616 combine_payload_bytes=266084352 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266425968 ingress_bytes=300687360 link_bytes=300687360 token_bytes=532510320 weight_movement_bytes=34603008 total_nvlink_bytes=567113328 weight_to_token_ratio=0.064981 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131462328e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64734 dup_rows=24212 dup_rows_remote=21218 dedup_ratio=0.246297 router_tokens_per_expert=[737, 742, 802, 796, 796, 729, 789, 762, 738, 763, 761, 742, 812, 798, 753, 733, 834, 759, 794, 745, 771, 812, 730, 731, 788, 754, 818, 769, 759, 761, 702, 761, 755, 710, 755, 824, 787, 783, 718, 726, 729, 742, 752, 772, 743, 779, 786, 708, 776, 784, 735, 778, 823, 818, 789, 773, 788, 787, 780, 766, 777, 811, 827, 730] payload_rows_per_dst=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] dup_rows_per_dst=[3010, 2952, 2994, 3040, 2936, 2914, 3156, 3210] combine_rows_per_src=[9296, 9248, 9358, 9184, 9180, 9108, 9396, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64790 dup_rows=24070 dup_rows_remote=20886 dedup_ratio=0.244853 router_tokens_per_expert=[788, 708, 782, 762, 735, 793, 826, 752, 772, 798, 759, 755, 784, 787, 766, 785, 780, 757, 744, 747, 727, 715, 740, 752, 733, 726, 728, 751, 810, 763, 723, 806, 772, 701, 782, 850, 741, 766, 755, 797, 774, 772, 743, 778, 748, 821, 806, 786, 765, 815, 774, 759, 820, 776, 810, 745, 741, 775, 761, 767, 741, 815, 744, 798] payload_rows_per_dst=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] dup_rows_per_dst=[3184, 2932, 3174, 2982, 2846, 2866, 2986, 3100] combine_rows_per_src=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344024 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969048 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594072 weight_movement_bytes=69206016 total_nvlink_bytes=602800088 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265355264 dispatch_meta_bytes=342952 combine_payload_bytes=265355264 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265698216 ingress_bytes=334561280 link_bytes=334561280 token_bytes=531053480 weight_movement_bytes=69206016 total_nvlink_bytes=600259496 weight_to_token_ratio=0.130318 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258927162e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=65174 dup_rows=23800 dup_rows_remote=20884 dedup_ratio=0.242106 router_tokens_per_expert=[788, 770, 748, 759, 783, 721, 769, 758, 740, 781, 776, 804, 779, 758, 774, 805, 798, 734, 760, 757, 738, 762, 756, 818, 741, 734, 757, 779, 796, 784, 780, 795, 770, 729, 706, 760, 794, 772, 782, 761, 783, 754, 732, 754, 759, 752, 808, 738, 825, 773, 787, 739, 777, 735, 780, 811, 777, 785, 707, 798, 796, 759, 768, 779] payload_rows_per_dst=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] dup_rows_per_dst=[2992, 3022, 2916, 2910, 2942, 2960, 3058, 3000] combine_rows_per_src=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265904128 dispatch_meta_bytes=344592 combine_payload_bytes=265904128 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266248720 ingress_bytes=335110144 link_bytes=335110144 token_bytes=532152848 weight_movement_bytes=69206016 total_nvlink_bytes=601358864 weight_to_token_ratio=0.130049 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260992493e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343064 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265894936 ingress_bytes=334757888 link_bytes=334757888 token_bytes=531446808 weight_movement_bytes=69206016 total_nvlink_bytes=600652824 weight_to_token_ratio=0.130222 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259666982e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64872 dup_rows=24114 dup_rows_remote=21094 dedup_ratio=0.245300 router_tokens_per_expert=[813, 773, 733, 774, 747, 732, 736, 753, 712, 807, 745, 774, 731, 792, 801, 807, 764, 748, 745, 805, 811, 769, 724, 758, 714, 794, 756, 773, 795, 805, 771, 800, 764, 764, 767, 766, 784, 775, 782, 729, 790, 765, 790, 790, 795, 808, 772, 753, 739, 781, 785, 731, 828, 743, 746, 726, 735, 785, 773, 756, 800, 742, 745, 781] payload_rows_per_dst=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] dup_rows_per_dst=[3020, 3020, 2996, 3042, 3002, 3052, 2946, 3036] combine_rows_per_src=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=265150464 dispatch_meta_bytes=343808 combine_payload_bytes=265150464 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265494272 ingress_bytes=334356480 link_bytes=334356480 token_bytes=530644736 weight_movement_bytes=69206016 total_nvlink_bytes=599850752 weight_to_token_ratio=0.130419 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258156516e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=342704 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265722544 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531102384 weight_movement_bytes=34603008 total_nvlink_bytes=565705392 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74528 payload_rows_remote=65142 dup_rows=23776 dup_rows_remote=20654 dedup_ratio=0.241862 router_tokens_per_expert=[760, 740, 775, 814, 753, 765, 778, 775, 749, 776, 793, 762, 801, 772, 789, 812, 766, 768, 745, 786, 734, 749, 780, 759, 773, 765, 810, 792, 777, 740, 727, 725, 775, 768, 780, 758, 760, 755, 768, 817, 763, 781, 775, 712, 794, 836, 727, 791, 770, 724, 750, 762, 804, 741, 773, 791, 737, 761, 790, 746, 743, 808, 733, 749] payload_rows_per_dst=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] dup_rows_per_dst=[3000, 3122, 2954, 2956, 2946, 2914, 3010, 2874] combine_rows_per_src=[9320, 9386, 9220, 9262, 9416, 9444, 9220, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74334 payload_rows_remote=65094 dup_rows=23970 dup_rows_remote=20912 dedup_ratio=0.243835 router_tokens_per_expert=[734, 716, 793, 769, 737, 736, 776, 761, 784, 791, 785, 772, 766, 726, 712, 792, 726, 797, 789, 777, 720, 785, 793, 791, 730, 797, 767, 766, 794, 769, 779, 853, 789, 751, 735, 771, 756, 820, 762, 765, 708, 821, 797, 768, 744, 766, 771, 794, 810, 738, 772, 732, 821, 790, 776, 759, 741, 718, 795, 794, 767, 787, 739, 712] payload_rows_per_dst=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] dup_rows_per_dst=[2846, 2928, 3058, 3080, 3058, 3074, 3046, 2880] combine_rows_per_src=[9198, 9328, 9298, 9430, 9240, 9264, 9350, 9226] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344232 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296936 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249640 weight_movement_bytes=34603008 total_nvlink_bytes=568852648 weight_to_token_ratio=0.064769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74204 payload_rows_remote=64848 dup_rows=24100 dup_rows_remote=20990 dedup_ratio=0.245158 router_tokens_per_expert=[742, 771, 742, 755, 799, 763, 741, 808, 740, 748, 772, 767, 771, 756, 742, 759, 813, 775, 764, 792, 793, 796, 763, 726, 794, 750, 766, 767, 765, 788, 752, 797, 764, 746, 802, 771, 751, 772, 812, 801, 770, 813, 749, 776, 765, 784, 768, 759, 809, 790, 775, 730, 781, 759, 778, 732, 738, 731, 752, 834, 715, 738, 765, 745] payload_rows_per_dst=[9212, 9178, 9306, 9262, 9432, 9344, 9114, 9356] dup_rows_per_dst=[2896, 2932, 3138, 2972, 3116, 3040, 2896, 3110] combine_rows_per_src=[9212, 9178, 9306, 9262, 9432, 9344, 9114, 9356] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343864 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059576 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775288 weight_movement_bytes=34603008 total_nvlink_bytes=566378296 weight_to_token_ratio=0.065071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65264 dup_rows=23788 dup_rows_remote=20796 dedup_ratio=0.241984 router_tokens_per_expert=[803, 686, 781, 770, 777, 763, 816, 770, 802, 747, 769, 744, 752, 768, 784, 709, 785, 807, 795, 712, 762, 761, 842, 731, 768, 771, 786, 717, 823, 779, 760, 718, 789, 785, 787, 848, 773, 774, 761, 752, 754, 740, 743, 748, 734, 813, 772, 737, 757, 764, 751, 764, 791, 756, 763, 777, 794, 773, 781, 847, 762, 691, 748, 765] payload_rows_per_dst=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] dup_rows_per_dst=[3032, 2838, 3106, 2992, 3134, 2870, 2886, 2930] combine_rows_per_src=[9300, 9312, 9284, 9252, 9404, 9212, 9360, 9392] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64890 dup_rows=23990 dup_rows_remote=20912 dedup_ratio=0.244039 router_tokens_per_expert=[748, 804, 759, 772, 783, 723, 757, 749, 815, 727, 817, 764, 805, 749, 744, 768, 777, 760, 802, 760, 766, 791, 748, 787, 737, 782, 773, 778, 757, 752, 773, 778, 747, 765, 756, 785, 803, 780, 777, 771, 769, 771, 767, 750, 761, 762, 763, 727, 779, 776, 795, 809, 761, 783, 748, 800, 713, 765, 806, 730, 749, 760, 771, 748] payload_rows_per_dst=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] dup_rows_per_dst=[3036, 3078, 3046, 2916, 3092, 2840, 3078, 2904] combine_rows_per_src=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266821632 dispatch_meta_bytes=343184 combine_payload_bytes=266821632 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267164816 ingress_bytes=336027648 link_bytes=336027648 token_bytes=533986448 weight_movement_bytes=69206016 total_nvlink_bytes=603192464 weight_to_token_ratio=0.129603 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264444987e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266625024 dispatch_meta_bytes=344024 combine_payload_bytes=266625024 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266969048 ingress_bytes=335831040 link_bytes=335831040 token_bytes=533594072 weight_movement_bytes=69206016 total_nvlink_bytes=602800088 weight_to_token_ratio=0.129698 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263705167e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65020 dup_rows=23978 dup_rows_remote=20894 dedup_ratio=0.243917 router_tokens_per_expert=[706, 791, 777, 742, 755, 760, 736, 783, 782, 760, 736, 733, 728, 763, 783, 712, 772, 772, 774, 758, 744, 735, 768, 799, 763, 712, 778, 756, 758, 774, 798, 807, 762, 770, 765, 800, 786, 736, 784, 792, 763, 746, 774, 723, 807, 760, 809, 796, 794, 744, 759, 796, 778, 753, 800, 806, 793, 777, 813, 752, 765, 811, 769, 754] payload_rows_per_dst=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] dup_rows_per_dst=[2842, 2840, 2946, 3006, 3084, 3050, 3088, 3122] combine_rows_per_src=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65274 dup_rows=23708 dup_rows_remote=20742 dedup_ratio=0.241170 router_tokens_per_expert=[785, 726, 832, 780, 775, 755, 802, 762, 715, 763, 784, 748, 757, 753, 741, 798, 731, 793, 749, 751, 800, 790, 719, 775, 730, 780, 779, 822, 785, 774, 752, 757, 737, 783, 794, 794, 758, 798, 795, 773, 766, 805, 743, 788, 757, 744, 786, 755, 765, 775, 724, 784, 760, 724, 759, 794, 780, 748, 764, 802, 731, 803, 723, 777] payload_rows_per_dst=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] dup_rows_per_dst=[3080, 2858, 2942, 2954, 3026, 2966, 2910, 2972] combine_rows_per_src=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74234 payload_rows_remote=64790 dup_rows=24070 dup_rows_remote=20886 dedup_ratio=0.244853 router_tokens_per_expert=[788, 708, 782, 762, 735, 793, 826, 752, 772, 798, 759, 755, 784, 787, 766, 785, 780, 757, 744, 747, 727, 715, 740, 752, 733, 726, 728, 751, 810, 763, 723, 806, 772, 701, 782, 850, 741, 766, 755, 797, 774, 772, 743, 778, 748, 821, 806, 786, 765, 815, 774, 759, 820, 776, 810, 745, 741, 775, 761, 767, 741, 815, 744, 798] payload_rows_per_dst=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] dup_rows_per_dst=[3184, 2932, 3174, 2982, 2846, 2866, 2986, 3100] combine_rows_per_src=[9444, 9168, 9266, 9384, 9140, 9336, 9312, 9184] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265617408 dispatch_meta_bytes=343352 combine_payload_bytes=265617408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265960760 ingress_bytes=300220416 link_bytes=300220416 token_bytes=531578168 weight_movement_bytes=34603008 total_nvlink_bytes=566181176 weight_to_token_ratio=0.065095 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129705256e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74464 payload_rows_remote=65120 dup_rows=23840 dup_rows_remote=20846 dedup_ratio=0.242513 router_tokens_per_expert=[797, 751, 775, 792, 782, 768, 729, 755, 803, 764, 747, 753, 756, 788, 765, 769, 732, 795, 796, 803, 773, 711, 731, 796, 712, 758, 703, 799, 767, 780, 786, 768, 774, 807, 784, 750, 775, 776, 804, 773, 755, 773, 753, 744, 753, 781, 803, 740, 768, 714, 792, 850, 781, 735, 748, 781, 766, 762, 752, 750, 776, 730, 810, 788] payload_rows_per_dst=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] dup_rows_per_dst=[2990, 2922, 2994, 2940, 3050, 2934, 2994, 3016] combine_rows_per_src=[9308, 9368, 9280, 9206, 9436, 9270, 9344, 9252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74596 payload_rows_remote=65274 dup_rows=23708 dup_rows_remote=20742 dedup_ratio=0.241170 router_tokens_per_expert=[785, 726, 832, 780, 775, 755, 802, 762, 715, 763, 784, 748, 757, 753, 741, 798, 731, 793, 749, 751, 800, 790, 719, 775, 730, 780, 779, 822, 785, 774, 752, 757, 737, 783, 794, 794, 758, 798, 795, 773, 766, 805, 743, 788, 757, 744, 786, 755, 765, 775, 724, 784, 760, 724, 759, 794, 780, 748, 764, 802, 731, 803, 723, 777] payload_rows_per_dst=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] dup_rows_per_dst=[3080, 2858, 2942, 2954, 3026, 2966, 2910, 2972] combine_rows_per_src=[9354, 9260, 9274, 9404, 9438, 9322, 9260, 9284] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267321344 dispatch_meta_bytes=344240 combine_payload_bytes=267321344 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267665584 ingress_bytes=371130368 link_bytes=371130368 token_bytes=534986928 weight_movement_bytes=103809024 total_nvlink_bytes=638795952 weight_to_token_ratio=0.194040 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396533696e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266132648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531922088 weight_movement_bytes=69206016 total_nvlink_bytes=601128104 weight_to_token_ratio=0.130106 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74504 payload_rows_remote=65174 dup_rows=23800 dup_rows_remote=20884 dedup_ratio=0.242106 router_tokens_per_expert=[788, 770, 748, 759, 783, 721, 769, 758, 740, 781, 776, 804, 779, 758, 774, 805, 798, 734, 760, 757, 738, 762, 756, 818, 741, 734, 757, 779, 796, 784, 780, 795, 770, 729, 706, 760, 794, 772, 782, 761, 783, 754, 732, 754, 759, 752, 808, 738, 825, 773, 787, 739, 777, 735, 780, 811, 777, 785, 707, 798, 796, 759, 768, 779] payload_rows_per_dst=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] dup_rows_per_dst=[2992, 3022, 2916, 2910, 2942, 2960, 3058, 3000] combine_rows_per_src=[9200, 9412, 9330, 9422, 9206, 9200, 9396, 9338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64668 dup_rows=24268 dup_rows_remote=21120 dedup_ratio=0.246867 router_tokens_per_expert=[797, 793, 762, 770, 785, 802, 749, 800, 790, 724, 757, 782, 761, 781, 755, 760, 717, 747, 773, 777, 756, 763, 758, 768, 728, 814, 792, 759, 730, 776, 728, 750, 771, 738, 753, 769, 774, 755, 753, 801, 794, 744, 771, 833, 734, 761, 794, 772, 758, 796, 807, 805, 804, 728, 754, 789, 763, 797, 752, 756, 774, 749, 742, 757] payload_rows_per_dst=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] dup_rows_per_dst=[3148, 3290, 3086, 2956, 2620, 3044, 3072, 3052] combine_rows_per_src=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343656 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266665576 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987496 weight_movement_bytes=69206016 total_nvlink_bytes=602193512 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64898 dup_rows=24212 dup_rows_remote=21202 dedup_ratio=0.246297 router_tokens_per_expert=[787, 781, 774, 787, 776, 729, 781, 836, 813, 740, 724, 767, 746, 758, 787, 801, 804, 758, 771, 747, 765, 736, 776, 745, 752, 729, 747, 794, 791, 806, 746, 808, 773, 750, 737, 766, 770, 779, 766, 774, 759, 775, 737, 735, 787, 755, 783, 759, 788, 765, 769, 745, 741, 819, 765, 746, 776, 769, 753, 767, 759, 759, 780, 784] payload_rows_per_dst=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] dup_rows_per_dst=[3046, 3072, 3010, 3048, 3002, 2966, 3032, 3036] combine_rows_per_src=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=265379840 dispatch_meta_bytes=342704 combine_payload_bytes=265379840 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265722544 ingress_bytes=299982848 link_bytes=299982848 token_bytes=531102384 weight_movement_bytes=34603008 total_nvlink_bytes=565705392 weight_to_token_ratio=0.065153 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128811306e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=267362304 dispatch_meta_bytes=344064 combine_payload_bytes=267362304 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267706368 ingress_bytes=336568320 link_bytes=336568320 token_bytes=535068672 weight_movement_bytes=69206016 total_nvlink_bytes=604274688 weight_to_token_ratio=0.129340 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.266479492e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65424 dup_rows=23738 dup_rows_remote=20782 dedup_ratio=0.241475 router_tokens_per_expert=[812, 769, 801, 749, 768, 792, 709, 815, 757, 742, 768, 741, 759, 767, 741, 774, 792, 749, 754, 780, 787, 798, 782, 782, 812, 775, 773, 749, 786, 805, 773, 767, 750, 785, 813, 712, 780, 786, 735, 763, 767, 770, 789, 778, 726, 766, 725, 764, 791, 749, 788, 754, 719, 811, 730, 774, 771, 761, 747, 776, 742, 760, 768, 774] payload_rows_per_dst=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] dup_rows_per_dst=[3030, 2956, 3056, 2996, 2934, 2808, 2992, 2966] combine_rows_per_src=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266952704 dispatch_meta_bytes=344232 combine_payload_bytes=266952704 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267296936 ingress_bytes=301555712 link_bytes=301555712 token_bytes=534249640 weight_movement_bytes=34603008 total_nvlink_bytes=568852648 weight_to_token_ratio=0.064769 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134729867e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=343152 combine_payload_bytes=264880128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265223280 ingress_bytes=334086144 link_bytes=334086144 token_bytes=530103408 weight_movement_bytes=69206016 total_nvlink_bytes=599309424 weight_to_token_ratio=0.130552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257139264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=266731520 dispatch_meta_bytes=343864 combine_payload_bytes=266731520 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=267075384 ingress_bytes=405143552 link_bytes=405143552 token_bytes=533806904 weight_movement_bytes=138412032 total_nvlink_bytes=672218936 weight_to_token_ratio=0.259292 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.524522569e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74190 payload_rows_remote=64872 dup_rows=24114 dup_rows_remote=21094 dedup_ratio=0.245300 router_tokens_per_expert=[813, 773, 733, 774, 747, 732, 736, 753, 712, 807, 745, 774, 731, 792, 801, 807, 764, 748, 745, 805, 811, 769, 724, 758, 714, 794, 756, 773, 795, 805, 771, 800, 764, 764, 767, 766, 784, 775, 782, 729, 790, 765, 790, 790, 795, 808, 772, 753, 739, 781, 785, 731, 828, 743, 746, 726, 735, 785, 773, 756, 800, 742, 745, 781] payload_rows_per_dst=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] dup_rows_per_dst=[3020, 3020, 2996, 3042, 3002, 3052, 2946, 3036] combine_rows_per_src=[9102, 9318, 9252, 9374, 9260, 9474, 9212, 9198] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74326 payload_rows_remote=65020 dup_rows=23978 dup_rows_remote=20894 dedup_ratio=0.243917 router_tokens_per_expert=[706, 791, 777, 742, 755, 760, 736, 783, 782, 760, 736, 733, 728, 763, 783, 712, 772, 772, 774, 758, 744, 735, 768, 799, 763, 712, 778, 756, 758, 774, 798, 807, 762, 770, 765, 800, 786, 736, 784, 792, 763, 746, 774, 723, 807, 760, 809, 796, 794, 744, 759, 796, 778, 753, 800, 806, 793, 777, 813, 752, 765, 811, 769, 754] payload_rows_per_dst=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] dup_rows_per_dst=[2842, 2840, 2946, 3006, 3084, 3050, 3088, 3122] combine_rows_per_src=[9258, 9154, 9298, 9286, 9306, 9306, 9372, 9346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=344400 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266166608 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988816 weight_movement_bytes=34603008 total_nvlink_bytes=566591824 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74278 payload_rows_remote=65034 dup_rows=24026 dup_rows_remote=21112 dedup_ratio=0.244405 router_tokens_per_expert=[786, 748, 767, 747, 751, 740, 796, 783, 755, 807, 754, 765, 797, 808, 722, 806, 765, 748, 790, 776, 753, 809, 779, 786, 765, 770, 773, 801, 760, 816, 756, 732, 755, 781, 815, 793, 768, 750, 743, 710, 745, 774, 754, 777, 746, 817, 736, 745, 787, 772, 745, 789, 764, 807, 734, 755, 751, 770, 757, 772, 733, 767, 783, 746] payload_rows_per_dst=[8976, 9230, 9394, 9494, 9288, 9292, 9360, 9244] dup_rows_per_dst=[2860, 2962, 3018, 3252, 2942, 3096, 2982, 2914] combine_rows_per_src=[8976, 9230, 9394, 9494, 9288, 9292, 9360, 9244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265715712 dispatch_meta_bytes=343864 combine_payload_bytes=265715712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266059576 ingress_bytes=300318720 link_bytes=300318720 token_bytes=531775288 weight_movement_bytes=34603008 total_nvlink_bytes=566378296 weight_to_token_ratio=0.065071 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130075166e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74350 payload_rows_remote=65182 dup_rows=23954 dup_rows_remote=21016 dedup_ratio=0.243673 router_tokens_per_expert=[756, 762, 775, 788, 762, 715, 792, 837, 760, 752, 806, 767, 789, 769, 807, 786, 778, 760, 784, 758, 808, 758, 750, 758, 772, 724, 766, 747, 735, 770, 745, 794, 704, 810, 730, 778, 793, 748, 759, 767, 806, 738, 710, 749, 804, 753, 770, 727, 811, 773, 821, 758, 786, 752, 817, 762, 794, 776, 779, 790, 725, 766, 711, 755] payload_rows_per_dst=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] dup_rows_per_dst=[3084, 2988, 3012, 2938, 2906, 2962, 3080, 2984] combine_rows_per_src=[9290, 9484, 9296, 9168, 9272, 9152, 9480, 9208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65310 dup_rows=23648 dup_rows_remote=20674 dedup_ratio=0.240560 router_tokens_per_expert=[743, 726, 770, 779, 768, 802, 750, 823, 789, 777, 809, 766, 763, 784, 752, 735, 747, 774, 752, 741, 769, 749, 790, 780, 767, 784, 786, 758, 739, 720, 774, 752, 799, 720, 786, 785, 791, 747, 783, 764, 788, 783, 750, 765, 795, 770, 761, 719, 784, 747, 790, 761, 753, 798, 753, 774, 821, 773, 749, 749, 781, 740, 751, 804] payload_rows_per_dst=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] dup_rows_per_dst=[2944, 3008, 2890, 2942, 2948, 2964, 2974, 2978] combine_rows_per_src=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344824 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268321528 ingress_bytes=406388736 link_bytes=406388736 token_bytes=536298232 weight_movement_bytes=138412032 total_nvlink_bytes=674710264 weight_to_token_ratio=0.258088 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529208097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266321920 dispatch_meta_bytes=343656 combine_payload_bytes=266321920 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266665576 ingress_bytes=335527936 link_bytes=335527936 token_bytes=532987496 weight_movement_bytes=69206016 total_nvlink_bytes=602193512 weight_to_token_ratio=0.129845 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262564611e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74036 payload_rows_remote=64668 dup_rows=24268 dup_rows_remote=21120 dedup_ratio=0.246867 router_tokens_per_expert=[797, 793, 762, 770, 785, 802, 749, 800, 790, 724, 757, 782, 761, 781, 755, 760, 717, 747, 773, 777, 756, 763, 758, 768, 728, 814, 792, 759, 730, 776, 728, 750, 771, 738, 753, 769, 774, 755, 753, 801, 794, 744, 771, 833, 734, 761, 794, 772, 758, 796, 807, 805, 804, 728, 754, 789, 763, 797, 752, 756, 774, 749, 742, 757] payload_rows_per_dst=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] dup_rows_per_dst=[3148, 3290, 3086, 2956, 2620, 3044, 3072, 3052] combine_rows_per_src=[9368, 9654, 9090, 9198, 8884, 9304, 9410, 9128] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65112 dup_rows=23930 dup_rows_remote=21062 dedup_ratio=0.243429 router_tokens_per_expert=[756, 788, 761, 771, 756, 732, 796, 782, 809, 734, 786, 785, 750, 778, 791, 771, 778, 800, 762, 769, 735, 764, 796, 855, 721, 752, 778, 742, 773, 775, 774, 790, 778, 701, 746, 785, 762, 803, 762, 737, 774, 748, 761, 752, 781, 718, 747, 784, 761, 740, 749, 774, 751, 762, 724, 763, 857, 800, 716, 775, 789, 772, 800, 770] payload_rows_per_dst=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] dup_rows_per_dst=[2992, 3090, 3028, 2924, 2964, 2868, 2908, 3156] combine_rows_per_src=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64832 dup_rows=24128 dup_rows_remote=21080 dedup_ratio=0.245443 router_tokens_per_expert=[741, 769, 797, 818, 786, 766, 775, 762, 742, 731, 725, 780, 783, 779, 786, 754, 752, 788, 808, 794, 758, 757, 728, 763, 769, 749, 738, 744, 758, 744, 792, 750, 738, 767, 746, 824, 780, 795, 750, 796, 799, 732, 771, 819, 714, 793, 818, 780, 791, 717, 761, 805, 741, 802, 780, 846, 737, 717, 766, 761, 749, 726, 743, 802] payload_rows_per_dst=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] dup_rows_per_dst=[3094, 2976, 3094, 2906, 3048, 3060, 3098, 2852] combine_rows_per_src=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266379264 dispatch_meta_bytes=344584 combine_payload_bytes=266379264 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266723848 ingress_bytes=300982272 link_bytes=300982272 token_bytes=533103112 weight_movement_bytes=34603008 total_nvlink_bytes=567706120 weight_to_token_ratio=0.064909 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132572058e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74374 payload_rows_remote=65112 dup_rows=23930 dup_rows_remote=21062 dedup_ratio=0.243429 router_tokens_per_expert=[756, 788, 761, 771, 756, 732, 796, 782, 809, 734, 786, 785, 750, 778, 791, 771, 778, 800, 762, 769, 735, 764, 796, 855, 721, 752, 778, 742, 773, 775, 774, 790, 778, 701, 746, 785, 762, 803, 762, 737, 774, 748, 761, 752, 781, 718, 747, 784, 761, 740, 749, 774, 751, 762, 724, 763, 857, 800, 716, 775, 789, 772, 800, 770] payload_rows_per_dst=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] dup_rows_per_dst=[2992, 3090, 3028, 2924, 2964, 2868, 2908, 3156] combine_rows_per_src=[9292, 9318, 9490, 9286, 9184, 9262, 9140, 9402] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=64890 dup_rows=23990 dup_rows_remote=20912 dedup_ratio=0.244039 router_tokens_per_expert=[748, 804, 759, 772, 783, 723, 757, 749, 815, 727, 817, 764, 805, 749, 744, 768, 777, 760, 802, 760, 766, 791, 748, 787, 737, 782, 773, 778, 757, 752, 773, 778, 747, 765, 756, 785, 803, 780, 777, 771, 769, 771, 767, 750, 761, 762, 763, 727, 779, 776, 795, 809, 761, 783, 748, 800, 713, 765, 806, 730, 749, 760, 771, 748] payload_rows_per_dst=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] dup_rows_per_dst=[3036, 3078, 3046, 2916, 3092, 2840, 3078, 2904] combine_rows_per_src=[9154, 9300, 9336, 9344, 9276, 9300, 9424, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266985472 dispatch_meta_bytes=344792 combine_payload_bytes=266985472 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267330264 ingress_bytes=301588480 link_bytes=301588480 token_bytes=534315736 weight_movement_bytes=34603008 total_nvlink_bytes=568918744 weight_to_token_ratio=0.064761 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134853170e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=343936 combine_payload_bytes=267509760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267853696 ingress_bytes=371318784 link_bytes=371318784 token_bytes=535363456 weight_movement_bytes=103809024 total_nvlink_bytes=639172480 weight_to_token_ratio=0.193904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397242691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64990 dup_rows=24054 dup_rows_remote=21058 dedup_ratio=0.244690 router_tokens_per_expert=[746, 795, 745, 783, 775, 742, 740, 786, 823, 818, 753, 750, 783, 726, 777, 760, 711, 751, 786, 788, 741, 717, 730, 754, 739, 744, 751, 749, 775, 788, 776, 738, 764, 809, 770, 756, 771, 745, 737, 815, 772, 770, 781, 803, 777, 759, 740, 789, 710, 765, 736, 789, 813, 811, 746, 767, 774, 777, 750, 792, 788, 823, 841, 772] payload_rows_per_dst=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] dup_rows_per_dst=[2996, 3050, 2844, 2890, 3154, 3104, 2832, 3184] combine_rows_per_src=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64820 dup_rows=24334 dup_rows_remote=21424 dedup_ratio=0.247538 router_tokens_per_expert=[780, 760, 773, 783, 822, 729, 775, 765, 759, 788, 766, 721, 777, 737, 771, 816, 795, 779, 793, 734, 719, 731, 752, 727, 802, 779, 773, 761, 813, 759, 784, 726, 802, 769, 722, 761, 782, 722, 804, 749, 759, 745, 800, 737, 786, 781, 799, 796, 816, 767, 746, 750, 745, 774, 770, 805, 754, 742, 793, 758, 777, 772, 745, 775] payload_rows_per_dst=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] dup_rows_per_dst=[3132, 3060, 2910, 3156, 2962, 3088, 3006, 3020] combine_rows_per_src=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=264880128 dispatch_meta_bytes=343152 combine_payload_bytes=264880128 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265223280 ingress_bytes=334086144 link_bytes=334086144 token_bytes=530103408 weight_movement_bytes=69206016 total_nvlink_bytes=599309424 weight_to_token_ratio=0.130552 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257139264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344696 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043448 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742200 weight_movement_bytes=34603008 total_nvlink_bytes=568345208 weight_to_token_ratio=0.064831 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74092 payload_rows_remote=64898 dup_rows=24212 dup_rows_remote=21202 dedup_ratio=0.246297 router_tokens_per_expert=[787, 781, 774, 787, 776, 729, 781, 836, 813, 740, 724, 767, 746, 758, 787, 801, 804, 758, 771, 747, 765, 736, 776, 745, 752, 729, 747, 794, 791, 806, 746, 808, 773, 750, 737, 766, 770, 779, 766, 774, 759, 775, 737, 735, 787, 755, 783, 759, 788, 765, 769, 745, 741, 819, 765, 746, 776, 769, 753, 767, 759, 759, 780, 784] payload_rows_per_dst=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] dup_rows_per_dst=[3046, 3072, 3010, 3048, 3002, 2966, 3032, 3036] combine_rows_per_src=[9456, 9200, 9194, 9298, 9228, 9214, 9244, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343648 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895520 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447392 weight_movement_bytes=34603008 total_nvlink_bytes=566050400 weight_to_token_ratio=0.065111 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65108 dup_rows=23948 dup_rows_remote=20858 dedup_ratio=0.243612 router_tokens_per_expert=[786, 748, 757, 768, 800, 761, 744, 773, 731, 829, 761, 767, 739, 767, 809, 766, 805, 795, 801, 739, 768, 754, 751, 750, 784, 769, 779, 777, 754, 766, 726, 777, 854, 749, 785, 811, 755, 754, 769, 777, 804, 767, 765, 735, 762, 743, 771, 766, 800, 756, 752, 766, 779, 808, 754, 704, 757, 756, 741, 732, 790, 772, 771, 746] payload_rows_per_dst=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] dup_rows_per_dst=[2978, 3090, 3006, 3036, 3054, 2956, 2956, 2872] combine_rows_per_src=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344192 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266543232 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742272 weight_movement_bytes=69206016 total_nvlink_bytes=601948288 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343208 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266132648 ingress_bytes=334995456 link_bytes=334995456 token_bytes=531922088 weight_movement_bytes=69206016 total_nvlink_bytes=601128104 weight_to_token_ratio=0.130106 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260560932e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74566 payload_rows_remote=65424 dup_rows=23738 dup_rows_remote=20782 dedup_ratio=0.241475 router_tokens_per_expert=[812, 769, 801, 749, 768, 792, 709, 815, 757, 742, 768, 741, 759, 767, 741, 774, 792, 749, 754, 780, 787, 798, 782, 782, 812, 775, 773, 749, 786, 805, 773, 767, 750, 785, 813, 712, 780, 786, 735, 763, 767, 770, 789, 778, 726, 766, 725, 764, 791, 749, 788, 754, 719, 811, 730, 774, 771, 761, 747, 776, 742, 760, 768, 774] payload_rows_per_dst=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] dup_rows_per_dst=[3030, 2956, 3056, 2996, 2934, 2808, 2992, 2966] combine_rows_per_src=[9400, 9142, 9392, 9484, 9314, 9362, 9240, 9232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344976 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847696 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531350416 weight_movement_bytes=34603008 total_nvlink_bytes=565953424 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74176 payload_rows_remote=64832 dup_rows=24128 dup_rows_remote=21080 dedup_ratio=0.245443 router_tokens_per_expert=[741, 769, 797, 818, 786, 766, 775, 762, 742, 731, 725, 780, 783, 779, 786, 754, 752, 788, 808, 794, 758, 757, 728, 763, 769, 749, 738, 744, 758, 744, 792, 750, 738, 767, 746, 824, 780, 795, 750, 796, 799, 732, 771, 819, 714, 793, 818, 780, 791, 717, 761, 805, 741, 802, 780, 846, 737, 717, 766, 761, 749, 726, 743, 802] payload_rows_per_dst=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] dup_rows_per_dst=[3094, 2976, 3094, 2906, 3048, 3060, 3098, 2852] combine_rows_per_src=[9334, 9184, 9202, 9182, 9344, 9392, 9388, 9150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74170 payload_rows_remote=64936 dup_rows=24134 dup_rows_remote=21056 dedup_ratio=0.245504 router_tokens_per_expert=[802, 767, 798, 799, 751, 726, 801, 787, 775, 745, 735, 761, 690, 757, 741, 812, 822, 791, 702, 720, 763, 771, 783, 783, 801, 787, 775, 755, 770, 796, 747, 834, 778, 747, 769, 734, 769, 728, 708, 836, 754, 812, 789, 730, 739, 772, 733, 771, 790, 780, 793, 811, 789, 724, 757, 774, 746, 783, 779, 804, 747, 735, 764, 760] payload_rows_per_dst=[8842, 9454, 8992, 9558, 9194, 9366, 9530, 9234] dup_rows_per_dst=[2740, 3168, 2768, 3334, 2944, 2906, 3196, 3078] combine_rows_per_src=[8842, 9454, 8992, 9558, 9194, 9366, 9530, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=265822208 dispatch_meta_bytes=344400 combine_payload_bytes=265822208 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266166608 ingress_bytes=300425216 link_bytes=300425216 token_bytes=531988816 weight_movement_bytes=34603008 total_nvlink_bytes=566591824 weight_to_token_ratio=0.065045 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130475902e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65220 dup_rows=24010 dup_rows_remote=21116 dedup_ratio=0.244242 router_tokens_per_expert=[801, 766, 786, 723, 793, 739, 763, 730, 768, 769, 785, 775, 781, 774, 827, 767, 785, 781, 749, 768, 718, 751, 798, 772, 756, 712, 734, 761, 747, 787, 716, 771, 755, 772, 778, 750, 773, 775, 804, 784, 808, 770, 779, 757, 782, 738, 767, 782, 803, 737, 731, 763, 767, 804, 752, 760, 728, 778, 747, 796, 829, 750, 802, 778] payload_rows_per_dst=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] dup_rows_per_dst=[2934, 3038, 2994, 2894, 3054, 3076, 2974, 3046] combine_rows_per_src=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64892 dup_rows=24226 dup_rows_remote=21226 dedup_ratio=0.246440 router_tokens_per_expert=[783, 757, 767, 789, 754, 754, 806, 792, 765, 815, 724, 723, 798, 768, 774, 786, 756, 801, 753, 742, 794, 748, 786, 758, 777, 741, 765, 771, 747, 709, 777, 788, 757, 785, 778, 739, 781, 773, 744, 790, 768, 776, 759, 715, 789, 784, 805, 758, 763, 804, 787, 752, 754, 733, 765, 735, 801, 838, 766, 782, 767, 764, 759, 713] payload_rows_per_dst=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] dup_rows_per_dst=[3086, 3026, 3046, 2962, 3000, 3026, 3000, 3080] combine_rows_per_src=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343864 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026232 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708600 weight_movement_bytes=34603008 total_nvlink_bytes=568311608 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344824 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=268321528 ingress_bytes=406388736 link_bytes=406388736 token_bytes=536298232 weight_movement_bytes=138412032 total_nvlink_bytes=674710264 weight_to_token_ratio=0.258088 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.529208097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74294 payload_rows_remote=65220 dup_rows=24010 dup_rows_remote=21116 dedup_ratio=0.244242 router_tokens_per_expert=[801, 766, 786, 723, 793, 739, 763, 730, 768, 769, 785, 775, 781, 774, 827, 767, 785, 781, 749, 768, 718, 751, 798, 772, 756, 712, 734, 761, 747, 787, 716, 771, 755, 772, 778, 750, 773, 775, 804, 784, 808, 770, 779, 757, 782, 738, 767, 782, 803, 737, 731, 763, 767, 804, 752, 760, 728, 778, 747, 796, 829, 750, 802, 778] payload_rows_per_dst=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] dup_rows_per_dst=[2934, 3038, 2994, 2894, 3054, 3076, 2974, 3046] combine_rows_per_src=[9268, 9454, 9250, 9074, 9328, 9290, 9260, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265551872 dispatch_meta_bytes=343648 combine_payload_bytes=265551872 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265895520 ingress_bytes=300154880 link_bytes=300154880 token_bytes=531447392 weight_movement_bytes=34603008 total_nvlink_bytes=566050400 weight_to_token_ratio=0.065111 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129458649e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343968 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321824 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299680 weight_movement_bytes=34603008 total_nvlink_bytes=566902688 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74250 payload_rows_remote=64990 dup_rows=24054 dup_rows_remote=21058 dedup_ratio=0.244690 router_tokens_per_expert=[746, 795, 745, 783, 775, 742, 740, 786, 823, 818, 753, 750, 783, 726, 777, 760, 711, 751, 786, 788, 741, 717, 730, 754, 739, 744, 751, 749, 775, 788, 776, 738, 764, 809, 770, 756, 771, 745, 737, 815, 772, 770, 781, 803, 777, 759, 740, 789, 710, 765, 736, 789, 813, 811, 746, 767, 774, 777, 750, 792, 788, 823, 841, 772] payload_rows_per_dst=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] dup_rows_per_dst=[2996, 3050, 2844, 2890, 3154, 3104, 2832, 3184] combine_rows_per_src=[9260, 9304, 9214, 9230, 9382, 9278, 9138, 9444] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=65138 dup_rows=23886 dup_rows_remote=20940 dedup_ratio=0.242981 router_tokens_per_expert=[769, 791, 744, 789, 798, 802, 751, 867, 783, 732, 754, 764, 771, 711, 765, 787, 762, 757, 775, 726, 781, 787, 802, 784, 785, 756, 793, 762, 762, 784, 768, 765, 753, 766, 757, 750, 736, 751, 777, 759, 720, 772, 757, 801, 775, 720, 820, 748, 796, 800, 774, 830, 791, 735, 732, 756, 777, 760, 799, 746, 759, 706, 791, 711] payload_rows_per_dst=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] dup_rows_per_dst=[3098, 2952, 2974, 3058, 2828, 2946, 3090, 2940] combine_rows_per_src=[9524, 9182, 9374, 9292, 9270, 9280, 9338, 9158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265977856 dispatch_meta_bytes=343968 combine_payload_bytes=265977856 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266321824 ingress_bytes=300580864 link_bytes=300580864 token_bytes=532299680 weight_movement_bytes=34603008 total_nvlink_bytes=566902688 weight_to_token_ratio=0.065007 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131061592e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64810 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[786, 824, 746, 802, 761, 796, 811, 779, 738, 709, 750, 789, 760, 730, 772, 748, 786, 765, 747, 746, 808, 788, 789, 803, 773, 834, 764, 779, 740, 771, 722, 735, 827, 782, 799, 816, 784, 791, 733, 735, 771, 750, 698, 827, 760, 733, 782, 752, 712, 783, 718, 759, 786, 780, 783, 763, 757, 778, 789, 803, 751, 762, 693, 744] payload_rows_per_dst=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] dup_rows_per_dst=[3128, 2854, 3060, 3012, 3074, 2934, 2980, 2992] combine_rows_per_src=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345344 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486464 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627584 weight_movement_bytes=34603008 total_nvlink_bytes=569230592 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266142104 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531939736 weight_movement_bytes=34603008 total_nvlink_bytes=566542744 weight_to_token_ratio=0.065051 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74656 payload_rows_remote=65310 dup_rows=23648 dup_rows_remote=20674 dedup_ratio=0.240560 router_tokens_per_expert=[743, 726, 770, 779, 768, 802, 750, 823, 789, 777, 809, 766, 763, 784, 752, 735, 747, 774, 752, 741, 769, 749, 790, 780, 767, 784, 786, 758, 739, 720, 774, 752, 799, 720, 786, 785, 791, 747, 783, 764, 788, 783, 750, 765, 795, 770, 761, 719, 784, 747, 790, 761, 753, 798, 753, 774, 821, 773, 749, 749, 781, 740, 751, 804] payload_rows_per_dst=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] dup_rows_per_dst=[2944, 3008, 2890, 2942, 2948, 2964, 2974, 2978] combine_rows_per_src=[9378, 9342, 9314, 9218, 9402, 9298, 9346, 9358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64912 dup_rows=24046 dup_rows_remote=20954 dedup_ratio=0.244609 router_tokens_per_expert=[790, 781, 777, 806, 749, 761, 751, 758, 762, 755, 757, 763, 767, 782, 779, 715, 779, 760, 747, 790, 765, 774, 798, 806, 762, 760, 747, 756, 740, 778, 744, 777, 765, 765, 745, 750, 757, 766, 744, 828, 746, 799, 764, 772, 789, 776, 778, 749, 796, 753, 739, 714, 746, 803, 798, 770, 756, 760, 788, 783, 790, 772, 784, 771] payload_rows_per_dst=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] dup_rows_per_dst=[3058, 2938, 3092, 2964, 3002, 3036, 2958, 2998] combine_rows_per_src=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267141120 dispatch_meta_bytes=345344 combine_payload_bytes=267141120 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267486464 ingress_bytes=301744128 link_bytes=301744128 token_bytes=534627584 weight_movement_bytes=34603008 total_nvlink_bytes=569230592 weight_to_token_ratio=0.064724 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135438861e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=266199040 dispatch_meta_bytes=344192 combine_payload_bytes=266199040 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266543232 ingress_bytes=335405056 link_bytes=335405056 token_bytes=532742272 weight_movement_bytes=69206016 total_nvlink_bytes=601948288 weight_to_token_ratio=0.129905 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262102223e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266805248 dispatch_meta_bytes=344312 combine_payload_bytes=266805248 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267149560 ingress_bytes=301408256 link_bytes=301408256 token_bytes=533954808 weight_movement_bytes=34603008 total_nvlink_bytes=568557816 weight_to_token_ratio=0.064805 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134175002e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74576 payload_rows_remote=65236 dup_rows=23728 dup_rows_remote=20804 dedup_ratio=0.241374 router_tokens_per_expert=[760, 753, 729, 764, 770, 771, 788, 777, 771, 757, 739, 748, 818, 775, 730, 792, 790, 770, 749, 716, 773, 832, 813, 778, 785, 740, 772, 766, 774, 751, 817, 753, 759, 761, 758, 768, 736, 790, 728, 778, 754, 768, 796, 744, 813, 767, 793, 772, 716, 735, 758, 757, 736, 731, 776, 834, 776, 799, 763, 807, 745, 783, 753, 777] payload_rows_per_dst=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] dup_rows_per_dst=[2924, 3008, 3072, 2990, 2856, 3084, 2854, 2940] combine_rows_per_src=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73970 payload_rows_remote=64820 dup_rows=24334 dup_rows_remote=21424 dedup_ratio=0.247538 router_tokens_per_expert=[780, 760, 773, 783, 822, 729, 775, 765, 759, 788, 766, 721, 777, 737, 771, 816, 795, 779, 793, 734, 719, 731, 752, 727, 802, 779, 773, 761, 813, 759, 784, 726, 802, 769, 722, 761, 782, 722, 804, 749, 759, 745, 800, 737, 786, 781, 799, 796, 816, 767, 746, 750, 745, 774, 770, 805, 754, 742, 793, 758, 777, 772, 745, 775] payload_rows_per_dst=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] dup_rows_per_dst=[3132, 3060, 2910, 3156, 2962, 3088, 3006, 3020] combine_rows_per_src=[9242, 9210, 9150, 9238, 9260, 9318, 9340, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343080 combine_payload_bytes=265461760 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265804840 ingress_bytes=334667776 link_bytes=334667776 token_bytes=531266600 weight_movement_bytes=69206016 total_nvlink_bytes=600472616 weight_to_token_ratio=0.130266 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259327898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64698 dup_rows=24222 dup_rows_remote=21142 dedup_ratio=0.246399 router_tokens_per_expert=[748, 763, 726, 796, 754, 764, 743, 738, 756, 809, 720, 820, 787, 769, 782, 789, 765, 752, 783, 781, 726, 738, 748, 773, 782, 778, 762, 779, 795, 770, 816, 770, 768, 777, 809, 735, 790, 757, 781, 775, 772, 751, 816, 787, 770, 724, 740, 778, 785, 728, 764, 794, 713, 762, 749, 789, 745, 765, 795, 776, 789, 732, 777, 777] payload_rows_per_dst=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] dup_rows_per_dst=[2876, 3080, 2926, 3112, 3110, 2990, 3046, 3082] combine_rows_per_src=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=267509760 dispatch_meta_bytes=343936 combine_payload_bytes=267509760 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267853696 ingress_bytes=371318784 link_bytes=371318784 token_bytes=535363456 weight_movement_bytes=103809024 total_nvlink_bytes=639172480 weight_to_token_ratio=0.193904 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.397242691e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74356 payload_rows_remote=65108 dup_rows=23948 dup_rows_remote=20858 dedup_ratio=0.243612 router_tokens_per_expert=[786, 748, 757, 768, 800, 761, 744, 773, 731, 829, 761, 767, 739, 767, 809, 766, 805, 795, 801, 739, 768, 754, 751, 750, 784, 769, 779, 777, 754, 766, 726, 777, 854, 749, 785, 811, 755, 754, 769, 777, 804, 767, 765, 735, 762, 743, 771, 766, 800, 756, 752, 766, 779, 808, 754, 704, 757, 756, 741, 732, 790, 772, 771, 746] payload_rows_per_dst=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] dup_rows_per_dst=[2978, 3090, 3006, 3036, 3054, 2956, 2956, 2872] combine_rows_per_src=[9296, 9248, 9320, 9228, 9454, 9270, 9282, 9258] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74270 payload_rows_remote=64810 dup_rows=24034 dup_rows_remote=20960 dedup_ratio=0.244486 router_tokens_per_expert=[786, 824, 746, 802, 761, 796, 811, 779, 738, 709, 750, 789, 760, 730, 772, 748, 786, 765, 747, 746, 808, 788, 789, 803, 773, 834, 764, 779, 740, 771, 722, 735, 827, 782, 799, 816, 784, 791, 733, 735, 771, 750, 698, 827, 760, 733, 782, 752, 712, 783, 718, 759, 786, 780, 783, 763, 757, 778, 789, 803, 751, 762, 693, 744] payload_rows_per_dst=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] dup_rows_per_dst=[3128, 2854, 3060, 3012, 3074, 2934, 2980, 2992] combine_rows_per_src=[9482, 9138, 9404, 9224, 9460, 9212, 9188, 9162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343464 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266223016 ingress_bytes=404291584 link_bytes=404291584 token_bytes=532102568 weight_movement_bytes=138412032 total_nvlink_bytes=670514600 weight_to_token_ratio=0.260123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521316682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74070 payload_rows_remote=64816 dup_rows=24234 dup_rows_remote=21246 dedup_ratio=0.246521 router_tokens_per_expert=[755, 759, 767, 852, 784, 817, 755, 751, 798, 722, 803, 733, 781, 757, 760, 761, 773, 756, 783, 802, 765, 758, 745, 751, 762, 763, 773, 728, 781, 779, 736, 751, 742, 771, 742, 811, 776, 788, 768, 773, 780, 809, 829, 720, 768, 762, 771, 728, 788, 707, 798, 753, 791, 810, 743, 742, 762, 810, 756, 783, 738, 790, 761, 721] payload_rows_per_dst=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] dup_rows_per_dst=[2790, 3006, 3030, 2984, 3094, 3382, 2960, 2988] combine_rows_per_src=[8974, 9224, 9236, 9162, 9248, 9668, 9304, 9254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344160 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267550816 ingress_bytes=371015680 link_bytes=371015680 token_bytes=534757472 weight_movement_bytes=103809024 total_nvlink_bytes=638566496 weight_to_token_ratio=0.194124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396102135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65008 dup_rows=24158 dup_rows_remote=21144 dedup_ratio=0.245748 router_tokens_per_expert=[745, 754, 780, 812, 765, 770, 806, 733, 739, 775, 767, 768, 779, 709, 784, 784, 792, 778, 754, 792, 776, 744, 733, 768, 753, 727, 797, 766, 761, 794, 756, 771, 773, 750, 773, 753, 787, 730, 741, 776, 787, 776, 812, 787, 739, 763, 773, 731, 766, 778, 765, 783, 692, 786, 719, 787, 765, 816, 765, 788, 784, 798, 796, 781] payload_rows_per_dst=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] dup_rows_per_dst=[3072, 2982, 2964, 2914, 3036, 3018, 3014, 3158] combine_rows_per_src=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64974 dup_rows=24124 dup_rows_remote=21242 dedup_ratio=0.245402 router_tokens_per_expert=[789, 733, 772, 774, 725, 758, 717, 760, 764, 763, 736, 735, 759, 721, 826, 808, 770, 769, 807, 741, 791, 795, 789, 739, 756, 716, 740, 765, 763, 792, 788, 724, 755, 792, 788, 734, 799, 810, 755, 803, 775, 782, 713, 740, 772, 773, 745, 778, 823, 772, 791, 772, 779, 736, 769, 751, 797, 792, 799, 786, 805, 744, 793, 744] payload_rows_per_dst=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] dup_rows_per_dst=[2920, 2974, 3102, 2882, 3106, 3054, 2994, 3092] combine_rows_per_src=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=265502720 dispatch_meta_bytes=344976 combine_payload_bytes=265502720 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265847696 ingress_bytes=300105728 link_bytes=300105728 token_bytes=531350416 weight_movement_bytes=34603008 total_nvlink_bytes=565953424 weight_to_token_ratio=0.065123 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129273694e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265003008 dispatch_meta_bytes=343360 combine_payload_bytes=265003008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265346368 ingress_bytes=299606016 link_bytes=299606016 token_bytes=530349376 weight_movement_bytes=34603008 total_nvlink_bytes=564952384 weight_to_token_ratio=0.065246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127393318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=266682368 dispatch_meta_bytes=343864 combine_payload_bytes=266682368 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267026232 ingress_bytes=301285376 link_bytes=301285376 token_bytes=533708600 weight_movement_bytes=34603008 total_nvlink_bytes=568311608 weight_to_token_ratio=0.064835 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133712614e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74180 payload_rows_remote=64974 dup_rows=24124 dup_rows_remote=21242 dedup_ratio=0.245402 router_tokens_per_expert=[789, 733, 772, 774, 725, 758, 717, 760, 764, 763, 736, 735, 759, 721, 826, 808, 770, 769, 807, 741, 791, 795, 789, 739, 756, 716, 740, 765, 763, 792, 788, 724, 755, 792, 788, 734, 799, 810, 755, 803, 775, 782, 713, 740, 772, 773, 745, 778, 823, 772, 791, 772, 779, 736, 769, 751, 797, 792, 799, 786, 805, 744, 793, 744] payload_rows_per_dst=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] dup_rows_per_dst=[2920, 2974, 3102, 2882, 3106, 3054, 2994, 3092] combine_rows_per_src=[9136, 9250, 9300, 9206, 9366, 9102, 9392, 9428] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265461760 dispatch_meta_bytes=343080 combine_payload_bytes=265461760 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=265804840 ingress_bytes=334667776 link_bytes=334667776 token_bytes=531266600 weight_movement_bytes=69206016 total_nvlink_bytes=600472616 weight_to_token_ratio=0.130266 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259327898e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344248 combine_payload_bytes=265486336 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265830584 ingress_bytes=403898368 link_bytes=403898368 token_bytes=531316920 weight_movement_bytes=138412032 total_nvlink_bytes=669728952 weight_to_token_ratio=0.260507 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519837042e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74576 payload_rows_remote=65236 dup_rows=23728 dup_rows_remote=20804 dedup_ratio=0.241374 router_tokens_per_expert=[760, 753, 729, 764, 770, 771, 788, 777, 771, 757, 739, 748, 818, 775, 730, 792, 790, 770, 749, 716, 773, 832, 813, 778, 785, 740, 772, 766, 774, 751, 817, 753, 759, 761, 758, 768, 736, 790, 728, 778, 754, 768, 796, 744, 813, 767, 793, 772, 716, 735, 758, 757, 736, 731, 776, 834, 776, 799, 763, 807, 745, 783, 753, 777] payload_rows_per_dst=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] dup_rows_per_dst=[2924, 3008, 3072, 2990, 2856, 3084, 2854, 2940] combine_rows_per_src=[9340, 9252, 9376, 9326, 9254, 9330, 9232, 9466] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74214 payload_rows_remote=64994 dup_rows=24090 dup_rows_remote=21262 dedup_ratio=0.245056 router_tokens_per_expert=[811, 786, 806, 821, 801, 791, 741, 768, 763, 772, 732, 781, 781, 714, 778, 772, 779, 764, 765, 752, 777, 772, 771, 814, 729, 755, 780, 770, 766, 745, 754, 775, 756, 795, 776, 803, 740, 762, 721, 771, 746, 731, 736, 771, 766, 717, 784, 773, 793, 756, 808, 779, 746, 724, 745, 799, 756, 809, 765, 772, 766, 773, 759, 768] payload_rows_per_dst=[9378, 9174, 9322, 9238, 9264, 9220, 9284, 9334] dup_rows_per_dst=[3272, 3012, 3066, 2910, 2984, 2828, 3016, 3002] combine_rows_per_src=[9378, 9174, 9322, 9238, 9264, 9220, 9284, 9334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65194 dup_rows=23854 dup_rows_remote=20980 dedup_ratio=0.242655 router_tokens_per_expert=[819, 802, 781, 793, 792, 806, 773, 743, 762, 772, 718, 763, 748, 783, 804, 787, 812, 747, 752, 749, 740, 736, 748, 754, 775, 758, 720, 757, 733, 750, 752, 750, 738, 739, 774, 773, 785, 744, 768, 744, 813, 752, 793, 754, 718, 748, 756, 785, 791, 801, 825, 754, 776, 803, 781, 778, 811, 780, 753, 757, 742, 803, 775, 759] payload_rows_per_dst=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] dup_rows_per_dst=[3154, 2948, 2898, 2842, 2874, 2926, 3156, 3056] combine_rows_per_src=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265486336 dispatch_meta_bytes=344248 combine_payload_bytes=265486336 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265830584 ingress_bytes=403898368 link_bytes=403898368 token_bytes=531316920 weight_movement_bytes=138412032 total_nvlink_bytes=669728952 weight_to_token_ratio=0.260507 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.519837042e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344608 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617376 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532890144 weight_movement_bytes=103809024 total_nvlink_bytes=636699168 weight_to_token_ratio=0.194804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74078 payload_rows_remote=64892 dup_rows=24226 dup_rows_remote=21226 dedup_ratio=0.246440 router_tokens_per_expert=[783, 757, 767, 789, 754, 754, 806, 792, 765, 815, 724, 723, 798, 768, 774, 786, 756, 801, 753, 742, 794, 748, 786, 758, 777, 741, 765, 771, 747, 709, 777, 788, 757, 785, 778, 739, 781, 773, 744, 790, 768, 776, 759, 715, 789, 784, 805, 758, 763, 804, 787, 752, 754, 733, 765, 735, 801, 838, 766, 782, 767, 764, 759, 713] payload_rows_per_dst=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] dup_rows_per_dst=[3086, 3026, 3046, 2962, 3000, 3026, 3000, 3080] combine_rows_per_src=[9318, 9280, 9230, 9188, 9294, 9282, 9186, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266133504 dispatch_meta_bytes=344864 combine_payload_bytes=266133504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266478368 ingress_bytes=300736512 link_bytes=300736512 token_bytes=532611872 weight_movement_bytes=34603008 total_nvlink_bytes=567214880 weight_to_token_ratio=0.064969 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131647283e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65178 dup_rows=23922 dup_rows_remote=20934 dedup_ratio=0.243347 router_tokens_per_expert=[794, 799, 779, 763, 734, 780, 795, 783, 732, 792, 760, 765, 771, 815, 756, 809, 772, 753, 757, 753, 773, 780, 768, 740, 798, 761, 794, 770, 763, 757, 739, 743, 747, 778, 730, 740, 779, 747, 802, 753, 744, 737, 762, 749, 784, 755, 803, 769, 715, 773, 837, 834, 807, 779, 730, 749, 769, 763, 773, 767, 724, 786, 769, 750] payload_rows_per_dst=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] dup_rows_per_dst=[3082, 3050, 2988, 2998, 2914, 2964, 3018, 2908] combine_rows_per_src=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=267206656 dispatch_meta_bytes=344160 combine_payload_bytes=267206656 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267550816 ingress_bytes=371015680 link_bytes=371015680 token_bytes=534757472 weight_movement_bytes=103809024 total_nvlink_bytes=638566496 weight_to_token_ratio=0.194124 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396102135e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265797632 dispatch_meta_bytes=344472 combine_payload_bytes=265797632 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266142104 ingress_bytes=300400640 link_bytes=300400640 token_bytes=531939736 weight_movement_bytes=34603008 total_nvlink_bytes=566542744 weight_to_token_ratio=0.065051 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130383424e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266215424 dispatch_meta_bytes=345024 combine_payload_bytes=266215424 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266560448 ingress_bytes=300818432 link_bytes=300818432 token_bytes=532775872 weight_movement_bytes=34603008 total_nvlink_bytes=567378880 weight_to_token_ratio=0.064949 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131955542e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344696 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379320 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413944 weight_movement_bytes=34603008 total_nvlink_bytes=569016952 weight_to_token_ratio=0.064749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64992 dup_rows=23964 dup_rows_remote=20944 dedup_ratio=0.243774 router_tokens_per_expert=[730, 786, 769, 794, 769, 773, 789, 742, 758, 749, 762, 755, 747, 771, 773, 768, 800, 745, 783, 749, 802, 765, 789, 775, 787, 808, 821, 806, 767, 771, 807, 759, 779, 784, 786, 738, 771, 721, 781, 761, 771, 733, 770, 755, 737, 730, 763, 738, 811, 755, 734, 757, 749, 731, 767, 784, 801, 783, 801, 758, 802, 724, 757, 751] payload_rows_per_dst=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] dup_rows_per_dst=[3020, 2998, 3134, 3328, 3016, 2806, 2866, 2796] combine_rows_per_src=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=65038 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[761, 721, 805, 699, 809, 758, 764, 732, 722, 759, 762, 796, 718, 792, 787, 750, 733, 783, 741, 759, 763, 736, 754, 826, 781, 794, 780, 790, 750, 728, 779, 798, 752, 805, 777, 740, 789, 751, 783, 785, 746, 747, 760, 780, 715, 764, 774, 848, 761, 715, 768, 755, 807, 775, 821, 783, 790, 793, 785, 739, 806, 760, 793, 755] payload_rows_per_dst=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] dup_rows_per_dst=[2928, 3042, 3006, 2994, 3048, 3008, 3042, 3068] combine_rows_per_src=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74258 payload_rows_remote=64912 dup_rows=24046 dup_rows_remote=20954 dedup_ratio=0.244609 router_tokens_per_expert=[790, 781, 777, 806, 749, 761, 751, 758, 762, 755, 757, 763, 767, 782, 779, 715, 779, 760, 747, 790, 765, 774, 798, 806, 762, 760, 747, 756, 740, 778, 744, 777, 765, 765, 745, 750, 757, 766, 744, 828, 746, 799, 764, 772, 789, 776, 778, 749, 796, 753, 739, 714, 746, 803, 798, 770, 756, 760, 788, 783, 790, 772, 784, 771] payload_rows_per_dst=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] dup_rows_per_dst=[3058, 2938, 3092, 2964, 3002, 3036, 2958, 2998] combine_rows_per_src=[9288, 9222, 9346, 9164, 9238, 9310, 9280, 9410] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343744 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550976 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758208 weight_movement_bytes=69206016 total_nvlink_bytes=601964224 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74082 payload_rows_remote=64698 dup_rows=24222 dup_rows_remote=21142 dedup_ratio=0.246399 router_tokens_per_expert=[748, 763, 726, 796, 754, 764, 743, 738, 756, 809, 720, 820, 787, 769, 782, 789, 765, 752, 783, 781, 726, 738, 748, 773, 782, 778, 762, 779, 795, 770, 816, 770, 768, 777, 809, 735, 790, 757, 781, 775, 772, 751, 816, 787, 770, 724, 740, 778, 785, 728, 764, 794, 713, 762, 749, 789, 745, 765, 795, 776, 789, 732, 777, 777] payload_rows_per_dst=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] dup_rows_per_dst=[2876, 3080, 2926, 3112, 3110, 2990, 3046, 3082] combine_rows_per_src=[9188, 9384, 9206, 9392, 9274, 9286, 9122, 9230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74450 payload_rows_remote=65194 dup_rows=23854 dup_rows_remote=20980 dedup_ratio=0.242655 router_tokens_per_expert=[819, 802, 781, 793, 792, 806, 773, 743, 762, 772, 718, 763, 748, 783, 804, 787, 812, 747, 752, 749, 740, 736, 748, 754, 775, 758, 720, 757, 733, 750, 752, 750, 738, 739, 774, 773, 785, 744, 768, 744, 813, 752, 793, 754, 718, 748, 756, 785, 791, 801, 825, 754, 776, 803, 781, 778, 811, 780, 753, 757, 742, 803, 775, 759] payload_rows_per_dst=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] dup_rows_per_dst=[3154, 2948, 2898, 2842, 2874, 2926, 3156, 3056] combine_rows_per_src=[9464, 9326, 9178, 9148, 9256, 9312, 9462, 9304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344448 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313536 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282624 weight_movement_bytes=34603008 total_nvlink_bytes=568885632 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74396 payload_rows_remote=64934 dup_rows=23908 dup_rows_remote=20842 dedup_ratio=0.243205 router_tokens_per_expert=[789, 752, 733, 764, 758, 769, 779, 779, 800, 819, 774, 759, 749, 755, 778, 742, 734, 774, 795, 755, 813, 773, 754, 822, 732, 797, 718, 780, 718, 740, 796, 695, 714, 737, 744, 786, 750, 731, 776, 772, 782, 778, 734, 808, 763, 744, 784, 784, 792, 772, 797, 793, 769, 775, 775, 787, 786, 766, 784, 771, 764, 788, 763, 788] payload_rows_per_dst=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] dup_rows_per_dst=[2926, 2910, 3062, 2640, 3082, 3040, 3182, 3066] combine_rows_per_src=[9320, 9284, 9374, 8890, 9316, 9408, 9342, 9462] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65144 dup_rows=23982 dup_rows_remote=21034 dedup_ratio=0.243958 router_tokens_per_expert=[738, 800, 752, 764, 794, 784, 761, 790, 790, 775, 805, 770, 781, 762, 813, 732, 760, 738, 773, 775, 765, 744, 788, 831, 791, 776, 820, 780, 768, 750, 768, 755, 742, 774, 759, 781, 764, 727, 771, 707, 746, 760, 775, 791, 770, 777, 771, 759, 753, 768, 744, 814, 727, 764, 749, 744, 719, 747, 795, 758, 779, 773, 808, 743] payload_rows_per_dst=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] dup_rows_per_dst=[3076, 3044, 2970, 3028, 2862, 2996, 2948, 3058] combine_rows_per_src=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344528 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266740176 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135824 weight_movement_bytes=103809024 total_nvlink_bytes=636944848 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=265003008 dispatch_meta_bytes=343360 combine_payload_bytes=265003008 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265346368 ingress_bytes=299606016 link_bytes=299606016 token_bytes=530349376 weight_movement_bytes=34603008 total_nvlink_bytes=564952384 weight_to_token_ratio=0.065246 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127393318e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73996 payload_rows_remote=64724 dup_rows=24308 dup_rows_remote=21244 dedup_ratio=0.247274 router_tokens_per_expert=[769, 788, 738, 715, 767, 798, 795, 766, 751, 778, 792, 777, 748, 794, 766, 723, 747, 773, 808, 777, 757, 719, 742, 733, 799, 792, 833, 755, 741, 715, 782, 751, 767, 823, 787, 746, 781, 754, 758, 771, 749, 792, 761, 820, 738, 792, 758, 766, 762, 786, 732, 751, 785, 789, 801, 777, 769, 793, 783, 750, 757, 792, 728, 745] payload_rows_per_dst=[9258, 9222, 9166, 9272, 9264, 9280, 9334, 9200] dup_rows_per_dst=[3014, 3036, 2946, 3064, 3110, 3072, 3032, 3034] combine_rows_per_src=[9258, 9222, 9166, 9272, 9264, 9280, 9334, 9200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=265879552 dispatch_meta_bytes=343464 combine_payload_bytes=265879552 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=266223016 ingress_bytes=404291584 link_bytes=404291584 token_bytes=532102568 weight_movement_bytes=138412032 total_nvlink_bytes=670514600 weight_to_token_ratio=0.260123 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.521316682e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65188 dup_rows=23942 dup_rows_remote=20968 dedup_ratio=0.243551 router_tokens_per_expert=[773, 766, 728, 773, 795, 759, 754, 796, 745, 765, 767, 781, 723, 765, 755, 782, 787, 792, 766, 732, 774, 763, 785, 794, 777, 792, 751, 790, 770, 771, 786, 761, 731, 730, 799, 791, 787, 749, 792, 804, 763, 736, 808, 714, 761, 770, 743, 779, 786, 762, 785, 785, 776, 746, 784, 758, 740, 787, 778, 767, 765, 773, 724, 761] payload_rows_per_dst=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] dup_rows_per_dst=[3012, 2848, 3068, 3090, 3040, 2974, 2944, 2966] combine_rows_per_src=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65112 dup_rows=23928 dup_rows_remote=21074 dedup_ratio=0.243408 router_tokens_per_expert=[779, 737, 754, 799, 790, 770, 705, 779, 778, 796, 763, 818, 783, 786, 750, 768, 731, 749, 790, 796, 744, 795, 802, 771, 789, 801, 821, 766, 732, 783, 783, 757, 759, 746, 794, 781, 766, 760, 716, 737, 765, 762, 786, 755, 752, 796, 766, 725, 761, 734, 744, 772, 785, 768, 755, 712, 769, 757, 806, 763, 778, 778, 764, 775] payload_rows_per_dst=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] dup_rows_per_dst=[2994, 3060, 3010, 3086, 2854, 2962, 2920, 3042] combine_rows_per_src=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343104 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266312768 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532282432 weight_movement_bytes=69206016 total_nvlink_bytes=601488448 weight_to_token_ratio=0.130017 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=267034624 dispatch_meta_bytes=344696 combine_payload_bytes=267034624 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267379320 ingress_bytes=301637632 link_bytes=301637632 token_bytes=534413944 weight_movement_bytes=34603008 total_nvlink_bytes=569016952 weight_to_token_ratio=0.064749 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135038125e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=64992 dup_rows=23964 dup_rows_remote=20944 dedup_ratio=0.243774 router_tokens_per_expert=[730, 786, 769, 794, 769, 773, 789, 742, 758, 749, 762, 755, 747, 771, 773, 768, 800, 745, 783, 749, 802, 765, 789, 775, 787, 808, 821, 806, 767, 771, 807, 759, 779, 784, 786, 738, 771, 721, 781, 761, 771, 733, 770, 755, 737, 730, 763, 738, 811, 755, 734, 757, 749, 731, 767, 784, 801, 783, 801, 758, 802, 724, 757, 751] payload_rows_per_dst=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] dup_rows_per_dst=[3020, 2998, 3134, 3328, 3016, 2806, 2866, 2796] combine_rows_per_src=[9348, 9318, 9440, 9600, 9394, 9188, 9102, 8950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74146 payload_rows_remote=65008 dup_rows=24158 dup_rows_remote=21144 dedup_ratio=0.245748 router_tokens_per_expert=[745, 754, 780, 812, 765, 770, 806, 733, 739, 775, 767, 768, 779, 709, 784, 784, 792, 778, 754, 792, 776, 744, 733, 768, 753, 727, 797, 766, 761, 794, 756, 771, 773, 750, 773, 753, 787, 730, 741, 776, 787, 776, 812, 787, 739, 763, 773, 731, 766, 778, 765, 783, 692, 786, 719, 787, 765, 816, 765, 788, 784, 798, 796, 781] payload_rows_per_dst=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] dup_rows_per_dst=[3072, 2982, 2964, 2914, 3036, 3018, 3014, 3158] combine_rows_per_src=[9258, 9228, 9310, 9336, 9130, 9318, 9138, 9428] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344712 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267174536 ingress_bytes=336035840 link_bytes=336035840 token_bytes=534004360 weight_movement_bytes=69206016 total_nvlink_bytes=603210376 weight_to_token_ratio=0.129598 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264475813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74362 payload_rows_remote=65188 dup_rows=23942 dup_rows_remote=20968 dedup_ratio=0.243551 router_tokens_per_expert=[773, 766, 728, 773, 795, 759, 754, 796, 745, 765, 767, 781, 723, 765, 755, 782, 787, 792, 766, 732, 774, 763, 785, 794, 777, 792, 751, 790, 770, 771, 786, 761, 731, 730, 799, 791, 787, 749, 792, 804, 763, 736, 808, 714, 761, 770, 743, 779, 786, 762, 785, 785, 776, 746, 784, 758, 740, 787, 778, 767, 765, 773, 724, 761] payload_rows_per_dst=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] dup_rows_per_dst=[3012, 2848, 3068, 3090, 3040, 2974, 2944, 2966] combine_rows_per_src=[9276, 9318, 9318, 9306, 9326, 9174, 9420, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=265109504 dispatch_meta_bytes=343872 combine_payload_bytes=265109504 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265453376 ingress_bytes=368918528 link_bytes=368918528 token_bytes=530562880 weight_movement_bytes=103809024 total_nvlink_bytes=634371904 weight_to_token_ratio=0.195658 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.388210720e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65026 dup_rows=23920 dup_rows_remote=21038 dedup_ratio=0.243327 router_tokens_per_expert=[807, 753, 764, 764, 751, 802, 751, 753, 735, 830, 778, 738, 764, 732, 742, 787, 756, 769, 754, 785, 747, 760, 753, 796, 769, 777, 761, 780, 769, 826, 737, 758, 780, 767, 760, 780, 757, 727, 787, 744, 794, 783, 759, 755, 760, 738, 802, 752, 755, 746, 758, 760, 804, 813, 770, 814, 745, 746, 770, 799, 742, 744, 792, 801] payload_rows_per_dst=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] dup_rows_per_dst=[2974, 2916, 2882, 2990, 3026, 3070, 3044, 3018] combine_rows_per_src=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266207232 dispatch_meta_bytes=343744 combine_payload_bytes=266207232 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266550976 ingress_bytes=335413248 link_bytes=335413248 token_bytes=532758208 weight_movement_bytes=69206016 total_nvlink_bytes=601964224 weight_to_token_ratio=0.129901 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262133049e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344608 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266617376 ingress_bytes=370081792 link_bytes=370081792 token_bytes=532890144 weight_movement_bytes=103809024 total_nvlink_bytes=636699168 weight_to_token_ratio=0.194804 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392587989e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=267010048 dispatch_meta_bytes=344624 combine_payload_bytes=267010048 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267354672 ingress_bytes=301613056 link_bytes=301613056 token_bytes=534364720 weight_movement_bytes=34603008 total_nvlink_bytes=568967728 weight_to_token_ratio=0.064755 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134945648e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65036 dup_rows=23858 dup_rows_remote=20750 dedup_ratio=0.242696 router_tokens_per_expert=[767, 733, 769, 784, 759, 788, 758, 753, 735, 827, 777, 770, 762, 748, 801, 750, 779, 788, 784, 773, 768, 770, 713, 743, 779, 782, 757, 780, 793, 741, 783, 789, 755, 805, 778, 778, 758, 767, 778, 796, 746, 803, 747, 786, 770, 756, 768, 769, 817, 701, 754, 776, 767, 697, 778, 747, 732, 763, 740, 808, 786, 772, 756, 795] payload_rows_per_dst=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] dup_rows_per_dst=[3108, 3032, 2952, 3024, 3006, 2912, 2874, 2950] combine_rows_per_src=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344744 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043496 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742248 weight_movement_bytes=34603008 total_nvlink_bytes=568345256 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65102 dup_rows=23962 dup_rows_remote=20952 dedup_ratio=0.243754 router_tokens_per_expert=[745, 737, 750, 776, 790, 748, 774, 823, 756, 746, 805, 767, 784, 778, 755, 734, 819, 813, 727, 781, 722, 728, 757, 738, 770, 770, 771, 761, 808, 812, 832, 760, 775, 762, 789, 793, 754, 770, 760, 762, 751, 801, 767, 733, 735, 785, 746, 762, 806, 785, 812, 758, 795, 742, 758, 760, 733, 793, 745, 752, 804, 740, 735, 752] payload_rows_per_dst=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] dup_rows_per_dst=[3008, 3010, 2928, 3158, 3046, 2898, 3074, 2840] combine_rows_per_src=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343144 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266730600 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118056 weight_movement_bytes=34603008 total_nvlink_bytes=567721064 weight_to_token_ratio=0.064907 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74382 payload_rows_remote=65178 dup_rows=23922 dup_rows_remote=20934 dedup_ratio=0.243347 router_tokens_per_expert=[794, 799, 779, 763, 734, 780, 795, 783, 732, 792, 760, 765, 771, 815, 756, 809, 772, 753, 757, 753, 773, 780, 768, 740, 798, 761, 794, 770, 763, 757, 739, 743, 747, 778, 730, 740, 779, 747, 802, 753, 744, 737, 762, 749, 784, 755, 803, 769, 715, 773, 837, 834, 807, 779, 730, 749, 769, 763, 773, 767, 724, 786, 769, 750] payload_rows_per_dst=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] dup_rows_per_dst=[3082, 3050, 2988, 2998, 2914, 2964, 3018, 2908] combine_rows_per_src=[9372, 9350, 9204, 9252, 9238, 9242, 9430, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74168 payload_rows_remote=65038 dup_rows=24136 dup_rows_remote=21094 dedup_ratio=0.245524 router_tokens_per_expert=[761, 721, 805, 699, 809, 758, 764, 732, 722, 759, 762, 796, 718, 792, 787, 750, 733, 783, 741, 759, 763, 736, 754, 826, 781, 794, 780, 790, 750, 728, 779, 798, 752, 805, 777, 740, 789, 751, 783, 785, 746, 747, 760, 780, 715, 764, 774, 848, 761, 715, 768, 755, 807, 775, 821, 783, 790, 793, 785, 739, 806, 760, 793, 755] payload_rows_per_dst=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] dup_rows_per_dst=[2928, 3042, 3006, 2994, 3048, 3008, 3042, 3068] combine_rows_per_src=[9170, 9130, 9184, 9406, 9316, 9260, 9328, 9374] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=73990 payload_rows_remote=64758 dup_rows=24314 dup_rows_remote=21276 dedup_ratio=0.247335 router_tokens_per_expert=[787, 748, 804, 767, 763, 798, 743, 793, 744, 756, 767, 747, 715, 767, 771, 766, 770, 794, 812, 743, 744, 765, 709, 736, 776, 775, 732, 747, 802, 852, 712, 775, 740, 809, 765, 789, 768, 733, 787, 743, 773, 739, 778, 776, 756, 793, 805, 801, 765, 783, 804, 757, 759, 783, 788, 743, 797, 742, 707, 814, 784, 737, 773, 781] payload_rows_per_dst=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] dup_rows_per_dst=[3122, 2786, 3220, 2924, 2568, 3388, 3268, 3038] combine_rows_per_src=[9284, 8850, 9356, 9046, 9000, 9754, 9468, 9232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65408 dup_rows=23626 dup_rows_remote=20764 dedup_ratio=0.240336 router_tokens_per_expert=[848, 765, 767, 805, 779, 758, 765, 787, 766, 735, 815, 772, 748, 784, 725, 799, 769, 746, 755, 788, 763, 741, 797, 753, 783, 739, 771, 750, 772, 775, 777, 812, 743, 738, 754, 806, 757, 757, 783, 748, 743, 762, 758, 794, 777, 758, 742, 772, 723, 717, 776, 773, 786, 795, 755, 741, 740, 801, 731, 798, 785, 772, 759, 799] payload_rows_per_dst=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] dup_rows_per_dst=[3074, 2912, 2942, 2906, 2834, 3028, 2862, 3068] combine_rows_per_src=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344216 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002008 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533659800 weight_movement_bytes=69206016 total_nvlink_bytes=602865816 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344256 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266690752 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037248 weight_movement_bytes=34603008 total_nvlink_bytes=567640256 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74376 payload_rows_remote=65112 dup_rows=23928 dup_rows_remote=21074 dedup_ratio=0.243408 router_tokens_per_expert=[779, 737, 754, 799, 790, 770, 705, 779, 778, 796, 763, 818, 783, 786, 750, 768, 731, 749, 790, 796, 744, 795, 802, 771, 789, 801, 821, 766, 732, 783, 783, 757, 759, 746, 794, 781, 766, 760, 716, 737, 765, 762, 786, 755, 752, 796, 766, 725, 761, 734, 744, 772, 785, 768, 755, 712, 769, 757, 806, 763, 778, 778, 764, 775] payload_rows_per_dst=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] dup_rows_per_dst=[2994, 3060, 3010, 3086, 2854, 2962, 2920, 3042] combine_rows_per_src=[9232, 9424, 9346, 9378, 9264, 9252, 9142, 9338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65198 dup_rows=23814 dup_rows_remote=20924 dedup_ratio=0.242249 router_tokens_per_expert=[753, 760, 794, 768, 801, 779, 725, 748, 757, 787, 773, 751, 715, 837, 792, 763, 760, 782, 776, 752, 790, 805, 792, 734, 767, 801, 735, 750, 753, 767, 751, 767, 749, 802, 760, 812, 793, 753, 744, 718, 814, 766, 792, 721, 768, 771, 758, 703, 770, 722, 782, 784, 760, 781, 752, 763, 758, 752, 781, 797, 786, 780, 771, 804] payload_rows_per_dst=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] dup_rows_per_dst=[2998, 3002, 3036, 2890, 2970, 2896, 3024, 2998] combine_rows_per_src=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65260 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[787, 735, 785, 783, 759, 736, 772, 779, 777, 743, 742, 798, 766, 748, 749, 775, 772, 782, 802, 748, 756, 830, 760, 790, 738, 777, 792, 790, 755, 761, 775, 817, 798, 772, 778, 776, 693, 822, 793, 792, 816, 734, 736, 753, 774, 711, 728, 734, 750, 787, 725, 723, 743, 772, 761, 809, 777, 782, 753, 774, 764, 804, 770, 769] payload_rows_per_dst=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] dup_rows_per_dst=[2964, 2912, 3082, 3100, 3094, 2864, 2902, 3018] combine_rows_per_src=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=13 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344448 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313536 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282624 weight_movement_bytes=34603008 total_nvlink_bytes=568885632 weight_to_token_ratio=0.064765 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64802 dup_rows=24172 dup_rows_remote=21092 dedup_ratio=0.245890 router_tokens_per_expert=[811, 754, 772, 756, 746, 774, 785, 774, 807, 810, 764, 743, 789, 788, 766, 803, 736, 740, 747, 749, 737, 756, 725, 778, 746, 796, 793, 763, 758, 768, 776, 718, 774, 777, 774, 788, 799, 767, 772, 754, 699, 787, 739, 712, 773, 750, 812, 781, 762, 815, 774, 781, 797, 759, 765, 786, 741, 777, 734, 774, 782, 811, 757, 751] payload_rows_per_dst=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] dup_rows_per_dst=[3060, 3116, 2824, 3042, 3080, 2938, 3096, 3016] combine_rows_per_src=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266395648 dispatch_meta_bytes=344528 combine_payload_bytes=266395648 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266740176 ingress_bytes=370204672 link_bytes=370204672 token_bytes=533135824 weight_movement_bytes=103809024 total_nvlink_bytes=636944848 weight_to_token_ratio=0.194714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393050377e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265248768 dispatch_meta_bytes=344136 combine_payload_bytes=265248768 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=265592904 ingress_bytes=403660800 link_bytes=403660800 token_bytes=530841672 weight_movement_bytes=138412032 total_nvlink_bytes=669253704 weight_to_token_ratio=0.260741 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.518943093e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74490 payload_rows_remote=65198 dup_rows=23814 dup_rows_remote=20924 dedup_ratio=0.242249 router_tokens_per_expert=[753, 760, 794, 768, 801, 779, 725, 748, 757, 787, 773, 751, 715, 837, 792, 763, 760, 782, 776, 752, 790, 805, 792, 734, 767, 801, 735, 750, 753, 767, 751, 767, 749, 802, 760, 812, 793, 753, 744, 718, 814, 766, 792, 721, 768, 771, 758, 703, 770, 722, 782, 784, 760, 781, 752, 763, 758, 752, 781, 797, 786, 780, 771, 804] payload_rows_per_dst=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] dup_rows_per_dst=[2998, 3002, 3036, 2890, 2970, 2896, 3024, 2998] combine_rows_per_src=[9258, 9348, 9346, 9292, 9292, 9290, 9204, 9460] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266698752 dispatch_meta_bytes=344744 combine_payload_bytes=266698752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267043496 ingress_bytes=301301760 link_bytes=301301760 token_bytes=533742248 weight_movement_bytes=34603008 total_nvlink_bytes=568345256 weight_to_token_ratio=0.064831 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133774266e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74446 payload_rows_remote=65036 dup_rows=23858 dup_rows_remote=20750 dedup_ratio=0.242696 router_tokens_per_expert=[767, 733, 769, 784, 759, 788, 758, 753, 735, 827, 777, 770, 762, 748, 801, 750, 779, 788, 784, 773, 768, 770, 713, 743, 779, 782, 757, 780, 793, 741, 783, 789, 755, 805, 778, 778, 758, 767, 778, 796, 746, 803, 747, 786, 770, 756, 768, 769, 817, 701, 754, 776, 767, 697, 778, 747, 732, 763, 740, 808, 786, 772, 756, 795] payload_rows_per_dst=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] dup_rows_per_dst=[3108, 3032, 2952, 3024, 3006, 2912, 2874, 2950] combine_rows_per_src=[9410, 9330, 9284, 9384, 9424, 9378, 9178, 9058] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=65144 dup_rows=23982 dup_rows_remote=21034 dedup_ratio=0.243958 router_tokens_per_expert=[738, 800, 752, 764, 794, 784, 761, 790, 790, 775, 805, 770, 781, 762, 813, 732, 760, 738, 773, 775, 765, 744, 788, 831, 791, 776, 820, 780, 768, 750, 768, 755, 742, 774, 759, 781, 764, 727, 771, 707, 746, 760, 775, 791, 770, 777, 771, 759, 753, 768, 744, 814, 727, 764, 749, 744, 719, 747, 795, 758, 779, 773, 808, 743] payload_rows_per_dst=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] dup_rows_per_dst=[3076, 3044, 2970, 3028, 2862, 2996, 2948, 3058] combine_rows_per_src=[9290, 9412, 9378, 9388, 9188, 9302, 9178, 9186] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267911168 dispatch_meta_bytes=344688 combine_payload_bytes=267911168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268255856 ingress_bytes=302514176 link_bytes=302514176 token_bytes=536167024 weight_movement_bytes=34603008 total_nvlink_bytes=570770032 weight_to_token_ratio=0.064538 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138336490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65010 dup_rows=24044 dup_rows_remote=21108 dedup_ratio=0.244588 router_tokens_per_expert=[704, 761, 803, 723, 771, 746, 802, 773, 794, 810, 773, 827, 778, 774, 798, 763, 785, 697, 794, 773, 741, 731, 808, 750, 733, 793, 759, 835, 817, 781, 770, 741, 855, 815, 749, 793, 758, 805, 775, 764, 770, 744, 766, 771, 751, 749, 751, 753, 767, 771, 771, 716, 718, 738, 771, 715, 766, 794, 751, 801, 733, 762, 722, 779] payload_rows_per_dst=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] dup_rows_per_dst=[2936, 3224, 2970, 3142, 3194, 2894, 2750, 2934] combine_rows_per_src=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=65260 dup_rows=23936 dup_rows_remote=21072 dedup_ratio=0.243490 router_tokens_per_expert=[787, 735, 785, 783, 759, 736, 772, 779, 777, 743, 742, 798, 766, 748, 749, 775, 772, 782, 802, 748, 756, 830, 760, 790, 738, 777, 792, 790, 755, 761, 775, 817, 798, 772, 778, 776, 693, 822, 793, 792, 816, 734, 736, 753, 774, 711, 728, 734, 750, 787, 725, 723, 743, 772, 761, 809, 777, 782, 753, 774, 764, 804, 770, 769] payload_rows_per_dst=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] dup_rows_per_dst=[2964, 2912, 3082, 3100, 3094, 2864, 2902, 3018] combine_rows_per_src=[9308, 9284, 9398, 9310, 9354, 9108, 9238, 9368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267051008 dispatch_meta_bytes=344488 combine_payload_bytes=267051008 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267395496 ingress_bytes=336257024 link_bytes=336257024 token_bytes=534446504 weight_movement_bytes=69206016 total_nvlink_bytes=603652520 weight_to_token_ratio=0.129491 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265308110e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64890 dup_rows=24048 dup_rows_remote=21036 dedup_ratio=0.244629 router_tokens_per_expert=[731, 736, 761, 761, 738, 768, 751, 757, 770, 757, 775, 789, 769, 800, 781, 754, 752, 763, 750, 759, 779, 812, 782, 792, 776, 716, 773, 724, 751, 786, 783, 735, 769, 773, 752, 776, 764, 775, 777, 793, 775, 771, 810, 765, 776, 757, 772, 782, 792, 758, 769, 713, 777, 749, 770, 784, 790, 783, 786, 822, 743, 747, 761, 790] payload_rows_per_dst=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] dup_rows_per_dst=[2858, 3046, 3012, 2964, 3028, 3120, 2924, 3096] combine_rows_per_src=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266387456 dispatch_meta_bytes=343144 combine_payload_bytes=266387456 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266730600 ingress_bytes=300990464 link_bytes=300990464 token_bytes=533118056 weight_movement_bytes=34603008 total_nvlink_bytes=567721064 weight_to_token_ratio=0.064907 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132602884e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266829824 dispatch_meta_bytes=344712 combine_payload_bytes=266829824 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267174536 ingress_bytes=336035840 link_bytes=336035840 token_bytes=534004360 weight_movement_bytes=69206016 total_nvlink_bytes=603210376 weight_to_token_ratio=0.129598 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264475813e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267304960 dispatch_meta_bytes=345328 combine_payload_bytes=267304960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267650288 ingress_bytes=301907968 link_bytes=301907968 token_bytes=534955248 weight_movement_bytes=34603008 total_nvlink_bytes=569558256 weight_to_token_ratio=0.064684 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136055378e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343576 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265772568 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531201560 weight_movement_bytes=69206016 total_nvlink_bytes=600407576 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65058 dup_rows=23960 dup_rows_remote=21062 dedup_ratio=0.243734 router_tokens_per_expert=[732, 767, 817, 761, 731, 812, 742, 823, 735, 746, 783, 812, 735, 760, 784, 737, 777, 732, 758, 716, 732, 768, 757, 787, 795, 740, 731, 793, 774, 749, 793, 764, 789, 769, 767, 764, 769, 795, 748, 725, 741, 790, 790, 798, 729, 781, 816, 764, 795, 770, 838, 773, 773, 727, 739, 816, 799, 739, 726, 791, 748, 763, 796, 781] payload_rows_per_dst=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] dup_rows_per_dst=[3052, 2898, 2856, 3016, 2998, 2972, 3156, 3012] combine_rows_per_src=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344472 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266625432 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532906392 weight_movement_bytes=34603008 total_nvlink_bytes=567509400 weight_to_token_ratio=0.064933 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74384 payload_rows_remote=65026 dup_rows=23920 dup_rows_remote=21038 dedup_ratio=0.243327 router_tokens_per_expert=[807, 753, 764, 764, 751, 802, 751, 753, 735, 830, 778, 738, 764, 732, 742, 787, 756, 769, 754, 785, 747, 760, 753, 796, 769, 777, 761, 780, 769, 826, 737, 758, 780, 767, 760, 780, 757, 727, 787, 744, 794, 783, 759, 755, 760, 738, 802, 752, 755, 746, 758, 760, 804, 813, 770, 814, 745, 746, 770, 799, 742, 744, 792, 801] payload_rows_per_dst=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] dup_rows_per_dst=[2974, 2916, 2882, 2990, 3026, 3070, 3044, 3018] combine_rows_per_src=[9316, 9296, 9358, 9364, 9178, 9216, 9396, 9260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74342 payload_rows_remote=65102 dup_rows=23962 dup_rows_remote=20952 dedup_ratio=0.243754 router_tokens_per_expert=[745, 737, 750, 776, 790, 748, 774, 823, 756, 746, 805, 767, 784, 778, 755, 734, 819, 813, 727, 781, 722, 728, 757, 738, 770, 770, 771, 761, 808, 812, 832, 760, 775, 762, 789, 793, 754, 770, 760, 762, 751, 801, 767, 733, 735, 785, 746, 762, 806, 785, 812, 758, 795, 742, 758, 760, 733, 793, 745, 752, 804, 740, 735, 752] payload_rows_per_dst=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] dup_rows_per_dst=[3008, 3010, 2928, 3158, 3046, 2898, 3074, 2840] combine_rows_per_src=[9278, 9240, 9242, 9410, 9284, 9262, 9358, 9268] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74418 payload_rows_remote=64822 dup_rows=23886 dup_rows_remote=20678 dedup_ratio=0.242981 router_tokens_per_expert=[738, 768, 797, 757, 749, 805, 805, 740, 759, 778, 761, 754, 804, 779, 799, 749, 745, 811, 772, 781, 758, 733, 742, 781, 811, 789, 768, 756, 793, 797, 774, 792, 786, 720, 775, 717, 747, 751, 778, 764, 723, 763, 791, 807, 737, 784, 755, 759, 777, 759, 769, 789, 776, 786, 779, 751, 738, 768, 791, 728, 711, 798, 768, 762] payload_rows_per_dst=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] dup_rows_per_dst=[3012, 3044, 2978, 2772, 2800, 2942, 3130, 3208] combine_rows_per_src=[9306, 9322, 9268, 9112, 9276, 9296, 9242, 9596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822048 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299616 weight_movement_bytes=34603008 total_nvlink_bytes=567902624 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65000 dup_rows=23998 dup_rows_remote=21070 dedup_ratio=0.244120 router_tokens_per_expert=[746, 774, 732, 791, 755, 771, 805, 751, 804, 796, 762, 737, 764, 763, 762, 753, 745, 770, 741, 762, 714, 723, 778, 801, 793, 791, 758, 813, 774, 823, 765, 744, 794, 769, 789, 732, 759, 773, 764, 783, 787, 775, 760, 776, 811, 810, 746, 770, 843, 781, 756, 744, 730, 742, 746, 775, 761, 724, 765, 776, 787, 739, 737, 787] payload_rows_per_dst=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] dup_rows_per_dst=[3052, 3066, 2926, 3118, 2970, 2996, 2928, 2942] combine_rows_per_src=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65232 dup_rows=23724 dup_rows_remote=20740 dedup_ratio=0.241333 router_tokens_per_expert=[769, 767, 762, 705, 764, 755, 758, 728, 752, 795, 747, 766, 795, 732, 763, 771, 813, 763, 735, 807, 783, 753, 718, 765, 785, 783, 736, 792, 751, 804, 771, 744, 723, 804, 762, 774, 768, 766, 743, 816, 804, 742, 786, 763, 745, 802, 765, 769, 778, 802, 742, 810, 822, 790, 785, 750, 800, 790, 751, 797, 745, 720, 762, 744] payload_rows_per_dst=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] dup_rows_per_dst=[2810, 2872, 2928, 2984, 3036, 2952, 3128, 3014] combine_rows_per_src=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74132 payload_rows_remote=64802 dup_rows=24172 dup_rows_remote=21092 dedup_ratio=0.245890 router_tokens_per_expert=[811, 754, 772, 756, 746, 774, 785, 774, 807, 810, 764, 743, 789, 788, 766, 803, 736, 740, 747, 749, 737, 756, 725, 778, 746, 796, 793, 763, 758, 768, 776, 718, 774, 777, 774, 788, 799, 767, 772, 754, 699, 787, 739, 712, 773, 750, 812, 781, 762, 815, 774, 781, 797, 759, 765, 786, 741, 777, 734, 774, 782, 811, 757, 751] payload_rows_per_dst=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] dup_rows_per_dst=[3060, 3116, 2824, 3042, 3080, 2938, 3096, 3016] combine_rows_per_src=[9284, 9424, 9112, 9194, 9330, 9168, 9382, 9238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343704 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133144 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531922584 weight_movement_bytes=34603008 total_nvlink_bytes=566525592 weight_to_token_ratio=0.065053 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64996 dup_rows=23972 dup_rows_remote=21066 dedup_ratio=0.243856 router_tokens_per_expert=[740, 760, 750, 756, 714, 768, 749, 770, 761, 753, 725, 764, 778, 814, 763, 716, 756, 768, 741, 798, 788, 826, 753, 748, 781, 777, 825, 758, 761, 719, 786, 796, 786, 765, 753, 788, 790, 816, 715, 758, 726, 803, 770, 733, 759, 791, 757, 782, 788, 809, 781, 813, 778, 746, 794, 728, 776, 746, 817, 786, 767, 747, 767, 755] payload_rows_per_dst=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] dup_rows_per_dst=[2938, 2932, 3056, 3094, 2950, 2906, 3110, 2986] combine_rows_per_src=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64866 dup_rows=23982 dup_rows_remote=20980 dedup_ratio=0.243958 router_tokens_per_expert=[757, 701, 789, 750, 776, 795, 736, 772, 741, 759, 777, 713, 759, 714, 820, 737, 762, 793, 765, 764, 747, 768, 790, 745, 777, 759, 792, 801, 771, 793, 802, 711, 774, 781, 810, 774, 787, 724, 787, 792, 804, 781, 785, 769, 810, 799, 778, 779, 776, 787, 739, 756, 752, 798, 721, 792, 769, 801, 735, 728, 749, 773, 726, 780] payload_rows_per_dst=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] dup_rows_per_dst=[2882, 2862, 3080, 3056, 3002, 3184, 2978, 2938] combine_rows_per_src=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=14 plans=1 dispatch_payload_bytes=266346496 dispatch_meta_bytes=344256 combine_payload_bytes=266346496 prefetch_bytes=0 reduce_bytes=34603008 egress_bytes=266690752 ingress_bytes=300949504 link_bytes=300949504 token_bytes=533037248 weight_movement_bytes=34603008 total_nvlink_bytes=567640256 weight_to_token_ratio=0.064917 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132448755e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266657792 dispatch_meta_bytes=344216 combine_payload_bytes=266657792 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267002008 ingress_bytes=335863808 link_bytes=335863808 token_bytes=533659800 weight_movement_bytes=69206016 total_nvlink_bytes=602865816 weight_to_token_ratio=0.129682 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263828470e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74580 payload_rows_remote=65232 dup_rows=23724 dup_rows_remote=20740 dedup_ratio=0.241333 router_tokens_per_expert=[769, 767, 762, 705, 764, 755, 758, 728, 752, 795, 747, 766, 795, 732, 763, 771, 813, 763, 735, 807, 783, 753, 718, 765, 785, 783, 736, 792, 751, 804, 771, 744, 723, 804, 762, 774, 768, 766, 743, 816, 804, 742, 786, 763, 745, 802, 765, 769, 778, 802, 742, 810, 822, 790, 785, 750, 800, 790, 751, 797, 745, 720, 762, 744] payload_rows_per_dst=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] dup_rows_per_dst=[2810, 2872, 2928, 2984, 3036, 2952, 3128, 3014] combine_rows_per_src=[9206, 9370, 9346, 9348, 9276, 9400, 9430, 9204] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265428992 dispatch_meta_bytes=343576 combine_payload_bytes=265428992 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265772568 ingress_bytes=334635008 link_bytes=334635008 token_bytes=531201560 weight_movement_bytes=69206016 total_nvlink_bytes=600407576 weight_to_token_ratio=0.130282 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259204595e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65010 dup_rows=24044 dup_rows_remote=21108 dedup_ratio=0.244588 router_tokens_per_expert=[704, 761, 803, 723, 771, 746, 802, 773, 794, 810, 773, 827, 778, 774, 798, 763, 785, 697, 794, 773, 741, 731, 808, 750, 733, 793, 759, 835, 817, 781, 770, 741, 855, 815, 749, 793, 758, 805, 775, 764, 770, 744, 766, 771, 751, 749, 751, 753, 767, 771, 771, 716, 718, 738, 771, 715, 766, 794, 751, 801, 733, 762, 722, 779] payload_rows_per_dst=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] dup_rows_per_dst=[2936, 3224, 2970, 3142, 3194, 2894, 2750, 2934] combine_rows_per_src=[9250, 9430, 9188, 9350, 9562, 9216, 9184, 9080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74678 payload_rows_remote=65408 dup_rows=23626 dup_rows_remote=20764 dedup_ratio=0.240336 router_tokens_per_expert=[848, 765, 767, 805, 779, 758, 765, 787, 766, 735, 815, 772, 748, 784, 725, 799, 769, 746, 755, 788, 763, 741, 797, 753, 783, 739, 771, 750, 772, 775, 777, 812, 743, 738, 754, 806, 757, 757, 783, 748, 743, 762, 758, 794, 777, 758, 742, 772, 723, 717, 776, 773, 786, 795, 755, 741, 740, 801, 731, 798, 785, 772, 759, 799] payload_rows_per_dst=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] dup_rows_per_dst=[3074, 2912, 2942, 2906, 2834, 3028, 2862, 3068] combine_rows_per_src=[9474, 9376, 9282, 9452, 9338, 9184, 9270, 9302] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74332 payload_rows_remote=64996 dup_rows=23972 dup_rows_remote=21066 dedup_ratio=0.243856 router_tokens_per_expert=[740, 760, 750, 756, 714, 768, 749, 770, 761, 753, 725, 764, 778, 814, 763, 716, 756, 768, 741, 798, 788, 826, 753, 748, 781, 777, 825, 758, 761, 719, 786, 796, 786, 765, 753, 788, 790, 816, 715, 758, 726, 803, 770, 733, 759, 791, 757, 782, 788, 809, 781, 813, 778, 746, 794, 728, 776, 746, 817, 786, 767, 747, 767, 755] payload_rows_per_dst=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] dup_rows_per_dst=[2938, 2932, 3056, 3094, 2950, 2906, 3110, 2986] combine_rows_per_src=[9076, 9216, 9300, 9312, 9392, 9336, 9364, 9336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265510912 dispatch_meta_bytes=342000 combine_payload_bytes=265510912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265852912 ingress_bytes=300113920 link_bytes=300113920 token_bytes=531363824 weight_movement_bytes=34603008 total_nvlink_bytes=565966832 weight_to_token_ratio=0.065121 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129304520e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64458 dup_rows=24278 dup_rows_remote=20984 dedup_ratio=0.246969 router_tokens_per_expert=[784, 783, 796, 748, 771, 775, 743, 806, 729, 750, 760, 780, 765, 787, 760, 726, 715, 791, 830, 751, 787, 785, 745, 790, 781, 775, 743, 761, 788, 800, 740, 741, 762, 762, 763, 802, 752, 749, 812, 742, 791, 737, 766, 768, 758, 770, 805, 762, 733, 823, 820, 794, 733, 748, 786, 737, 758, 824, 731, 756, 775, 746, 758, 743] payload_rows_per_dst=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] dup_rows_per_dst=[3294, 3360, 3360, 2874, 3066, 2716, 2556, 3052] combine_rows_per_src=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344280 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266584280 ingress_bytes=370049024 link_bytes=370049024 token_bytes=532824280 weight_movement_bytes=103809024 total_nvlink_bytes=636633304 weight_to_token_ratio=0.194828 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392464686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267190272 dispatch_meta_bytes=343888 combine_payload_bytes=267190272 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267534160 ingress_bytes=370999296 link_bytes=370999296 token_bytes=534724432 weight_movement_bytes=103809024 total_nvlink_bytes=638533456 weight_to_token_ratio=0.194136 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.396040483e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65224 dup_rows=23976 dup_rows_remote=20954 dedup_ratio=0.243896 router_tokens_per_expert=[825, 778, 725, 797, 734, 753, 760, 789, 718, 842, 767, 781, 740, 736, 755, 754, 755, 750, 757, 729, 795, 735, 765, 777, 776, 738, 797, 802, 749, 742, 768, 818, 753, 760, 772, 763, 768, 757, 758, 785, 805, 735, 790, 749, 749, 751, 747, 846, 785, 787, 780, 770, 758, 790, 792, 786, 746, 775, 757, 685, 771, 778, 789, 808] payload_rows_per_dst=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] dup_rows_per_dst=[3054, 2896, 3022, 3014, 2954, 2994, 3058, 2984] combine_rows_per_src=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=266280960 dispatch_meta_bytes=344472 combine_payload_bytes=266280960 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266625432 ingress_bytes=300883968 link_bytes=300883968 token_bytes=532906392 weight_movement_bytes=34603008 total_nvlink_bytes=567509400 weight_to_token_ratio=0.064933 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132202148e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=267911168 dispatch_meta_bytes=344688 combine_payload_bytes=267911168 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=268255856 ingress_bytes=302514176 link_bytes=302514176 token_bytes=536167024 weight_movement_bytes=34603008 total_nvlink_bytes=570770032 weight_to_token_ratio=0.064538 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.035000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.138336490e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=344248 combine_payload_bytes=266223616 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266567864 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791480 weight_movement_bytes=103809024 total_nvlink_bytes=636600504 weight_to_token_ratio=0.194840 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343384 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034520 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531725656 weight_movement_bytes=34603008 total_nvlink_bytes=566328664 weight_to_token_ratio=0.065077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65166 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[762, 792, 766, 770, 749, 748, 768, 692, 776, 781, 791, 742, 763, 767, 783, 728, 829, 766, 755, 779, 848, 770, 726, 747, 708, 779, 785, 773, 774, 767, 789, 769, 806, 785, 805, 737, 761, 734, 754, 798, 783, 795, 765, 711, 748, 783, 754, 750, 786, 791, 782, 732, 816, 783, 740, 792, 785, 777, 745, 763, 765, 749, 776, 759] payload_rows_per_dst=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] dup_rows_per_dst=[2818, 2944, 2990, 3070, 2980, 2916, 3144, 2958] combine_rows_per_src=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264019968 dispatch_meta_bytes=341768 combine_payload_bytes=264019968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264361736 ingress_bytes=298622976 link_bytes=298622976 token_bytes=528381704 weight_movement_bytes=34603008 total_nvlink_bytes=562984712 weight_to_token_ratio=0.065489 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123694218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74256 payload_rows_remote=64890 dup_rows=24048 dup_rows_remote=21036 dedup_ratio=0.244629 router_tokens_per_expert=[731, 736, 761, 761, 738, 768, 751, 757, 770, 757, 775, 789, 769, 800, 781, 754, 752, 763, 750, 759, 779, 812, 782, 792, 776, 716, 773, 724, 751, 786, 783, 735, 769, 773, 752, 776, 764, 775, 777, 793, 775, 771, 810, 765, 776, 757, 772, 782, 792, 758, 769, 713, 777, 749, 770, 784, 790, 783, 786, 822, 743, 747, 761, 790] payload_rows_per_dst=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] dup_rows_per_dst=[2858, 3046, 3012, 2964, 3028, 3120, 2924, 3096] combine_rows_per_src=[9148, 9344, 9366, 9124, 9330, 9296, 9300, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74344 payload_rows_remote=65058 dup_rows=23960 dup_rows_remote=21062 dedup_ratio=0.243734 router_tokens_per_expert=[732, 767, 817, 761, 731, 812, 742, 823, 735, 746, 783, 812, 735, 760, 784, 737, 777, 732, 758, 716, 732, 768, 757, 787, 795, 740, 731, 793, 774, 749, 793, 764, 789, 769, 767, 764, 769, 795, 748, 725, 741, 790, 790, 798, 729, 781, 816, 764, 795, 770, 838, 773, 773, 727, 739, 816, 799, 739, 726, 791, 748, 763, 796, 781] payload_rows_per_dst=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] dup_rows_per_dst=[3052, 2898, 2856, 3016, 2998, 2972, 3156, 3012] combine_rows_per_src=[9318, 9286, 9198, 9262, 9254, 9446, 9306, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344712 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267502216 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659720 weight_movement_bytes=34603008 total_nvlink_bytes=569262728 weight_to_token_ratio=0.064720 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74322 payload_rows_remote=64866 dup_rows=23982 dup_rows_remote=20980 dedup_ratio=0.243958 router_tokens_per_expert=[757, 701, 789, 750, 776, 795, 736, 772, 741, 759, 777, 713, 759, 714, 820, 737, 762, 793, 765, 764, 747, 768, 790, 745, 777, 759, 792, 801, 771, 793, 802, 711, 774, 781, 810, 774, 787, 724, 787, 792, 804, 781, 785, 769, 810, 799, 778, 779, 776, 787, 739, 756, 752, 798, 721, 792, 769, 801, 735, 728, 749, 773, 726, 780] payload_rows_per_dst=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] dup_rows_per_dst=[2882, 2862, 3080, 3056, 3002, 3184, 2978, 2938] combine_rows_per_src=[9270, 9178, 9188, 9356, 9456, 9426, 9264, 9184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266477568 dispatch_meta_bytes=344480 combine_payload_bytes=266477568 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266822048 ingress_bytes=301080576 link_bytes=301080576 token_bytes=533299616 weight_movement_bytes=34603008 total_nvlink_bytes=567902624 weight_to_token_ratio=0.064885 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132941969e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74224 payload_rows_remote=64842 dup_rows=24080 dup_rows_remote=21034 dedup_ratio=0.244954 router_tokens_per_expert=[776, 754, 742, 829, 782, 774, 776, 794, 773, 798, 825, 781, 739, 758, 749, 720, 785, 781, 803, 780, 732, 768, 793, 775, 769, 812, 731, 771, 776, 727, 751, 773, 733, 726, 799, 747, 732, 773, 740, 798, 749, 747, 787, 727, 810, 778, 764, 751, 742, 790, 736, 778, 774, 763, 751, 763, 775, 768, 818, 777, 788, 720, 804, 747] payload_rows_per_dst=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] dup_rows_per_dst=[2986, 3114, 3160, 2756, 2720, 2950, 3348, 3046] combine_rows_per_src=[9182, 9458, 9274, 9094, 8948, 9276, 9610, 9382] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344168 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264104 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184040 weight_movement_bytes=34603008 total_nvlink_bytes=568787048 weight_to_token_ratio=0.064777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65074 dup_rows=24028 dup_rows_remote=21164 dedup_ratio=0.244425 router_tokens_per_expert=[758, 788, 804, 751, 787, 774, 782, 759, 785, 785, 761, 735, 781, 768, 733, 727, 768, 750, 790, 741, 775, 799, 758, 764, 770, 852, 801, 784, 701, 756, 728, 781, 757, 750, 780, 776, 776, 807, 807, 819, 780, 777, 744, 788, 749, 759, 757, 742, 716, 807, 783, 724, 813, 745, 709, 736, 763, 742, 762, 743, 770, 792, 773, 810] payload_rows_per_dst=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] dup_rows_per_dst=[3162, 2914, 3032, 3040, 3092, 2908, 2864, 3016] combine_rows_per_src=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=265691136 dispatch_meta_bytes=343384 combine_payload_bytes=265691136 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266034520 ingress_bytes=300294144 link_bytes=300294144 token_bytes=531725656 weight_movement_bytes=34603008 total_nvlink_bytes=566328664 weight_to_token_ratio=0.065077 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129982688e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74330 payload_rows_remote=65052 dup_rows=23974 dup_rows_remote=20902 dedup_ratio=0.243876 router_tokens_per_expert=[766, 774, 781, 780, 755, 759, 805, 821, 750, 745, 747, 799, 819, 741, 762, 804, 739, 783, 759, 793, 763, 777, 758, 754, 769, 783, 771, 814, 774, 743, 782, 739, 737, 765, 766, 784, 768, 819, 763, 782, 738, 794, 752, 740, 762, 801, 745, 779, 749, 798, 782, 723, 736, 735, 752, 739, 777, 767, 796, 753, 719, 786, 762, 774] payload_rows_per_dst=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] dup_rows_per_dst=[3072, 3000, 2980, 3072, 3060, 2922, 2874, 2994] combine_rows_per_src=[9410, 9334, 9272, 9278, 9308, 9300, 9154, 9274] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74026 payload_rows_remote=64458 dup_rows=24278 dup_rows_remote=20984 dedup_ratio=0.246969 router_tokens_per_expert=[784, 783, 796, 748, 771, 775, 743, 806, 729, 750, 760, 780, 765, 787, 760, 726, 715, 791, 830, 751, 787, 785, 745, 790, 781, 775, 743, 761, 788, 800, 740, 741, 762, 762, 763, 802, 752, 749, 812, 742, 791, 737, 766, 768, 758, 770, 805, 762, 733, 823, 820, 794, 733, 748, 786, 737, 758, 824, 731, 756, 775, 746, 758, 743] payload_rows_per_dst=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] dup_rows_per_dst=[3294, 3360, 3360, 2874, 3066, 2716, 2556, 3052] combine_rows_per_src=[9568, 9496, 9552, 9216, 9240, 8914, 8910, 9130] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74306 payload_rows_remote=65000 dup_rows=23998 dup_rows_remote=21070 dedup_ratio=0.244120 router_tokens_per_expert=[746, 774, 732, 791, 755, 771, 805, 751, 804, 796, 762, 737, 764, 763, 762, 753, 745, 770, 741, 762, 714, 723, 778, 801, 793, 791, 758, 813, 774, 823, 765, 744, 794, 769, 789, 732, 759, 773, 764, 783, 787, 775, 760, 776, 811, 810, 746, 770, 843, 781, 756, 744, 730, 742, 746, 775, 761, 724, 765, 776, 787, 739, 737, 787] payload_rows_per_dst=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] dup_rows_per_dst=[3052, 3066, 2926, 3118, 2970, 2996, 2928, 2942] combine_rows_per_src=[9198, 9216, 9142, 9404, 9356, 9474, 9306, 9210] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65132 dup_rows=23834 dup_rows_remote=20880 dedup_ratio=0.242452 router_tokens_per_expert=[761, 776, 817, 806, 787, 719, 787, 760, 760, 760, 794, 763, 780, 726, 759, 775, 808, 766, 734, 794, 738, 754, 760, 778, 763, 734, 801, 750, 757, 777, 761, 802, 730, 787, 726, 772, 791, 777, 791, 803, 744, 775, 787, 736, 750, 775, 791, 788, 749, 768, 771, 724, 766, 780, 742, 743, 728, 738, 807, 811, 801, 767, 756, 771] payload_rows_per_dst=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] dup_rows_per_dst=[3072, 2894, 2942, 3016, 3022, 2954, 2946, 2988] combine_rows_per_src=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64908 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[829, 797, 762, 770, 754, 778, 752, 747, 791, 743, 758, 767, 811, 751, 759, 776, 766, 751, 721, 754, 785, 773, 748, 716, 783, 789, 807, 783, 765, 747, 790, 767, 777, 786, 773, 768, 763, 747, 733, 780, 778, 769, 729, 795, 768, 771, 802, 788, 782, 726, 739, 738, 796, 767, 762, 799, 744, 738, 765, 780, 787, 765, 767, 780] payload_rows_per_dst=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] dup_rows_per_dst=[3062, 3084, 2876, 3088, 3008, 3090, 2926, 3016] combine_rows_per_src=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=15 plans=1 dispatch_payload_bytes=265789440 dispatch_meta_bytes=343704 combine_payload_bytes=265789440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266133144 ingress_bytes=300392448 link_bytes=300392448 token_bytes=531922584 weight_movement_bytes=34603008 total_nvlink_bytes=566525592 weight_to_token_ratio=0.065053 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130352598e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74470 payload_rows_remote=65132 dup_rows=23834 dup_rows_remote=20880 dedup_ratio=0.242452 router_tokens_per_expert=[761, 776, 817, 806, 787, 719, 787, 760, 760, 760, 794, 763, 780, 726, 759, 775, 808, 766, 734, 794, 738, 754, 760, 778, 763, 734, 801, 750, 757, 777, 761, 802, 730, 787, 726, 772, 791, 777, 791, 803, 744, 775, 787, 736, 750, 775, 791, 788, 749, 768, 771, 724, 766, 780, 742, 743, 728, 738, 807, 811, 801, 767, 756, 771] payload_rows_per_dst=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] dup_rows_per_dst=[3072, 2894, 2942, 3016, 3022, 2954, 2946, 2988] combine_rows_per_src=[9354, 9340, 9322, 9274, 9332, 9338, 9140, 9370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20928 dedup_ratio=0.243774 router_tokens_per_expert=[778, 778, 755, 778, 764, 706, 768, 733, 780, 749, 772, 753, 725, 771, 802, 753, 794, 797, 784, 757, 750, 727, 781, 739, 826, 758, 781, 735, 810, 743, 819, 766, 784, 769, 765, 788, 778, 755, 784, 811, 796, 804, 772, 750, 765, 770, 758, 731, 814, 759, 720, 758, 792, 781, 750, 760, 759, 781, 760, 780, 738, 796, 728, 764] payload_rows_per_dst=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] dup_rows_per_dst=[2878, 2868, 3036, 3124, 3140, 2988, 3032, 2898] combine_rows_per_src=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265592832 dispatch_meta_bytes=343504 combine_payload_bytes=265592832 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265936336 ingress_bytes=334798848 link_bytes=334798848 token_bytes=531529168 weight_movement_bytes=69206016 total_nvlink_bytes=600735184 weight_to_token_ratio=0.130202 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259821112e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65164 dup_rows=23812 dup_rows_remote=20910 dedup_ratio=0.242228 router_tokens_per_expert=[751, 743, 805, 776, 746, 778, 787, 709, 782, 755, 812, 735, 740, 818, 753, 720, 785, 757, 807, 735, 752, 804, 766, 710, 787, 735, 743, 760, 759, 781, 782, 767, 722, 793, 768, 803, 795, 739, 774, 750, 764, 795, 802, 765, 780, 840, 795, 737, 785, 798, 737, 769, 772, 790, 776, 691, 760, 760, 770, 822, 760, 733, 785, 782] payload_rows_per_dst=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] dup_rows_per_dst=[2902, 2950, 3028, 2974, 2992, 3026, 3002, 2938] combine_rows_per_src=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344952 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888056 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431160 weight_movement_bytes=34603008 total_nvlink_bytes=568034168 weight_to_token_ratio=0.064869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266452992 dispatch_meta_bytes=343816 combine_payload_bytes=266452992 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266796808 ingress_bytes=370262016 link_bytes=370262016 token_bytes=533249800 weight_movement_bytes=103809024 total_nvlink_bytes=637058824 weight_to_token_ratio=0.194672 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393266158e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=264019968 dispatch_meta_bytes=341768 combine_payload_bytes=264019968 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264361736 ingress_bytes=298622976 link_bytes=298622976 token_bytes=528381704 weight_movement_bytes=34603008 total_nvlink_bytes=562984712 weight_to_token_ratio=0.065489 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.123694218e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=344280 combine_payload_bytes=266240000 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266584280 ingress_bytes=370049024 link_bytes=370049024 token_bytes=532824280 weight_movement_bytes=103809024 total_nvlink_bytes=636633304 weight_to_token_ratio=0.194828 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392464686e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74484 payload_rows_remote=65166 dup_rows=23820 dup_rows_remote=20876 dedup_ratio=0.242310 router_tokens_per_expert=[762, 792, 766, 770, 749, 748, 768, 692, 776, 781, 791, 742, 763, 767, 783, 728, 829, 766, 755, 779, 848, 770, 726, 747, 708, 779, 785, 773, 774, 767, 789, 769, 806, 785, 805, 737, 761, 734, 754, 798, 783, 795, 765, 711, 748, 783, 754, 750, 786, 791, 782, 732, 816, 783, 740, 792, 785, 777, 745, 763, 765, 749, 776, 759] payload_rows_per_dst=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] dup_rows_per_dst=[2818, 2944, 2990, 3070, 2980, 2916, 3144, 2958] combine_rows_per_src=[9276, 9318, 9450, 9218, 9380, 9262, 9300, 9280] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266780672 dispatch_meta_bytes=344048 combine_payload_bytes=266780672 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267124720 ingress_bytes=335986688 link_bytes=335986688 token_bytes=533905392 weight_movement_bytes=69206016 total_nvlink_bytes=603111408 weight_to_token_ratio=0.129622 gemm_rows_defined=49408 gemm_rows_charged=51200 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.035000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264290858e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344200 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207368 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070536 weight_movement_bytes=69206016 total_nvlink_bytes=601276552 weight_to_token_ratio=0.130069 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65178 dup_rows=23934 dup_rows_remote=20990 dedup_ratio=0.243469 router_tokens_per_expert=[755, 805, 754, 771, 785, 779, 731, 776, 780, 714, 725, 768, 789, 760, 739, 793, 780, 776, 837, 768, 768, 727, 786, 749, 780, 766, 756, 746, 702, 777, 738, 797, 722, 775, 830, 778, 751, 760, 768, 786, 815, 760, 752, 782, 767, 736, 774, 738, 755, 784, 780, 755, 791, 763, 772, 806, 756, 804, 773, 788, 767, 769, 744, 774] payload_rows_per_dst=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] dup_rows_per_dst=[3006, 2944, 3052, 2908, 3010, 2932, 3092, 2990] combine_rows_per_src=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344296 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267256040 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167784 weight_movement_bytes=69206016 total_nvlink_bytes=603373800 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74328 payload_rows_remote=65224 dup_rows=23976 dup_rows_remote=20954 dedup_ratio=0.243896 router_tokens_per_expert=[825, 778, 725, 797, 734, 753, 760, 789, 718, 842, 767, 781, 740, 736, 755, 754, 755, 750, 757, 729, 795, 735, 765, 777, 776, 738, 797, 802, 749, 742, 768, 818, 753, 760, 772, 763, 768, 757, 758, 785, 805, 735, 790, 749, 749, 751, 747, 846, 785, 787, 780, 770, 758, 790, 792, 786, 746, 775, 757, 685, 771, 778, 789, 808] payload_rows_per_dst=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] dup_rows_per_dst=[3054, 2896, 3022, 3014, 2954, 2994, 3058, 2984] combine_rows_per_src=[9268, 9290, 9104, 9366, 9278, 9350, 9438, 9234] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344672 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313760 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282848 weight_movement_bytes=34603008 total_nvlink_bytes=568885856 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344184 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067512 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790840 weight_movement_bytes=34603008 total_nvlink_bytes=568393848 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74154 payload_rows_remote=64908 dup_rows=24150 dup_rows_remote=21142 dedup_ratio=0.245667 router_tokens_per_expert=[829, 797, 762, 770, 754, 778, 752, 747, 791, 743, 758, 767, 811, 751, 759, 776, 766, 751, 721, 754, 785, 773, 748, 716, 783, 789, 807, 783, 765, 747, 790, 767, 777, 786, 773, 768, 763, 747, 733, 780, 778, 769, 729, 795, 768, 771, 802, 788, 782, 726, 739, 738, 796, 767, 762, 799, 744, 738, 765, 780, 787, 765, 767, 780] payload_rows_per_dst=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] dup_rows_per_dst=[3062, 3084, 2876, 3088, 3008, 3090, 2926, 3016] combine_rows_per_src=[9316, 9228, 9152, 9374, 9246, 9310, 9292, 9236] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344168 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267264104 ingress_bytes=301522944 link_bytes=301522944 token_bytes=534184040 weight_movement_bytes=34603008 total_nvlink_bytes=568787048 weight_to_token_ratio=0.064777 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134606564e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=265863168 dispatch_meta_bytes=344200 combine_payload_bytes=265863168 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266207368 ingress_bytes=335069184 link_bytes=335069184 token_bytes=532070536 weight_movement_bytes=69206016 total_nvlink_bytes=601276552 weight_to_token_ratio=0.130069 gemm_rows_defined=50048 gemm_rows_charged=51200 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.022500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.260838364e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74124 payload_rows_remote=64900 dup_rows=24180 dup_rows_remote=21008 dedup_ratio=0.245972 router_tokens_per_expert=[793, 735, 798, 768, 757, 737, 760, 811, 756, 790, 755, 826, 768, 731, 806, 759, 771, 716, 794, 793, 772, 690, 773, 775, 759, 743, 794, 748, 783, 790, 781, 784, 795, 752, 802, 763, 797, 785, 748, 789, 739, 754, 754, 746, 750, 797, 774, 774, 735, 778, 775, 727, 745, 807, 781, 731, 747, 767, 763, 768, 805, 719, 825, 744] payload_rows_per_dst=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] dup_rows_per_dst=[3006, 3042, 2918, 3140, 2978, 2858, 3066, 3172] combine_rows_per_src=[9286, 9340, 9216, 9224, 9388, 9142, 9304, 9224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64886 dup_rows=24132 dup_rows_remote=21122 dedup_ratio=0.245483 router_tokens_per_expert=[802, 747, 782, 767, 740, 755, 741, 750, 763, 756, 746, 789, 824, 782, 764, 772, 777, 776, 781, 766, 801, 723, 795, 767, 788, 787, 722, 746, 785, 801, 785, 775, 776, 775, 730, 777, 715, 767, 773, 798, 778, 775, 772, 777, 774, 788, 785, 746, 775, 787, 726, 718, 787, 788, 826, 741, 776, 779, 726, 753, 767, 729, 781, 732] payload_rows_per_dst=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] dup_rows_per_dst=[2990, 3094, 3092, 3084, 2974, 2984, 3010, 2904] combine_rows_per_src=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74522 payload_rows_remote=65164 dup_rows=23782 dup_rows_remote=20802 dedup_ratio=0.241923 router_tokens_per_expert=[808, 782, 785, 720, 731, 751, 754, 792, 809, 834, 753, 742, 783, 746, 739, 721, 775, 788, 806, 792, 776, 746, 760, 722, 801, 741, 774, 746, 782, 743, 782, 800, 809, 751, 749, 727, 781, 755, 770, 779, 774, 738, 816, 757, 749, 797, 806, 761, 730, 799, 768, 728, 748, 802, 718, 746, 832, 768, 797, 782, 762, 759, 751, 759] payload_rows_per_dst=[9316, 9248, 9346, 9358, 9340, 9396, 9148, 9370] dup_rows_per_dst=[2930, 3006, 2984, 2980, 2902, 3000, 2930, 3050] combine_rows_per_src=[9316, 9248, 9346, 9358, 9340, 9396, 9148, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74492 payload_rows_remote=65164 dup_rows=23812 dup_rows_remote=20910 dedup_ratio=0.242228 router_tokens_per_expert=[751, 743, 805, 776, 746, 778, 787, 709, 782, 755, 812, 735, 740, 818, 753, 720, 785, 757, 807, 735, 752, 804, 766, 710, 787, 735, 743, 760, 759, 781, 782, 767, 722, 793, 768, 803, 795, 739, 774, 750, 764, 795, 802, 765, 780, 840, 795, 737, 785, 798, 737, 769, 772, 790, 776, 691, 760, 760, 770, 822, 760, 733, 785, 782] payload_rows_per_dst=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] dup_rows_per_dst=[2902, 2950, 3028, 2974, 2992, 3026, 3002, 2938] combine_rows_per_src=[9328, 9280, 9310, 9268, 9296, 9476, 9234, 9300] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=65074 dup_rows=24028 dup_rows_remote=21164 dedup_ratio=0.244425 router_tokens_per_expert=[758, 788, 804, 751, 787, 774, 782, 759, 785, 785, 761, 735, 781, 768, 733, 727, 768, 750, 790, 741, 775, 799, 758, 764, 770, 852, 801, 784, 701, 756, 728, 781, 757, 750, 780, 776, 776, 807, 807, 819, 780, 777, 744, 788, 749, 759, 757, 742, 716, 807, 783, 724, 813, 745, 709, 736, 763, 742, 762, 743, 770, 792, 773, 810] payload_rows_per_dst=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] dup_rows_per_dst=[3162, 2914, 3032, 3040, 3092, 2908, 2864, 3016] combine_rows_per_src=[9244, 9236, 9258, 9306, 9452, 9284, 9202, 9294] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74290 payload_rows_remote=64880 dup_rows=24014 dup_rows_remote=20936 dedup_ratio=0.244283 router_tokens_per_expert=[772, 778, 745, 795, 744, 796, 755, 792, 756, 725, 799, 750, 750, 789, 764, 757, 825, 771, 757, 791, 828, 743, 764, 749, 747, 819, 755, 742, 756, 757, 739, 785, 773, 720, 783, 801, 758, 744, 764, 762, 738, 819, 779, 782, 785, 773, 807, 761, 761, 752, 765, 756, 731, 774, 754, 720, 777, 768, 782, 796, 751, 799, 772, 750] payload_rows_per_dst=[9350, 9220, 9314, 9284, 9192, 9410, 9150, 9370] dup_rows_per_dst=[3004, 2960, 3142, 2916, 3018, 3078, 2876, 3020] combine_rows_per_src=[9350, 9220, 9314, 9284, 9192, 9410, 9150, 9370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64732 dup_rows=24184 dup_rows_remote=21148 dedup_ratio=0.246012 router_tokens_per_expert=[794, 759, 735, 770, 768, 776, 790, 791, 764, 724, 744, 751, 778, 810, 782, 766, 711, 803, 816, 764, 737, 776, 746, 781, 755, 765, 770, 807, 747, 767, 753, 767, 804, 771, 779, 760, 795, 753, 773, 777, 748, 732, 817, 737, 793, 778, 755, 784, 767, 783, 740, 800, 750, 799, 777, 787, 762, 775, 746, 721, 780, 743, 769, 730] payload_rows_per_dst=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] dup_rows_per_dst=[3078, 2976, 2966, 3132, 3036, 3036, 3036, 2924] combine_rows_per_src=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=16 plans=1 dispatch_payload_bytes=267157504 dispatch_meta_bytes=344712 combine_payload_bytes=267157504 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267502216 ingress_bytes=301760512 link_bytes=301760512 token_bytes=534659720 weight_movement_bytes=34603008 total_nvlink_bytes=569262728 weight_to_token_ratio=0.064720 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135500513e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65078 dup_rows=23946 dup_rows_remote=20898 dedup_ratio=0.243591 router_tokens_per_expert=[781, 719, 785, 704, 762, 746, 779, 786, 768, 738, 737, 790, 774, 743, 774, 734, 763, 770, 758, 779, 791, 785, 755, 763, 742, 788, 816, 757, 750, 787, 770, 740, 774, 796, 775, 754, 756, 742, 744, 792, 785, 750, 764, 814, 746, 757, 758, 772, 775, 781, 778, 787, 735, 787, 748, 763, 801, 789, 828, 765, 736, 821, 753, 792] payload_rows_per_dst=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] dup_rows_per_dst=[2932, 2940, 3048, 2964, 2972, 2982, 3018, 3090] combine_rows_per_src=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=343632 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266174032 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532004432 weight_movement_bytes=34603008 total_nvlink_bytes=566607440 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344032 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117088 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890144 weight_movement_bytes=34603008 total_nvlink_bytes=566493152 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65054 dup_rows=23792 dup_rows_remote=20808 dedup_ratio=0.242025 router_tokens_per_expert=[775, 756, 812, 791, 775, 763, 744, 805, 751, 776, 760, 795, 728, 805, 769, 755, 758, 782, 745, 770, 800, 751, 776, 799, 774, 771, 718, 810, 751, 750, 792, 764, 822, 762, 749, 773, 728, 776, 757, 750, 757, 777, 783, 763, 802, 731, 774, 753, 736, 787, 775, 767, 776, 764, 789, 810, 732, 799, 770, 737, 720, 739, 774, 749] payload_rows_per_dst=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] dup_rows_per_dst=[2984, 3172, 2820, 3046, 2934, 2944, 3092, 2800] combine_rows_per_src=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=343864 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267255608 ingress_bytes=301514752 link_bytes=301514752 token_bytes=534167352 weight_movement_bytes=34603008 total_nvlink_bytes=568770360 weight_to_token_ratio=0.064779 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134575738e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64840 dup_rows=24060 dup_rows_remote=21118 dedup_ratio=0.244751 router_tokens_per_expert=[768, 786, 772, 775, 796, 784, 784, 768, 762, 813, 818, 748, 734, 735, 811, 752, 821, 761, 750, 770, 800, 785, 777, 778, 791, 766, 743, 766, 790, 715, 753, 772, 794, 768, 751, 753, 785, 803, 777, 703, 725, 762, 795, 748, 761, 768, 755, 773, 732, 773, 705, 771, 751, 769, 776, 796, 743, 829, 726, 761, 713, 773, 790, 779] payload_rows_per_dst=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] dup_rows_per_dst=[3174, 2942, 3118, 3028, 2944, 2950, 2898, 3006] combine_rows_per_src=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266911744 dispatch_meta_bytes=344296 combine_payload_bytes=266911744 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267256040 ingress_bytes=336117760 link_bytes=336117760 token_bytes=534167784 weight_movement_bytes=69206016 total_nvlink_bytes=603373800 weight_to_token_ratio=0.129559 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264784071e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266543104 dispatch_meta_bytes=344952 combine_payload_bytes=266543104 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266888056 ingress_bytes=301146112 link_bytes=301146112 token_bytes=533431160 weight_movement_bytes=34603008 total_nvlink_bytes=568034168 weight_to_token_ratio=0.064869 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133188575e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265748480 dispatch_meta_bytes=343264 combine_payload_bytes=265748480 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266091744 ingress_bytes=369557504 link_bytes=369557504 token_bytes=531840224 weight_movement_bytes=103809024 total_nvlink_bytes=635649248 weight_to_token_ratio=0.195188 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390615136e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=343520 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265485792 ingress_bytes=334348288 link_bytes=334348288 token_bytes=530628064 weight_movement_bytes=69206016 total_nvlink_bytes=599834080 weight_to_token_ratio=0.130423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343448 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266804632 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265816 weight_movement_bytes=69206016 total_nvlink_bytes=602471832 weight_to_token_ratio=0.129778 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343832 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265928472 ingress_bytes=334790656 link_bytes=334790656 token_bytes=531513112 weight_movement_bytes=69206016 total_nvlink_bytes=600719128 weight_to_token_ratio=0.130206 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259790286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74340 payload_rows_remote=65118 dup_rows=23964 dup_rows_remote=20928 dedup_ratio=0.243774 router_tokens_per_expert=[778, 778, 755, 778, 764, 706, 768, 733, 780, 749, 772, 753, 725, 771, 802, 753, 794, 797, 784, 757, 750, 727, 781, 739, 826, 758, 781, 735, 810, 743, 819, 766, 784, 769, 765, 788, 778, 755, 784, 811, 796, 804, 772, 750, 765, 770, 758, 731, 814, 759, 720, 758, 792, 781, 750, 760, 759, 781, 760, 780, 738, 796, 728, 764] payload_rows_per_dst=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] dup_rows_per_dst=[2878, 2868, 3036, 3124, 3140, 2988, 3032, 2898] combine_rows_per_src=[9242, 9342, 9222, 9352, 9328, 9304, 9236, 9314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343904 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903392 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462880 weight_movement_bytes=103809024 total_nvlink_bytes=637271904 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74370 payload_rows_remote=65178 dup_rows=23934 dup_rows_remote=20990 dedup_ratio=0.243469 router_tokens_per_expert=[755, 805, 754, 771, 785, 779, 731, 776, 780, 714, 725, 768, 789, 760, 739, 793, 780, 776, 837, 768, 768, 727, 786, 749, 780, 766, 756, 746, 702, 777, 738, 797, 722, 775, 830, 778, 751, 760, 768, 786, 815, 760, 752, 782, 767, 736, 774, 738, 755, 784, 780, 755, 791, 763, 772, 806, 756, 804, 773, 788, 767, 769, 744, 774] payload_rows_per_dst=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] dup_rows_per_dst=[3006, 2944, 3052, 2908, 3010, 2932, 3092, 2990] combine_rows_per_src=[9306, 9192, 9330, 9216, 9330, 9316, 9320, 9360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74120 payload_rows_remote=64732 dup_rows=24184 dup_rows_remote=21148 dedup_ratio=0.246012 router_tokens_per_expert=[794, 759, 735, 770, 768, 776, 790, 791, 764, 724, 744, 751, 778, 810, 782, 766, 711, 803, 816, 764, 737, 776, 746, 781, 755, 765, 770, 807, 747, 767, 753, 767, 804, 771, 779, 760, 795, 753, 773, 777, 748, 732, 817, 737, 793, 778, 755, 784, 767, 783, 740, 800, 750, 799, 777, 787, 762, 775, 746, 721, 780, 743, 769, 730] payload_rows_per_dst=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] dup_rows_per_dst=[3078, 2976, 2966, 3132, 3036, 3036, 3036, 2924] combine_rows_per_src=[9288, 9262, 9302, 9130, 9388, 9252, 9370, 9128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266969088 dispatch_meta_bytes=344672 combine_payload_bytes=266969088 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267313760 ingress_bytes=301572096 link_bytes=301572096 token_bytes=534282848 weight_movement_bytes=34603008 total_nvlink_bytes=568885856 weight_to_token_ratio=0.064765 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134791519e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=64572 dup_rows=23910 dup_rows_remote=20526 dedup_ratio=0.243225 router_tokens_per_expert=[757, 802, 728, 801, 777, 780, 738, 765, 759, 772, 783, 806, 752, 823, 773, 730, 771, 740, 794, 760, 766, 774, 787, 737, 770, 821, 713, 807, 780, 805, 795, 719, 788, 780, 791, 781, 748, 747, 764, 763, 732, 751, 782, 777, 772, 773, 788, 712, 785, 750, 784, 701, 788, 759, 829, 729, 789, 731, 768, 745, 754, 780, 759, 767] payload_rows_per_dst=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] dup_rows_per_dst=[2830, 2564, 3064, 3078, 3036, 2898, 3056, 3384] combine_rows_per_src=[9148, 8812, 9280, 9342, 9336, 9276, 9378, 9822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20774 dedup_ratio=0.242635 router_tokens_per_expert=[815, 798, 759, 794, 776, 706, 765, 766, 735, 762, 795, 764, 762, 783, 758, 744, 757, 770, 751, 733, 760, 770, 811, 804, 817, 784, 784, 762, 758, 830, 722, 772, 763, 729, 747, 809, 765, 769, 786, 757, 770, 755, 775, 743, 713, 774, 720, 831, 785, 763, 752, 783, 766, 760, 778, 829, 744, 786, 790, 727, 763, 768, 751, 734] payload_rows_per_dst=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] dup_rows_per_dst=[2996, 2952, 2932, 3094, 2996, 2904, 3078, 2900] combine_rows_per_src=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74460 payload_rows_remote=65154 dup_rows=23844 dup_rows_remote=20824 dedup_ratio=0.242554 router_tokens_per_expert=[804, 740, 772, 793, 749, 757, 800, 720, 823, 754, 754, 737, 788, 758, 793, 795, 789, 752, 772, 728, 754, 781, 742, 715, 771, 788, 766, 744, 758, 802, 728, 806, 775, 740, 785, 778, 817, 729, 768, 720, 786, 744, 751, 749, 749, 734, 766, 788, 792, 730, 803, 806, 782, 790, 789, 750, 747, 756, 799, 762, 789, 740, 812, 793] payload_rows_per_dst=[9330, 9384, 9192, 9306, 9310, 9204, 9376, 9358] dup_rows_per_dst=[2940, 3020, 2874, 3020, 2914, 2930, 3108, 3038] combine_rows_per_src=[9330, 9384, 9192, 9306, 9310, 9204, 9376, 9358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265142272 dispatch_meta_bytes=343520 combine_payload_bytes=265142272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265485792 ingress_bytes=334348288 link_bytes=334348288 token_bytes=530628064 weight_movement_bytes=69206016 total_nvlink_bytes=599834080 weight_to_token_ratio=0.130423 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258125690e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74512 payload_rows_remote=65054 dup_rows=23792 dup_rows_remote=20808 dedup_ratio=0.242025 router_tokens_per_expert=[775, 756, 812, 791, 775, 763, 744, 805, 751, 776, 760, 795, 728, 805, 769, 755, 758, 782, 745, 770, 800, 751, 776, 799, 774, 771, 718, 810, 751, 750, 792, 764, 822, 762, 749, 773, 728, 776, 757, 750, 757, 777, 783, 763, 802, 731, 774, 753, 736, 787, 775, 767, 776, 764, 789, 810, 732, 799, 770, 737, 720, 739, 774, 749] payload_rows_per_dst=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] dup_rows_per_dst=[2984, 3172, 2820, 3046, 2934, 2944, 3092, 2800] combine_rows_per_src=[9458, 9434, 9214, 9292, 9300, 9258, 9316, 9240] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64886 dup_rows=24132 dup_rows_remote=21122 dedup_ratio=0.245483 router_tokens_per_expert=[802, 747, 782, 767, 740, 755, 741, 750, 763, 756, 746, 789, 824, 782, 764, 772, 777, 776, 781, 766, 801, 723, 795, 767, 788, 787, 722, 746, 785, 801, 785, 775, 776, 775, 730, 777, 715, 767, 773, 798, 778, 775, 772, 777, 774, 788, 785, 746, 775, 787, 726, 718, 787, 788, 826, 741, 776, 779, 726, 753, 767, 729, 781, 732] payload_rows_per_dst=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] dup_rows_per_dst=[2990, 3094, 3092, 3084, 2974, 2984, 3010, 2904] combine_rows_per_src=[9178, 9298, 9280, 9294, 9248, 9406, 9286, 9182] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74192 payload_rows_remote=64900 dup_rows=24112 dup_rows_remote=21270 dedup_ratio=0.245280 router_tokens_per_expert=[774, 785, 722, 784, 755, 771, 764, 771, 801, 758, 792, 831, 805, 812, 761, 804, 772, 791, 804, 787, 765, 800, 765, 772, 788, 762, 761, 768, 757, 755, 737, 766, 763, 799, 720, 774, 742, 716, 752, 761, 793, 730, 773, 778, 749, 735, 743, 766, 790, 771, 793, 739, 779, 792, 767, 755, 760, 732, 772, 758, 771, 767, 743, 729] payload_rows_per_dst=[9150, 9514, 9430, 9302, 9138, 9292, 9280, 9086] dup_rows_per_dst=[3102, 3214, 3082, 2886, 2916, 2842, 3092, 2978] combine_rows_per_src=[9150, 9514, 9430, 9302, 9138, 9292, 9280, 9086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64742 dup_rows=24044 dup_rows_remote=20990 dedup_ratio=0.244588 router_tokens_per_expert=[726, 792, 757, 778, 735, 788, 761, 740, 728, 765, 815, 730, 725, 828, 799, 726, 787, 790, 730, 767, 707, 719, 814, 748, 771, 774, 766, 775, 794, 806, 746, 777, 788, 796, 742, 765, 779, 803, 803, 810, 792, 787, 780, 732, 757, 762, 780, 779, 784, 819, 778, 824, 753, 773, 761, 771, 745, 759, 785, 715, 751, 766, 674, 775] payload_rows_per_dst=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] dup_rows_per_dst=[2936, 3016, 2994, 3090, 3054, 2958, 3108, 2888] combine_rows_per_src=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=17 plans=1 dispatch_payload_bytes=266723328 dispatch_meta_bytes=344184 combine_payload_bytes=266723328 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267067512 ingress_bytes=301326336 link_bytes=301326336 token_bytes=533790840 weight_movement_bytes=34603008 total_nvlink_bytes=568393848 weight_to_token_ratio=0.064825 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133866744e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65024 dup_rows=24012 dup_rows_remote=21050 dedup_ratio=0.244263 router_tokens_per_expert=[756, 745, 750, 816, 775, 777, 785, 787, 763, 759, 735, 783, 818, 796, 812, 750, 754, 759, 783, 746, 790, 761, 773, 749, 795, 783, 794, 748, 755, 767, 764, 783, 773, 753, 769, 755, 771, 769, 734, 730, 749, 759, 742, 781, 803, 736, 761, 763, 773, 763, 807, 749, 773, 761, 812, 716, 704, 744, 727, 753, 812, 794, 780, 825] payload_rows_per_dst=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] dup_rows_per_dst=[2940, 3150, 2962, 3044, 2912, 2994, 3060, 2950] combine_rows_per_src=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=264486912 dispatch_meta_bytes=340392 combine_payload_bytes=264486912 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264827304 ingress_bytes=299089920 link_bytes=299089920 token_bytes=529314216 weight_movement_bytes=34603008 total_nvlink_bytes=563917224 weight_to_token_ratio=0.065373 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125451290e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343488 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266984896 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626304 weight_movement_bytes=69206016 total_nvlink_bytes=602832320 weight_to_token_ratio=0.129690 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65160 dup_rows=23744 dup_rows_remote=20700 dedup_ratio=0.241536 router_tokens_per_expert=[794, 777, 756, 787, 748, 757, 796, 806, 775, 763, 722, 758, 738, 796, 764, 766, 763, 770, 741, 773, 800, 760, 767, 742, 781, 752, 781, 773, 764, 763, 823, 734, 790, 749, 802, 833, 761, 763, 772, 758, 805, 790, 765, 783, 775, 725, 778, 747, 735, 753, 782, 728, 832, 767, 760, 776, 764, 742, 773, 754, 762, 776, 727, 735] payload_rows_per_dst=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] dup_rows_per_dst=[3044, 3082, 2886, 2836, 3092, 3348, 2674, 2782] combine_rows_per_src=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343912 combine_payload_bytes=266870784 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=267214696 ingress_bytes=336076800 link_bytes=336076800 token_bytes=534085480 weight_movement_bytes=69206016 total_nvlink_bytes=603291496 weight_to_token_ratio=0.129579 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264629942e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266461184 dispatch_meta_bytes=343448 combine_payload_bytes=266461184 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266804632 ingress_bytes=335667200 link_bytes=335667200 token_bytes=533265816 weight_movement_bytes=69206016 total_nvlink_bytes=602471832 weight_to_token_ratio=0.129778 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263088650e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=344032 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266117088 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890144 weight_movement_bytes=34603008 total_nvlink_bytes=566493152 weight_to_token_ratio=0.065057 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74244 payload_rows_remote=64840 dup_rows=24060 dup_rows_remote=21118 dedup_ratio=0.244751 router_tokens_per_expert=[768, 786, 772, 775, 796, 784, 784, 768, 762, 813, 818, 748, 734, 735, 811, 752, 821, 761, 750, 770, 800, 785, 777, 778, 791, 766, 743, 766, 790, 715, 753, 772, 794, 768, 751, 753, 785, 803, 777, 703, 725, 762, 795, 748, 761, 768, 755, 773, 732, 773, 705, 771, 751, 769, 776, 796, 743, 829, 726, 761, 713, 773, 790, 779] payload_rows_per_dst=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] dup_rows_per_dst=[3174, 2942, 3118, 3028, 2944, 2950, 2898, 3006] combine_rows_per_src=[9292, 9404, 9366, 9164, 9324, 9224, 9248, 9222] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265830400 dispatch_meta_bytes=344680 combine_payload_bytes=265830400 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266175080 ingress_bytes=300433408 link_bytes=300433408 token_bytes=532005480 weight_movement_bytes=34603008 total_nvlink_bytes=566608488 weight_to_token_ratio=0.065043 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130506727e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65162 dup_rows=23878 dup_rows_remote=20830 dedup_ratio=0.242900 router_tokens_per_expert=[763, 858, 788, 731, 755, 812, 793, 803, 762, 749, 803, 732, 774, 766, 797, 773, 777, 752, 773, 787, 789, 757, 769, 781, 753, 809, 768, 786, 790, 805, 744, 765, 797, 708, 778, 760, 750, 735, 805, 794, 713, 767, 748, 761, 720, 764, 778, 744, 860, 763, 771, 698, 743, 744, 761, 775, 782, 709, 774, 773, 754, 755, 748, 756] payload_rows_per_dst=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] dup_rows_per_dst=[3088, 3048, 3022, 3036, 2952, 2850, 2940, 2942] combine_rows_per_src=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=342928 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265526160 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530709392 weight_movement_bytes=34603008 total_nvlink_bytes=565312400 weight_to_token_ratio=0.065201 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343440 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267238800 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534134160 weight_movement_bytes=138412032 total_nvlink_bytes=672546192 weight_to_token_ratio=0.259133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343968 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247520 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151072 weight_movement_bytes=34603008 total_nvlink_bytes=568754080 weight_to_token_ratio=0.064781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344296 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682600 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020904 weight_movement_bytes=34603008 total_nvlink_bytes=567623912 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=64742 dup_rows=24044 dup_rows_remote=20990 dedup_ratio=0.244588 router_tokens_per_expert=[726, 792, 757, 778, 735, 788, 761, 740, 728, 765, 815, 730, 725, 828, 799, 726, 787, 790, 730, 767, 707, 719, 814, 748, 771, 774, 766, 775, 794, 806, 746, 777, 788, 796, 742, 765, 779, 803, 803, 810, 792, 787, 780, 732, 757, 762, 780, 779, 784, 819, 778, 824, 753, 773, 761, 771, 745, 759, 785, 715, 751, 766, 674, 775] payload_rows_per_dst=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] dup_rows_per_dst=[2936, 3016, 2994, 3090, 3054, 2958, 3108, 2888] combine_rows_per_src=[9218, 9216, 9130, 9328, 9518, 9380, 9418, 9052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343832 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265928472 ingress_bytes=334790656 link_bytes=334790656 token_bytes=531513112 weight_movement_bytes=69206016 total_nvlink_bytes=600719128 weight_to_token_ratio=0.130206 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.259790286e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74172 payload_rows_remote=64728 dup_rows=24132 dup_rows_remote=20932 dedup_ratio=0.245483 router_tokens_per_expert=[718, 788, 768, 768, 775, 790, 804, 772, 755, 713, 713, 775, 774, 782, 745, 766, 777, 811, 813, 763, 761, 773, 722, 738, 751, 751, 745, 759, 798, 798, 724, 739, 747, 783, 815, 749, 750, 783, 757, 773, 795, 767, 761, 769, 777, 808, 788, 746, 750, 716, 781, 758, 774, 762, 761, 797, 788, 782, 818, 778, 827, 736, 755, 772] payload_rows_per_dst=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] dup_rows_per_dst=[2702, 3294, 2888, 2986, 2896, 3172, 2994, 3200] combine_rows_per_src=[8976, 9440, 9296, 9068, 9170, 9556, 9222, 9444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64886 dup_rows=24122 dup_rows_remote=21096 dedup_ratio=0.245382 router_tokens_per_expert=[767, 775, 765, 754, 776, 809, 784, 788, 736, 745, 801, 800, 748, 764, 751, 776, 762, 787, 766, 752, 807, 770, 750, 769, 778, 747, 754, 800, 777, 787, 763, 752, 762, 795, 781, 750, 809, 759, 787, 760, 769, 783, 764, 757, 740, 766, 773, 756, 782, 831, 762, 735, 714, 773, 790, 774, 772, 738, 753, 737, 773, 750, 772, 725] payload_rows_per_dst=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] dup_rows_per_dst=[3104, 2910, 3062, 2982, 3074, 2972, 3026, 2992] combine_rows_per_src=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=65078 dup_rows=23946 dup_rows_remote=20898 dedup_ratio=0.243591 router_tokens_per_expert=[781, 719, 785, 704, 762, 746, 779, 786, 768, 738, 737, 790, 774, 743, 774, 734, 763, 770, 758, 779, 791, 785, 755, 763, 742, 788, 816, 757, 750, 787, 770, 740, 774, 796, 775, 754, 756, 742, 744, 792, 785, 750, 764, 814, 746, 757, 758, 772, 775, 781, 778, 787, 735, 787, 748, 763, 801, 789, 828, 765, 736, 821, 753, 792] payload_rows_per_dst=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] dup_rows_per_dst=[2932, 2940, 3048, 2964, 2972, 2982, 3018, 3090] combine_rows_per_src=[9192, 9176, 9280, 9336, 9294, 9310, 9290, 9480] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74300 payload_rows_remote=64940 dup_rows=24004 dup_rows_remote=21006 dedup_ratio=0.244181 router_tokens_per_expert=[765, 725, 774, 776, 805, 793, 749, 769, 739, 766, 770, 801, 778, 730, 749, 739, 715, 778, 765, 748, 763, 777, 758, 749, 714, 768, 712, 779, 738, 803, 820, 845, 797, 776, 730, 792, 805, 746, 755, 771, 756, 783, 755, 787, 748, 748, 763, 792, 768, 793, 794, 807, 729, 811, 733, 775, 767, 783, 784, 750, 781, 764, 803, 746] payload_rows_per_dst=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] dup_rows_per_dst=[3002, 2968, 2970, 2998, 2962, 2970, 3130, 3004] combine_rows_per_src=[9310, 9176, 9136, 9360, 9382, 9294, 9290, 9352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=265183232 dispatch_meta_bytes=342928 combine_payload_bytes=265183232 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265526160 ingress_bytes=299786240 link_bytes=299786240 token_bytes=530709392 weight_movement_bytes=34603008 total_nvlink_bytes=565312400 weight_to_token_ratio=0.065201 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.128071486e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74560 payload_rows_remote=65160 dup_rows=23744 dup_rows_remote=20700 dedup_ratio=0.241536 router_tokens_per_expert=[794, 777, 756, 787, 748, 757, 796, 806, 775, 763, 722, 758, 738, 796, 764, 766, 763, 770, 741, 773, 800, 760, 767, 742, 781, 752, 781, 773, 764, 763, 823, 734, 790, 749, 802, 833, 761, 763, 772, 758, 805, 790, 765, 783, 775, 725, 778, 747, 735, 753, 782, 728, 832, 767, 760, 776, 764, 742, 773, 754, 762, 776, 727, 735] payload_rows_per_dst=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] dup_rows_per_dst=[3044, 3082, 2886, 2836, 3092, 3348, 2674, 2782] combine_rows_per_src=[9400, 9498, 9346, 9088, 9364, 9652, 8928, 9284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74452 payload_rows_remote=65098 dup_rows=23852 dup_rows_remote=20774 dedup_ratio=0.242635 router_tokens_per_expert=[815, 798, 759, 794, 776, 706, 765, 766, 735, 762, 795, 764, 762, 783, 758, 744, 757, 770, 751, 733, 760, 770, 811, 804, 817, 784, 784, 762, 758, 830, 722, 772, 763, 729, 747, 809, 765, 769, 786, 757, 770, 755, 775, 743, 713, 774, 720, 831, 785, 763, 752, 783, 766, 760, 778, 829, 744, 786, 790, 727, 763, 768, 751, 734] payload_rows_per_dst=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] dup_rows_per_dst=[2996, 2952, 2932, 3094, 2996, 2904, 3078, 2900] combine_rows_per_src=[9362, 9254, 9380, 9364, 9254, 9258, 9354, 9226] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65122 dup_rows=23768 dup_rows_remote=20858 dedup_ratio=0.241781 router_tokens_per_expert=[739, 751, 755, 834, 738, 776, 760, 801, 729, 789, 796, 748, 753, 760, 767, 712, 767, 783, 796, 814, 791, 734, 748, 750, 789, 755, 775, 762, 793, 766, 773, 750, 774, 776, 765, 753, 722, 782, 772, 764, 747, 738, 824, 809, 788, 751, 745, 760, 727, 842, 761, 788, 783, 765, 776, 797, 778, 730, 756, 751, 752, 795, 766, 761] payload_rows_per_dst=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] dup_rows_per_dst=[3030, 2878, 3018, 2946, 3062, 2910, 3070, 2854] combine_rows_per_src=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64846 dup_rows=24102 dup_rows_remote=21074 dedup_ratio=0.245178 router_tokens_per_expert=[787, 771, 758, 740, 776, 769, 733, 755, 791, 757, 768, 778, 704, 817, 771, 754, 796, 764, 785, 839, 766, 741, 738, 758, 776, 755, 735, 748, 806, 774, 784, 724, 793, 762, 827, 795, 735, 749, 738, 793, 747, 825, 700, 800, 777, 784, 712, 768, 811, 753, 805, 787, 762, 734, 761, 796, 797, 733, 796, 749, 779, 733, 756, 777] payload_rows_per_dst=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] dup_rows_per_dst=[3092, 2960, 3044, 2986, 3028, 2998, 3004, 2990] combine_rows_per_src=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74536 payload_rows_remote=65122 dup_rows=23768 dup_rows_remote=20858 dedup_ratio=0.241781 router_tokens_per_expert=[739, 751, 755, 834, 738, 776, 760, 801, 729, 789, 796, 748, 753, 760, 767, 712, 767, 783, 796, 814, 791, 734, 748, 750, 789, 755, 775, 762, 793, 766, 773, 750, 774, 776, 765, 753, 722, 782, 772, 764, 747, 738, 824, 809, 788, 751, 745, 760, 727, 842, 761, 788, 783, 765, 776, 797, 778, 730, 756, 751, 752, 795, 766, 761] payload_rows_per_dst=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] dup_rows_per_dst=[3030, 2878, 3018, 2946, 3062, 2910, 3070, 2854] combine_rows_per_src=[9278, 9230, 9348, 9380, 9154, 9414, 9408, 9324] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74680 payload_rows_remote=65424 dup_rows=23624 dup_rows_remote=20754 dedup_ratio=0.240316 router_tokens_per_expert=[784, 781, 785, 803, 766, 760, 678, 758, 792, 783, 816, 752, 806, 785, 718, 738, 751, 744, 758, 768, 747, 802, 763, 730, 747, 762, 730, 789, 763, 712, 816, 767, 792, 788, 752, 776, 819, 831, 794, 764, 729, 712, 763, 844, 787, 762, 777, 808, 757, 745, 789, 740, 754, 774, 770, 751, 801, 753, 766, 762, 777, 776, 753, 732] payload_rows_per_dst=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] dup_rows_per_dst=[2986, 3010, 2870, 2836, 3106, 3054, 2886, 2876] combine_rows_per_src=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265125888 dispatch_meta_bytes=342640 combine_payload_bytes=265125888 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265468528 ingress_bytes=299728896 link_bytes=299728896 token_bytes=530594416 weight_movement_bytes=34603008 total_nvlink_bytes=565197424 weight_to_token_ratio=0.065216 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.127855705e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64744 dup_rows=24170 dup_rows_remote=21004 dedup_ratio=0.245870 router_tokens_per_expert=[777, 772, 775, 790, 768, 750, 787, 751, 766, 760, 749, 755, 713, 755, 793, 761, 770, 823, 772, 778, 781, 778, 763, 751, 757, 818, 770, 754, 803, 785, 755, 785, 766, 760, 736, 773, 739, 745, 790, 800, 798, 750, 742, 774, 778, 728, 761, 765, 755, 765, 738, 729, 761, 789, 762, 731, 737, 799, 807, 801, 770, 728, 838, 772] payload_rows_per_dst=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] dup_rows_per_dst=[3166, 2962, 3102, 3128, 2902, 2954, 2882, 3074] combine_rows_per_src=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343928 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116984 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890040 weight_movement_bytes=34603008 total_nvlink_bytes=566493048 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=18 plans=1 dispatch_payload_bytes=266559488 dispatch_meta_bytes=343904 combine_payload_bytes=266559488 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266903392 ingress_bytes=370368512 link_bytes=370368512 token_bytes=533462880 weight_movement_bytes=103809024 total_nvlink_bytes=637271904 weight_to_token_ratio=0.194595 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.393666894e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=343784 combine_payload_bytes=265994240 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266338024 ingress_bytes=369803264 link_bytes=369803264 token_bytes=532332264 weight_movement_bytes=103809024 total_nvlink_bytes=636141288 weight_to_token_ratio=0.195008 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391539911e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65158 dup_rows=23830 dup_rows_remote=20746 dedup_ratio=0.242411 router_tokens_per_expert=[752, 770, 790, 749, 782, 764, 720, 729, 741, 739, 785, 782, 804, 792, 770, 787, 748, 792, 780, 778, 801, 772, 767, 749, 772, 766, 780, 729, 775, 752, 719, 774, 735, 739, 756, 742, 724, 835, 776, 773, 742, 808, 758, 766, 749, 780, 761, 780, 810, 814, 755, 759, 812, 739, 788, 728, 757, 790, 757, 780, 794, 759, 749, 827] payload_rows_per_dst=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] dup_rows_per_dst=[3004, 3084, 3006, 2884, 2854, 2886, 3016, 3096] combine_rows_per_src=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343440 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=103809024 egress_bytes=267238800 ingress_bytes=405307392 link_bytes=405307392 token_bytes=534134160 weight_movement_bytes=138412032 total_nvlink_bytes=672546192 weight_to_token_ratio=0.259133 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.525139086e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=343488 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266984896 ingress_bytes=335847424 link_bytes=335847424 token_bytes=533626304 weight_movement_bytes=69206016 total_nvlink_bytes=602832320 weight_to_token_ratio=0.129690 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263766819e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74426 payload_rows_remote=65162 dup_rows=23878 dup_rows_remote=20830 dedup_ratio=0.242900 router_tokens_per_expert=[763, 858, 788, 731, 755, 812, 793, 803, 762, 749, 803, 732, 774, 766, 797, 773, 777, 752, 773, 787, 789, 757, 769, 781, 753, 809, 768, 786, 790, 805, 744, 765, 797, 708, 778, 760, 750, 735, 805, 794, 713, 767, 748, 761, 720, 764, 778, 744, 860, 763, 771, 698, 743, 744, 761, 775, 782, 709, 774, 773, 754, 755, 748, 756] payload_rows_per_dst=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] dup_rows_per_dst=[3088, 3048, 3022, 3036, 2952, 2850, 2940, 2942] combine_rows_per_src=[9518, 9264, 9348, 9404, 9302, 9140, 9290, 9160] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266739712 dispatch_meta_bytes=343920 combine_payload_bytes=266739712 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267083632 ingress_bytes=301342720 link_bytes=301342720 token_bytes=533823344 weight_movement_bytes=34603008 total_nvlink_bytes=568426352 weight_to_token_ratio=0.064821 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133928395e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343680 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531562112 weight_movement_bytes=103809024 total_nvlink_bytes=635371136 weight_to_token_ratio=0.195290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=342992 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265534416 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530725840 weight_movement_bytes=69206016 total_nvlink_bytes=599931856 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343616 combine_payload_bytes=266887168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267230784 ingress_bytes=370696192 link_bytes=370696192 token_bytes=534117952 weight_movement_bytes=103809024 total_nvlink_bytes=637926976 weight_to_token_ratio=0.194356 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394899927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344712 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321416 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298120 weight_movement_bytes=69206016 total_nvlink_bytes=605504136 weight_to_token_ratio=0.129044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74202 payload_rows_remote=64846 dup_rows=24102 dup_rows_remote=21074 dedup_ratio=0.245178 router_tokens_per_expert=[787, 771, 758, 740, 776, 769, 733, 755, 791, 757, 768, 778, 704, 817, 771, 754, 796, 764, 785, 839, 766, 741, 738, 758, 776, 755, 735, 748, 806, 774, 784, 724, 793, 762, 827, 795, 735, 749, 738, 793, 747, 825, 700, 800, 777, 784, 712, 768, 811, 753, 805, 787, 762, 734, 761, 796, 797, 733, 796, 749, 779, 733, 756, 777] payload_rows_per_dst=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] dup_rows_per_dst=[3092, 2960, 3044, 2986, 3028, 2998, 3004, 2990] combine_rows_per_src=[9086, 9320, 9330, 9218, 9356, 9228, 9414, 9250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=266903552 dispatch_meta_bytes=343968 combine_payload_bytes=266903552 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267247520 ingress_bytes=301506560 link_bytes=301506560 token_bytes=534151072 weight_movement_bytes=34603008 total_nvlink_bytes=568754080 weight_to_token_ratio=0.064781 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134544912e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74236 payload_rows_remote=64588 dup_rows=24068 dup_rows_remote=20734 dedup_ratio=0.244832 router_tokens_per_expert=[780, 762, 772, 717, 732, 810, 777, 742, 751, 804, 769, 751, 791, 775, 768, 770, 802, 813, 757, 808, 755, 751, 771, 755, 798, 793, 741, 739, 763, 753, 781, 764, 764, 736, 805, 754, 756, 752, 764, 787, 747, 744, 789, 774, 743, 755, 769, 757, 785, 754, 709, 740, 760, 779, 777, 769, 853, 785, 753, 804, 792, 769, 782, 730] payload_rows_per_dst=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] dup_rows_per_dst=[3058, 2800, 3308, 2958, 2868, 2926, 2816, 3334] combine_rows_per_src=[9126, 9062, 9600, 9270, 9170, 9238, 9122, 9648] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64934 dup_rows=24142 dup_rows_remote=21056 dedup_ratio=0.245585 router_tokens_per_expert=[769, 755, 747, 758, 774, 804, 788, 780, 781, 744, 782, 744, 764, 755, 792, 796, 710, 770, 740, 770, 757, 693, 795, 783, 781, 788, 754, 752, 760, 773, 740, 804, 775, 781, 750, 783, 732, 790, 740, 784, 776, 792, 759, 762, 797, 800, 752, 753, 739, 757, 797, 782, 756, 767, 796, 763, 788, 785, 792, 818, 763, 761, 707, 752] payload_rows_per_dst=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] dup_rows_per_dst=[3014, 3074, 2856, 3014, 2964, 3020, 3086, 3114] combine_rows_per_src=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74292 payload_rows_remote=65024 dup_rows=24012 dup_rows_remote=21050 dedup_ratio=0.244263 router_tokens_per_expert=[756, 745, 750, 816, 775, 777, 785, 787, 763, 759, 735, 783, 818, 796, 812, 750, 754, 759, 783, 746, 790, 761, 773, 749, 795, 783, 794, 748, 755, 767, 764, 783, 773, 753, 769, 755, 771, 769, 734, 730, 749, 759, 742, 781, 803, 736, 761, 763, 773, 763, 807, 749, 773, 761, 812, 716, 704, 744, 727, 753, 812, 794, 780, 825] payload_rows_per_dst=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] dup_rows_per_dst=[2940, 3150, 2962, 3044, 2912, 2994, 3060, 2950] combine_rows_per_src=[9442, 9282, 9268, 9334, 9196, 9194, 9248, 9328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74394 payload_rows_remote=65000 dup_rows=23910 dup_rows_remote=20928 dedup_ratio=0.243225 router_tokens_per_expert=[782, 759, 797, 773, 777, 738, 747, 758, 780, 774, 728, 816, 800, 765, 783, 793, 763, 756, 794, 777, 742, 819, 741, 758, 771, 772, 788, 768, 755, 825, 743, 766, 755, 790, 730, 730, 732, 755, 797, 768, 713, 791, 789, 787, 742, 765, 789, 780, 773, 754, 734, 758, 810, 792, 770, 802, 757, 752, 776, 780, 734, 690, 705, 844] payload_rows_per_dst=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] dup_rows_per_dst=[2994, 3054, 3062, 2982, 2912, 2944, 3110, 2852] combine_rows_per_src=[9268, 9424, 9238, 9394, 9202, 9368, 9276, 9224] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265609216 dispatch_meta_bytes=343680 combine_payload_bytes=265609216 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=265952896 ingress_bytes=369418240 link_bytes=369418240 token_bytes=531562112 weight_movement_bytes=103809024 total_nvlink_bytes=635371136 weight_to_token_ratio=0.195290 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390091097e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74134 payload_rows_remote=64744 dup_rows=24170 dup_rows_remote=21004 dedup_ratio=0.245870 router_tokens_per_expert=[777, 772, 775, 790, 768, 750, 787, 751, 766, 760, 749, 755, 713, 755, 793, 761, 770, 823, 772, 778, 781, 778, 763, 751, 757, 818, 770, 754, 803, 785, 755, 785, 766, 760, 736, 773, 739, 745, 790, 800, 798, 750, 742, 774, 778, 728, 761, 765, 755, 765, 738, 729, 761, 789, 762, 731, 737, 799, 807, 801, 770, 728, 838, 772] payload_rows_per_dst=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] dup_rows_per_dst=[3166, 2962, 3102, 3128, 2902, 2954, 2882, 3074] combine_rows_per_src=[9390, 9142, 9350, 9306, 9100, 9238, 9178, 9430] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74182 payload_rows_remote=64886 dup_rows=24122 dup_rows_remote=21096 dedup_ratio=0.245382 router_tokens_per_expert=[767, 775, 765, 754, 776, 809, 784, 788, 736, 745, 801, 800, 748, 764, 751, 776, 762, 787, 766, 752, 807, 770, 750, 769, 778, 747, 754, 800, 777, 787, 763, 752, 762, 795, 781, 750, 809, 759, 787, 760, 769, 783, 764, 757, 740, 766, 773, 756, 782, 831, 762, 735, 714, 773, 790, 774, 772, 738, 753, 737, 773, 750, 772, 725] payload_rows_per_dst=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] dup_rows_per_dst=[3104, 2910, 3062, 2982, 3074, 2972, 3026, 2992] combine_rows_per_src=[9332, 9332, 9264, 9334, 9332, 9244, 9296, 9048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74358 payload_rows_remote=64982 dup_rows=23946 dup_rows_remote=21008 dedup_ratio=0.243591 router_tokens_per_expert=[836, 793, 743, 786, 756, 790, 770, 796, 769, 767, 715, 793, 773, 812, 781, 795, 737, 756, 796, 769, 778, 749, 789, 729, 756, 737, 765, 785, 762, 747, 791, 769, 802, 850, 722, 795, 774, 774, 786, 767, 755, 748, 756, 792, 797, 792, 745, 772, 771, 702, 769, 783, 753, 778, 759, 723, 807, 742, 717, 764, 771, 713, 729, 754] payload_rows_per_dst=[9420, 9262, 9332, 9196, 9358, 9376, 9224, 9190] dup_rows_per_dst=[3120, 3148, 2874, 3028, 3182, 2938, 2852, 2804] combine_rows_per_src=[9420, 9262, 9332, 9196, 9358, 9376, 9224, 9190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65160 dup_rows=23872 dup_rows_remote=20754 dedup_ratio=0.242839 router_tokens_per_expert=[742, 814, 736, 751, 704, 724, 799, 792, 778, 773, 775, 753, 815, 754, 767, 772, 771, 744, 777, 753, 713, 740, 752, 750, 781, 778, 788, 757, 736, 791, 796, 767, 759, 734, 798, 762, 760, 821, 779, 782, 717, 803, 804, 784, 770, 754, 739, 784, 781, 793, 830, 761, 735, 794, 748, 767, 756, 784, 751, 759, 767, 752, 819, 762] payload_rows_per_dst=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] dup_rows_per_dst=[2928, 3012, 2768, 2984, 3118, 3052, 3008, 3002] combine_rows_per_src=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20914 dedup_ratio=0.243490 router_tokens_per_expert=[775, 780, 762, 732, 773, 767, 774, 778, 718, 778, 761, 759, 781, 790, 802, 758, 749, 761, 779, 735, 767, 792, 780, 806, 780, 787, 767, 768, 781, 754, 763, 762, 814, 752, 749, 734, 801, 773, 752, 786, 740, 792, 751, 742, 772, 757, 750, 806, 823, 741, 763, 805, 755, 787, 764, 782, 729, 775, 766, 759, 737, 742, 745, 789] payload_rows_per_dst=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] dup_rows_per_dst=[3022, 2974, 3014, 3036, 2974, 2886, 3062, 2968] combine_rows_per_src=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65268 dup_rows=23788 dup_rows_remote=20894 dedup_ratio=0.241984 router_tokens_per_expert=[706, 814, 802, 741, 775, 778, 765, 768, 753, 746, 785, 733, 747, 773, 764, 784, 759, 755, 763, 780, 747, 779, 742, 746, 786, 785, 770, 788, 756, 778, 746, 754, 772, 781, 801, 796, 719, 817, 766, 783, 768, 736, 790, 778, 789, 802, 759, 811, 751, 780, 773, 804, 767, 725, 808, 753, 716, 762, 727, 757, 771, 778, 774, 770] payload_rows_per_dst=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] dup_rows_per_dst=[3062, 2916, 2894, 2924, 3076, 3078, 3002, 2836] combine_rows_per_src=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=264552448 dispatch_meta_bytes=341288 combine_payload_bytes=264552448 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264893736 ingress_bytes=299155456 link_bytes=299155456 token_bytes=529446184 weight_movement_bytes=34603008 total_nvlink_bytes=564049192 weight_to_token_ratio=0.065357 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.125697897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343960 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313624 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283288 weight_movement_bytes=69206016 total_nvlink_bytes=601489304 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64996 dup_rows=24036 dup_rows_remote=20970 dedup_ratio=0.244507 router_tokens_per_expert=[735, 767, 754, 792, 802, 801, 787, 746, 761, 806, 793, 725, 773, 770, 760, 781, 748, 799, 816, 813, 812, 790, 765, 794, 771, 767, 805, 734, 752, 727, 768, 734, 728, 804, 763, 744, 725, 722, 783, 732, 786, 772, 744, 787, 733, 768, 771, 787, 795, 757, 742, 820, 720, 798, 776, 813, 761, 771, 707, 802, 737, 770, 758, 728] payload_rows_per_dst=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] dup_rows_per_dst=[3004, 3066, 3206, 2950, 2874, 2960, 3070, 2906] combine_rows_per_src=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=19 plans=1 dispatch_payload_bytes=266338304 dispatch_meta_bytes=344296 combine_payload_bytes=266338304 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266682600 ingress_bytes=300941312 link_bytes=300941312 token_bytes=533020904 weight_movement_bytes=34603008 total_nvlink_bytes=567623912 weight_to_token_ratio=0.064919 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132417929e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266240000 dispatch_meta_bytes=343712 combine_payload_bytes=266240000 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266583712 ingress_bytes=335446016 link_bytes=335446016 token_bytes=532823712 weight_movement_bytes=69206016 total_nvlink_bytes=602029728 weight_to_token_ratio=0.129885 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262256353e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265191424 dispatch_meta_bytes=342992 combine_payload_bytes=265191424 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265534416 ingress_bytes=334397440 link_bytes=334397440 token_bytes=530725840 weight_movement_bytes=69206016 total_nvlink_bytes=599931856 weight_to_token_ratio=0.130399 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258310646e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343928 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266116984 ingress_bytes=300376064 link_bytes=300376064 token_bytes=531890040 weight_movement_bytes=34603008 total_nvlink_bytes=566493048 weight_to_token_ratio=0.065057 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130290947e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74474 payload_rows_remote=65158 dup_rows=23830 dup_rows_remote=20746 dedup_ratio=0.242411 router_tokens_per_expert=[752, 770, 790, 749, 782, 764, 720, 729, 741, 739, 785, 782, 804, 792, 770, 787, 748, 792, 780, 778, 801, 772, 767, 749, 772, 766, 780, 729, 775, 752, 719, 774, 735, 739, 756, 742, 724, 835, 776, 773, 742, 808, 758, 766, 749, 780, 761, 780, 810, 814, 755, 759, 812, 739, 788, 728, 757, 790, 757, 780, 794, 759, 749, 827] payload_rows_per_dst=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] dup_rows_per_dst=[3004, 3084, 3006, 2884, 2854, 2886, 3016, 3096] combine_rows_per_src=[9108, 9316, 9368, 9250, 9306, 9402, 9394, 9330] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266166272 dispatch_meta_bytes=343960 combine_payload_bytes=266166272 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266510232 ingress_bytes=335372288 link_bytes=335372288 token_bytes=532676504 weight_movement_bytes=69206016 total_nvlink_bytes=601882520 weight_to_token_ratio=0.129921 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261978920e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343656 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239016 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134376 weight_movement_bytes=69206016 total_nvlink_bytes=603340392 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343480 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403256 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463032 weight_movement_bytes=34603008 total_nvlink_bytes=567066040 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343864 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567480 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791096 weight_movement_bytes=103809024 total_nvlink_bytes=636600120 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=344648 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267682376 ingress_bytes=301940736 link_bytes=301940736 token_bytes=535020104 weight_movement_bytes=34603008 total_nvlink_bytes=569623112 weight_to_token_ratio=0.064676 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74432 payload_rows_remote=65160 dup_rows=23872 dup_rows_remote=20754 dedup_ratio=0.242839 router_tokens_per_expert=[742, 814, 736, 751, 704, 724, 799, 792, 778, 773, 775, 753, 815, 754, 767, 772, 771, 744, 777, 753, 713, 740, 752, 750, 781, 778, 788, 757, 736, 791, 796, 767, 759, 734, 798, 762, 760, 821, 779, 782, 717, 803, 804, 784, 770, 754, 739, 784, 781, 793, 830, 761, 735, 794, 748, 767, 756, 784, 751, 759, 767, 752, 819, 762] payload_rows_per_dst=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] dup_rows_per_dst=[2928, 3012, 2768, 2984, 3118, 3052, 3008, 3002] combine_rows_per_src=[9196, 9362, 9232, 9404, 9272, 9258, 9410, 9298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266887168 dispatch_meta_bytes=343616 combine_payload_bytes=266887168 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=267230784 ingress_bytes=370696192 link_bytes=370696192 token_bytes=534117952 weight_movement_bytes=103809024 total_nvlink_bytes=637926976 weight_to_token_ratio=0.194356 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.032500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394899927e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74276 payload_rows_remote=64612 dup_rows=24028 dup_rows_remote=20504 dedup_ratio=0.244425 router_tokens_per_expert=[773, 788, 750, 746, 779, 801, 741, 761, 811, 786, 767, 765, 798, 754, 793, 783, 764, 784, 791, 781, 802, 794, 742, 810, 758, 742, 716, 761, 753, 707, 748, 772, 779, 760, 762, 729, 787, 804, 748, 731, 746, 777, 749, 786, 809, 759, 777, 804, 800, 749, 771, 765, 762, 744, 759, 773, 746, 790, 746, 797, 772, 732, 748, 770] payload_rows_per_dst=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] dup_rows_per_dst=[2698, 3058, 2726, 2834, 3004, 3052, 3132, 3524] combine_rows_per_src=[9088, 9346, 8824, 9080, 9294, 9374, 9606, 9664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65180 dup_rows=23862 dup_rows_remote=20832 dedup_ratio=0.242737 router_tokens_per_expert=[796, 743, 790, 800, 768, 776, 801, 744, 760, 763, 795, 766, 752, 805, 800, 714, 731, 754, 719, 739, 740, 772, 796, 721, 814, 773, 825, 783, 780, 760, 753, 761, 748, 733, 730, 752, 790, 721, 760, 803, 728, 767, 808, 742, 781, 772, 786, 775, 784, 789, 781, 768, 728, 770, 756, 770, 784, 799, 772, 750, 790, 731, 828, 762] payload_rows_per_dst=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] dup_rows_per_dst=[3032, 3000, 2862, 3050, 2920, 2974, 3030, 2994] combine_rows_per_src=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65206 dup_rows=23996 dup_rows_remote=21090 dedup_ratio=0.244100 router_tokens_per_expert=[776, 741, 803, 745, 711, 762, 795, 743, 799, 751, 779, 784, 773, 768, 748, 766, 755, 770, 799, 809, 795, 767, 795, 804, 745, 766, 752, 777, 731, 715, 731, 787, 852, 801, 806, 799, 780, 800, 749, 749, 735, 750, 779, 757, 767, 752, 774, 797, 746, 767, 734, 771, 779, 750, 755, 795, 747, 754, 736, 752, 803, 754, 790, 730] payload_rows_per_dst=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] dup_rows_per_dst=[2866, 3074, 3102, 2906, 3200, 2924, 3014, 2910] combine_rows_per_src=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266895360 dispatch_meta_bytes=343656 combine_payload_bytes=266895360 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267239016 ingress_bytes=336101376 link_bytes=336101376 token_bytes=534134376 weight_movement_bytes=69206016 total_nvlink_bytes=603340392 weight_to_token_ratio=0.129567 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264722420e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74162 payload_rows_remote=64934 dup_rows=24142 dup_rows_remote=21056 dedup_ratio=0.245585 router_tokens_per_expert=[769, 755, 747, 758, 774, 804, 788, 780, 781, 744, 782, 744, 764, 755, 792, 796, 710, 770, 740, 770, 757, 693, 795, 783, 781, 788, 754, 752, 760, 773, 740, 804, 775, 781, 750, 783, 732, 790, 740, 784, 776, 792, 759, 762, 797, 800, 752, 753, 739, 757, 797, 782, 756, 767, 796, 763, 788, 785, 792, 818, 763, 761, 707, 752] payload_rows_per_dst=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] dup_rows_per_dst=[3014, 3074, 2856, 3014, 2964, 3020, 3086, 3114] combine_rows_per_src=[9336, 9242, 9180, 9290, 9306, 9362, 9228, 9218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74368 payload_rows_remote=64956 dup_rows=23936 dup_rows_remote=20914 dedup_ratio=0.243490 router_tokens_per_expert=[775, 780, 762, 732, 773, 767, 774, 778, 718, 778, 761, 759, 781, 790, 802, 758, 749, 761, 779, 735, 767, 792, 780, 806, 780, 787, 767, 768, 781, 754, 763, 762, 814, 752, 749, 734, 801, 773, 752, 786, 740, 792, 751, 742, 772, 757, 750, 806, 823, 741, 763, 805, 755, 787, 764, 782, 729, 775, 766, 759, 737, 742, 745, 789] payload_rows_per_dst=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] dup_rows_per_dst=[3022, 2974, 3014, 3036, 2974, 2886, 3062, 2968] combine_rows_per_src=[9412, 9320, 9324, 9288, 9348, 9182, 9378, 9116] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74128 payload_rows_remote=64940 dup_rows=24176 dup_rows_remote=21174 dedup_ratio=0.245931 router_tokens_per_expert=[775, 745, 749, 771, 774, 757, 800, 746, 790, 755, 786, 783, 776, 795, 822, 750, 798, 785, 751, 745, 817, 775, 779, 752, 745, 777, 778, 813, 792, 794, 718, 792, 732, 749, 778, 786, 763, 769, 782, 771, 750, 763, 754, 745, 801, 789, 732, 761, 786, 752, 764, 781, 740, 796, 752, 720, 757, 716, 762, 803, 781, 731, 761, 740] payload_rows_per_dst=[9214, 9438, 9386, 9302, 9226, 9188, 9198, 9176] dup_rows_per_dst=[3020, 3076, 3018, 3116, 3034, 3002, 2984, 2926] combine_rows_per_src=[9214, 9438, 9386, 9302, 9226, 9188, 9198, 9176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=65206 dup_rows=23996 dup_rows_remote=21090 dedup_ratio=0.244100 router_tokens_per_expert=[776, 741, 803, 745, 711, 762, 795, 743, 799, 751, 779, 784, 773, 768, 748, 766, 755, 770, 799, 809, 795, 767, 795, 804, 745, 766, 752, 777, 731, 715, 731, 787, 852, 801, 806, 799, 780, 800, 749, 749, 735, 750, 779, 757, 767, 752, 774, 797, 746, 767, 734, 771, 779, 750, 755, 795, 747, 754, 736, 752, 803, 754, 790, 730] payload_rows_per_dst=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] dup_rows_per_dst=[2866, 3074, 3102, 2906, 3200, 2924, 3014, 2910] combine_rows_per_src=[9286, 9262, 9486, 9102, 9472, 9298, 9180, 9222] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74680 payload_rows_remote=65424 dup_rows=23624 dup_rows_remote=20754 dedup_ratio=0.240316 router_tokens_per_expert=[784, 781, 785, 803, 766, 760, 678, 758, 792, 783, 816, 752, 806, 785, 718, 738, 751, 744, 758, 768, 747, 802, 763, 730, 747, 762, 730, 789, 763, 712, 816, 767, 792, 788, 752, 776, 819, 831, 794, 764, 729, 712, 763, 844, 787, 762, 777, 808, 757, 745, 789, 740, 754, 774, 770, 751, 801, 753, 766, 762, 777, 776, 753, 732] payload_rows_per_dst=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] dup_rows_per_dst=[2986, 3010, 2870, 2836, 3106, 3054, 2886, 2876] combine_rows_per_src=[9244, 9370, 9256, 9336, 9526, 9310, 9274, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65210 dup_rows=23824 dup_rows_remote=20826 dedup_ratio=0.242350 router_tokens_per_expert=[752, 768, 778, 815, 795, 775, 772, 744, 732, 769, 738, 748, 770, 750, 723, 774, 753, 706, 779, 791, 795, 740, 790, 780, 790, 773, 795, 789, 757, 786, 794, 743, 782, 808, 741, 780, 769, 765, 757, 814, 766, 796, 796, 742, 754, 790, 738, 788, 729, 779, 736, 792, 763, 749, 752, 783, 723, 757, 761, 771, 759, 755, 762, 831] payload_rows_per_dst=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] dup_rows_per_dst=[2946, 2880, 2998, 3012, 3068, 3052, 2918, 2950] combine_rows_per_src=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264650752 dispatch_meta_bytes=340464 combine_payload_bytes=264650752 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=264991216 ingress_bytes=299253760 link_bytes=299253760 token_bytes=529641968 weight_movement_bytes=34603008 total_nvlink_bytes=564244976 weight_to_token_ratio=0.065333 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.126067807e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64676 dup_rows=24190 dup_rows_remote=20990 dedup_ratio=0.246073 router_tokens_per_expert=[807, 797, 787, 764, 775, 775, 784, 779, 737, 835, 746, 776, 733, 768, 768, 752, 787, 755, 825, 763, 762, 747, 752, 770, 786, 795, 759, 766, 745, 785, 788, 704, 776, 795, 806, 772, 802, 820, 768, 780, 773, 777, 819, 737, 755, 772, 743, 749, 731, 752, 774, 739, 710, 763, 771, 776, 709, 755, 757, 775, 756, 737, 775, 756] payload_rows_per_dst=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] dup_rows_per_dst=[3172, 3010, 3090, 3016, 3200, 2982, 2862, 2858] combine_rows_per_src=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65012 dup_rows=23950 dup_rows_remote=20898 dedup_ratio=0.243632 router_tokens_per_expert=[764, 829, 747, 835, 747, 735, 809, 721, 736, 762, 759, 739, 749, 765, 791, 791, 777, 796, 781, 723, 739, 773, 769, 732, 781, 774, 762, 773, 792, 756, 766, 793, 778, 808, 793, 784, 757, 700, 791, 740, 795, 794, 829, 754, 786, 812, 734, 750, 774, 746, 779, 783, 746, 787, 759, 755, 774, 815, 684, 788, 755, 737, 762, 737] payload_rows_per_dst=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] dup_rows_per_dst=[3052, 2912, 3078, 3206, 2852, 3024, 2934, 2892] combine_rows_per_src=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344048 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321328 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298608 weight_movement_bytes=34603008 total_nvlink_bytes=568901616 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=265969664 dispatch_meta_bytes=343960 combine_payload_bytes=265969664 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266313624 ingress_bytes=335175680 link_bytes=335175680 token_bytes=532283288 weight_movement_bytes=69206016 total_nvlink_bytes=601489304 weight_to_token_ratio=0.130017 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261239100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=345184 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267428960 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534512736 weight_movement_bytes=34603008 total_nvlink_bytes=569115744 weight_to_token_ratio=0.064737 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64924 dup_rows=23986 dup_rows_remote=20998 dedup_ratio=0.243998 router_tokens_per_expert=[773, 767, 735, 741, 768, 791, 776, 830, 753, 760, 766, 771, 791, 769, 815, 766, 782, 757, 753, 821, 710, 789, 760, 790, 762, 772, 778, 791, 723, 779, 756, 768, 746, 777, 806, 765, 835, 750, 811, 814, 720, 758, 701, 796, 775, 712, 786, 766, 747, 760, 734, 757, 768, 826, 760, 771, 764, 781, 727, 743, 762, 758, 765, 748] payload_rows_per_dst=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] dup_rows_per_dst=[2904, 2988, 3002, 2974, 3192, 2930, 2974, 3022] combine_rows_per_src=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=266059776 dispatch_meta_bytes=343480 combine_payload_bytes=266059776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266403256 ingress_bytes=300662784 link_bytes=300662784 token_bytes=532463032 weight_movement_bytes=34603008 total_nvlink_bytes=567066040 weight_to_token_ratio=0.064987 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131369851e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74268 payload_rows_remote=64996 dup_rows=24036 dup_rows_remote=20970 dedup_ratio=0.244507 router_tokens_per_expert=[735, 767, 754, 792, 802, 801, 787, 746, 761, 806, 793, 725, 773, 770, 760, 781, 748, 799, 816, 813, 812, 790, 765, 794, 771, 767, 805, 734, 752, 727, 768, 734, 728, 804, 763, 744, 725, 722, 783, 732, 786, 772, 744, 787, 733, 768, 771, 787, 795, 757, 742, 820, 720, 798, 776, 813, 761, 771, 707, 802, 737, 770, 758, 728] payload_rows_per_dst=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] dup_rows_per_dst=[3004, 3066, 3206, 2950, 2874, 2960, 3070, 2906] combine_rows_per_src=[9364, 9272, 9468, 9166, 9128, 9336, 9372, 9162] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=265994240 dispatch_meta_bytes=344456 combine_payload_bytes=265994240 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266338696 ingress_bytes=335200256 link_bytes=335200256 token_bytes=532332936 weight_movement_bytes=69206016 total_nvlink_bytes=601538952 weight_to_token_ratio=0.130005 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.261331577e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267083776 dispatch_meta_bytes=345184 combine_payload_bytes=267083776 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267428960 ingress_bytes=301686784 link_bytes=301686784 token_bytes=534512736 weight_movement_bytes=34603008 total_nvlink_bytes=569115744 weight_to_token_ratio=0.064737 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.135223080e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=20 plans=1 dispatch_payload_bytes=267976704 dispatch_meta_bytes=344712 combine_payload_bytes=267976704 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=268321416 ingress_bytes=337182720 link_bytes=337182720 token_bytes=536298120 weight_movement_bytes=69206016 total_nvlink_bytes=605504136 weight_to_token_ratio=0.129044 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.268791430e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344144 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444304 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544464 weight_movement_bytes=69206016 total_nvlink_bytes=603750480 weight_to_token_ratio=0.129467 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74114 payload_rows_remote=64676 dup_rows=24190 dup_rows_remote=20990 dedup_ratio=0.246073 router_tokens_per_expert=[807, 797, 787, 764, 775, 775, 784, 779, 737, 835, 746, 776, 733, 768, 768, 752, 787, 755, 825, 763, 762, 747, 752, 770, 786, 795, 759, 766, 745, 785, 788, 704, 776, 795, 806, 772, 802, 820, 768, 780, 773, 777, 819, 737, 755, 772, 743, 749, 731, 752, 774, 739, 710, 763, 771, 776, 709, 755, 757, 775, 756, 737, 775, 756] payload_rows_per_dst=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] dup_rows_per_dst=[3172, 3010, 3090, 3016, 3200, 2982, 2862, 2858] combine_rows_per_src=[9364, 9220, 9232, 9240, 9438, 9268, 9170, 9182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266223616 dispatch_meta_bytes=343864 combine_payload_bytes=266223616 prefetch_bytes=0 reduce_bytes=103809024 egress_bytes=266567480 ingress_bytes=370032640 link_bytes=370032640 token_bytes=532791096 weight_movement_bytes=103809024 total_nvlink_bytes=636600120 weight_to_token_ratio=0.194840 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.392403034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=264912896 dispatch_meta_bytes=342664 combine_payload_bytes=264912896 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265255560 ingress_bytes=334118912 link_bytes=334118912 token_bytes=530168456 weight_movement_bytes=69206016 total_nvlink_bytes=599374472 weight_to_token_ratio=0.130536 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.257262567e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343640 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266632792 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532921944 weight_movement_bytes=69206016 total_nvlink_bytes=602127960 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343688 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266272392 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532201096 weight_movement_bytes=103809024 total_nvlink_bytes=636010120 weight_to_token_ratio=0.195056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74550 payload_rows_remote=65086 dup_rows=23754 dup_rows_remote=20644 dedup_ratio=0.241638 router_tokens_per_expert=[752, 801, 839, 772, 760, 723, 783, 817, 760, 722, 769, 750, 770, 769, 794, 731, 720, 763, 807, 789, 718, 805, 747, 780, 769, 807, 717, 765, 796, 764, 752, 727, 799, 781, 775, 729, 748, 830, 753, 742, 766, 766, 782, 750, 769, 791, 728, 775, 786, 729, 747, 787, 757, 768, 744, 725, 808, 776, 761, 790, 776, 807, 781, 788] payload_rows_per_dst=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] dup_rows_per_dst=[3020, 2868, 3010, 2916, 2918, 2962, 2950, 3110] combine_rows_per_src=[9382, 9204, 9306, 9278, 9252, 9292, 9372, 9464] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64882 dup_rows=23996 dup_rows_remote=21070 dedup_ratio=0.244100 router_tokens_per_expert=[754, 795, 750, 738, 765, 740, 779, 787, 747, 724, 820, 747, 827, 776, 761, 755, 746, 762, 785, 786, 730, 765, 744, 760, 838, 763, 767, 784, 789, 800, 758, 776, 751, 747, 763, 801, 759, 759, 814, 759, 755, 754, 775, 778, 729, 755, 755, 738, 780, 775, 783, 772, 746, 763, 770, 787, 747, 786, 727, 765, 821, 781, 796, 743] payload_rows_per_dst=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] dup_rows_per_dst=[2978, 3038, 2960, 3218, 3074, 2834, 2926, 2968] combine_rows_per_src=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74442 payload_rows_remote=65180 dup_rows=23862 dup_rows_remote=20832 dedup_ratio=0.242737 router_tokens_per_expert=[796, 743, 790, 800, 768, 776, 801, 744, 760, 763, 795, 766, 752, 805, 800, 714, 731, 754, 719, 739, 740, 772, 796, 721, 814, 773, 825, 783, 780, 760, 753, 761, 748, 733, 730, 752, 790, 721, 760, 803, 728, 767, 808, 742, 781, 772, 786, 775, 784, 789, 781, 768, 728, 770, 756, 770, 784, 799, 772, 750, 790, 731, 828, 762] payload_rows_per_dst=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] dup_rows_per_dst=[3032, 3000, 2862, 3050, 2920, 2974, 3030, 2994] combine_rows_per_src=[9404, 9310, 9082, 9448, 9154, 9344, 9262, 9438] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74354 payload_rows_remote=65012 dup_rows=23950 dup_rows_remote=20898 dedup_ratio=0.243632 router_tokens_per_expert=[764, 829, 747, 835, 747, 735, 809, 721, 736, 762, 759, 739, 749, 765, 791, 791, 777, 796, 781, 723, 739, 773, 769, 732, 781, 774, 762, 773, 792, 756, 766, 793, 778, 808, 793, 784, 757, 700, 791, 740, 795, 794, 829, 754, 786, 812, 734, 750, 774, 746, 779, 783, 746, 787, 759, 755, 774, 815, 684, 788, 755, 737, 762, 737] payload_rows_per_dst=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] dup_rows_per_dst=[3052, 2912, 3078, 3206, 2852, 3024, 2934, 2892] combine_rows_per_src=[9342, 9252, 9476, 9434, 9076, 9238, 9324, 9212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=65008 dup_rows=24058 dup_rows_remote=21102 dedup_ratio=0.244731 router_tokens_per_expert=[734, 780, 798, 753, 784, 757, 775, 750, 747, 835, 774, 769, 746, 784, 776, 791, 738, 785, 752, 796, 720, 758, 816, 774, 783, 743, 785, 752, 690, 790, 778, 776, 786, 736, 757, 748, 785, 833, 734, 751, 799, 801, 730, 774, 799, 745, 758, 750, 747, 758, 768, 825, 777, 756, 690, 790, 760, 761, 752, 758, 774, 792, 796, 773] payload_rows_per_dst=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] dup_rows_per_dst=[2966, 3172, 2996, 2956, 3036, 2974, 2976, 2982] combine_rows_per_src=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74408 payload_rows_remote=65140 dup_rows=23896 dup_rows_remote=20924 dedup_ratio=0.243083 router_tokens_per_expert=[780, 787, 738, 774, 739, 728, 772, 812, 800, 792, 791, 722, 759, 775, 772, 738, 798, 761, 743, 749, 782, 760, 789, 728, 768, 779, 749, 788, 775, 776, 727, 774, 711, 782, 779, 806, 753, 735, 731, 762, 740, 765, 770, 830, 799, 750, 724, 742, 757, 765, 789, 757, 792, 802, 795, 772, 755, 749, 800, 785, 778, 766, 791, 795] payload_rows_per_dst=[9238, 9338, 9220, 9300, 9296, 9268, 9396, 9352] dup_rows_per_dst=[3022, 2960, 3000, 2972, 2822, 2972, 3062, 3086] combine_rows_per_src=[9238, 9338, 9220, 9300, 9296, 9268, 9396, 9352] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74246 payload_rows_remote=65008 dup_rows=24058 dup_rows_remote=21102 dedup_ratio=0.244731 router_tokens_per_expert=[734, 780, 798, 753, 784, 757, 775, 750, 747, 835, 774, 769, 746, 784, 776, 791, 738, 785, 752, 796, 720, 758, 816, 774, 783, 743, 785, 752, 690, 790, 778, 776, 786, 736, 757, 748, 785, 833, 734, 751, 799, 801, 730, 774, 799, 745, 758, 750, 747, 758, 768, 825, 777, 756, 690, 790, 760, 761, 752, 758, 774, 792, 796, 773] payload_rows_per_dst=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] dup_rows_per_dst=[2966, 3172, 2996, 2956, 3036, 2974, 2976, 2982] combine_rows_per_src=[9296, 9272, 9282, 9238, 9224, 9338, 9246, 9350] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266592256 dispatch_meta_bytes=342920 combine_payload_bytes=266592256 prefetch_bytes=0 reduce_bytes=69206016 egress_bytes=266935176 ingress_bytes=335798272 link_bytes=335798272 token_bytes=533527432 weight_movement_bytes=69206016 total_nvlink_bytes=602733448 weight_to_token_ratio=0.129714 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.027500 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.263581864e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343808 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100480 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857152 weight_movement_bytes=34603008 total_nvlink_bytes=566460160 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64840 dup_rows=24214 dup_rows_remote=21156 dedup_ratio=0.246318 router_tokens_per_expert=[767, 730, 757, 758, 772, 796, 748, 774, 754, 774, 755, 716, 753, 797, 796, 747, 790, 766, 750, 802, 758, 783, 755, 750, 793, 747, 778, 777, 776, 769, 753, 734, 840, 802, 793, 762, 789, 786, 733, 773, 753, 782, 744, 782, 790, 728, 745, 759, 777, 807, 729, 791, 755, 746, 802, 731, 778, 736, 730, 786, 838, 759, 766, 785] payload_rows_per_dst=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] dup_rows_per_dst=[3054, 3018, 3058, 3012, 3196, 2882, 3004, 2990] combine_rows_per_src=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266977280 dispatch_meta_bytes=344048 combine_payload_bytes=266977280 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267321328 ingress_bytes=301580288 link_bytes=301580288 token_bytes=534298608 weight_movement_bytes=34603008 total_nvlink_bytes=568901616 weight_to_token_ratio=0.064763 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134822345e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=266289152 dispatch_meta_bytes=343640 combine_payload_bytes=266289152 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=266632792 ingress_bytes=335495168 link_bytes=335495168 token_bytes=532921944 weight_movement_bytes=69206016 total_nvlink_bytes=602127960 weight_to_token_ratio=0.129861 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.262441308e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344440 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617208 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889976 weight_movement_bytes=34603008 total_nvlink_bytes=567492984 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65134 dup_rows=24044 dup_rows_remote=20982 dedup_ratio=0.244588 router_tokens_per_expert=[773, 772, 758, 713, 845, 791, 782, 763, 763, 796, 761, 777, 759, 759, 749, 791, 744, 771, 800, 785, 728, 790, 747, 798, 810, 732, 775, 732, 802, 748, 797, 736, 776, 755, 779, 782, 752, 714, 763, 773, 713, 796, 764, 815, 821, 734, 756, 760, 785, 742, 755, 777, 774, 747, 739, 768, 799, 693, 781, 827, 780, 775, 721, 789] payload_rows_per_dst=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] dup_rows_per_dst=[3104, 3032, 3016, 2958, 3062, 2990, 2846, 3036] combine_rows_per_src=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65166 dup_rows=23860 dup_rows_remote=20934 dedup_ratio=0.242716 router_tokens_per_expert=[709, 769, 758, 763, 760, 780, 755, 802, 780, 762, 755, 782, 767, 759, 765, 783, 773, 770, 705, 815, 766, 792, 797, 770, 714, 817, 745, 731, 776, 787, 762, 719, 762, 782, 808, 752, 735, 809, 774, 735, 747, 784, 817, 770, 802, 788, 793, 741, 794, 761, 775, 718, 762, 793, 758, 766, 810, 759, 741, 751, 777, 772, 779, 749] payload_rows_per_dst=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] dup_rows_per_dst=[2926, 3038, 3036, 2916, 2866, 3334, 2930, 2814] combine_rows_per_src=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74516 payload_rows_remote=65268 dup_rows=23788 dup_rows_remote=20894 dedup_ratio=0.241984 router_tokens_per_expert=[706, 814, 802, 741, 775, 778, 765, 768, 753, 746, 785, 733, 747, 773, 764, 784, 759, 755, 763, 780, 747, 779, 742, 746, 786, 785, 770, 788, 756, 778, 746, 754, 772, 781, 801, 796, 719, 817, 766, 783, 768, 736, 790, 778, 789, 802, 759, 811, 751, 780, 773, 804, 767, 725, 808, 753, 716, 762, 727, 757, 771, 778, 774, 770] payload_rows_per_dst=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] dup_rows_per_dst=[3062, 2916, 2894, 2924, 3076, 3078, 3002, 2836] combine_rows_per_src=[9236, 9254, 9248, 9402, 9394, 9388, 9320, 9274] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266813440 dispatch_meta_bytes=344256 combine_payload_bytes=266813440 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267157696 ingress_bytes=301416448 link_bytes=301416448 token_bytes=533971136 weight_movement_bytes=34603008 total_nvlink_bytes=568574144 weight_to_token_ratio=0.064803 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134205828e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65064 dup_rows=24086 dup_rows_remote=21080 dedup_ratio=0.245015 router_tokens_per_expert=[727, 734, 819, 751, 770, 730, 775, 778, 793, 769, 827, 723, 751, 735, 709, 773, 771, 791, 771, 752, 771, 800, 759, 827, 798, 760, 761, 709, 775, 794, 756, 774, 725, 798, 785, 780, 783, 752, 792, 771, 755, 794, 784, 737, 749, 719, 777, 783, 754, 771, 793, 737, 811, 781, 739, 777, 806, 725, 827, 774, 738, 792, 738, 772] payload_rows_per_dst=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] dup_rows_per_dst=[2910, 3006, 3076, 2988, 3070, 3006, 3008, 3022] combine_rows_per_src=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266272768 dispatch_meta_bytes=344440 combine_payload_bytes=266272768 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266617208 ingress_bytes=300875776 link_bytes=300875776 token_bytes=532889976 weight_movement_bytes=34603008 total_nvlink_bytes=567492984 weight_to_token_ratio=0.064935 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.132171323e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74318 payload_rows_remote=64924 dup_rows=23986 dup_rows_remote=20998 dedup_ratio=0.243998 router_tokens_per_expert=[773, 767, 735, 741, 768, 791, 776, 830, 753, 760, 766, 771, 791, 769, 815, 766, 782, 757, 753, 821, 710, 789, 760, 790, 762, 772, 778, 791, 723, 779, 756, 768, 746, 777, 806, 765, 835, 750, 811, 814, 720, 758, 701, 796, 775, 712, 786, 766, 747, 760, 734, 757, 768, 826, 760, 771, 764, 781, 727, 743, 762, 758, 765, 748] payload_rows_per_dst=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] dup_rows_per_dst=[2904, 2988, 3002, 2974, 3192, 2930, 2974, 3022] combine_rows_per_src=[9458, 9394, 9322, 9284, 9416, 9098, 9272, 9074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74260 payload_rows_remote=65134 dup_rows=24044 dup_rows_remote=20982 dedup_ratio=0.244588 router_tokens_per_expert=[773, 772, 758, 713, 845, 791, 782, 763, 763, 796, 761, 777, 759, 759, 749, 791, 744, 771, 800, 785, 728, 790, 747, 798, 810, 732, 775, 732, 802, 748, 797, 736, 776, 755, 779, 782, 752, 714, 763, 773, 713, 796, 764, 815, 821, 734, 756, 760, 785, 742, 755, 777, 774, 747, 739, 768, 799, 693, 781, 827, 780, 775, 721, 789] payload_rows_per_dst=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] dup_rows_per_dst=[3104, 3032, 3016, 2958, 3062, 2990, 2846, 3036] combine_rows_per_src=[9290, 9278, 9310, 9306, 9126, 9328, 9328, 9294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343984 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928624 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513264 weight_movement_bytes=34603008 total_nvlink_bytes=566116272 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344400 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267264336 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534184272 weight_movement_bytes=69206016 total_nvlink_bytes=603390288 weight_to_token_ratio=0.129555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344464 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267133328 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533922192 weight_movement_bytes=103809024 total_nvlink_bytes=637731216 weight_to_token_ratio=0.194427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=21 plans=1 dispatch_payload_bytes=267337728 dispatch_meta_bytes=344648 combine_payload_bytes=267337728 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267682376 ingress_bytes=301940736 link_bytes=301940736 token_bytes=535020104 weight_movement_bytes=34603008 total_nvlink_bytes=569623112 weight_to_token_ratio=0.064676 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.136178681e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344576 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846720 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348864 weight_movement_bytes=34603008 total_nvlink_bytes=567951872 weight_to_token_ratio=0.064879 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265928704 dispatch_meta_bytes=343688 combine_payload_bytes=265928704 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266272392 ingress_bytes=369737728 link_bytes=369737728 token_bytes=532201096 weight_movement_bytes=103809024 total_nvlink_bytes=636010120 weight_to_token_ratio=0.195056 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.391293304e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74562 payload_rows_remote=65048 dup_rows=23742 dup_rows_remote=20616 dedup_ratio=0.241516 router_tokens_per_expert=[779, 735, 774, 769, 751, 727, 740, 773, 783, 783, 728, 743, 769, 762, 776, 760, 801, 781, 781, 767, 781, 743, 801, 716, 777, 749, 818, 763, 746, 793, 794, 780, 773, 774, 749, 789, 785, 740, 751, 742, 764, 750, 766, 810, 792, 752, 748, 817, 714, 727, 763, 787, 726, 766, 740, 764, 785, 782, 822, 828, 809, 770, 760, 764] payload_rows_per_dst=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] dup_rows_per_dst=[2858, 2944, 3012, 2880, 2966, 3016, 2940, 3126] combine_rows_per_src=[9238, 9264, 9330, 9164, 9330, 9382, 9340, 9514] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64986 dup_rows=24152 dup_rows_remote=21038 dedup_ratio=0.245687 router_tokens_per_expert=[765, 775, 725, 756, 765, 800, 735, 741, 796, 762, 735, 804, 769, 803, 776, 781, 725, 740, 742, 753, 763, 767, 777, 785, 706, 743, 773, 743, 776, 793, 762, 789, 788, 768, 783, 769, 756, 774, 789, 732, 761, 802, 750, 774, 778, 800, 786, 788, 762, 783, 764, 795, 778, 752, 815, 691, 758, 736, 752, 821, 748, 751, 774, 849] payload_rows_per_dst=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] dup_rows_per_dst=[2936, 3104, 2920, 2946, 3006, 3096, 3114, 3030] combine_rows_per_src=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263422144 prefetch_bytes=726663168 reduce_bytes=899678208 weight_movement_bytes=1626341376 egress_bytes=6135666880 ingress_bytes=7754096640 link_bytes=7754096640 total_nvlink_bytes=13889763520 weight_to_token_ratio=0.132617 prefetch_share_of_ingress=0.093713 dedup_ratio=0.243723 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=26 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.268610496e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266788864 dispatch_meta_bytes=344464 combine_payload_bytes=266788864 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=267133328 ingress_bytes=370597888 link_bytes=370597888 token_bytes=533922192 weight_movement_bytes=103809024 total_nvlink_bytes=637731216 weight_to_token_ratio=0.194427 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.394530017e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74308 payload_rows_remote=64882 dup_rows=23996 dup_rows_remote=21070 dedup_ratio=0.244100 router_tokens_per_expert=[754, 795, 750, 738, 765, 740, 779, 787, 747, 724, 820, 747, 827, 776, 761, 755, 746, 762, 785, 786, 730, 765, 744, 760, 838, 763, 767, 784, 789, 800, 758, 776, 751, 747, 763, 801, 759, 759, 814, 759, 755, 754, 775, 778, 729, 755, 755, 738, 780, 775, 783, 772, 746, 763, 770, 787, 747, 786, 727, 765, 821, 781, 796, 743] payload_rows_per_dst=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] dup_rows_per_dst=[2978, 3038, 2960, 3218, 3074, 2834, 2926, 2968] combine_rows_per_src=[9238, 9276, 9196, 9332, 9232, 9244, 9426, 9364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74444 payload_rows_remote=65166 dup_rows=23860 dup_rows_remote=20934 dedup_ratio=0.242716 router_tokens_per_expert=[709, 769, 758, 763, 760, 780, 755, 802, 780, 762, 755, 782, 767, 759, 765, 783, 773, 770, 705, 815, 766, 792, 797, 770, 714, 817, 745, 731, 776, 787, 762, 719, 762, 782, 808, 752, 735, 809, 774, 735, 747, 784, 817, 770, 802, 788, 793, 741, 794, 761, 775, 718, 762, 793, 758, 766, 810, 759, 741, 751, 777, 772, 779, 749] payload_rows_per_dst=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] dup_rows_per_dst=[2926, 3038, 3036, 2916, 2866, 3334, 2930, 2814] combine_rows_per_src=[9278, 9262, 9340, 9402, 9232, 9574, 9324, 9032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64886 dup_rows=24078 dup_rows_remote=21088 dedup_ratio=0.244934 router_tokens_per_expert=[760, 796, 778, 734, 780, 766, 750, 807, 788, 774, 738, 755, 758, 791, 737, 733, 803, 802, 733, 805, 763, 713, 794, 737, 731, 778, 787, 785, 792, 797, 765, 730, 764, 775, 726, 796, 789, 772, 772, 756, 732, 709, 779, 780, 780, 783, 788, 742, 727, 764, 799, 776, 797, 742, 829, 791, 765, 789, 753, 780, 768, 781, 756, 732] payload_rows_per_dst=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] dup_rows_per_dst=[3000, 2934, 3054, 2990, 3072, 2954, 3092, 2982] combine_rows_per_src=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74212 payload_rows_remote=64782 dup_rows=24092 dup_rows_remote=21064 dedup_ratio=0.245076 router_tokens_per_expert=[767, 743, 729, 732, 811, 721, 790, 773, 737, 783, 781, 750, 812, 777, 816, 778, 744, 776, 756, 782, 760, 808, 768, 761, 747, 754, 778, 760, 812, 768, 833, 755, 766, 763, 758, 765, 756, 767, 765, 747, 768, 792, 770, 763, 794, 775, 798, 769, 746, 734, 708, 770, 772, 746, 736, 739, 807, 770, 782, 779, 801, 755, 773, 756] payload_rows_per_dst=[9206, 9330, 9332, 9336, 9184, 9430, 9112, 9282] dup_rows_per_dst=[2926, 3138, 2978, 3078, 2990, 3028, 2790, 3164] combine_rows_per_src=[9206, 9330, 9332, 9336, 9184, 9430, 9112, 9282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74226 payload_rows_remote=64886 dup_rows=24078 dup_rows_remote=21088 dedup_ratio=0.244934 router_tokens_per_expert=[760, 796, 778, 734, 780, 766, 750, 807, 788, 774, 738, 755, 758, 791, 737, 733, 803, 802, 733, 805, 763, 713, 794, 737, 731, 778, 787, 785, 792, 797, 765, 730, 764, 775, 726, 796, 789, 772, 772, 756, 732, 709, 779, 780, 780, 783, 788, 742, 727, 764, 799, 776, 797, 742, 829, 791, 765, 789, 753, 780, 768, 781, 756, 732] payload_rows_per_dst=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] dup_rows_per_dst=[3000, 2934, 3054, 2990, 3072, 2954, 3092, 2982] combine_rows_per_src=[9342, 9214, 9246, 9340, 9228, 9232, 9358, 9266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266436608 dispatch_meta_bytes=342656 combine_payload_bytes=266436608 prefetch_bytes=0 reduce_bytes=138412032 egress_bytes=266779264 ingress_bytes=404848640 link_bytes=404848640 token_bytes=533215872 weight_movement_bytes=138412032 total_nvlink_bytes=671627904 weight_to_token_ratio=0.259580 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.030000 gemm_active_groups=9 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.523412839e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65098 dup_rows=23990 dup_rows_remote=21080 dedup_ratio=0.244039 router_tokens_per_expert=[773, 770, 790, 802, 717, 755, 769, 744, 754, 780, 742, 805, 785, 753, 725, 760, 783, 777, 761, 764, 756, 726, 762, 734, 801, 796, 741, 768, 736, 777, 778, 783, 764, 778, 772, 758, 731, 768, 747, 773, 776, 756, 766, 795, 758, 774, 775, 741, 747, 797, 774, 782, 816, 769, 760, 790, 738, 787, 760, 796, 822, 729, 811, 775] payload_rows_per_dst=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] dup_rows_per_dst=[2940, 3018, 2910, 3020, 2990, 2912, 3076, 3124] combine_rows_per_src=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265756672 dispatch_meta_bytes=343808 combine_payload_bytes=265756672 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266100480 ingress_bytes=300359680 link_bytes=300359680 token_bytes=531857152 weight_movement_bytes=34603008 total_nvlink_bytes=566460160 weight_to_token_ratio=0.065061 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.130229295e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344096 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526752 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709408 weight_movement_bytes=34603008 total_nvlink_bytes=567312416 weight_to_token_ratio=0.064957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64356 dup_rows=24224 dup_rows_remote=20922 dedup_ratio=0.246419 router_tokens_per_expert=[770, 781, 756, 765, 734, 767, 789, 788, 758, 763, 793, 782, 757, 761, 742, 743, 804, 775, 779, 760, 811, 749, 785, 795, 771, 738, 777, 746, 784, 803, 757, 781, 739, 784, 764, 745, 802, 750, 788, 783, 743, 775, 737, 784, 774, 740, 762, 745, 808, 720, 743, 757, 743, 758, 775, 760, 755, 754, 785, 794, 797, 729, 768, 827] payload_rows_per_dst=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] dup_rows_per_dst=[3302, 3110, 3266, 3012, 3024, 2910, 3006, 2594] combine_rows_per_src=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65154 dup_rows=23756 dup_rows_remote=20798 dedup_ratio=0.241659 router_tokens_per_expert=[781, 750, 749, 816, 721, 711, 777, 783, 760, 751, 695, 789, 776, 743, 812, 773, 748, 707, 799, 756, 735, 707, 751, 763, 780, 795, 766, 731, 763, 722, 836, 812, 781, 801, 776, 760, 756, 796, 750, 756, 783, 802, 788, 782, 740, 792, 731, 750, 818, 736, 774, 769, 816, 766, 797, 758, 813, 780, 820, 793, 732, 791, 772, 715] payload_rows_per_dst=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] dup_rows_per_dst=[2870, 2992, 2806, 2996, 2958, 3072, 3044, 3018] combine_rows_per_src=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343384 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265690456 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531037528 weight_movement_bytes=69206016 total_nvlink_bytes=600243544 weight_to_token_ratio=0.130322 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=266919936 dispatch_meta_bytes=344400 combine_payload_bytes=266919936 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267264336 ingress_bytes=336125952 link_bytes=336125952 token_bytes=534184272 weight_movement_bytes=69206016 total_nvlink_bytes=603390288 weight_to_token_ratio=0.129555 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.264814897e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265773056 dispatch_meta_bytes=343896 combine_payload_bytes=265773056 prefetch_bytes=34603008 reduce_bytes=69206016 egress_bytes=266116952 ingress_bytes=369582080 link_bytes=369582080 token_bytes=531890008 weight_movement_bytes=103809024 total_nvlink_bytes=635699032 weight_to_token_ratio=0.195170 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.390707613e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74480 payload_rows_remote=65210 dup_rows=23824 dup_rows_remote=20826 dedup_ratio=0.242350 router_tokens_per_expert=[752, 768, 778, 815, 795, 775, 772, 744, 732, 769, 738, 748, 770, 750, 723, 774, 753, 706, 779, 791, 795, 740, 790, 780, 790, 773, 795, 789, 757, 786, 794, 743, 782, 808, 741, 780, 769, 765, 757, 814, 766, 796, 796, 742, 754, 790, 738, 788, 729, 779, 736, 792, 763, 749, 752, 783, 723, 757, 761, 771, 759, 755, 762, 831] payload_rows_per_dst=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] dup_rows_per_dst=[2946, 2880, 2998, 3012, 3068, 3052, 2918, 2950] combine_rows_per_src=[9452, 9128, 9270, 9442, 9364, 9288, 9248, 9288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=265347072 dispatch_meta_bytes=343384 combine_payload_bytes=265347072 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=265690456 ingress_bytes=334553088 link_bytes=334553088 token_bytes=531037528 weight_movement_bytes=69206016 total_nvlink_bytes=600243544 weight_to_token_ratio=0.130322 gemm_rows_defined=49664 gemm_rows_charged=51200 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.030000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.258896336e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12229498048 prefetch_bytes=588251136 reduce_bytes=795869184 weight_movement_bytes=1384120320 egress_bytes=6118692032 ingress_bytes=7494926336 link_bytes=7494926336 total_nvlink_bytes=13613618368 weight_to_token_ratio=0.113179 prefetch_share_of_ingress=0.078487 dedup_ratio=0.244248 prefetch_slots=34 unused_prefetch_slots=334 slot_utilization=0.092391 reduce_slots=23 remote_demand=17 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28448 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.226208887e-04 prefetch_bytes_per_gemm_flop=9.624094203e-06 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264298040 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=6136108600 ingress_bytes=7581515776 link_bytes=7581515776 total_nvlink_bytes=13717624376 weight_to_token_ratio=0.118501 prefetch_share_of_ingress=0.095847 dedup_ratio=0.244106 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240375370e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74218 payload_rows_remote=65064 dup_rows=24086 dup_rows_remote=21080 dedup_ratio=0.245015 router_tokens_per_expert=[727, 734, 819, 751, 770, 730, 775, 778, 793, 769, 827, 723, 751, 735, 709, 773, 771, 791, 771, 752, 771, 800, 759, 827, 798, 760, 761, 709, 775, 794, 756, 774, 725, 798, 785, 780, 783, 752, 792, 771, 755, 794, 784, 737, 749, 719, 777, 783, 754, 771, 793, 737, 811, 781, 739, 777, 806, 725, 827, 774, 738, 792, 738, 772] payload_rows_per_dst=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] dup_rows_per_dst=[2910, 3006, 3076, 2988, 3070, 3006, 3008, 3022] combine_rows_per_src=[9258, 9154, 9408, 9266, 9302, 9190, 9318, 9322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344712 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266986120 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627528 weight_movement_bytes=34603008 total_nvlink_bytes=568230536 weight_to_token_ratio=0.064845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74548 payload_rows_remote=65154 dup_rows=23756 dup_rows_remote=20798 dedup_ratio=0.241659 router_tokens_per_expert=[781, 750, 749, 816, 721, 711, 777, 783, 760, 751, 695, 789, 776, 743, 812, 773, 748, 707, 799, 756, 735, 707, 751, 763, 780, 795, 766, 731, 763, 722, 836, 812, 781, 801, 776, 760, 756, 796, 750, 756, 783, 802, 788, 782, 740, 792, 731, 750, 818, 736, 774, 769, 816, 766, 797, 758, 813, 780, 820, 793, 732, 791, 772, 715] payload_rows_per_dst=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] dup_rows_per_dst=[2870, 2992, 2806, 2996, 2958, 3072, 3044, 3018] combine_rows_per_src=[9306, 9206, 9126, 9414, 9394, 9264, 9424, 9414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266884200 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6137400424 ingress_bytes=7513604096 link_bytes=7513604096 total_nvlink_bytes=13651004520 weight_to_token_ratio=0.112834 prefetch_share_of_ingress=0.096713 dedup_ratio=0.243711 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229264666e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263002624 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6135460352 ingress_bytes=7407853568 link_bytes=7407853568 total_nvlink_bytes=13543313920 weight_to_token_ratio=0.104404 prefetch_share_of_ingress=0.093422 dedup_ratio=0.243895 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211963330e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263602176 dispatch_meta_bytes=341112 combine_payload_bytes=263602176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263943288 ingress_bytes=298205184 link_bytes=298205184 token_bytes=527545464 weight_movement_bytes=34603008 total_nvlink_bytes=562148472 weight_to_token_ratio=0.065592 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122122100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255248568 prefetch_bytes=726663168 reduce_bytes=484442112 weight_movement_bytes=1211105280 egress_bytes=6131581112 ingress_bytes=7334772736 link_bytes=7334772736 total_nvlink_bytes=13466353848 weight_to_token_ratio=0.098823 prefetch_share_of_ingress=0.099071 dedup_ratio=0.244246 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=14 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200006926e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74152 payload_rows_remote=64986 dup_rows=24152 dup_rows_remote=21038 dedup_ratio=0.245687 router_tokens_per_expert=[765, 775, 725, 756, 765, 800, 735, 741, 796, 762, 735, 804, 769, 803, 776, 781, 725, 740, 742, 753, 763, 767, 777, 785, 706, 743, 773, 743, 776, 793, 762, 789, 788, 768, 783, 769, 756, 774, 789, 732, 761, 802, 750, 774, 778, 800, 786, 788, 762, 783, 764, 795, 778, 752, 815, 691, 758, 736, 752, 821, 748, 751, 774, 849] payload_rows_per_dst=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] dup_rows_per_dst=[2936, 3104, 2920, 2946, 3006, 3096, 3114, 3030] combine_rows_per_src=[9188, 9348, 9184, 9224, 9312, 9382, 9166, 9348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266502144 dispatch_meta_bytes=344576 combine_payload_bytes=266502144 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266846720 ingress_bytes=301105152 link_bytes=301105152 token_bytes=533348864 weight_movement_bytes=34603008 total_nvlink_bytes=567951872 weight_to_token_ratio=0.064879 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133034446e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266870784 dispatch_meta_bytes=343808 combine_payload_bytes=266870784 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=267214592 ingress_bytes=301473792 link_bytes=301473792 token_bytes=534085376 weight_movement_bytes=34603008 total_nvlink_bytes=568688384 weight_to_token_ratio=0.064789 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.134421609e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263002624 prefetch_bytes=692060160 reduce_bytes=588251136 weight_movement_bytes=1280311296 egress_bytes=6135460352 ingress_bytes=7407853568 link_bytes=7407853568 total_nvlink_bytes=13543313920 weight_to_token_ratio=0.104404 prefetch_share_of_ingress=0.093422 dedup_ratio=0.243895 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=17 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.211963330e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12266884200 prefetch_bytes=726663168 reduce_bytes=657457152 weight_movement_bytes=1384120320 egress_bytes=6137400424 ingress_bytes=7513604096 link_bytes=7513604096 total_nvlink_bytes=13651004520 weight_to_token_ratio=0.112834 prefetch_share_of_ingress=0.096713 dedup_ratio=0.243711 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=19 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.229264666e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=22 plans=1 dispatch_payload_bytes=267100160 dispatch_meta_bytes=344144 combine_payload_bytes=267100160 prefetch_bytes=34603008 reduce_bytes=34603008 egress_bytes=267444304 ingress_bytes=336306176 link_bytes=336306176 token_bytes=534544464 weight_movement_bytes=69206016 total_nvlink_bytes=603750480 weight_to_token_ratio=0.129467 gemm_rows_defined=49920 gemm_rows_charged=51200 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.025000 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.265493065e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12236651648 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6122273920 ingress_bytes=7533101056 link_bytes=7533101056 total_nvlink_bytes=13655374976 weight_to_token_ratio=0.115940 prefetch_share_of_ingress=0.091869 dedup_ratio=0.244171 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232454470e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74080 payload_rows_remote=64356 dup_rows=24224 dup_rows_remote=20922 dedup_ratio=0.246419 router_tokens_per_expert=[770, 781, 756, 765, 734, 767, 789, 788, 758, 763, 793, 782, 757, 761, 742, 743, 804, 775, 779, 760, 811, 749, 785, 795, 771, 738, 777, 746, 784, 803, 757, 781, 739, 784, 764, 745, 802, 750, 788, 783, 743, 775, 737, 784, 774, 740, 762, 745, 808, 720, 743, 757, 743, 758, 775, 760, 755, 754, 785, 794, 797, 729, 768, 827] payload_rows_per_dst=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] dup_rows_per_dst=[3302, 3110, 3266, 3012, 3024, 2910, 3006, 2594] combine_rows_per_src=[9724, 9352, 9440, 9310, 9286, 9060, 9122, 8786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12263422144 prefetch_bytes=726663168 reduce_bytes=899678208 weight_movement_bytes=1626341376 egress_bytes=6135666880 ingress_bytes=7754096640 link_bytes=7754096640 total_nvlink_bytes=13889763520 weight_to_token_ratio=0.132617 prefetch_share_of_ingress=0.093713 dedup_ratio=0.243723 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=26 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028478 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.268610496e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12249490744 prefetch_bytes=692060160 reduce_bytes=692060160 weight_movement_bytes=1384120320 egress_bytes=6128698680 ingress_bytes=7504912384 link_bytes=7504912384 total_nvlink_bytes=13633611064 weight_to_token_ratio=0.112994 prefetch_share_of_ingress=0.092214 dedup_ratio=0.244219 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=20 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.029130 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.227842657e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74090 payload_rows_remote=64840 dup_rows=24214 dup_rows_remote=21156 dedup_ratio=0.246318 router_tokens_per_expert=[767, 730, 757, 758, 772, 796, 748, 774, 754, 774, 755, 716, 753, 797, 796, 747, 790, 766, 750, 802, 758, 783, 755, 750, 793, 747, 778, 777, 776, 769, 753, 734, 840, 802, 793, 762, 789, 786, 733, 773, 753, 782, 744, 782, 790, 728, 745, 759, 777, 807, 729, 791, 755, 746, 802, 731, 778, 736, 730, 786, 838, 759, 766, 785] payload_rows_per_dst=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] dup_rows_per_dst=[3054, 3018, 3058, 3012, 3196, 2882, 3004, 2990] combine_rows_per_src=[9150, 9166, 9250, 9242, 9360, 9284, 9272, 9366] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=23 plans=1 dispatch_payload_bytes=265584640 dispatch_meta_bytes=343984 combine_payload_bytes=265584640 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=265928624 ingress_bytes=300187648 link_bytes=300187648 token_bytes=531513264 weight_movement_bytes=34603008 total_nvlink_bytes=566116272 weight_to_token_ratio=0.065103 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.129581952e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 E=64 B=8 K=6 R=8 NvS=51184 hidden_size=2048 assignments=49152 payload_rows=74314 payload_rows_remote=65098 dup_rows=23990 dup_rows_remote=21080 dedup_ratio=0.244039 router_tokens_per_expert=[773, 770, 790, 802, 717, 755, 769, 744, 754, 780, 742, 805, 785, 753, 725, 760, 783, 777, 761, 764, 756, 726, 762, 734, 801, 796, 741, 768, 736, 777, 778, 783, 764, 778, 772, 758, 731, 768, 747, 773, 776, 756, 766, 795, 758, 774, 775, 741, 747, 797, 774, 782, 816, 769, 760, 790, 738, 787, 760, 796, 822, 729, 811, 775] payload_rows_per_dst=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] dup_rows_per_dst=[2940, 3018, 2910, 3020, 2990, 2912, 3076, 3124] combine_rows_per_src=[9300, 9190, 9216, 9340, 9192, 9370, 9394, 9312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=17301504 grad_bytes_per_expert=34603008 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266641408 dispatch_meta_bytes=344712 combine_payload_bytes=266641408 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266986120 ingress_bytes=301244416 link_bytes=301244416 token_bytes=533627528 weight_movement_bytes=34603008 total_nvlink_bytes=568230536 weight_to_token_ratio=0.064845 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.133558485e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12255248568 prefetch_bytes=726663168 reduce_bytes=484442112 weight_movement_bytes=1211105280 egress_bytes=6131581112 ingress_bytes=7334772736 link_bytes=7334772736 total_nvlink_bytes=13466353848 weight_to_token_ratio=0.098823 prefetch_share_of_ingress=0.099071 dedup_ratio=0.244246 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=14 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.200006926e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none - [2026-08-26 16:56:42] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 805.4 | throughput per GPU (TFLOP/s/GPU): 158.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.618065E+00 | loss scale: 1.0 | grad norm: 47.305 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 16:56:42 -[rank0]:[W826 16:56:44.048677672 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 16:56:45.092944786 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 16:56:45.278174907 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 16:56:45.280425487 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 16:56:45.551550311 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 16:56:45.630757492 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 16:56:45.632653250 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 16:56:45.636200815 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 16:56:51.592000 287395 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:56:51.592000 287395 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 287492 closing signal SIGTERM +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=266182656 dispatch_meta_bytes=344096 combine_payload_bytes=266182656 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=266526752 ingress_bytes=300785664 link_bytes=300785664 token_bytes=532709408 weight_movement_bytes=34603008 total_nvlink_bytes=567312416 weight_to_token_ratio=0.064957 gemm_rows_defined=49792 gemm_rows_charged=51200 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.027500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.131832238e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=24 plans=1 dispatch_payload_bytes=263602176 dispatch_meta_bytes=341112 combine_payload_bytes=263602176 prefetch_bytes=34603008 reduce_bytes=0 egress_bytes=263943288 ingress_bytes=298205184 link_bytes=298205184 token_bytes=527545464 weight_movement_bytes=34603008 total_nvlink_bytes=562148472 weight_to_token_ratio=0.065592 gemm_rows_defined=49536 gemm_rows_charged=51200 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.032500 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=885837004800 gemm_flops_step=2657511014400 bytes_per_gemm_flop=1.122122100e-04 prefetch_bytes_per_gemm_flop=1.302083333e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12236651648 prefetch_bytes=692060160 reduce_bytes=726663168 weight_movement_bytes=1418723328 egress_bytes=6122273920 ingress_bytes=7533101056 link_bytes=7533101056 total_nvlink_bytes=13655374976 weight_to_token_ratio=0.115940 prefetch_share_of_ingress=0.091869 dedup_ratio=0.244171 prefetch_slots=40 unused_prefetch_slots=328 slot_utilization=0.108696 reduce_slots=21 remote_demand=20 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028370 gemm_inner_pad_upper_bound=28829 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.232454470e-04 prefetch_bytes_per_gemm_flop=1.132246377e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=23 plans=23 dispatch_calls=46 combine_calls=46 prefetch_calls=46 reduce_calls=23 token_bytes=12264298040 prefetch_bytes=726663168 reduce_bytes=726663168 weight_movement_bytes=1453326336 egress_bytes=6136108600 ingress_bytes=7581515776 link_bytes=7581515776 total_nvlink_bytes=13717624376 weight_to_token_ratio=0.118501 prefetch_share_of_ingress=0.095847 dedup_ratio=0.244106 prefetch_slots=42 unused_prefetch_slots=326 slot_utilization=0.114130 reduce_slots=21 remote_demand=21 gemm_rows_charged=1177600 gemm_tail_pad_fraction=0.028804 gemm_inner_pad_upper_bound=28956 gemm_flops_step=61122753331200 bytes_per_gemm_flop=1.240375370e-04 prefetch_bytes_per_gemm_flop=1.188858696e-05 anomalies=none + [2026-08-26 18:08:48] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 790.8 | throughput per GPU (TFLOP/s/GPU): 161.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.618065E+00 | loss scale: 1.0 | grad norm: 47.305 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-26 18:08:48 +[rank0]:[W826 18:08:49.746565345 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W826 18:08:50.689938570 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W826 18:08:50.689947988 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W826 18:08:50.897727914 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W826 18:08:50.898201811 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W826 18:08:51.332236719 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W826 18:08:51.334423137 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W826 18:08:51.349025771 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0826 18:08:58.251000 459728 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:08:58.252000 459728 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 459825 closing signal SIGTERM +W0826 18:08:58.252000 459728 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 459828 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -7027,4 +7028,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 287395 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 459728 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv index 0c99cb182f653a2d710d6ff82acf1bc15a005409..83a394f9a9755064af42e52717145e8da49e9ec1 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv @@ -14,7 +14,7 @@ 5, 0 6, 0 7, 0 -0, 4 +0, 1 1, 1 2, 1 3, 1 @@ -37,156 +37,60 @@ 4, 4 5, 4 6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 7, 4 -0, 725 -1, 725 -2, 725 -3, 725 -4, 725 -5, 725 -6, 725 -7, 725 -0, 1083 -1, 1081 -2, 1083 -3, 1083 -4, 1081 -5, 1083 -6, 1081 -7, 1081 -0, 1863 -1, 1863 -2, 1863 -3, 1863 -4, 1863 -5, 1863 -6, 1863 -7, 1863 -0, 6795 -1, 6791 -2, 6801 -3, 6811 -4, 6813 -5, 6789 -6, 6779 -7, 6797 -0, 58379 -1, 57837 -2, 57837 -3, 58517 -4, 58565 -5, 58493 -6, 58581 -7, 58535 -0, 59427 -1, 58873 -2, 58869 -3, 59577 -4, 59581 -5, 59577 -6, 59577 -7, 59587 -0, 59719 -1, 59151 -2, 59151 -3, 59853 -4, 59851 -5, 59851 -6, 59851 -7, 59851 -0, 60421 -1, 59851 -2, 59851 -3, 60551 -4, 60551 -5, 60551 -6, 60551 -7, 60551 -0, 87081 -1, 86879 -2, 90725 -3, 87037 -4, 88613 -5, 88575 -6, 89863 -7, 86317 -0, 117979 -1, 118017 -2, 117995 -3, 118023 -4, 118031 -5, 118033 -6, 118031 -7, 118041 -0, 118939 -1, 118985 -2, 118967 -3, 119001 -4, 119001 -5, 119001 -6, 119001 -7, 119001 -0, 118939 -1, 118985 -2, 118967 -3, 119001 -4, 119001 -5, 119001 -6, 119001 -7, 119001 -0, 119213 -1, 119257 -2, 119239 -3, 119273 -4, 119273 -5, 119273 -6, 119273 -7, 119273 -0, 128731 -1, 128367 -2, 128533 -3, 128585 -4, 128531 -5, 128497 -6, 128519 -7, 128517 -0, 141327 -1, 141555 -2, 141529 -3, 141355 -4, 141355 -5, 141355 -6, 141355 -7, 141355 -0, 141327 -1, 141555 -2, 141529 -3, 141355 -4, 141355 -5, 141355 -6, 141355 -7, 141355 -0, 140745 -1, 141555 -2, 141225 -3, 140931 -4, 140819 -5, 140709 -6, 140709 -7, 140709 -0, 139407 -1, 139697 -2, 138963 -3, 139177 -4, 139177 -5, 139177 -6, 138789 -7, 138853 -0, 12601 -1, 12604 -2, 12604 -3, 12732 -4, 138787 -5, 13244 -6, 12604 -7, 12604 +0, 821 +1, 819 +2, 819 +3, 821 +4, 819 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1919 +1, 1919 +2, 1919 +3, 1919 +4, 1919 +5, 1919 +6, 1919 +7, 1919 +0, 7279 +1, 7279 +2, 7279 +3, 7279 +4, 7279 +5, 7279 +6, 7279 +7, 7279 +0, 58511 +1, 57863 +2, 57885 +3, 58603 +4, 58561 +5, 58591 +6, 58549 +7, 58561 +0, 59095 +1, 58527 +2, 58527 +3, 59227 +4, 59227 +5, 59227 +6, 59227 +7, 59227 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log index 3605df7423954a723783f524174a24719f416727..6e51fdbf41d1ebd52d962d4f6747432915a9a279 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log @@ -1,5 +1,5 @@ -[16:56:53] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:56:53] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[18:08:59] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:08:59] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN2_57B / moonep (slots/rank=8 policy=matched) E=64 K=8 H'=2560 m=8 @@ -8,12 +8,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 16:56:55.100000 290089 torch/distributed/run.py:766] -W0826 16:56:55.100000 290089 torch/distributed/run.py:766] ***************************************** -W0826 16:56:55.100000 290089 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:56:55.100000 290089 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 18:09:00.771000 462358 torch/distributed/run.py:766] +W0826 18:09:00.771000 462358 torch/distributed/run.py:766] ***************************************** +W0826 18:09:00.771000 462358 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:09:00.771000 462358 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -22,11 +20,11 @@ W0826 16:56:55.100000 290089 torch/distributed/run.py:766] ********************* warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -577,54 +575,56 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 5 is connected to 76 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1[Gloo] Rank is connected to [Gloo] Rank 073 is connected to peer ranks. is connected to 7Expected number of connected peer ranks is : 7 peer ranks. 7 peer ranks. Expected number of connected peer ranks is : -Expected number of connected peer ranks is : 77 +[Gloo] Rank [Gloo] Rank 4 is connected to 27 is connected to peer ranks. 7 peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... -[Gloo] Rank [Gloo] Rank 0 is connected to 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. -Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0[Gloo] Rank 0 is connected to is connected to 000 is connected to peer ranks. peer ranks. 0Expected number of connected peer ranks is : Expected number of connected peer ranks is : peer ranks. 00Expected number of connected peer ranks is : + +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank3]:[W826 16:57:13.849136060 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:57:13.849151024 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:57:13.849162466 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:57:13.849970708 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:57:13.852642041 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:57:13.852721283 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:57:13.854587480 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:09:19.798476614 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:09:19.798600798 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:09:19.798663642 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:09:19.798723933 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:09:19.798801193 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:09:19.800080667 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:09:19.800148669 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' ->>> done with dataset index builder. Compilation time: 0.035 seconds +>>> done with dataset index builder. Compilation time: 0.034 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:57:13.017491400 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.568 seconds -time to initialize megatron (seconds): 7.063 -[after megatron is initialized] datetime: 2026-08-26 16:57:19 +[rank0]:[W826 18:09:19.947251625 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.938 seconds +time to initialize megatron (seconds): 7.101 +[after megatron is initialized] datetime: 2026-08-26 18:09:25 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -653,71 +653,71 @@ Params for bucket 2 (73403904 elements, 73403904 padded size): Params for bucket 3 (146800640 elements, 146800640 padded size): module.decoder.layers.9.mlp.shared_experts.linear_fc1.weight Params for bucket 4 (103001600 elements, 103001600 padded size): - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.self_attention.linear_proj.weight - module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight module.decoder.layers.9.mlp.router.weight - module.decoder.layers.9.self_attention.linear_qkv.bias module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_qkv.bias + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight Params for bucket 5 (146800640 elements, 146800640 padded size): module.decoder.layers.8.mlp.shared_experts.linear_fc1.weight Params for bucket 6 (103001600 elements, 103001600 padded size): + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.self_attention.linear_qkv.bias module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.linear_qkv.bias module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.7.mlp.shared_experts.linear_fc2.weight Params for bucket 7 (146800640 elements, 146800640 padded size): module.decoder.layers.7.mlp.shared_experts.linear_fc1.weight Params for bucket 8 (103001600 elements, 103001600 padded size): + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_qkv.bias + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_proj.weight module.decoder.layers.6.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.pre_mlp_layernorm.weight Params for bucket 9 (146800640 elements, 146800640 padded size): module.decoder.layers.6.mlp.shared_experts.linear_fc1.weight Params for bucket 10 (103001600 elements, 103001600 padded size): - module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.linear_qkv.bias - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.5.mlp.shared_experts.linear_fc2.weight Params for bucket 11 (146800640 elements, 146800640 padded size): module.decoder.layers.5.mlp.shared_experts.linear_fc1.weight Params for bucket 12 (103001600 elements, 103001600 padded size): module.decoder.layers.5.mlp.router.weight - module.decoder.layers.5.self_attention.linear_qkv.bias - module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_proj.weight module.decoder.layers.4.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.bias Params for bucket 13 (146800640 elements, 146800640 padded size): module.decoder.layers.4.mlp.shared_experts.linear_fc1.weight Params for bucket 14 (103001600 elements, 103001600 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.linear_qkv.bias module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_proj.weight module.decoder.layers.3.mlp.shared_experts.linear_fc2.weight - module.decoder.layers.4.mlp.router.weight - module.decoder.layers.4.self_attention.linear_qkv.weight - module.decoder.layers.4.pre_mlp_layernorm.weight Params for bucket 15 (146800640 elements, 146800640 padded size): module.decoder.layers.3.mlp.shared_experts.linear_fc1.weight Params for bucket 16 (103001600 elements, 103001600 padded size): - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.3.self_attention.linear_qkv.bias + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.2.mlp.shared_experts.linear_fc2.weight Params for bucket 17 (146800640 elements, 146800640 padded size): @@ -726,42 +726,42 @@ Params for bucket 18 (103001600 elements, 103001600 padded size): module.decoder.layers.2.mlp.router.weight module.decoder.layers.2.self_attention.linear_qkv.bias module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.2.self_attention.linear_proj.weight - module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.mlp.shared_experts.linear_fc2.weight Params for bucket 19 (146800640 elements, 146800640 padded size): module.decoder.layers.1.mlp.shared_experts.linear_fc1.weight Params for bucket 20 (103001600 elements, 103001600 padded size): - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.1.mlp.router.weight module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.bias module.decoder.layers.1.self_attention.linear_proj.weight module.decoder.layers.0.mlp.shared_experts.linear_fc2.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_qkv.weight Params for bucket 21 (146800640 elements, 146800640 padded size): module.decoder.layers.0.mlp.shared_experts.linear_fc1.weight Params for bucket 22 (209890816 elements, 209890816 padded size): - module.decoder.layers.0.self_attention.linear_qkv.bias - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.self_attention.linear_proj.weight module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.linear_qkv.bias module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 44 Params for bucket 1 (45875200 elements, 45875200 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight6 - module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 Params for bucket 2 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 Params for bucket 3 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 @@ -772,37 +772,37 @@ Params for bucket 4 (55050240 elements, 55050240 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 Params for bucket 5 (45875200 elements, 45875200 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight6 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight5 Params for bucket 6 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 Params for bucket 7 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight7 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 Params for bucket 8 (55050240 elements, 55050240 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 Params for bucket 9 (45875200 elements, 45875200 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight1 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight7 Params for bucket 10 (45875200 elements, 45875200 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 Params for bucket 11 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 Params for bucket 12 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 @@ -813,151 +813,151 @@ Params for bucket 13 (55050240 elements, 55050240 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 Params for bucket 14 (45875200 elements, 45875200 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 Params for bucket 15 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 Params for bucket 16 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 module.decoder.layers.6.mlp.experts.linear_fc1.weight5 Params for bucket 17 (55050240 elements, 55050240 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight3 - module.decoder.layers.6.mlp.experts.linear_fc1.weight2 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 Params for bucket 18 (45875200 elements, 45875200 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 Params for bucket 19 (45875200 elements, 45875200 padded size): - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 20 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 Params for bucket 21 (55050240 elements, 55050240 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight3 Params for bucket 22 (45875200 elements, 45875200 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 Params for bucket 23 (45875200 elements, 45875200 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 Params for bucket 24 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 Params for bucket 25 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 Params for bucket 26 (55050240 elements, 55050240 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 27 (45875200 elements, 45875200 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 Params for bucket 28 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight7 - module.decoder.layers.3.mlp.experts.linear_fc2.weight0 Params for bucket 29 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 Params for bucket 30 (55050240 elements, 55050240 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight3 - module.decoder.layers.3.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 31 (45875200 elements, 45875200 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 Params for bucket 32 (45875200 elements, 45875200 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight0 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 - module.decoder.layers.2.mlp.experts.linear_fc2.weight1 Params for bucket 33 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight6 module.decoder.layers.2.mlp.experts.linear_fc1.weight5 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 Params for bucket 34 (55050240 elements, 55050240 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight2 Params for bucket 35 (45875200 elements, 45875200 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 Params for bucket 36 (45875200 elements, 45875200 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 Params for bucket 37 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 Params for bucket 38 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 39 (55050240 elements, 55050240 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 Params for bucket 40 (45875200 elements, 45875200 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 Params for bucket 41 (45875200 elements, 45875200 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 Params for bucket 42 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 - module.decoder.layers.0.mlp.experts.linear_fc1.weight4 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 Params for bucket 43 (55050240 elements, 55050240 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 Params for bucket 44 (18350080 elements, 18350080 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:57:22 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:09:28 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -965,7 +965,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -978,14 +978,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:57:22 +[after dataloaders are built] datetime: 2026-08-26 18:09:28 done with setup ... -training ... -Overwriting rerun_state_machine.current_iteration from -1 to 0... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2479.63, 2494.90) - train/valid/test-data-iterators-setup ..........: (59.84, 69.90) -[before the start of training step] datetime: 2026-08-26 16:57:22 + model-and-optimizer-setup ......................: (2631.16, 2638.71) + train/valid/test-data-iterators-setup ..........: (56.31, 69.36)training ... + +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-26 18:09:28 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=3584, E=64, K=8, NvS=34800, NvS_padded=34816 sm_count : num_sms=32, num_sms_dedup=148 @@ -993,2548 +993,3 @@ INFO:moonep.api:MoonEP comm buffer sizes: meta_buffer_size : 2.00 MiB local_temp_buffer_size : 1.26 MiB total_size : 241.26 MiB -WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 16:57:44] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 22495.7 | throughput per GPU (TFLOP/s/GPU): 5.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.097850E+01 | loss scale: 1.0 | grad norm: 29.821 | number of skipped iterations: 0 | number of nan iterations: 0 | -Number of parameters in transformer block in billions: 20.11 -Number of parameters in embedding layers in billions: 0.36 -Total number of parameters in billions: 20.47 -Number of parameters in most loaded shard in billions: 20.4724 -Theoretical memory footprints: weight and optimizer=146430.14 MB -[Rank 0] (after 1 iterations) memory (MB) | allocated: 104985.1162109375 | max allocated: 104985.11669921875 | reserved: 108632.0 | max reserved: 108632.0 - [2026-08-26 16:57:45] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 436.1 | throughput per GPU (TFLOP/s/GPU): 283.2 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.097848E+01 | loss scale: 1.0 | grad norm: 30.558 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46488 payload_rows_remote=40946 dup_rows=19048 dup_rows_remote=17264 dedup_ratio=0.290649 router_tokens_per_expert=[363, 132, 574, 35, 27, 18, 1073, 15, 1237, 928, 216, 95, 1212, 217, 272, 296, 49, 37, 671, 509, 452, 89, 261, 27, 202, 1095, 346, 1058, 846, 983, 214, 53, 523, 96, 1498, 590, 635, 15, 73, 233, 406, 623, 1209, 63, 660, 148, 303, 360, 1247, 730, 115, 51, 715, 882, 3160, 1770, 306, 444, 285, 262, 814, 728, 40, 182] payload_rows_per_dst=[5296, 6224, 7170, 6212, 5542, 5368, 5148, 5528] dup_rows_per_dst=[1672, 2722, 3340, 3382, 1784, 2176, 1546, 2426] combine_rows_per_src=[5296, 6224, 7170, 6212, 5542, 5368, 5148, 5528] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45166 payload_rows_remote=39130 dup_rows=20370 dup_rows_remote=18268 dedup_ratio=0.310822 router_tokens_per_expert=[402, 143, 597, 10, 164, 19, 919, 94, 1156, 630, 268, 273, 1363, 159, 51, 408, 158, 70, 295, 227, 919, 163, 105, 24, 246, 1168, 343, 711, 453, 544, 315, 30, 1084, 664, 1150, 482, 717, 36, 109, 741, 690, 621, 1209, 79, 808, 215, 189, 219, 1246, 390, 174, 37, 360, 1228, 3260, 1434, 231, 457, 222, 59, 813, 616, 78, 723] payload_rows_per_dst=[5428, 5050, 4920, 5286, 6644, 5578, 6036, 6224] dup_rows_per_dst=[1760, 2004, 1762, 2334, 4884, 2482, 2102, 3042] combine_rows_per_src=[5428, 5050, 4920, 5286, 6644, 5578, 6036, 6224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45560 payload_rows_remote=39644 dup_rows=19976 dup_rows_remote=17172 dedup_ratio=0.304810 router_tokens_per_expert=[506, 199, 361, 18, 31, 30, 1119, 23, 1043, 941, 188, 103, 1321, 143, 106, 425, 58, 58, 455, 564, 661, 22, 188, 57, 447, 1128, 368, 824, 1018, 428, 334, 49, 517, 278, 1265, 429, 687, 47, 163, 474, 523, 1230, 795, 155, 970, 264, 63, 360, 1516, 734, 211, 49, 604, 955, 2508, 1094, 410, 649, 280, 53, 542, 1225, 141, 361] payload_rows_per_dst=[5526, 5978, 6398, 5898, 5340, 5916, 4338, 6166] dup_rows_per_dst=[2080, 2562, 2744, 3294, 2380, 2804, 768, 3344] combine_rows_per_src=[5526, 5978, 6398, 5898, 5340, 5916, 4338, 6166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45802 payload_rows_remote=40382 dup_rows=19734 dup_rows_remote=17688 dedup_ratio=0.301117 router_tokens_per_expert=[683, 165, 1161, 13, 155, 2, 742, 75, 925, 872, 193, 192, 1267, 229, 32, 365, 126, 19, 391, 370, 790, 30, 223, 48, 327, 636, 348, 775, 715, 529, 344, 59, 664, 388, 722, 323, 661, 48, 143, 494, 551, 997, 1202, 211, 1244, 204, 74, 206, 1442, 372, 215, 73, 453, 834, 2452, 809, 643, 839, 455, 106, 646, 1649, 190, 657] payload_rows_per_dst=[6108, 5698, 6240, 5420, 4936, 6392, 4504, 6504] dup_rows_per_dst=[2768, 2452, 2658, 2046, 1950, 2986, 1008, 3866] combine_rows_per_src=[6108, 5698, 6240, 5420, 4936, 6392, 4504, 6504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43514 payload_rows_remote=37942 dup_rows=22022 dup_rows_remote=19894 dedup_ratio=0.336029 router_tokens_per_expert=[371, 172, 995, 17, 131, 10, 1085, 180, 881, 1102, 196, 291, 900, 167, 354, 394, 90, 27, 503, 267, 234, 71, 184, 75, 256, 1358, 403, 862, 496, 886, 280, 31, 310, 134, 1404, 280, 1044, 26, 170, 552, 446, 678, 1490, 168, 671, 149, 129, 430, 1267, 686, 135, 28, 1069, 1901, 2482, 1269, 361, 526, 191, 68, 501, 643, 85, 206] payload_rows_per_dst=[5736, 5030, 2596, 4954, 6156, 5758, 7712, 5572] dup_rows_per_dst=[2208, 1740, 306, 1528, 3484, 2564, 8064, 2128] combine_rows_per_src=[5736, 5030, 2596, 4954, 6156, 5758, 7712, 5572] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46194 payload_rows_remote=39852 dup_rows=19342 dup_rows_remote=16930 dedup_ratio=0.295135 router_tokens_per_expert=[365, 117, 971, 25, 154, 40, 378, 111, 662, 1060, 394, 331, 860, 188, 187, 473, 63, 60, 523, 188, 511, 100, 259, 49, 335, 1233, 652, 951, 414, 406, 283, 92, 530, 198, 928, 556, 784, 48, 62, 683, 345, 1076, 1269, 214, 1003, 127, 285, 290, 1520, 458, 101, 13, 836, 1582, 2624, 1249, 337, 525, 281, 110, 561, 1393, 88, 257] payload_rows_per_dst=[5546, 5776, 6342, 5834, 5326, 6224, 6042, 5104] dup_rows_per_dst=[1816, 2534, 2412, 2898, 2252, 2994, 2436, 2000] combine_rows_per_src=[5546, 5776, 6342, 5834, 5326, 6224, 6042, 5104] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46016 payload_rows_remote=40264 dup_rows=19520 dup_rows_remote=17056 dedup_ratio=0.297852 router_tokens_per_expert=[743, 164, 911, 8, 147, 6, 909, 101, 961, 1237, 215, 116, 927, 236, 84, 332, 76, 68, 364, 197, 357, 76, 590, 59, 663, 1462, 326, 771, 330, 702, 215, 132, 882, 247, 567, 269, 639, 196, 80, 739, 417, 1144, 637, 96, 798, 290, 59, 280, 1899, 283, 103, 20, 846, 1593, 2701, 1011, 378, 364, 174, 76, 430, 1564, 130, 371] payload_rows_per_dst=[6524, 5752, 6530, 6022, 5146, 5488, 5644, 4910] dup_rows_per_dst=[3252, 2464, 2446, 3180, 2092, 1954, 2068, 2064] combine_rows_per_src=[6524, 5752, 6530, 6022, 5146, 5488, 5644, 4910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=293500928 dispatch_meta_bytes=232840 combine_payload_bytes=293500928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293733768 ingress_bytes=403601408 link_bytes=403601408 token_bytes=587234696 weight_movement_bytes=110100480 total_nvlink_bytes=697335176 weight_to_token_ratio=0.187490 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.019292295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284168192 dispatch_meta_bytes=227264 combine_payload_bytes=284168192 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=284395456 ingress_bytes=504369152 link_bytes=504369152 token_bytes=568563648 weight_movement_bytes=220200960 total_nvlink_bytes=788764608 weight_to_token_ratio=0.387293 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.771809098e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280483840 dispatch_meta_bytes=229592 combine_payload_bytes=280483840 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=280713432 ingress_bytes=720885760 link_bytes=720885760 token_bytes=561197272 weight_movement_bytes=440401920 total_nvlink_bytes=1001599192 weight_to_token_ratio=0.784754 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253738902e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271968256 dispatch_meta_bytes=231344 combine_payload_bytes=271968256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272199600 ingress_bytes=382068736 link_bytes=382068736 token_bytes=544167856 weight_movement_bytes=110100480 total_nvlink_bytes=654268336 weight_to_token_ratio=0.202328 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.644803714e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289458176 dispatch_meta_bytes=232280 combine_payload_bytes=289458176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289690456 ingress_bytes=509659136 link_bytes=509659136 token_bytes=579148632 weight_movement_bytes=220200960 total_nvlink_bytes=799349592 weight_to_token_ratio=0.380215 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.863810620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285659136 dispatch_meta_bytes=227128 combine_payload_bytes=285659136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285886264 ingress_bytes=395759616 link_bytes=395759616 token_bytes=571545400 weight_movement_bytes=110100480 total_nvlink_bytes=681645880 weight_to_token_ratio=0.192636 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=18048 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.882910635e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=288612352 dispatch_meta_bytes=229280 combine_payload_bytes=288612352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=288841632 ingress_bytes=508813312 link_bytes=508813312 token_bytes=577453984 weight_movement_bytes=220200960 total_nvlink_bytes=797654944 weight_to_token_ratio=0.381331 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.849100350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47346 payload_rows_remote=39998 dup_rows=18190 dup_rows_remote=13880 dedup_ratio=0.277557 router_tokens_per_expert=[72, 63, 1055, 170, 31, 25, 591, 3, 1651, 1530, 342, 13, 1908, 2976, 26, 2112, 1, 105, 86, 1010, 1, 157, 0, 694, 148, 684, 16, 559, 1709, 46, 60, 15, 2454, 923, 0, 173, 228, 61, 6, 19, 15, 10, 766, 146, 314, 3, 14, 174, 1119, 1926, 745, 62, 558, 290, 131, 21, 26, 71, 1446, 22, 404, 1848, 389, 545] payload_rows_per_dst=[5654, 5512, 6282, 4960, 6000, 7348, 6350, 5240] dup_rows_per_dst=[2062, 1612, 2050, 1514, 1728, 4310, 3354, 1560] combine_rows_per_src=[5654, 5512, 6282, 4960, 6000, 7348, 6350, 5240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47678 payload_rows_remote=40372 dup_rows=17858 dup_rows_remote=16118 dedup_ratio=0.272491 router_tokens_per_expert=[4, 34, 1732, 854, 17, 31, 541, 28, 988, 712, 308, 24, 2021, 3074, 20, 1793, 0, 83, 36, 944, 41, 219, 14, 815, 66, 320, 8, 841, 1220, 170, 75, 41, 3505, 731, 1, 35, 200, 6, 37, 8, 2, 19, 1302, 98, 140, 1, 0, 271, 1765, 1205, 91, 350, 273, 265, 26, 86, 34, 139, 469, 47, 2306, 1992, 128, 162] payload_rows_per_dst=[6608, 4854, 6160, 4324, 7306, 7258, 5738, 5430] dup_rows_per_dst=[3858, 1174, 3288, 1158, 1740, 3116, 2384, 1140] combine_rows_per_src=[6608, 4854, 6160, 4324, 7306, 7258, 5738, 5430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46574 payload_rows_remote=39858 dup_rows=18962 dup_rows_remote=15862 dedup_ratio=0.289337 router_tokens_per_expert=[678, 185, 516, 8, 113, 20, 995, 68, 702, 1276, 287, 70, 1372, 210, 162, 477, 89, 15, 548, 250, 380, 108, 196, 29, 187, 921, 389, 890, 516, 347, 192, 53, 525, 424, 1108, 400, 1028, 13, 71, 241, 688, 600, 1079, 106, 835, 212, 52, 202, 2325, 394, 175, 25, 415, 1249, 3221, 1802, 441, 379, 188, 141, 1000, 870, 87, 223] payload_rows_per_dst=[6716, 6382, 7184, 5104, 5302, 4958, 5984, 4944] dup_rows_per_dst=[3100, 3168, 2488, 2144, 2318, 1894, 2136, 1714] combine_rows_per_src=[6716, 6382, 7184, 5104, 5302, 4958, 5984, 4944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46702 payload_rows_remote=40800 dup_rows=18834 dup_rows_remote=16424 dedup_ratio=0.287384 router_tokens_per_expert=[77, 6, 783, 393, 90, 37, 328, 4, 1546, 1480, 333, 0, 1883, 1749, 22, 1586, 0, 407, 34, 628, 2, 917, 17, 633, 106, 359, 2, 1144, 1342, 211, 3, 139, 3233, 733, 1, 201, 47, 0, 19, 115, 5, 7, 727, 68, 339, 1, 11, 275, 1318, 1529, 141, 647, 360, 78, 69, 14, 49, 50, 1764, 107, 1613, 2330, 212, 444] payload_rows_per_dst=[5368, 5266, 5878, 5060, 6972, 4956, 5902, 7300] dup_rows_per_dst=[1854, 1496, 2570, 1552, 1726, 1408, 2410, 5818] combine_rows_per_src=[5368, 5266, 5878, 5060, 6972, 4956, 5902, 7300] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44450 payload_rows_remote=37034 dup_rows=21086 dup_rows_remote=14172 dedup_ratio=0.321747 router_tokens_per_expert=[17, 13, 1177, 340, 43, 27, 452, 27, 1020, 659, 214, 17, 1199, 1898, 29, 1766, 0, 364, 19, 472, 20, 476, 34, 888, 151, 523, 4, 1138, 1443, 198, 14, 31, 2383, 786, 4, 95, 117, 3, 97, 44, 9, 7, 1616, 175, 335, 0, 2, 164, 1396, 1863, 294, 426, 860, 1344, 59, 50, 45, 117, 719, 151, 1689, 2589, 131, 525] payload_rows_per_dst=[3688, 3492, 5964, 5166, 5770, 5912, 7042, 7416] dup_rows_per_dst=[504, 386, 2114, 1838, 1288, 2500, 5542, 6914] combine_rows_per_src=[3688, 3492, 5964, 5166, 5770, 5912, 7042, 7416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286705664 dispatch_meta_bytes=215512 combine_payload_bytes=286705664 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=286921176 ingress_bytes=506906624 link_bytes=506906624 token_bytes=573626840 weight_movement_bytes=220200960 total_nvlink_bytes=793827800 weight_to_token_ratio=0.383875 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.815939909e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47562 payload_rows_remote=41344 dup_rows=17974 dup_rows_remote=16002 dedup_ratio=0.274261 router_tokens_per_expert=[42, 14, 1608, 491, 29, 48, 434, 78, 993, 1372, 305, 15, 1989, 1743, 162, 2360, 1, 342, 64, 381, 4, 539, 12, 392, 235, 96, 12, 288, 1161, 339, 68, 98, 2777, 724, 1, 279, 282, 6, 174, 12, 17, 0, 1658, 117, 573, 2, 14, 252, 1773, 1892, 213, 133, 382, 726, 92, 6, 16, 38, 1502, 118, 458, 2449, 104, 263] payload_rows_per_dst=[6558, 6718, 6218, 6230, 6440, 5924, 5120, 4354] dup_rows_per_dst=[3828, 2954, 1972, 2148, 2070, 2828, 1530, 644] combine_rows_per_src=[6558, 6718, 6218, 6230, 6440, 5924, 5120, 4354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48658 payload_rows_remote=41894 dup_rows=16878 dup_rows_remote=13786 dedup_ratio=0.257538 router_tokens_per_expert=[79, 62, 1294, 398, 4, 87, 387, 1, 978, 661, 317, 92, 2727, 1986, 9, 2644, 1, 175, 79, 410, 5, 78, 7, 866, 200, 282, 3, 1250, 1557, 79, 7, 17, 2984, 735, 0, 137, 187, 7, 183, 10, 4, 1, 562, 69, 318, 0, 0, 469, 1326, 1861, 940, 48, 398, 98, 19, 6, 26, 24, 2070, 71, 983, 1704, 370, 416] payload_rows_per_dst=[5724, 6798, 6522, 6764, 6580, 5270, 4894, 6106] dup_rows_per_dst=[2308, 2770, 2008, 3092, 1906, 1548, 1432, 1814] combine_rows_per_src=[5724, 6798, 6522, 6764, 6580, 5270, 4894, 6106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=289386496 dispatch_meta_bytes=225960 combine_payload_bytes=289386496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=289612456 ingress_bytes=509587456 link_bytes=509587456 token_bytes=578998952 weight_movement_bytes=220200960 total_nvlink_bytes=799199912 weight_to_token_ratio=0.380313 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.862563987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=292454400 dispatch_meta_bytes=228896 combine_payload_bytes=292454400 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=292683296 ingress_bytes=512655360 link_bytes=512655360 token_bytes=585137696 weight_movement_bytes=220200960 total_nvlink_bytes=805338656 weight_to_token_ratio=0.376323 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.915919884e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285702144 dispatch_meta_bytes=222880 combine_payload_bytes=285702144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285925024 ingress_bytes=395802624 link_bytes=395802624 token_bytes=571627168 weight_movement_bytes=110100480 total_nvlink_bytes=681727648 weight_to_token_ratio=0.192609 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.883658615e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265459712 dispatch_meta_bytes=204824 combine_payload_bytes=265459712 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=265664536 ingress_bytes=485660672 link_bytes=485660672 token_bytes=531124248 weight_movement_bytes=220200960 total_nvlink_bytes=751325208 weight_to_token_ratio=0.414594 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=11264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.446437861e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48722 payload_rows_remote=41724 dup_rows=16814 dup_rows_remote=13348 dedup_ratio=0.256561 router_tokens_per_expert=[34, 11, 1738, 289, 39, 34, 699, 51, 1569, 1546, 318, 14, 1725, 1744, 60, 1997, 2, 83, 108, 389, 25, 638, 7, 397, 302, 82, 49, 484, 1917, 50, 15, 17, 3511, 675, 76, 23, 253, 72, 241, 5, 48, 2, 672, 303, 185, 21, 37, 249, 1946, 1428, 153, 62, 220, 952, 112, 48, 9, 22, 1953, 59, 348, 2393, 112, 145] payload_rows_per_dst=[6780, 6998, 5798, 6428, 7340, 4992, 5600, 4786] dup_rows_per_dst=[3796, 3466, 1494, 2260, 2372, 1530, 1386, 510] combine_rows_per_src=[6780, 6998, 5798, 6428, 7340, 4992, 5600, 4786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296353792 dispatch_meta_bytes=229384 combine_payload_bytes=296353792 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=296583176 ingress_bytes=516554752 link_bytes=516554752 token_bytes=592936968 weight_movement_bytes=220200960 total_nvlink_bytes=813137928 weight_to_token_ratio=0.371373 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.983736724e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=300296192 dispatch_meta_bytes=222720 combine_payload_bytes=300296192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300518912 ingress_bytes=410396672 link_bytes=410396672 token_bytes=600815104 weight_movement_bytes=110100480 total_nvlink_bytes=710915584 weight_to_token_ratio=0.183252 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.137473113e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 2, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46892 payload_rows_remote=40726 dup_rows=18644 dup_rows_remote=17174 dedup_ratio=0.284485 router_tokens_per_expert=[1261, 995, 73, 19, 35, 201, 1193, 5, 5, 2069, 12, 2, 146, 200, 3025, 0, 0, 334, 0, 593, 217, 1239, 4, 1, 1425, 0, 194, 1964, 37, 11, 126, 707, 1346, 932, 12, 217, 1144, 3, 1429, 30, 397, 125, 0, 7, 593, 2690, 2, 4, 1, 0, 1792, 0, 491, 4, 198, 0, 812, 2, 16, 30, 863, 1, 2936, 598] payload_rows_per_dst=[6000, 706, 6576, 6618, 6580, 6166, 7364, 6882] dup_rows_per_dst=[1564, 24, 2338, 2310, 3646, 1470, 3658, 3634] combine_rows_per_src=[6000, 706, 6576, 6618, 6580, 6166, 7364, 6882] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=299077632 dispatch_meta_bytes=220288 combine_payload_bytes=299077632 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=299297920 ingress_bytes=959680512 link_bytes=959680512 token_bytes=598375552 weight_movement_bytes=660602880 total_nvlink_bytes=1258978432 weight_to_token_ratio=1.103994 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.669042251e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47242 payload_rows_remote=43104 dup_rows=18294 dup_rows_remote=17448 dedup_ratio=0.279144 router_tokens_per_expert=[448, 2130, 121, 26, 5, 162, 693, 3, 0, 3036, 12, 12, 335, 529, 2698, 0, 0, 243, 0, 1214, 58, 1499, 116, 36, 2156, 9, 1543, 1305, 1, 2, 19, 1138, 411, 378, 7, 134, 339, 1, 1212, 10, 656, 97, 0, 28, 1148, 2080, 0, 0, 33, 0, 1139, 1, 964, 0, 224, 0, 772, 0, 24, 4, 758, 0, 2407, 392] payload_rows_per_dst=[5850, 4802, 6002, 7452, 4138, 5866, 7098, 6034] dup_rows_per_dst=[1540, 856, 2520, 4894, 846, 2152, 3020, 2466] combine_rows_per_src=[5850, 4802, 6002, 7452, 4138, 5866, 7098, 6034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46934 payload_rows_remote=41024 dup_rows=18602 dup_rows_remote=16738 dedup_ratio=0.283844 router_tokens_per_expert=[1826, 854, 113, 5, 0, 185, 386, 0, 67, 3346, 0, 28, 83, 681, 2153, 0, 0, 290, 13, 248, 20, 2478, 12, 10, 1564, 1, 458, 1011, 0, 15, 45, 960, 275, 908, 61, 118, 120, 0, 3153, 25, 217, 248, 0, 85, 934, 2334, 56, 1, 3, 0, 995, 0, 496, 4, 236, 0, 1499, 3, 4, 9, 1145, 1, 2879, 107] payload_rows_per_dst=[5568, 1664, 7768, 6064, 6880, 6004, 5910, 7076] dup_rows_per_dst=[1170, 54, 5066, 2044, 2126, 1746, 1864, 4532] combine_rows_per_src=[5568, 1664, 7768, 6064, 6880, 6004, 5910, 7076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=291923968 dispatch_meta_bytes=231600 combine_payload_bytes=291923968 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292155568 ingress_bytes=402024448 link_bytes=402024448 token_bytes=584079536 weight_movement_bytes=110100480 total_nvlink_bytes=694180016 weight_to_token_ratio=0.188503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.991866367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47016 payload_rows_remote=40492 dup_rows=18520 dup_rows_remote=15686 dedup_ratio=0.282593 router_tokens_per_expert=[676, 1715, 44, 4, 13, 238, 330, 2, 17, 2708, 4, 20, 646, 1147, 3177, 3, 0, 550, 0, 82, 32, 2022, 5, 15, 1385, 0, 245, 1896, 1, 12, 23, 1435, 1285, 442, 11, 209, 397, 14, 1571, 3, 396, 435, 11, 72, 982, 2456, 0, 0, 3, 0, 1527, 0, 758, 0, 638, 3, 462, 7, 20, 20, 884, 0, 1527, 188] payload_rows_per_dst=[5258, 3396, 7342, 5832, 3982, 7104, 7578, 6524] dup_rows_per_dst=[786, 278, 3486, 1664, 740, 4098, 4634, 2834] combine_rows_per_src=[5258, 3396, 7342, 5832, 3982, 7104, 7578, 6524] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46202 payload_rows_remote=41846 dup_rows=19334 dup_rows_remote=18454 dedup_ratio=0.295013 router_tokens_per_expert=[501, 1218, 133, 5, 18, 435, 856, 2, 0, 2563, 26, 2, 92, 937, 2968, 0, 5, 545, 6, 846, 13, 1077, 113, 13, 498, 0, 634, 2791, 15, 29, 36, 1153, 1099, 646, 8, 856, 425, 3, 2136, 31, 315, 135, 0, 3, 472, 2189, 0, 0, 0, 0, 2453, 43, 221, 13, 332, 1, 913, 1, 112, 4, 1267, 1, 800, 759] payload_rows_per_dst=[5854, 7778, 4356, 5848, 6864, 5274, 5368, 4860] dup_rows_per_dst=[2082, 7542, 880, 2320, 3544, 954, 758, 1254] combine_rows_per_src=[5854, 7778, 4356, 5848, 6864, 5274, 5368, 4860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=308969472 dispatch_meta_bytes=242208 combine_payload_bytes=308969472 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=309211680 ingress_bytes=529170432 link_bytes=529170432 token_bytes=618181152 weight_movement_bytes=220200960 total_nvlink_bytes=838382112 weight_to_token_ratio=0.356208 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.203144148e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=294060032 dispatch_meta_bytes=231048 combine_payload_bytes=294060032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294291080 ingress_bytes=404160512 link_bytes=404160512 token_bytes=588351112 weight_movement_bytes=110100480 total_nvlink_bytes=698451592 weight_to_token_ratio=0.187134 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.029016033e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47678 payload_rows_remote=41812 dup_rows=17858 dup_rows_remote=16052 dedup_ratio=0.272491 router_tokens_per_expert=[1646, 1398, 34, 19, 4, 93, 554, 0, 1, 1810, 12, 26, 102, 274, 1735, 0, 0, 851, 0, 738, 143, 1800, 1, 1, 1814, 0, 137, 1218, 0, 4, 73, 590, 263, 1483, 1, 288, 123, 2, 1934, 25, 334, 849, 0, 2, 1050, 1802, 2, 0, 0, 0, 1987, 0, 336, 0, 211, 0, 2219, 1, 5, 38, 1647, 0, 2846, 242] payload_rows_per_dst=[7730, 5666, 5418, 5866, 6168, 6066, 4722, 6042] dup_rows_per_dst=[5458, 1704, 1650, 1806, 2070, 2012, 896, 2262] combine_rows_per_src=[7730, 5666, 5418, 5866, 6168, 6066, 4722, 6042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=290246656 dispatch_meta_bytes=224712 combine_payload_bytes=290246656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290471368 ingress_bytes=510447616 link_bytes=510447616 token_bytes=580718024 weight_movement_bytes=220200960 total_nvlink_bytes=800918984 weight_to_token_ratio=0.379187 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.877523584e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=299952128 dispatch_meta_bytes=241200 combine_payload_bytes=299952128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300193328 ingress_bytes=410052608 link_bytes=410052608 token_bytes=600145456 weight_movement_bytes=110100480 total_nvlink_bytes=710245936 weight_to_token_ratio=0.183456 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.131489274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45906 payload_rows_remote=38084 dup_rows=19630 dup_rows_remote=11712 dedup_ratio=0.299530 router_tokens_per_expert=[1254, 548, 232, 0, 0, 191, 425, 15, 2, 1373, 6, 0, 139, 1570, 3125, 2, 63, 107, 32, 1205, 124, 1106, 3, 37, 1775, 8, 120, 1653, 45, 273, 13, 1064, 513, 1323, 2, 114, 1585, 5, 957, 26, 572, 39, 1, 5, 242, 2421, 8, 0, 10, 7, 1932, 5, 1026, 68, 808, 2, 569, 19, 39, 242, 1096, 0, 1889, 733] payload_rows_per_dst=[6492, 7822, 4444, 5342, 6182, 5452, 5692, 4480] dup_rows_per_dst=[2616, 7918, 910, 1254, 2868, 1124, 2024, 916] combine_rows_per_src=[6492, 7822, 4444, 5342, 6182, 5452, 5692, 4480] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46356 payload_rows_remote=40874 dup_rows=19180 dup_rows_remote=17716 dedup_ratio=0.292664 router_tokens_per_expert=[2, 1, 682, 86, 11, 26, 843, 1, 918, 1485, 155, 10, 1759, 988, 6, 1379, 1, 236, 101, 370, 1, 454, 8, 1059, 129, 279, 2, 1112, 2448, 137, 31, 13, 3201, 802, 0, 221, 366, 13, 14, 0, 57, 0, 1453, 82, 608, 0, 5, 125, 1922, 2370, 359, 951, 218, 140, 55, 0, 11, 9, 1321, 46, 886, 1821, 500, 479] payload_rows_per_dst=[5482, 7184, 5538, 7522, 5076, 4858, 5582, 5114] dup_rows_per_dst=[1464, 4856, 1680, 5520, 1222, 1778, 1270, 1390] combine_rows_per_src=[5482, 7184, 5538, 7522, 5076, 4858, 5582, 5114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=299708416 dispatch_meta_bytes=231456 combine_payload_bytes=299708416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=299939872 ingress_bytes=630009856 link_bytes=630009856 token_bytes=599648288 weight_movement_bytes=330301440 total_nvlink_bytes=929949728 weight_to_token_ratio=0.550825 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.095690758e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52676 payload_rows_remote=46056 dup_rows=12860 dup_rows_remote=11452 dedup_ratio=0.196228 router_tokens_per_expert=[0, 17, 80, 174, 2874, 271, 3026, 64, 40, 0, 43, 131, 3253, 3366, 142, 17, 167, 32, 3399, 95, 166, 0, 43, 6, 6, 334, 650, 155, 0, 56, 371, 0, 21, 2, 18, 1121, 4, 3981, 587, 22, 1, 2838, 16, 58, 172, 0, 928, 1, 1884, 7, 298, 9, 165, 0, 195, 6, 327, 136, 17, 3, 4, 1, 71, 897] payload_rows_per_dst=[6182, 6926, 6996, 7136, 3276, 6620, 7438, 8102] dup_rows_per_dst=[778, 552, 820, 2514, 274, 1408, 3742, 2772] combine_rows_per_src=[6182, 6926, 6996, 7136, 3276, 6620, 7438, 8102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51402 payload_rows_remote=44338 dup_rows=14134 dup_rows_remote=10198 dedup_ratio=0.215668 router_tokens_per_expert=[2, 226, 115, 44, 2016, 7, 1904, 5, 2, 0, 0, 190, 3642, 3849, 18, 3, 279, 3, 1687, 2, 17, 0, 148, 5, 281, 98, 959, 3, 0, 22, 198, 0, 0, 0, 1, 2381, 62, 3661, 568, 10, 0, 3928, 32, 1, 175, 0, 1185, 3, 2553, 3, 56, 0, 17, 0, 350, 3, 109, 162, 1, 0, 2, 0, 56, 1724] payload_rows_per_dst=[5110, 7726, 3978, 7644, 7064, 8010, 6622, 5248] dup_rows_per_dst=[624, 398, 304, 2762, 3936, 2638, 2246, 1226] combine_rows_per_src=[5110, 7726, 3978, 7644, 7064, 8010, 6622, 5248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=272986112 dispatch_meta_bytes=199184 combine_payload_bytes=272986112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=273185296 ingress_bytes=603287552 link_bytes=603287552 token_bytes=546171408 weight_movement_bytes=330301440 total_nvlink_bytes=876472848 weight_to_token_ratio=0.604758 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.049216277e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=292984832 dispatch_meta_bytes=234360 combine_payload_bytes=292984832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293219192 ingress_bytes=403085312 link_bytes=403085312 token_bytes=586204024 weight_movement_bytes=110100480 total_nvlink_bytes=696304504 weight_to_token_ratio=0.187819 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.010316537e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47022 payload_rows_remote=39474 dup_rows=18514 dup_rows_remote=16508 dedup_ratio=0.282501 router_tokens_per_expert=[0, 10, 489, 21, 2149, 6, 3359, 8, 1, 1, 0, 545, 2322, 3520, 60, 1, 136, 9, 2524, 50, 65, 0, 20, 2, 36, 82, 636, 12, 6, 20, 340, 3, 18, 0, 91, 1374, 2, 4079, 1096, 3, 0, 3671, 1, 27, 1703, 0, 816, 0, 1256, 5, 90, 9, 53, 0, 3, 2, 52, 4, 0, 3, 0, 1, 183, 1793] payload_rows_per_dst=[4848, 2858, 5354, 5574, 4554, 8104, 7548, 8182] dup_rows_per_dst=[518, 540, 258, 1340, 614, 9190, 2006, 4048] combine_rows_per_src=[4848, 2858, 5354, 5574, 4554, 8104, 7548, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330129408 dispatch_meta_bytes=230032 combine_payload_bytes=330129408 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=330359440 ingress_bytes=550330368 link_bytes=550330368 token_bytes=660488848 weight_movement_bytes=220200960 total_nvlink_bytes=880689808 weight_to_token_ratio=0.333391 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.571150237e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47434 payload_rows_remote=40416 dup_rows=18102 dup_rows_remote=14022 dedup_ratio=0.276215 router_tokens_per_expert=[9, 77, 229, 53, 3116, 4, 2710, 19, 15, 0, 66, 154, 3136, 2959, 145, 171, 859, 3, 2144, 23, 26, 0, 15, 10, 95, 335, 1188, 0, 0, 107, 15, 1, 0, 0, 116, 1715, 0, 3772, 35, 325, 19, 2469, 49, 1, 195, 0, 442, 2, 2830, 9, 85, 0, 51, 0, 229, 126, 479, 31, 0, 0, 41, 0, 6, 2057] payload_rows_per_dst=[7878, 7294, 7018, 5130, 8054, 1342, 6024, 4694] dup_rows_per_dst=[4556, 2936, 4080, 1414, 3872, 74, 636, 534] combine_rows_per_src=[7878, 7294, 7018, 5130, 8054, 1342, 6024, 4694] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46096 payload_rows_remote=38098 dup_rows=19440 dup_rows_remote=16438 dedup_ratio=0.296631 router_tokens_per_expert=[3, 31, 78, 224, 2048, 9, 2744, 7, 6, 0, 4, 148, 3417, 3407, 128, 5, 633, 14, 2737, 28, 51, 0, 87, 6, 116, 340, 533, 20, 1, 55, 12, 0, 1, 1, 152, 1987, 2, 3968, 517, 78, 2, 3466, 5, 10, 203, 8, 1573, 0, 1612, 13, 149, 10, 355, 0, 167, 65, 227, 115, 2, 0, 2, 1, 122, 1063] payload_rows_per_dst=[3368, 578, 5968, 7226, 5530, 8122, 7306, 7998] dup_rows_per_dst=[216, 4, 1144, 1762, 1162, 9226, 2924, 3002] combine_rows_per_src=[3368, 578, 5968, 7226, 5530, 8122, 7306, 7998] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=317814784 dispatch_meta_bytes=218144 combine_payload_bytes=317814784 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=318032928 ingress_bytes=648116224 link_bytes=648116224 token_bytes=635847712 weight_movement_bytes=330301440 total_nvlink_bytes=966149152 weight_to_token_ratio=0.519466 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.127180710e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=282949632 dispatch_meta_bytes=223928 combine_payload_bytes=282949632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=283173560 ingress_bytes=393050112 link_bytes=393050112 token_bytes=566123192 weight_movement_bytes=110100480 total_nvlink_bytes=676223672 weight_to_token_ratio=0.194481 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.835787904e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50278 payload_rows_remote=42502 dup_rows=15258 dup_rows_remote=10580 dedup_ratio=0.232819 router_tokens_per_expert=[0, 39, 76, 85, 1733, 135, 3590, 2, 3, 0, 1, 32, 3347, 2639, 306, 20, 99, 2, 2465, 55, 36, 0, 36, 8, 6, 171, 2507, 3, 0, 10, 183, 0, 1, 0, 4, 1045, 0, 3614, 549, 59, 0, 3728, 1, 2, 365, 0, 461, 0, 2147, 4, 79, 2, 173, 0, 45, 8, 119, 2, 0, 0, 9, 0, 29, 2733] payload_rows_per_dst=[7822, 5716, 6948, 7776, 7806, 3966, 4654, 5590] dup_rows_per_dst=[3498, 286, 3022, 4678, 2738, 580, 262, 194] combine_rows_per_src=[7822, 5716, 6948, 7776, 7806, 3966, 4654, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=289701888 dispatch_meta_bytes=217752 combine_payload_bytes=289701888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289919640 ingress_bytes=399802368 link_bytes=399802368 token_bytes=579621528 weight_movement_bytes=110100480 total_nvlink_bytes=689722008 weight_to_token_ratio=0.189952 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.953220741e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=273086464 dispatch_meta_bytes=218144 combine_payload_bytes=273086464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=273304608 ingress_bytes=383186944 link_bytes=383186944 token_bytes=546391072 weight_movement_bytes=110100480 total_nvlink_bytes=656491552 weight_to_token_ratio=0.201505 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.664251190e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49514 payload_rows_remote=42112 dup_rows=16022 dup_rows_remote=11464 dedup_ratio=0.244476 router_tokens_per_expert=[20, 193, 21, 82, 2148, 93, 2558, 11, 83, 2, 33, 36, 2969, 2834, 1, 22, 751, 99, 1397, 21, 18, 1, 1, 9, 0, 37, 2069, 45, 0, 650, 227, 32, 78, 1, 0, 472, 52, 3391, 1191, 139, 2, 1940, 0, 10, 51, 4, 1607, 0, 2996, 18, 529, 105, 2, 1, 584, 0, 134, 125, 7, 87, 21, 14, 14, 2730] payload_rows_per_dst=[7158, 7402, 6300, 5080, 7538, 3324, 6808, 5904] dup_rows_per_dst=[3094, 4558, 2174, 1040, 3110, 24, 1662, 360] combine_rows_per_src=[7158, 7402, 6300, 5080, 7538, 3324, 6808, 5904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=304654336 dispatch_meta_bytes=212328 combine_payload_bytes=304654336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=304866664 ingress_bytes=414754816 link_bytes=414754816 token_bytes=609521000 weight_movement_bytes=110100480 total_nvlink_bytes=719621480 weight_to_token_ratio=0.180634 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=17536 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.213268405e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46586 payload_rows_remote=40810 dup_rows=18950 dup_rows_remote=17158 dedup_ratio=0.289154 router_tokens_per_expert=[1341, 717, 23, 43, 29, 43, 540, 0, 0, 3409, 8, 31, 39, 241, 3011, 0, 0, 724, 0, 219, 174, 2269, 1, 3, 1628, 0, 155, 924, 1, 4, 114, 1108, 441, 1030, 3, 615, 591, 34, 1831, 7, 310, 1040, 5, 3, 786, 2879, 0, 1, 15, 0, 1713, 1, 557, 1, 188, 0, 648, 2, 97, 24, 1721, 0, 1048, 378] payload_rows_per_dst=[5776, 7872, 5552, 4952, 6246, 7106, 4260, 4822] dup_rows_per_dst=[1792, 7454, 1228, 1270, 2656, 2942, 690, 918] combine_rows_per_src=[5776, 7872, 5552, 4952, 6246, 7106, 4260, 4822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50026 payload_rows_remote=43200 dup_rows=15510 dup_rows_remote=13070 dedup_ratio=0.236664 router_tokens_per_expert=[1, 1407, 0, 1, 22, 9, 1126, 670, 1, 1, 0, 103, 1, 267, 0, 31, 612, 173, 660, 1938, 1152, 1283, 267, 2428, 117, 0, 7, 1, 506, 0, 43, 0, 273, 2, 223, 2787, 0, 0, 300, 1976, 1, 2135, 244, 20, 2087, 9, 0, 137, 32, 3348, 2, 271, 93, 1, 93, 60, 179, 2, 0, 2179, 22, 941, 68, 2456] payload_rows_per_dst=[5800, 6446, 5540, 5484, 5202, 6826, 7152, 7576] dup_rows_per_dst=[1950, 2340, 2230, 776, 1078, 2440, 648, 4048] combine_rows_per_src=[5800, 6446, 5540, 5484, 5202, 6826, 7152, 7576] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47668 payload_rows_remote=40760 dup_rows=17868 dup_rows_remote=16300 dedup_ratio=0.272644 router_tokens_per_expert=[0, 1482, 0, 0, 2, 6, 1740, 422, 0, 2, 0, 529, 0, 1357, 0, 325, 355, 257, 2289, 447, 516, 1442, 458, 2026, 852, 0, 53, 32, 1007, 0, 428, 0, 27, 0, 418, 3123, 0, 0, 173, 497, 0, 435, 53, 66, 2771, 4, 0, 411, 2, 2907, 0, 765, 28, 3, 25, 477, 155, 2, 3, 672, 6, 704, 3, 3011] payload_rows_per_dst=[7100, 6162, 4836, 6934, 6908, 6184, 6788, 2756] dup_rows_per_dst=[4782, 3210, 1220, 3832, 1568, 1296, 1626, 334] combine_rows_per_src=[7100, 6162, 4836, 6934, 6908, 6184, 6788, 2756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47802 payload_rows_remote=42836 dup_rows=17734 dup_rows_remote=17370 dedup_ratio=0.270599 router_tokens_per_expert=[4, 1380, 7, 0, 9, 27, 93, 1315, 0, 0, 1, 114, 0, 1100, 0, 169, 262, 282, 895, 1901, 597, 899, 777, 1856, 179, 2, 12, 2, 902, 1, 99, 0, 653, 0, 362, 2992, 2, 0, 452, 1502, 0, 888, 550, 2, 3269, 0, 0, 6, 9, 2088, 6, 136, 330, 0, 62, 34, 288, 1, 0, 1706, 0, 1793, 63, 2689] payload_rows_per_dst=[4770, 5058, 6834, 6354, 4898, 7234, 4966, 7688] dup_rows_per_dst=[900, 1422, 4392, 1418, 1044, 2196, 364, 5998] combine_rows_per_src=[4770, 5058, 6834, 6354, 4898, 7234, 4966, 7688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=301858816 dispatch_meta_bytes=214304 combine_payload_bytes=301858816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=302073120 ingress_bytes=632160256 link_bytes=632160256 token_bytes=603931936 weight_movement_bytes=330301440 total_nvlink_bytes=934233376 weight_to_token_ratio=0.546918 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.099430658e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49508 payload_rows_remote=42770 dup_rows=16028 dup_rows_remote=12618 dedup_ratio=0.244568 router_tokens_per_expert=[0, 1180, 0, 13, 13, 23, 650, 245, 4, 6, 3, 105, 2, 1084, 29, 32, 2203, 229, 1438, 1222, 1261, 1189, 192, 2179, 317, 7, 8, 2, 138, 0, 20, 16, 130, 8, 180, 1821, 0, 0, 1471, 1569, 5, 667, 546, 92, 2151, 25, 0, 2, 178, 2719, 4, 575, 369, 74, 1, 9, 428, 0, 0, 914, 18, 1415, 43, 3544] payload_rows_per_dst=[5416, 6470, 6738, 7188, 6980, 5420, 6474, 4822] dup_rows_per_dst=[1708, 2862, 3410, 916, 3378, 1556, 1384, 814] combine_rows_per_src=[5416, 6470, 6738, 7188, 6980, 5420, 6474, 4822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=292526080 dispatch_meta_bytes=231872 combine_payload_bytes=292526080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292757952 ingress_bytes=402626560 link_bytes=402626560 token_bytes=585284032 weight_movement_bytes=110100480 total_nvlink_bytes=695384512 weight_to_token_ratio=0.188115 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=9600 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.002338085e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=309657600 dispatch_meta_bytes=225080 combine_payload_bytes=309657600 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=309882680 ingress_bytes=639959040 link_bytes=639959040 token_bytes=619540280 weight_movement_bytes=330301440 total_nvlink_bytes=949841720 weight_to_token_ratio=0.533140 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.112994026e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45492 payload_rows_remote=38070 dup_rows=20044 dup_rows_remote=11746 dedup_ratio=0.305847 router_tokens_per_expert=[0, 1454, 265, 14, 369, 29, 174, 743, 2, 0, 5, 23, 0, 903, 1, 95, 700, 348, 1982, 473, 597, 324, 203, 2225, 57, 0, 7, 0, 746, 14, 350, 9, 40, 0, 763, 1818, 11, 1, 1038, 1389, 3, 416, 462, 66, 3669, 7, 1, 48, 8, 2758, 0, 689, 94, 13, 147, 219, 1000, 0, 0, 1491, 5, 1549, 0, 2951] payload_rows_per_dst=[4952, 5442, 6288, 3540, 5172, 5842, 6834, 7422] dup_rows_per_dst=[1144, 1066, 2966, 734, 1312, 1294, 3230, 8298] combine_rows_per_src=[4952, 5442, 6288, 3540, 5172, 5842, 6834, 7422] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50596 payload_rows_remote=43670 dup_rows=14940 dup_rows_remote=13908 dedup_ratio=0.227966 router_tokens_per_expert=[0, 1638, 1, 1, 5, 0, 295, 1447, 0, 1, 0, 568, 1, 851, 0, 72, 333, 4, 1442, 685, 321, 1212, 519, 2838, 231, 0, 36, 1, 397, 1, 36, 0, 6, 0, 570, 2107, 0, 0, 358, 1000, 0, 1173, 519, 12, 3516, 17, 0, 143, 2, 3619, 1, 183, 278, 4, 47, 564, 50, 1, 0, 1779, 21, 532, 53, 3277] payload_rows_per_dst=[6640, 7030, 4074, 6926, 6194, 7642, 7832, 4258] dup_rows_per_dst=[3018, 3002, 704, 1032, 1888, 3118, 1564, 614] combine_rows_per_src=[6640, 7030, 4074, 6926, 6194, 7642, 7832, 4258] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=292167680 dispatch_meta_bytes=228240 combine_payload_bytes=292167680 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=292395920 ingress_bytes=512368640 link_bytes=512368640 token_bytes=584563600 weight_movement_bytes=220200960 total_nvlink_bytes=804764560 weight_to_token_ratio=0.376693 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.910933351e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=307048448 dispatch_meta_bytes=240824 combine_payload_bytes=307048448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=307289272 ingress_bytes=417148928 link_bytes=417148928 token_bytes=614337720 weight_movement_bytes=110100480 total_nvlink_bytes=724438200 weight_to_token_ratio=0.179218 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.254905950e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=306575360 dispatch_meta_bytes=221552 combine_payload_bytes=306575360 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=306796912 ingress_bytes=746977280 link_bytes=746977280 token_bytes=613372272 weight_movement_bytes=440401920 total_nvlink_bytes=1053774192 weight_to_token_ratio=0.718001 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.299116347e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=272885760 dispatch_meta_bytes=199264 combine_payload_bytes=272885760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=273085024 ingress_bytes=493086720 link_bytes=493086720 token_bytes=545970784 weight_movement_bytes=220200960 total_nvlink_bytes=766171744 weight_to_token_ratio=0.403320 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.575589049e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=313026560 dispatch_meta_bytes=230312 combine_payload_bytes=313026560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=313256872 ingress_bytes=423127040 link_bytes=423127040 token_bytes=626283432 weight_movement_bytes=110100480 total_nvlink_bytes=736383912 weight_to_token_ratio=0.175800 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.358875150e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48926 payload_rows_remote=43334 dup_rows=16610 dup_rows_remote=14632 dedup_ratio=0.253448 router_tokens_per_expert=[2, 1576, 0, 9, 10, 12, 830, 588, 7, 0, 5, 121, 70, 232, 29, 246, 471, 139, 1512, 1521, 665, 1474, 1173, 1554, 488, 20, 55, 1, 2164, 6, 87, 14, 325, 24, 197, 2273, 66, 3, 271, 1207, 80, 526, 999, 1, 1593, 213, 4, 61, 14, 2599, 61, 418, 138, 0, 23, 193, 2021, 26, 4, 881, 98, 111, 235, 3022] payload_rows_per_dst=[6484, 5592, 5774, 7442, 6520, 5480, 6204, 5430] dup_rows_per_dst=[2594, 1978, 2070, 4272, 2212, 1474, 688, 1322] combine_rows_per_src=[6484, 5592, 5774, 7442, 6520, 5480, 6204, 5430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49754 payload_rows_remote=42000 dup_rows=15782 dup_rows_remote=12504 dedup_ratio=0.240814 router_tokens_per_expert=[0, 1421, 102, 22, 135, 1037, 1, 3662, 2, 1358, 1, 199, 1296, 991, 3080, 23, 7, 16, 111, 1317, 214, 0, 7, 2257, 187, 3, 2, 49, 0, 3, 72, 1427, 933, 0, 91, 2198, 30, 1110, 0, 1289, 7, 181, 0, 472, 1190, 0, 1, 3, 5, 40, 25, 81, 750, 3575, 3, 0, 4, 2, 861, 281, 39, 196, 384, 15] payload_rows_per_dst=[4394, 6010, 6194, 4376, 6254, 7754, 7604, 7168] dup_rows_per_dst=[1042, 1730, 1664, 594, 3564, 3278, 1354, 2556] combine_rows_per_src=[4394, 6010, 6194, 4376, 6254, 7754, 7604, 7168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46052 payload_rows_remote=38834 dup_rows=19484 dup_rows_remote=16816 dedup_ratio=0.297302 router_tokens_per_expert=[0, 1, 149, 334, 2961, 236, 1238, 24, 16, 0, 0, 71, 3223, 2424, 23, 14, 207, 0, 2505, 155, 58, 2, 7, 0, 27, 83, 2092, 8, 0, 13, 7, 0, 5, 0, 66, 1629, 6, 3808, 287, 313, 0, 2652, 198, 6, 256, 0, 989, 5, 2294, 3, 111, 3, 181, 0, 217, 20, 1525, 32, 19, 43, 0, 0, 55, 2167] payload_rows_per_dst=[7218, 8164, 7344, 4358, 4898, 2650, 5226, 6194] dup_rows_per_dst=[2668, 9598, 3828, 102, 1110, 258, 432, 1488] combine_rows_per_src=[7218, 8164, 7344, 4358, 4898, 2650, 5226, 6194] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=310618112 dispatch_meta_bytes=231864 combine_payload_bytes=310618112 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=310849976 ingress_bytes=530819072 link_bytes=530819072 token_bytes=621468088 weight_movement_bytes=220200960 total_nvlink_bytes=841669048 weight_to_token_ratio=0.354324 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.040441 gemm_active_groups=11 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.231816709e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49462 payload_rows_remote=43480 dup_rows=16074 dup_rows_remote=15880 dedup_ratio=0.245270 router_tokens_per_expert=[42, 257, 26, 330, 320, 1155, 0, 3920, 0, 2454, 2, 1132, 143, 1156, 3039, 1, 0, 7, 0, 267, 111, 0, 94, 3002, 820, 0, 0, 39, 0, 2, 374, 1410, 1712, 0, 10, 2620, 0, 785, 1, 1052, 0, 13, 0, 145, 1645, 1, 0, 1, 21, 34, 13, 23, 179, 2814, 4, 0, 0, 9, 1048, 383, 32, 60, 60, 0] payload_rows_per_dst=[3620, 6806, 6294, 4552, 7250, 8026, 5982, 6932] dup_rows_per_dst=[640, 2970, 668, 738, 5110, 3424, 194, 2330] combine_rows_per_src=[3620, 6806, 6294, 4552, 7250, 8026, 5982, 6932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48548 payload_rows_remote=43080 dup_rows=16988 dup_rows_remote=15578 dedup_ratio=0.259216 router_tokens_per_expert=[14, 834, 237, 113, 1046, 335, 0, 3759, 0, 2340, 127, 140, 229, 542, 2606, 146, 0, 9, 129, 1030, 633, 3, 7, 1628, 209, 0, 1, 157, 0, 4, 311, 1851, 1539, 6, 553, 1376, 0, 353, 21, 1307, 0, 377, 23, 1070, 1570, 2, 0, 20, 12, 7, 220, 616, 718, 2763, 0, 0, 1, 106, 315, 191, 307, 64, 791, 0] payload_rows_per_dst=[7928, 5808, 5468, 5800, 5340, 4948, 6700, 6556] dup_rows_per_dst=[4748, 1240, 1410, 2018, 2218, 1176, 1972, 2206] combine_rows_per_src=[7928, 5808, 5468, 5800, 5340, 4948, 6700, 6556] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49242 payload_rows_remote=46652 dup_rows=16294 dup_rows_remote=16066 dedup_ratio=0.248627 router_tokens_per_expert=[0, 67, 12, 1, 711, 1696, 0, 2861, 0, 3222, 0, 282, 271, 1024, 3841, 71, 0, 75, 38, 99, 98, 0, 175, 2913, 288, 7, 0, 0, 1, 0, 131, 1561, 367, 0, 138, 2281, 0, 761, 0, 143, 0, 10, 0, 2298, 337, 0, 0, 6, 10, 6, 7, 351, 1187, 3298, 0, 0, 0, 0, 752, 265, 947, 36, 65, 58] payload_rows_per_dst=[4604, 7342, 6056, 6396, 2590, 7106, 7160, 7988] dup_rows_per_dst=[624, 2736, 740, 2142, 228, 3326, 2558, 3940] combine_rows_per_src=[4604, 7342, 6056, 6396, 2590, 7106, 7160, 7988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=301056000 dispatch_meta_bytes=218016 combine_payload_bytes=301056000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301274016 ingress_bytes=411156480 link_bytes=411156480 token_bytes=602330016 weight_movement_bytes=110100480 total_nvlink_bytes=712430496 weight_to_token_ratio=0.182791 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.150687423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=278362112 dispatch_meta_bytes=222600 combine_payload_bytes=278362112 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278584712 ingress_bytes=498563072 link_bytes=498563072 token_bytes=556946824 weight_movement_bytes=220200960 total_nvlink_bytes=777147784 weight_to_token_ratio=0.395372 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.670831817e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49128 payload_rows_remote=41622 dup_rows=16408 dup_rows_remote=12672 dedup_ratio=0.250366 router_tokens_per_expert=[3, 462, 19, 69, 181, 1027, 6, 3399, 0, 1689, 1, 179, 207, 997, 3410, 3, 2, 7, 4, 882, 235, 3, 38, 3793, 245, 3, 1, 49, 0, 0, 12, 2616, 1371, 0, 91, 2176, 4, 704, 5, 301, 0, 20, 0, 693, 1216, 0, 0, 3, 5, 0, 3, 84, 1144, 3194, 1, 0, 3, 9, 1155, 26, 819, 33, 166, 0] payload_rows_per_dst=[3174, 3972, 7706, 5558, 6690, 7366, 7156, 7506] dup_rows_per_dst=[360, 654, 2222, 294, 3326, 3296, 2520, 3736] combine_rows_per_src=[3174, 3972, 7706, 5558, 6690, 7366, 7156, 7506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48598 payload_rows_remote=41746 dup_rows=16938 dup_rows_remote=14310 dedup_ratio=0.258453 router_tokens_per_expert=[59, 412, 33, 3, 485, 2061, 0, 3788, 0, 814, 0, 115, 674, 943, 2353, 5, 0, 4, 22, 580, 1, 0, 66, 3646, 28, 1, 3, 0, 0, 1, 575, 2865, 828, 0, 6, 1267, 0, 1465, 0, 1931, 0, 11, 0, 1589, 806, 3, 0, 0, 65, 8, 78, 237, 1877, 1698, 23, 0, 6, 16, 847, 152, 154, 30, 132, 2] payload_rows_per_dst=[8034, 4176, 7414, 6852, 6050, 4248, 6092, 5732] dup_rows_per_dst=[5648, 926, 1224, 2628, 2410, 570, 1880, 1652] combine_rows_per_src=[8034, 4176, 7414, 6852, 6050, 4248, 6092, 5732] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=311664640 dispatch_meta_bytes=237440 combine_payload_bytes=311664640 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=311902080 ingress_bytes=531865600 link_bytes=531865600 token_bytes=623566720 weight_movement_bytes=220200960 total_nvlink_bytes=843767680 weight_to_token_ratio=0.353131 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.250017553e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=308797440 dispatch_meta_bytes=234632 combine_payload_bytes=308797440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309032072 ingress_bytes=418897920 link_bytes=418897920 token_bytes=617829512 weight_movement_bytes=110100480 total_nvlink_bytes=727929992 weight_to_token_ratio=0.178205 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.285323797e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=334401536 dispatch_meta_bytes=250872 combine_payload_bytes=334401536 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334652408 ingress_bytes=554602496 link_bytes=554602496 token_bytes=669053944 weight_movement_bytes=220200960 total_nvlink_bytes=889254904 weight_to_token_ratio=0.329123 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.645449570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=298346496 dispatch_meta_bytes=217176 combine_payload_bytes=298346496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298563672 ingress_bytes=408446976 link_bytes=408446976 token_bytes=596910168 weight_movement_bytes=110100480 total_nvlink_bytes=707010648 weight_to_token_ratio=0.184451 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.103564692e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=299235328 dispatch_meta_bytes=224224 combine_payload_bytes=299235328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299459552 ingress_bytes=409335808 link_bytes=409335808 token_bytes=598694880 weight_movement_bytes=110100480 total_nvlink_bytes=708795360 weight_to_token_ratio=0.183901 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.119022943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47562 payload_rows_remote=41208 dup_rows=17974 dup_rows_remote=15640 dedup_ratio=0.274261 router_tokens_per_expert=[3, 287, 6, 10, 1702, 359, 73, 3972, 2, 1033, 49, 354, 63, 427, 3119, 80, 40, 278, 175, 457, 52, 0, 66, 2060, 97, 117, 0, 291, 2, 16, 2362, 2012, 107, 11, 35, 921, 10, 915, 24, 1941, 0, 531, 0, 981, 855, 10, 1, 0, 538, 52, 1471, 111, 779, 2231, 2, 6, 0, 5, 281, 1047, 35, 46, 258, 0] payload_rows_per_dst=[8026, 6354, 5368, 7202, 5302, 3984, 7146, 4180] dup_rows_per_dst=[4798, 2334, 888, 4434, 784, 772, 3234, 730] combine_rows_per_src=[8026, 6354, 5368, 7202, 5302, 3984, 7146, 4180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51386 payload_rows_remote=47376 dup_rows=14150 dup_rows_remote=13662 dedup_ratio=0.215912 router_tokens_per_expert=[2978, 503, 0, 0, 1412, 135, 2, 8, 32, 0, 0, 33, 13, 2064, 0, 1, 0, 1, 244, 720, 2, 3273, 66, 56, 1277, 2783, 178, 20, 0, 0, 2, 2741, 3, 1, 0, 210, 0, 1456, 0, 0, 4, 288, 369, 3680, 0, 0, 1588, 0, 2, 36, 7, 0, 34, 662, 241, 558, 0, 249, 3, 45, 819, 3830, 139, 0] payload_rows_per_dst=[3646, 7394, 7168, 6386, 6836, 4010, 7910, 8036] dup_rows_per_dst=[474, 2848, 1556, 2134, 1986, 488, 2530, 2134] combine_rows_per_src=[3646, 7394, 7168, 6386, 6836, 4010, 7910, 8036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50290 payload_rows_remote=44054 dup_rows=15246 dup_rows_remote=13204 dedup_ratio=0.232635 router_tokens_per_expert=[0, 2319, 2, 2, 22, 16, 18, 467, 0, 3, 0, 96, 6, 167, 0, 12, 1019, 59, 1293, 1410, 689, 1437, 45, 2034, 97, 0, 9, 0, 255, 0, 32, 0, 8, 0, 266, 2321, 0, 0, 708, 910, 2, 1892, 413, 73, 3529, 19, 14, 184, 2, 3795, 20, 525, 464, 1, 44, 40, 9, 6, 0, 1335, 9, 700, 19, 3951] payload_rows_per_dst=[6236, 5958, 5998, 7936, 6716, 5744, 7972, 3730] dup_rows_per_dst=[2042, 1498, 2462, 752, 4466, 1790, 1810, 426] combine_rows_per_src=[6236, 5958, 5998, 7936, 6716, 5744, 7972, 3730] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=295378944 dispatch_meta_bytes=227392 combine_payload_bytes=295378944 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=295606336 ingress_bytes=735780864 link_bytes=735780864 token_bytes=590985280 weight_movement_bytes=440401920 total_nvlink_bytes=1031387200 weight_to_token_ratio=0.745199 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279643938e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49546 payload_rows_remote=41496 dup_rows=15990 dup_rows_remote=12414 dedup_ratio=0.243988 router_tokens_per_expert=[1544, 478, 1, 0, 1147, 654, 6, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 1, 0, 271, 929, 1, 1882, 3, 1, 2705, 3761, 169, 8, 0, 0, 0, 3336, 0, 0, 0, 114, 0, 745, 2, 0, 14, 72, 399, 3818, 0, 0, 577, 0, 4, 4, 9, 3, 397, 20, 97, 1461, 0, 62, 0, 76, 753, 3476, 43, 1] payload_rows_per_dst=[3994, 7816, 5240, 7886, 7042, 2088, 8050, 7430] dup_rows_per_dst=[472, 2720, 936, 5400, 1352, 36, 3576, 1498] combine_rows_per_src=[3994, 7816, 5240, 7886, 7042, 2088, 8050, 7430] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47398 payload_rows_remote=41744 dup_rows=18138 dup_rows_remote=17062 dedup_ratio=0.276764 router_tokens_per_expert=[2191, 2054, 0, 0, 1843, 1499, 2, 2, 3, 4, 2, 2, 78, 2341, 0, 2, 13, 6, 36, 1802, 0, 1083, 395, 30, 2629, 1558, 82, 182, 30, 0, 0, 2131, 17, 28, 2, 75, 0, 103, 0, 4, 45, 1125, 61, 2848, 1, 0, 920, 0, 0, 37, 1, 4, 8, 154, 3, 778, 6, 1224, 0, 54, 1458, 3652, 152, 8] payload_rows_per_dst=[7480, 5246, 5654, 6586, 4480, 5472, 5618, 6862] dup_rows_per_dst=[6502, 818, 1076, 2376, 240, 3260, 856, 3010] combine_rows_per_src=[7480, 5246, 5654, 6586, 4480, 5472, 5618, 6862] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51098 payload_rows_remote=44424 dup_rows=14438 dup_rows_remote=13440 dedup_ratio=0.220306 router_tokens_per_expert=[2116, 2440, 0, 0, 1492, 58, 0, 6, 0, 0, 0, 0, 23, 3207, 0, 67, 1, 0, 149, 1437, 0, 1895, 24, 69, 1390, 1929, 36, 0, 0, 1, 0, 3115, 9, 0, 0, 638, 0, 74, 0, 0, 3, 475, 1324, 3601, 1, 0, 138, 0, 0, 0, 1, 0, 112, 8, 61, 1815, 2, 747, 0, 245, 119, 3922, 17, 1] payload_rows_per_dst=[6456, 7412, 5682, 5424, 6674, 3644, 7910, 7896] dup_rows_per_dst=[1536, 3414, 1468, 1288, 998, 238, 3286, 2210] combine_rows_per_src=[6456, 7412, 5682, 5424, 6674, 3644, 7910, 7896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=339591168 dispatch_meta_bytes=244152 combine_payload_bytes=339591168 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=339835320 ingress_bytes=559792128 link_bytes=559792128 token_bytes=679426488 weight_movement_bytes=220200960 total_nvlink_bytes=899627448 weight_to_token_ratio=0.324098 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.735705806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=315779072 dispatch_meta_bytes=229032 combine_payload_bytes=315779072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316008104 ingress_bytes=425879552 link_bytes=425879552 token_bytes=631787176 weight_movement_bytes=110100480 total_nvlink_bytes=741887656 weight_to_token_ratio=0.174268 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.406745861e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51946 payload_rows_remote=44366 dup_rows=13590 dup_rows_remote=8628 dedup_ratio=0.207367 router_tokens_per_expert=[1600, 671, 0, 1, 1945, 187, 0, 0, 1, 0, 0, 3, 9, 2061, 0, 3, 0, 0, 273, 1848, 0, 1943, 34, 0, 2859, 3230, 166, 16, 0, 0, 0, 3700, 0, 0, 0, 128, 0, 319, 0, 0, 4, 331, 385, 3484, 0, 0, 1615, 0, 2, 0, 1, 0, 91, 172, 15, 927, 0, 176, 0, 223, 532, 3751, 62, 0] payload_rows_per_dst=[4974, 5840, 6514, 7580, 7482, 4360, 7290, 7906] dup_rows_per_dst=[634, 1514, 1682, 4962, 812, 310, 2094, 1582] combine_rows_per_src=[4974, 5840, 6514, 7580, 7482, 4360, 7290, 7906] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47342 payload_rows_remote=39258 dup_rows=18194 dup_rows_remote=13014 dedup_ratio=0.277618 router_tokens_per_expert=[504, 483, 1, 3, 942, 85, 6, 1, 14, 8, 0, 8, 8, 3635, 0, 0, 4, 5, 117, 576, 1, 3714, 241, 0, 1564, 3684, 0, 14, 0, 2, 0, 3620, 4, 2, 0, 255, 0, 87, 1, 1, 1, 857, 21, 3201, 6, 8, 1487, 0, 1, 5, 8, 7, 143, 28, 203, 1512, 5, 281, 0, 18, 1305, 3847, 234, 0] payload_rows_per_dst=[1154, 7496, 7672, 3482, 7858, 4048, 7548, 8084] dup_rows_per_dst=[4, 858, 1644, 266, 6862, 712, 2668, 5180] combine_rows_per_src=[1154, 7496, 7672, 3482, 7858, 4048, 7548, 8084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=297443328 dispatch_meta_bytes=215640 combine_payload_bytes=297443328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297658968 ingress_bytes=407543808 link_bytes=407543808 token_bytes=595102296 weight_movement_bytes=110100480 total_nvlink_bytes=705202776 weight_to_token_ratio=0.185011 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.087857116e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=299220992 dispatch_meta_bytes=235224 combine_payload_bytes=299220992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=299456216 ingress_bytes=409321472 link_bytes=409321472 token_bytes=598677208 weight_movement_bytes=110100480 total_nvlink_bytes=708777688 weight_to_token_ratio=0.183906 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.118773616e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318431232 dispatch_meta_bytes=231456 combine_payload_bytes=318431232 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=318662688 ingress_bytes=538632192 link_bytes=538632192 token_bytes=637093920 weight_movement_bytes=220200960 total_nvlink_bytes=857294880 weight_to_token_ratio=0.345633 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=24192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.367699717e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=318015488 dispatch_meta_bytes=211976 combine_payload_bytes=318015488 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=318227464 ingress_bytes=758417408 link_bytes=758417408 token_bytes=636242952 weight_movement_bytes=440401920 total_nvlink_bytes=1076644872 weight_to_token_ratio=0.692191 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.319012611e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45380 payload_rows_remote=39416 dup_rows=20156 dup_rows_remote=19868 dedup_ratio=0.307556 router_tokens_per_expert=[1031, 2704, 0, 25, 1845, 208, 93, 16, 1, 0, 1, 0, 239, 2797, 0, 88, 8, 3, 27, 1190, 0, 3301, 8, 69, 1790, 1455, 7, 271, 9, 27, 1, 2482, 0, 104, 0, 145, 1, 233, 8, 6, 1263, 1152, 5, 2157, 0, 11, 1298, 91, 10, 57, 0, 2, 0, 61, 119, 1679, 1, 164, 26, 173, 675, 3625, 5, 1] payload_rows_per_dst=[7426, 5964, 7394, 5586, 5422, 7954, 3668, 1966] dup_rows_per_dst=[4418, 288, 1818, 1534, 536, 11250, 188, 124] combine_rows_per_src=[7426, 5964, 7394, 5586, 5422, 7954, 3668, 1966] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=281401344 dispatch_meta_bytes=209088 combine_payload_bytes=281401344 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281610432 ingress_bytes=501602304 link_bytes=501602304 token_bytes=563011776 weight_movement_bytes=220200960 total_nvlink_bytes=783212736 weight_to_token_ratio=0.391113 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=20864 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.723689061e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51248 payload_rows_remote=45524 dup_rows=14288 dup_rows_remote=12864 dedup_ratio=0.218018 router_tokens_per_expert=[142, 433, 2, 1531, 9, 0, 0, 943, 1, 2923, 10, 1, 436, 65, 2223, 0, 0, 1, 77, 1345, 5, 1371, 61, 2, 1107, 5, 3955, 196, 30, 773, 744, 2114, 1015, 26, 38, 9, 8, 15, 132, 0, 0, 6, 0, 1941, 6, 1441, 179, 1, 495, 1, 7, 9, 262, 1, 3, 4027, 2275, 0, 37, 199, 80, 4, 2, 44] payload_rows_per_dst=[4880, 6318, 6818, 4774, 8004, 5724, 8114, 6616] dup_rows_per_dst=[1240, 554, 3352, 936, 2392, 1424, 1496, 2894] combine_rows_per_src=[4880, 6318, 6818, 4774, 8004, 5724, 8114, 6616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45862 payload_rows_remote=39712 dup_rows=19674 dup_rows_remote=17928 dedup_ratio=0.300201 router_tokens_per_expert=[0, 656, 20, 36, 176, 604, 0, 2456, 0, 2492, 0, 58, 622, 403, 2616, 48, 0, 261, 35, 57, 26, 1, 73, 3618, 31, 0, 1, 157, 44, 89, 619, 3377, 1458, 0, 5, 1658, 175, 410, 9, 2153, 0, 145, 0, 2521, 381, 34, 0, 50, 175, 4, 34, 406, 900, 2493, 19, 0, 6, 0, 160, 522, 66, 90, 317, 1] payload_rows_per_dst=[6150, 7386, 7686, 7648, 5854, 5502, 3464, 2172] dup_rows_per_dst=[1746, 5092, 4534, 4304, 2566, 760, 520, 152] combine_rows_per_src=[6150, 7386, 7686, 7648, 5854, 5502, 3464, 2172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=282533888 dispatch_meta_bytes=237136 combine_payload_bytes=282533888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282771024 ingress_bytes=392634368 link_bytes=392634368 token_bytes=565304912 weight_movement_bytes=110100480 total_nvlink_bytes=675405392 weight_to_token_ratio=0.194763 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.828557432e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48180 payload_rows_remote=43482 dup_rows=17356 dup_rows_remote=16224 dedup_ratio=0.264832 router_tokens_per_expert=[47, 601, 2, 811, 2, 16, 2, 268, 3, 2349, 0, 4, 165, 24, 3562, 15, 1, 0, 102, 1267, 0, 1506, 24, 15, 161, 5, 3386, 464, 21, 1210, 292, 2594, 1879, 47, 0, 7, 11, 0, 99, 1, 1, 18, 5, 1303, 12, 1771, 245, 17, 429, 0, 49, 0, 866, 20, 3, 4015, 1792, 3, 4, 704, 125, 291, 50, 82] payload_rows_per_dst=[3672, 7810, 4698, 7292, 3962, 5464, 8112, 7170] dup_rows_per_dst=[816, 4434, 1132, 2998, 126, 1280, 2652, 3918] combine_rows_per_src=[3672, 7810, 4698, 7292, 3962, 5464, 8112, 7170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50480 payload_rows_remote=42320 dup_rows=15056 dup_rows_remote=13216 dedup_ratio=0.229736 router_tokens_per_expert=[234, 425, 0, 2344, 5, 0, 31, 970, 2, 2176, 0, 5, 773, 93, 3094, 0, 29, 0, 57, 1880, 0, 865, 0, 0, 1555, 0, 3845, 467, 197, 1464, 344, 1679, 956, 1, 0, 25, 0, 0, 52, 0, 0, 2, 0, 339, 53, 1613, 455, 0, 12, 0, 15, 0, 10, 0, 17, 4079, 2366, 0, 10, 81, 78, 12, 57, 1] payload_rows_per_dst=[6016, 5364, 7544, 4910, 7820, 4416, 8160, 6250] dup_rows_per_dst=[2002, 734, 4306, 1410, 1938, 508, 1840, 2318] combine_rows_per_src=[6016, 5364, 7544, 4910, 7820, 4416, 8160, 6250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=326316032 dispatch_meta_bytes=233552 combine_payload_bytes=326316032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=326549584 ingress_bytes=436416512 link_bytes=436416512 token_bytes=652865616 weight_movement_bytes=110100480 total_nvlink_bytes=762966096 weight_to_token_ratio=0.168642 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.590000926e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52042 payload_rows_remote=43856 dup_rows=13494 dup_rows_remote=12546 dedup_ratio=0.205902 router_tokens_per_expert=[15, 51, 0, 2631, 0, 1, 0, 2846, 0, 1207, 0, 1, 551, 111, 2937, 0, 3, 0, 194, 1555, 0, 660, 11, 0, 717, 0, 4089, 9, 785, 537, 250, 1755, 225, 73, 107, 0, 0, 0, 73, 0, 0, 17, 0, 3581, 9, 928, 352, 0, 162, 0, 313, 0, 344, 0, 2, 3948, 1074, 0, 5, 477, 96, 44, 16, 6] payload_rows_per_dst=[7594, 6472, 4692, 3906, 8186, 7498, 8034, 5660] dup_rows_per_dst=[3494, 1842, 1454, 690, 948, 2276, 1504, 1286] combine_rows_per_src=[7594, 6472, 4692, 3906, 8186, 7498, 8034, 5660] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=284655616 dispatch_meta_bytes=230560 combine_payload_bytes=284655616 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=284886176 ingress_bytes=504856576 link_bytes=504856576 token_bytes=569541792 weight_movement_bytes=220200960 total_nvlink_bytes=789742752 weight_to_token_ratio=0.386628 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.780286203e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48786 payload_rows_remote=44068 dup_rows=16750 dup_rows_remote=14226 dedup_ratio=0.255585 router_tokens_per_expert=[227, 37, 1, 1102, 1, 0, 0, 1395, 1, 2048, 0, 4, 780, 13, 2547, 0, 52, 0, 316, 1247, 0, 1044, 2, 0, 457, 0, 4031, 139, 150, 1211, 1178, 1801, 1285, 46, 4, 0, 0, 0, 46, 0, 0, 34, 0, 1084, 8, 2312, 485, 13, 42, 0, 29, 0, 691, 1, 0, 4023, 2320, 0, 5, 166, 182, 83, 67, 58] payload_rows_per_dst=[4644, 7252, 4220, 4718, 7364, 6086, 8090, 6412] dup_rows_per_dst=[882, 3534, 1102, 2524, 2488, 1786, 1482, 2952] combine_rows_per_src=[4644, 7252, 4220, 4718, 7364, 6086, 8090, 6412] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=311678976 dispatch_meta_bytes=238824 combine_payload_bytes=311678976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311917800 ingress_bytes=421779456 link_bytes=421779456 token_bytes=623596776 weight_movement_bytes=110100480 total_nvlink_bytes=733697256 weight_to_token_ratio=0.176557 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.335438448e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=303349760 dispatch_meta_bytes=222144 combine_payload_bytes=303349760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=303571904 ingress_bytes=523550720 link_bytes=523550720 token_bytes=606921664 weight_movement_bytes=220200960 total_nvlink_bytes=827122624 weight_to_token_ratio=0.362816 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.105408114e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48896 payload_rows_remote=42568 dup_rows=16640 dup_rows_remote=14060 dedup_ratio=0.253906 router_tokens_per_expert=[1, 417, 0, 1343, 1, 0, 3, 269, 0, 2087, 1, 0, 1050, 45, 1607, 0, 1, 1, 873, 1267, 1, 2101, 2, 0, 1618, 1, 3765, 76, 715, 451, 1655, 2173, 1447, 202, 3, 4, 2, 0, 89, 0, 5, 10, 0, 1343, 24, 1084, 583, 0, 130, 0, 13, 0, 240, 8, 6, 3770, 1872, 0, 0, 284, 91, 28, 2, 4] payload_rows_per_dst=[3684, 5388, 6998, 5822, 7888, 5000, 7788, 6328] dup_rows_per_dst=[384, 978, 4708, 2308, 3136, 1098, 1448, 2580] combine_rows_per_src=[3684, 5388, 6998, 5822, 7888, 5000, 7788, 6328] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=314359808 dispatch_meta_bytes=225608 combine_payload_bytes=314359808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314585416 ingress_bytes=424460288 link_bytes=424460288 token_bytes=628945224 weight_movement_bytes=110100480 total_nvlink_bytes=739045704 weight_to_token_ratio=0.175056 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=19200 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.382062526e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=315879424 dispatch_meta_bytes=233176 combine_payload_bytes=315879424 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=316112600 ingress_bytes=866381824 link_bytes=866381824 token_bytes=631992024 weight_movement_bytes=550502400 total_nvlink_bytes=1182494424 weight_to_token_ratio=0.871059 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.506780487e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=305127424 dispatch_meta_bytes=226512 combine_payload_bytes=305127424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=305353936 ingress_bytes=415227904 link_bytes=415227904 token_bytes=610481360 weight_movement_bytes=110100480 total_nvlink_bytes=720581840 weight_to_token_ratio=0.180350 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.221496183e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49358 payload_rows_remote=42948 dup_rows=16178 dup_rows_remote=14566 dedup_ratio=0.246857 router_tokens_per_expert=[45, 311, 1, 1875, 22, 3, 33, 888, 1, 2890, 2, 0, 76, 13, 1025, 4, 11, 1, 26, 2790, 5, 961, 45, 75, 14, 4, 3603, 1471, 485, 583, 493, 2169, 2357, 12, 8, 65, 25, 23, 20, 17, 0, 0, 4, 761, 27, 171, 353, 18, 969, 5, 409, 1, 300, 4, 194, 3635, 2342, 0, 12, 941, 11, 32, 44, 83] payload_rows_per_dst=[8042, 6410, 6502, 6902, 4970, 7416, 3316, 5800] dup_rows_per_dst=[5520, 1612, 1326, 3536, 84, 2392, 578, 1130] combine_rows_per_src=[8042, 6410, 6502, 6902, 4970, 7416, 3316, 5800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48316 payload_rows_remote=44002 dup_rows=17220 dup_rows_remote=16394 dedup_ratio=0.262756 router_tokens_per_expert=[2, 342, 2213, 82, 6, 2137, 12, 4, 2704, 1270, 0, 3, 0, 52, 0, 2108, 7, 14, 1, 0, 28, 233, 3475, 8, 1, 0, 76, 0, 4, 2031, 872, 0, 0, 1, 26, 0, 2, 15, 73, 1849, 4, 94, 0, 846, 1404, 0, 103, 2, 41, 44, 0, 124, 3654, 1509, 1, 2295, 652, 4, 132, 536, 485, 1172, 14, 1] payload_rows_per_dst=[6602, 6528, 7086, 5092, 5348, 4314, 6848, 6498] dup_rows_per_dst=[2680, 1530, 446, 876, 1572, 826, 5580, 3710] combine_rows_per_src=[6602, 6528, 7086, 5092, 5348, 4314, 6848, 6498] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46568 payload_rows_remote=38488 dup_rows=18968 dup_rows_remote=13874 dedup_ratio=0.289429 router_tokens_per_expert=[25, 493, 2340, 17, 0, 2271, 7, 12, 1325, 453, 79, 4, 1, 0, 2, 2140, 7, 42, 0, 0, 99, 2021, 3753, 665, 2, 1, 910, 10, 39, 2859, 1619, 14, 10, 0, 0, 41, 19, 0, 10, 1774, 12, 38, 1, 1535, 182, 11, 22, 17, 38, 1, 0, 334, 3165, 1634, 78, 408, 201, 6, 22, 765, 558, 461, 215, 0] payload_rows_per_dst=[6896, 6202, 8080, 7240, 7420, 5418, 1620, 3692] dup_rows_per_dst=[3434, 1806, 5094, 3668, 2618, 1486, 98, 764] combine_rows_per_src=[6896, 6202, 8080, 7240, 7420, 5418, 1620, 3692] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=307851264 dispatch_meta_bytes=230056 combine_payload_bytes=307851264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=308081320 ingress_bytes=528052224 link_bytes=528052224 token_bytes=615932584 weight_movement_bytes=220200960 total_nvlink_bytes=836133544 weight_to_token_ratio=0.357508 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.183696672e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43642 payload_rows_remote=35522 dup_rows=21894 dup_rows_remote=13648 dedup_ratio=0.334076 router_tokens_per_expert=[3285, 803, 0, 0, 3314, 458, 5, 29, 1, 15, 0, 14, 173, 2737, 0, 14, 4, 0, 178, 81, 1, 3692, 14, 19, 889, 2655, 41, 54, 0, 0, 0, 2985, 1, 0, 0, 292, 0, 200, 0, 0, 60, 623, 261, 3759, 7, 2, 370, 0, 69, 18, 1, 0, 259, 104, 26, 695, 24, 113, 0, 22, 537, 3751, 113, 0] payload_rows_per_dst=[8120, 5808, 7514, 3884, 6322, 8188, 2238, 1568] dup_rows_per_dst=[8246, 100, 2518, 762, 634, 9478, 106, 50] combine_rows_per_src=[8120, 5808, 7514, 3884, 6322, 8188, 2238, 1568] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=315406336 dispatch_meta_bytes=241584 combine_payload_bytes=315406336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315647920 ingress_bytes=425506816 link_bytes=425506816 token_bytes=631054256 weight_movement_bytes=110100480 total_nvlink_bytes=741154736 weight_to_token_ratio=0.174471 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=10368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.400263369e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45974 payload_rows_remote=37790 dup_rows=19562 dup_rows_remote=10036 dedup_ratio=0.298492 router_tokens_per_expert=[0, 1095, 90, 45, 1, 566, 0, 0, 1449, 1831, 0, 0, 0, 136, 0, 1804, 4, 0, 0, 11, 208, 71, 3617, 18, 2, 0, 1403, 20, 0, 1569, 251, 0, 0, 0, 306, 17, 29, 0, 17, 1073, 20, 2719, 1, 1961, 479, 2, 21, 0, 145, 89, 0, 29, 3954, 2497, 0, 1575, 79, 31, 25, 486, 778, 2244, 0, 0] payload_rows_per_dst=[2400, 5482, 7328, 5434, 2728, 7212, 8184, 7206] dup_rows_per_dst=[62, 1350, 530, 1056, 156, 3194, 9526, 3688] combine_rows_per_src=[2400, 5482, 7328, 5434, 2728, 7212, 8184, 7206] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50344 payload_rows_remote=42376 dup_rows=15192 dup_rows_remote=10714 dedup_ratio=0.231812 router_tokens_per_expert=[0, 1489, 808, 382, 0, 1067, 0, 0, 382, 1531, 2, 0, 0, 6, 0, 3441, 29, 3, 0, 3, 5, 3, 4044, 4, 0, 0, 604, 0, 8, 1453, 509, 10, 0, 0, 13, 9, 4, 0, 17, 2731, 1, 421, 0, 559, 1915, 19, 0, 0, 0, 41, 0, 304, 3449, 2044, 55, 2098, 4, 4, 0, 819, 423, 1999, 56, 0] payload_rows_per_dst=[5996, 5448, 8090, 4514, 7968, 6888, 4562, 6878] dup_rows_per_dst=[1496, 1174, 92, 654, 4478, 3030, 434, 3834] combine_rows_per_src=[5996, 5448, 8090, 4514, 7968, 6888, 4562, 6878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48846 payload_rows_remote=41622 dup_rows=16690 dup_rows_remote=13334 dedup_ratio=0.254669 router_tokens_per_expert=[0, 2219, 286, 96, 0, 736, 0, 0, 1736, 946, 0, 0, 0, 419, 0, 1865, 0, 0, 0, 2, 94, 169, 4031, 134, 7, 0, 1646, 13, 0, 2705, 919, 0, 0, 0, 3, 56, 24, 0, 82, 1842, 1, 156, 8, 1049, 1373, 3, 208, 0, 32, 3, 0, 256, 3987, 2702, 0, 976, 14, 44, 0, 1251, 298, 370, 7, 0] payload_rows_per_dst=[5648, 6928, 8072, 7224, 8064, 7046, 2368, 3496] dup_rows_per_dst=[1026, 3004, 788, 3356, 3924, 3954, 166, 472] combine_rows_per_src=[5648, 6928, 8072, 7224, 8064, 7046, 2368, 3496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=275881984 dispatch_meta_bytes=209448 combine_payload_bytes=275881984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=276091432 ingress_bytes=496082944 link_bytes=496082944 token_bytes=551973416 weight_movement_bytes=220200960 total_nvlink_bytes=772174376 weight_to_token_ratio=0.398934 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.627698312e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44958 payload_rows_remote=38068 dup_rows=20578 dup_rows_remote=17232 dedup_ratio=0.313995 router_tokens_per_expert=[1, 1197, 2428, 2, 0, 1534, 9, 0, 1713, 1094, 6, 1, 0, 2, 1, 2302, 0, 0, 0, 1, 106, 137, 2879, 4, 20, 3, 1570, 0, 0, 3190, 760, 101, 0, 0, 33, 14, 2, 0, 0, 1164, 51, 474, 0, 599, 504, 5, 26, 0, 206, 6, 28, 209, 3769, 2537, 5, 1259, 173, 16, 29, 106, 285, 2167, 39, 1] payload_rows_per_dst=[6278, 4758, 6018, 7700, 2410, 2808, 8096, 6890] dup_rows_per_dst=[996, 876, 236, 3588, 16, 510, 11010, 3346] combine_rows_per_src=[6278, 4758, 6018, 7700, 2410, 2808, 8096, 6890] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=254621696 dispatch_meta_bytes=196680 combine_payload_bytes=254621696 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=254818376 ingress_bytes=584923136 link_bytes=584923136 token_bytes=509440072 weight_movement_bytes=330301440 total_nvlink_bytes=839741512 weight_to_token_ratio=0.648362 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.017277537e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=270878720 dispatch_meta_bytes=191304 combine_payload_bytes=270878720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=271070024 ingress_bytes=601180160 link_bytes=601180160 token_bytes=541948744 weight_movement_bytes=330301440 total_nvlink_bytes=872250184 weight_to_token_ratio=0.609470 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=10112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045551175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=303751168 dispatch_meta_bytes=212360 combine_payload_bytes=303751168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=303963528 ingress_bytes=413851648 link_bytes=413851648 token_bytes=607714696 weight_movement_bytes=110100480 total_nvlink_bytes=717815176 weight_to_token_ratio=0.181171 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.197560828e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=298346496 dispatch_meta_bytes=219824 combine_payload_bytes=298346496 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=298566320 ingress_bytes=518547456 link_bytes=518547456 token_bytes=596912816 weight_movement_bytes=220200960 total_nvlink_bytes=817113776 weight_to_token_ratio=0.368900 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.018393124e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49050 payload_rows_remote=40986 dup_rows=16486 dup_rows_remote=12972 dedup_ratio=0.251556 router_tokens_per_expert=[18, 67, 0, 1845, 43, 190, 2, 3, 1, 14, 646, 37, 6, 1443, 59, 7, 7, 0, 5, 30, 27, 2, 3, 412, 2074, 43, 450, 3712, 0, 2, 3, 364, 3492, 0, 30, 1493, 165, 79, 0, 2, 157, 6, 0, 532, 4018, 0, 81, 995, 30, 78, 597, 0, 1732, 1, 24, 1, 1886, 1, 1, 3127, 12, 335, 1159, 1219] payload_rows_per_dst=[7886, 3906, 6518, 5120, 3408, 8064, 7780, 6368] dup_rows_per_dst=[3874, 520, 708, 752, 130, 3514, 4130, 2858] combine_rows_per_src=[7886, 3906, 6518, 5120, 3408, 8064, 7780, 6368] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=272871424 dispatch_meta_bytes=221200 combine_payload_bytes=272871424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=273092624 ingress_bytes=382971904 link_bytes=382971904 token_bytes=545964048 weight_movement_bytes=110100480 total_nvlink_bytes=656064528 weight_to_token_ratio=0.201663 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.660511291e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49376 payload_rows_remote=44334 dup_rows=16160 dup_rows_remote=15536 dedup_ratio=0.246582 router_tokens_per_expert=[0, 1311, 1746, 414, 0, 3110, 3, 0, 422, 1770, 0, 34, 14, 4, 2, 465, 1, 0, 12, 0, 37, 237, 3227, 62, 4, 62, 9, 2, 46, 3699, 1827, 1, 6, 1, 400, 11, 0, 174, 684, 306, 19, 76, 6, 1688, 668, 4, 39, 24, 6, 8, 281, 1696, 1916, 1333, 44, 566, 953, 33, 62, 19, 1969, 1195, 27, 33] payload_rows_per_dst=[7656, 5042, 6952, 7472, 5670, 5842, 4462, 6280] dup_rows_per_dst=[5512, 624, 456, 3328, 1326, 1872, 740, 2302] combine_rows_per_src=[7656, 5042, 6952, 7472, 5670, 5842, 4462, 6280] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293787648 dispatch_meta_bytes=215832 combine_payload_bytes=293787648 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=294003480 ingress_bytes=513988608 link_bytes=513988608 token_bytes=587791128 weight_movement_bytes=220200960 total_nvlink_bytes=807992088 weight_to_token_ratio=0.374625 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.939107259e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45654 payload_rows_remote=38032 dup_rows=19882 dup_rows_remote=15894 dedup_ratio=0.303375 router_tokens_per_expert=[86, 23, 2, 2002, 100, 256, 0, 13, 1, 490, 210, 1, 0, 1921, 43, 22, 1, 2, 28, 3, 166, 9, 1, 23, 262, 183, 27, 3574, 2, 1, 0, 291, 3176, 2, 9, 385, 1414, 59, 0, 64, 2, 29, 4, 231, 4065, 0, 43, 1274, 27, 90, 355, 0, 2312, 1, 471, 5, 2283, 0, 0, 3289, 4, 1380, 386, 1665] payload_rows_per_dst=[7806, 8170, 7622, 4270, 7630, 3048, 5566, 1542] dup_rows_per_dst=[4306, 5336, 3988, 524, 4506, 118, 956, 148] combine_rows_per_src=[7806, 8170, 7622, 4270, 7630, 3048, 5566, 1542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=317786112 dispatch_meta_bytes=239480 combine_payload_bytes=317786112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=318025592 ingress_bytes=537987072 link_bytes=537987072 token_bytes=635811704 weight_movement_bytes=220200960 total_nvlink_bytes=856012664 weight_to_token_ratio=0.346330 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.356480019e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45936 payload_rows_remote=37890 dup_rows=19600 dup_rows_remote=15436 dedup_ratio=0.299072 router_tokens_per_expert=[0, 135, 0, 982, 154, 70, 1, 0, 10, 1, 46, 0, 1, 793, 208, 27, 0, 0, 16, 0, 21, 0, 0, 179, 387, 2, 383, 3386, 0, 2, 3, 2062, 3864, 0, 1, 51, 603, 6, 0, 95, 0, 95, 1, 540, 4087, 0, 166, 546, 79, 24, 900, 0, 1190, 8, 36, 4, 2077, 0, 0, 3134, 0, 3631, 985, 1776] payload_rows_per_dst=[5988, 2074, 6348, 5704, 1484, 8184, 8046, 8108] dup_rows_per_dst=[1814, 98, 352, 1628, 28, 2686, 4164, 8830] combine_rows_per_src=[5988, 2074, 6348, 5704, 1484, 8184, 8046, 8108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46904 payload_rows_remote=38758 dup_rows=18632 dup_rows_remote=14112 dedup_ratio=0.284302 router_tokens_per_expert=[0, 312, 74, 1245, 1, 2, 0, 677, 0, 451, 4, 1, 550, 0, 1091, 0, 125, 0, 381, 1485, 0, 1283, 68, 0, 1833, 0, 4022, 798, 11, 1210, 1469, 2404, 3007, 13, 64, 1, 0, 1, 29, 0, 0, 8, 0, 201, 4, 2460, 79, 8, 814, 0, 3, 3, 639, 0, 2, 3824, 1600, 0, 1, 260, 212, 2, 35, 1] payload_rows_per_dst=[8146, 5464, 5202, 7336, 6126, 8094, 2678, 3858] dup_rows_per_dst=[4520, 1370, 1482, 5474, 104, 5074, 244, 364] combine_rows_per_src=[8146, 5464, 5202, 7336, 6126, 8094, 2678, 3858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48076 payload_rows_remote=43066 dup_rows=17460 dup_rows_remote=15934 dedup_ratio=0.266418 router_tokens_per_expert=[0, 88, 0, 879, 1705, 54, 0, 0, 0, 126, 555, 6, 29, 97, 502, 27, 0, 0, 57, 1, 9, 1, 0, 39, 1439, 147, 988, 3339, 0, 0, 0, 640, 4073, 205, 23, 871, 831, 13, 0, 0, 4, 797, 10, 13, 4096, 3, 14, 878, 1, 2, 122, 0, 1003, 1, 72, 0, 2732, 0, 0, 2729, 0, 807, 2142, 598] payload_rows_per_dst=[7550, 7222, 5570, 5156, 5010, 4520, 6154, 6894] dup_rows_per_dst=[4580, 2456, 102, 1272, 1526, 1334, 1744, 4446] combine_rows_per_src=[7550, 7222, 5570, 5156, 5010, 4520, 6154, 6894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51342 payload_rows_remote=45898 dup_rows=14194 dup_rows_remote=13186 dedup_ratio=0.216583 router_tokens_per_expert=[0, 24, 0, 636, 489, 37, 0, 12, 0, 79, 30, 3, 0, 533, 315, 16, 0, 2, 16, 30, 68, 17, 0, 2, 1470, 6, 347, 3529, 0, 0, 3, 1400, 3256, 7, 37, 1915, 467, 0, 0, 85, 194, 51, 0, 1823, 4088, 0, 118, 1819, 91, 3, 664, 0, 1363, 0, 1, 0, 1341, 0, 0, 2931, 0, 1104, 471, 1875] payload_rows_per_dst=[7380, 8184, 6560, 5444, 7762, 6590, 3834, 5588] dup_rows_per_dst=[2074, 1944, 1078, 1008, 3772, 2596, 410, 1312] combine_rows_per_src=[7380, 8184, 6560, 5444, 7762, 6590, 3834, 5588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6159766656 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3081024640 ingress_bytes=4950450176 link_bytes=4950450176 total_nvlink_bytes=8031474816 weight_to_token_ratio=0.303860 prefetch_share_of_ingress=0.200164 dedup_ratio=0.248880 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.609647065e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=272613376 dispatch_meta_bytes=215704 combine_payload_bytes=272613376 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=272829080 ingress_bytes=492814336 link_bytes=492814336 token_bytes=545442456 weight_movement_bytes=220200960 total_nvlink_bytes=765643416 weight_to_token_ratio=0.403711 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.570851843e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44882 payload_rows_remote=37124 dup_rows=20654 dup_rows_remote=13078 dedup_ratio=0.315155 router_tokens_per_expert=[1, 115, 0, 2485, 34, 337, 0, 0, 0, 2, 155, 4, 0, 382, 92, 5, 13, 0, 25, 2, 50, 4, 0, 126, 370, 204, 2215, 3267, 0, 1, 0, 1166, 3505, 16, 45, 663, 813, 6, 0, 9, 6, 79, 7, 57, 4048, 4, 9, 908, 20, 47, 323, 0, 1528, 1, 79, 1, 1984, 1, 0, 1872, 0, 2616, 1643, 1423] payload_rows_per_dst=[5332, 1206, 3974, 7912, 2862, 8138, 7700, 7758] dup_rows_per_dst=[612, 74, 210, 6534, 242, 2098, 3308, 7576] combine_rows_per_src=[5332, 1206, 3974, 7912, 2862, 8138, 7700, 7758] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=271595520 dispatch_meta_bytes=213304 combine_payload_bytes=271595520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271808824 ingress_bytes=381696000 link_bytes=381696000 token_bytes=543404344 weight_movement_bytes=110100480 total_nvlink_bytes=653504824 weight_to_token_ratio=0.202612 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.638321222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=277817344 dispatch_meta_bytes=211480 combine_payload_bytes=277817344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278028824 ingress_bytes=387917824 link_bytes=387917824 token_bytes=555846168 weight_movement_bytes=110100480 total_nvlink_bytes=665946648 weight_to_token_ratio=0.198077 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.746528975e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=308697088 dispatch_meta_bytes=236000 combine_payload_bytes=308697088 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=308933088 ingress_bytes=528898048 link_bytes=528898048 token_bytes=617630176 weight_movement_bytes=220200960 total_nvlink_bytes=837831136 weight_to_token_ratio=0.356526 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.198406943e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=328996864 dispatch_meta_bytes=236336 combine_payload_bytes=328996864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=329233200 ingress_bytes=549197824 link_bytes=549197824 token_bytes=658230064 weight_movement_bytes=220200960 total_nvlink_bytes=878431024 weight_to_token_ratio=0.334535 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=9472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.551453435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5895140992 prefetch_bytes=1321205760 reduce_bytes=440401920 weight_movement_bytes=1761607680 egress_bytes=2948705920 ingress_bytes=4708042752 link_bytes=4708042752 total_nvlink_bytes=7656748672 weight_to_token_ratio=0.298824 prefetch_share_of_ingress=0.280627 dedup_ratio=0.277881 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=4 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=12827 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.188060686e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=266104832 dispatch_meta_bytes=200808 combine_payload_bytes=266104832 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=266305640 ingress_bytes=816607232 link_bytes=816607232 token_bytes=532410472 weight_movement_bytes=550502400 total_nvlink_bytes=1082912872 weight_to_token_ratio=1.033981 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=8960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.420214285e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5826090760 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2914162440 ingress_bytes=4893736960 link_bytes=4893736960 total_nvlink_bytes=7807899400 weight_to_token_ratio=0.340161 prefetch_share_of_ingress=0.202484 dedup_ratio=0.275171 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=12573 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.511013455e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49164 payload_rows_remote=40986 dup_rows=16372 dup_rows_remote=11498 dedup_ratio=0.249817 router_tokens_per_expert=[128, 4, 1, 166, 279, 36, 0, 2, 2, 128, 166, 9, 17, 2035, 5, 100, 2, 14, 549, 0, 71, 267, 161, 11, 333, 125, 266, 965, 37, 4, 6, 2892, 3561, 670, 33, 529, 1249, 31, 1, 75, 6, 16, 63, 1171, 4064, 8, 182, 1677, 194, 18, 114, 11, 3673, 1, 525, 100, 222, 9, 0, 3270, 28, 299, 1274, 913] payload_rows_per_dst=[2980, 8178, 7362, 6918, 8086, 5252, 7770, 2618] dup_rows_per_dst=[182, 4874, 1772, 2234, 4810, 994, 1502, 4] combine_rows_per_src=[2980, 8178, 7362, 6918, 8086, 5252, 7770, 2618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6165264072 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3083783880 ingress_bytes=5063288832 link_bytes=5063288832 total_nvlink_bytes=8147072712 weight_to_token_ratio=0.321447 prefetch_share_of_ingress=0.217449 dedup_ratio=0.250366 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=12700 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.805892047e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6137489192 prefetch_bytes=770703360 reduce_bytes=1651507200 weight_movement_bytes=2422210560 egress_bytes=3069871912 ingress_bytes=5489827840 link_bytes=5489827840 total_nvlink_bytes=8559699752 weight_to_token_ratio=0.394658 prefetch_share_of_ingress=0.140388 dedup_ratio=0.248459 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=15 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=12192 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.547713535e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5597149808 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=2799651440 ingress_bytes=4779307008 link_bytes=4779307008 total_nvlink_bytes=7578958448 weight_to_token_ratio=0.354075 prefetch_share_of_ingress=0.207332 dedup_ratio=0.295389 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.312000948e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=293787648 dispatch_meta_bytes=209936 combine_payload_bytes=293787648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=293997584 ingress_bytes=403888128 link_bytes=403888128 token_bytes=587785232 weight_movement_bytes=110100480 total_nvlink_bytes=697885712 weight_to_token_ratio=0.187314 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.024278828e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46686 payload_rows_remote=40240 dup_rows=18850 dup_rows_remote=16442 dedup_ratio=0.287628 router_tokens_per_expert=[10, 349, 1004, 126, 1, 2658, 0, 5, 1786, 1392, 1, 1, 0, 178, 0, 1443, 2, 105, 0, 1, 5, 46, 3230, 19, 2, 0, 513, 0, 7, 2591, 1643, 2, 0, 0, 0, 0, 53, 1, 74, 2232, 49, 1018, 0, 343, 1472, 1, 234, 2, 0, 3, 0, 15, 3559, 2449, 65, 838, 44, 30, 5, 281, 398, 2375, 106, 1] payload_rows_per_dst=[6446, 7690, 6176, 3930, 7960, 7182, 1828, 5474] dup_rows_per_dst=[2408, 7094, 92, 404, 3878, 3954, 14, 1006] combine_rows_per_src=[6446, 7690, 6176, 3930, 7960, 7182, 1828, 5474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5943220680 prefetch_bytes=1211105280 reduce_bytes=1651507200 weight_movement_bytes=2862612480 egress_bytes=2972729800 ingress_bytes=5833103360 link_bytes=5833103360 total_nvlink_bytes=8805833160 weight_to_token_ratio=0.481660 prefetch_share_of_ingress=0.207626 dedup_ratio=0.267694 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=15 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.014472612e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=288440320 dispatch_meta_bytes=226728 combine_payload_bytes=288440320 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=288667048 ingress_bytes=508641280 link_bytes=508641280 token_bytes=577107368 weight_movement_bytes=220200960 total_nvlink_bytes=797308328 weight_to_token_ratio=0.381560 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.846108430e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44588 payload_rows_remote=41646 dup_rows=20948 dup_rows_remote=20718 dedup_ratio=0.319641 router_tokens_per_expert=[0, 20, 0, 207, 16, 406, 1, 4, 0, 39, 112, 5, 13, 1400, 22, 10, 0, 0, 253, 3, 145, 42, 44, 29, 1119, 129, 1094, 932, 0, 0, 2, 572, 3515, 13, 345, 1575, 772, 2, 0, 11, 527, 15, 0, 2479, 4046, 0, 12, 1167, 8, 61, 24, 0, 1128, 13, 0, 2, 2538, 0, 0, 3632, 0, 907, 743, 2614] payload_rows_per_dst=[2942, 8120, 7936, 7238, 7926, 6496, 2444, 1486] dup_rows_per_dst=[230, 3174, 5436, 3822, 6354, 1904, 28, 0] combine_rows_per_src=[2942, 8120, 7936, 7238, 7926, 6496, 2444, 1486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=298518528 dispatch_meta_bytes=249456 combine_payload_bytes=298518528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298767984 ingress_bytes=408619008 link_bytes=408619008 token_bytes=597286512 weight_movement_bytes=110100480 total_nvlink_bytes=707386992 weight_to_token_ratio=0.184334 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.106556612e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=5741071136 prefetch_bytes=880803840 reduce_bytes=770703360 weight_movement_bytes=1651507200 egress_bytes=2871663392 ingress_bytes=4520914944 link_bytes=4520914944 total_nvlink_bytes=7392578336 weight_to_token_ratio=0.287665 prefetch_share_of_ingress=0.194829 dedup_ratio=0.292694 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=7 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=12446 gemm_flops_step=57498874675200 bytes_per_gemm_flop=7.862614650e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:57:45] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 370.3 | throughput per GPU (TFLOP/s/GPU): 333.4 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.533543E+00 | loss scale: 1.0 | grad norm: 31.985 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45358 payload_rows_remote=39402 dup_rows=20178 dup_rows_remote=17262 dedup_ratio=0.307892 router_tokens_per_expert=[382, 65, 387, 42, 116, 18, 1172, 611, 556, 403, 240, 377, 243, 292, 31, 393, 117, 204, 1348, 95, 379, 83, 124, 27, 1143, 985, 201, 1367, 66, 1253, 393, 69, 476, 83, 664, 1284, 288, 39, 45, 190, 740, 1362, 1085, 229, 1275, 756, 134, 353, 424, 1158, 715, 36, 795, 1304, 1583, 1397, 804, 564, 330, 44, 311, 769, 237, 112] payload_rows_per_dst=[5676, 5698, 5376, 5646, 5956, 6272, 6058, 4676] dup_rows_per_dst=[2704, 2538, 1928, 2574, 2916, 3046, 2806, 1666] combine_rows_per_src=[5676, 5698, 5376, 5646, 5956, 6272, 6058, 4676] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45300 payload_rows_remote=39604 dup_rows=20236 dup_rows_remote=17612 dedup_ratio=0.308777 router_tokens_per_expert=[232, 89, 690, 76, 159, 34, 2038, 375, 334, 504, 216, 262, 167, 119, 14, 329, 189, 306, 1324, 118, 253, 103, 179, 25, 831, 1285, 260, 1063, 110, 1183, 473, 18, 369, 66, 728, 1515, 137, 11, 62, 586, 642, 589, 954, 48, 1741, 438, 361, 283, 744, 1235, 827, 20, 757, 1251, 1726, 1065, 1111, 601, 182, 25, 272, 573, 415, 76] payload_rows_per_dst=[6266, 5504, 5830, 5696, 6112, 4888, 6176, 4828] dup_rows_per_dst=[3250, 1838, 2646, 2624, 2962, 1742, 3492, 1682] combine_rows_per_src=[6266, 5504, 5830, 5696, 6112, 4888, 6176, 4828] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45704 payload_rows_remote=39946 dup_rows=19832 dup_rows_remote=16822 dedup_ratio=0.302612 router_tokens_per_expert=[323, 104, 436, 23, 23, 39, 1462, 407, 774, 744, 189, 148, 406, 203, 5, 420, 203, 165, 1085, 132, 250, 120, 48, 65, 681, 811, 181, 1017, 254, 1024, 369, 17, 540, 60, 582, 1768, 96, 4, 65, 213, 875, 724, 1224, 68, 1616, 611, 383, 455, 1252, 914, 500, 31, 619, 1068, 2298, 1241, 876, 824, 270, 55, 363, 637, 317, 91] payload_rows_per_dst=[5594, 6792, 5428, 5040, 6188, 4706, 5758, 6198] dup_rows_per_dst=[2522, 3582, 1940, 1634, 2502, 1690, 3010, 2952] combine_rows_per_src=[5594, 6792, 5428, 5040, 6188, 4706, 5758, 6198] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44494 payload_rows_remote=37848 dup_rows=21042 dup_rows_remote=16960 dedup_ratio=0.321075 router_tokens_per_expert=[538, 106, 578, 22, 14, 23, 1222, 254, 728, 410, 123, 275, 272, 202, 5, 263, 278, 40, 1329, 102, 367, 373, 125, 22, 591, 1096, 169, 1305, 190, 837, 309, 12, 489, 142, 773, 1432, 64, 0, 58, 399, 778, 723, 1341, 54, 1586, 421, 386, 583, 554, 1034, 284, 63, 523, 1221, 1324, 2333, 1009, 676, 167, 41, 522, 1050, 462, 96] payload_rows_per_dst=[4724, 4354, 5058, 4820, 6230, 5260, 7402, 6646] dup_rows_per_dst=[1108, 1480, 1810, 1588, 3094, 2206, 5674, 4082] combine_rows_per_src=[4724, 4354, 5058, 4820, 6230, 5260, 7402, 6646] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45420 payload_rows_remote=39768 dup_rows=20116 dup_rows_remote=17646 dedup_ratio=0.306946 router_tokens_per_expert=[342, 148, 464, 12, 19, 38, 1137, 470, 774, 274, 190, 488, 247, 215, 5, 469, 145, 180, 1267, 126, 206, 192, 109, 159, 680, 778, 329, 1014, 82, 766, 420, 26, 444, 136, 404, 1049, 149, 21, 126, 775, 518, 1380, 1102, 70, 1677, 858, 283, 527, 500, 1406, 422, 55, 840, 1219, 1406, 1214, 1039, 786, 95, 87, 269, 1525, 401, 214] payload_rows_per_dst=[5308, 5658, 5652, 5602, 5582, 6284, 5392, 5942] dup_rows_per_dst=[2380, 2478, 2470, 2344, 2654, 3192, 1708, 2890] combine_rows_per_src=[5308, 5658, 5652, 5602, 5582, 6284, 5392, 5942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38920 dup_rows=20968 dup_rows_remote=18506 dedup_ratio=0.319946 router_tokens_per_expert=[351, 204, 1099, 15, 46, 14, 1338, 691, 943, 550, 224, 171, 172, 170, 14, 399, 101, 206, 1566, 287, 343, 240, 120, 89, 703, 1072, 282, 1123, 289, 826, 382, 43, 476, 54, 581, 1058, 115, 4, 40, 469, 350, 1381, 963, 163, 1318, 683, 268, 420, 450, 891, 488, 45, 505, 671, 1412, 1080, 1168, 1073, 214, 77, 823, 821, 537, 97] payload_rows_per_dst=[5974, 5648, 5782, 5916, 5394, 5854, 3756, 6244] dup_rows_per_dst=[3702, 2462, 2758, 3160, 2346, 2602, 562, 3376] combine_rows_per_src=[5974, 5648, 5782, 5916, 5394, 5854, 3756, 6244] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45192 payload_rows_remote=39250 dup_rows=20344 dup_rows_remote=17002 dedup_ratio=0.310425 router_tokens_per_expert=[322, 113, 454, 15, 102, 55, 1133, 519, 1024, 489, 179, 168, 187, 224, 21, 446, 144, 202, 1369, 108, 334, 65, 270, 87, 786, 552, 226, 690, 273, 1112, 586, 55, 363, 393, 258, 1461, 110, 23, 120, 600, 697, 903, 884, 113, 1807, 920, 701, 424, 598, 1379, 319, 90, 557, 706, 1771, 1828, 730, 551, 274, 48, 331, 1071, 289, 139] payload_rows_per_dst=[6012, 6112, 6066, 5274, 5488, 5942, 5390, 4908] dup_rows_per_dst=[3070, 2906, 2706, 1906, 2548, 3342, 1908, 1958] combine_rows_per_src=[6012, 6112, 6066, 5274, 5488, 5942, 5390, 4908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282433536 dispatch_meta_bytes=226656 combine_payload_bytes=282433536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282660192 ingress_bytes=392534016 link_bytes=392534016 token_bytes=565093728 weight_movement_bytes=110100480 total_nvlink_bytes=675194208 weight_to_token_ratio=0.194836 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.826812146e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283881472 dispatch_meta_bytes=228864 combine_payload_bytes=283881472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=284110336 ingress_bytes=504082432 link_bytes=504082432 token_bytes=567991808 weight_movement_bytes=220200960 total_nvlink_bytes=788192768 weight_to_token_ratio=0.387683 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.766822566e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271294464 dispatch_meta_bytes=219232 combine_payload_bytes=271294464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=271513696 ingress_bytes=381394944 link_bytes=381394944 token_bytes=542808160 weight_movement_bytes=110100480 total_nvlink_bytes=652908640 weight_to_token_ratio=0.202835 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.633085363e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286332928 dispatch_meta_bytes=227072 combine_payload_bytes=286332928 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=286560000 ingress_bytes=726734848 link_bytes=726734848 token_bytes=572892928 weight_movement_bytes=440401920 total_nvlink_bytes=1013294848 weight_to_token_ratio=0.768733 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=8704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.263911428e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285057024 dispatch_meta_bytes=229656 combine_payload_bytes=285057024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285286680 ingress_bytes=395157504 link_bytes=395157504 token_bytes=570343704 weight_movement_bytes=110100480 total_nvlink_bytes=680444184 weight_to_token_ratio=0.193042 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.872438917e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278978560 dispatch_meta_bytes=229704 combine_payload_bytes=278978560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279208264 ingress_bytes=389079040 link_bytes=389079040 token_bytes=558186824 weight_movement_bytes=110100480 total_nvlink_bytes=668287304 weight_to_token_ratio=0.197247 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.766724431e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43830 payload_rows_remote=38708 dup_rows=21706 dup_rows_remote=19582 dedup_ratio=0.331207 router_tokens_per_expert=[310, 150, 747, 24, 23, 33, 1185, 377, 316, 793, 359, 402, 179, 231, 10, 462, 112, 248, 1711, 115, 794, 84, 168, 103, 968, 526, 292, 1633, 219, 698, 607, 31, 120, 52, 374, 1043, 303, 37, 138, 715, 621, 920, 1152, 114, 1708, 426, 270, 656, 1240, 882, 146, 63, 604, 1265, 1491, 774, 993, 902, 396, 68, 596, 310, 293, 186] payload_rows_per_dst=[5122, 5630, 6288, 6768, 4160, 5742, 4890, 5230] dup_rows_per_dst=[2124, 2856, 3798, 4944, 1404, 2768, 1554, 2258] combine_rows_per_src=[5122, 5630, 6288, 6768, 4160, 5742, 4890, 5230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281344000 dispatch_meta_bytes=225008 combine_payload_bytes=281344000 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281569008 ingress_bytes=611645440 link_bytes=611645440 token_bytes=562913008 weight_movement_bytes=330301440 total_nvlink_bytes=893214448 weight_to_token_ratio=0.586772 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063752019e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277458944 dispatch_meta_bytes=233160 combine_payload_bytes=277458944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277692104 ingress_bytes=387559424 link_bytes=387559424 token_bytes=555151048 weight_movement_bytes=110100480 total_nvlink_bytes=665251528 weight_to_token_ratio=0.198325 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.051471 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.740295809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50544 payload_rows_remote=44028 dup_rows=14992 dup_rows_remote=13492 dedup_ratio=0.228760 router_tokens_per_expert=[0, 1368, 4039, 1138, 4, 17, 1466, 99, 889, 1399, 1230, 0, 636, 2, 0, 13, 0, 45, 624, 53, 15, 1518, 0, 40, 341, 377, 3, 13, 2809, 20, 126, 10, 2998, 341, 0, 0, 0, 5, 0, 0, 9, 0, 3724, 0, 116, 0, 0, 0, 1244, 647, 2, 208, 0, 1214, 2, 9, 12, 0, 1035, 0, 15, 1649, 786, 458] payload_rows_per_dst=[5880, 6202, 7614, 6226, 6516, 7488, 5930, 4688] dup_rows_per_dst=[2304, 2136, 3726, 1172, 1500, 210, 2792, 1152] combine_rows_per_src=[5880, 6202, 7614, 6226, 6516, 7488, 5930, 4688] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48894 payload_rows_remote=43110 dup_rows=16642 dup_rows_remote=14870 dedup_ratio=0.253937 router_tokens_per_expert=[0, 259, 3941, 1702, 7, 18, 1448, 514, 834, 1562, 970, 5, 649, 0, 2, 280, 0, 94, 1138, 44, 22, 1515, 0, 84, 501, 735, 0, 0, 2277, 0, 264, 1, 1603, 450, 0, 0, 24, 3, 4, 3, 85, 0, 3842, 1, 128, 1, 0, 0, 654, 707, 11, 284, 9, 1009, 13, 52, 93, 0, 1494, 5, 44, 1758, 1242, 383] payload_rows_per_dst=[5694, 6170, 8052, 5784, 3948, 7706, 6140, 5400] dup_rows_per_dst=[2202, 2434, 5624, 1772, 226, 408, 2326, 1650] combine_rows_per_src=[5694, 6170, 8052, 5784, 3948, 7706, 6140, 5400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47456 payload_rows_remote=39914 dup_rows=18080 dup_rows_remote=12930 dedup_ratio=0.275879 router_tokens_per_expert=[0, 307, 4065, 1403, 5, 15, 2260, 124, 249, 1321, 802, 0, 323, 7, 2, 11, 1, 145, 347, 196, 9, 1760, 0, 38, 501, 996, 1, 29, 2677, 0, 182, 0, 1985, 332, 0, 0, 0, 4, 0, 0, 7, 0, 3853, 5, 241, 1, 0, 2, 518, 944, 8, 305, 3, 617, 0, 2, 104, 0, 2386, 0, 57, 1520, 1390, 708] payload_rows_per_dst=[3392, 5670, 4308, 6508, 8166, 7806, 4064, 7542] dup_rows_per_dst=[316, 1788, 684, 2264, 4606, 2542, 730, 5150] combine_rows_per_src=[3392, 5670, 4308, 6508, 8166, 7806, 4064, 7542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48016 payload_rows_remote=43928 dup_rows=17520 dup_rows_remote=16736 dedup_ratio=0.267334 router_tokens_per_expert=[0, 351, 4002, 1312, 0, 3, 2567, 106, 247, 2045, 1072, 0, 426, 4, 0, 19, 0, 65, 312, 85, 0, 1751, 0, 38, 405, 493, 0, 11, 2746, 12, 205, 0, 2787, 255, 2, 0, 15, 2, 0, 0, 47, 1, 3776, 2, 89, 0, 0, 0, 413, 578, 5, 511, 0, 914, 0, 15, 4, 0, 1988, 0, 20, 1106, 1276, 685] payload_rows_per_dst=[3228, 7582, 4144, 6252, 8134, 7630, 4088, 6958] dup_rows_per_dst=[316, 5178, 358, 1492, 5992, 200, 784, 3200] combine_rows_per_src=[3228, 7582, 4144, 6252, 8134, 7630, 4088, 6958] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49574 payload_rows_remote=41522 dup_rows=15962 dup_rows_remote=10972 dedup_ratio=0.243561 router_tokens_per_expert=[0, 435, 3906, 1126, 7, 9, 2399, 49, 671, 1648, 949, 1, 252, 1, 22, 29, 4, 229, 217, 61, 0, 2005, 2, 97, 1070, 293, 0, 31, 2374, 0, 257, 3, 1793, 274, 0, 5, 0, 5, 0, 0, 112, 0, 3564, 1, 115, 0, 0, 0, 464, 755, 0, 609, 3, 719, 1, 33, 83, 0, 2546, 7, 34, 1987, 1370, 141] payload_rows_per_dst=[6102, 5784, 8052, 6012, 3972, 7224, 6758, 5670] dup_rows_per_dst=[1948, 1362, 4990, 2044, 182, 360, 3502, 1574] combine_rows_per_src=[6102, 5784, 8052, 6012, 3972, 7224, 6758, 5670] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49562 payload_rows_remote=44854 dup_rows=15974 dup_rows_remote=15192 dedup_ratio=0.243744 router_tokens_per_expert=[0, 271, 4035, 940, 1, 181, 2318, 59, 465, 1361, 574, 5, 302, 6, 0, 32, 0, 49, 1064, 93, 2, 1048, 5, 24, 401, 982, 7, 25, 2785, 2, 198, 1, 2402, 570, 0, 0, 13, 5, 0, 0, 27, 0, 3540, 4, 106, 0, 4, 0, 606, 442, 1, 190, 3, 1076, 0, 1, 7, 0, 2119, 1, 68, 2151, 1098, 1098] payload_rows_per_dst=[7130, 4708, 6030, 6584, 5314, 7118, 6334, 6344] dup_rows_per_dst=[4618, 782, 2402, 2218, 666, 244, 2542, 2502] combine_rows_per_src=[7130, 4708, 6030, 6584, 5314, 7118, 6334, 6344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315592704 dispatch_meta_bytes=230080 combine_payload_bytes=315592704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315822784 ingress_bytes=425693184 link_bytes=425693184 token_bytes=631415488 weight_movement_bytes=110100480 total_nvlink_bytes=741515968 weight_to_token_ratio=0.174371 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.403504615e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49136 payload_rows_remote=41404 dup_rows=16400 dup_rows_remote=15926 dedup_ratio=0.250244 router_tokens_per_expert=[0, 654, 3841, 791, 3, 38, 2809, 119, 235, 1335, 1054, 1, 425, 4, 0, 124, 0, 56, 205, 356, 6, 1158, 0, 27, 235, 571, 19, 13, 2573, 14, 122, 1, 1628, 513, 0, 0, 3, 0, 0, 4, 11, 0, 3854, 0, 234, 4, 0, 0, 264, 947, 7, 225, 31, 1796, 0, 1, 48, 0, 2564, 0, 17, 897, 2398, 533] payload_rows_per_dst=[5356, 5792, 3266, 5942, 7916, 7732, 6530, 6602] dup_rows_per_dst=[1570, 2466, 350, 1154, 4062, 474, 3252, 3072] combine_rows_per_src=[5356, 5792, 3266, 5942, 7916, 7732, 6530, 6602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=309012480 dispatch_meta_bytes=231920 combine_payload_bytes=309012480 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309244400 ingress_bytes=419112960 link_bytes=419112960 token_bytes=618256880 weight_movement_bytes=110100480 total_nvlink_bytes=728357360 weight_to_token_ratio=0.178082 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=20736 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.289063697e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=286103552 dispatch_meta_bytes=211376 combine_payload_bytes=286103552 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=286314928 ingress_bytes=616404992 link_bytes=616404992 token_bytes=572418480 weight_movement_bytes=330301440 total_nvlink_bytes=902719920 weight_to_token_ratio=0.577028 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072029662e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=314875904 dispatch_meta_bytes=242656 combine_payload_bytes=314875904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=315118560 ingress_bytes=424976384 link_bytes=424976384 token_bytes=629994464 weight_movement_bytes=110100480 total_nvlink_bytes=740094944 weight_to_token_ratio=0.174764 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.391038284e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297629696 dispatch_meta_bytes=209976 combine_payload_bytes=297629696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297839672 ingress_bytes=407730176 link_bytes=407730176 token_bytes=595469368 weight_movement_bytes=110100480 total_nvlink_bytes=705569848 weight_to_token_ratio=0.184897 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.091098362e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=321513472 dispatch_meta_bytes=240184 combine_payload_bytes=321513472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=321753656 ingress_bytes=431613952 link_bytes=431613952 token_bytes=643267128 weight_movement_bytes=110100480 total_nvlink_bytes=753367608 weight_to_token_ratio=0.171158 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.506476508e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296783872 dispatch_meta_bytes=229320 combine_payload_bytes=296783872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297013192 ingress_bytes=406884352 link_bytes=406884352 token_bytes=593797064 weight_movement_bytes=110100480 total_nvlink_bytes=703897544 weight_to_token_ratio=0.185418 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.076388091e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47150 payload_rows_remote=38996 dup_rows=18386 dup_rows_remote=11060 dedup_ratio=0.280548 router_tokens_per_expert=[0, 683, 3991, 1253, 3, 18, 1559, 233, 451, 1673, 898, 0, 469, 9, 1, 240, 1, 200, 933, 21, 0, 1806, 0, 7, 520, 1125, 0, 8, 2404, 4, 122, 2, 1858, 308, 0, 0, 2, 0, 0, 0, 5, 0, 3787, 2, 402, 0, 1, 0, 593, 1240, 4, 198, 11, 972, 0, 0, 45, 0, 1864, 1, 107, 1267, 1242, 225] payload_rows_per_dst=[8154, 5796, 4824, 6660, 4168, 7666, 6396, 3486] dup_rows_per_dst=[7326, 1686, 1112, 3452, 168, 728, 3368, 546] combine_rows_per_src=[8154, 5796, 4824, 6660, 4168, 7666, 6396, 3486] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54628 payload_rows_remote=48046 dup_rows=10908 dup_rows_remote=8430 dedup_ratio=0.166443 router_tokens_per_expert=[230, 3756, 1100, 90, 180, 0, 109, 0, 0, 2, 0, 0, 25, 70, 3669, 0, 1, 1, 0, 3974, 0, 0, 0, 0, 2, 716, 0, 1082, 0, 21, 0, 218, 105, 3005, 0, 2194, 0, 0, 1143, 1088, 215, 0, 0, 0, 3431, 41, 1, 0, 0, 0, 3978, 0, 932, 0, 33, 0, 0, 0, 443, 1, 912, 0, 0, 0] payload_rows_per_dst=[3168, 7358, 7950, 7848, 6582, 6962, 7978, 6782] dup_rows_per_dst=[250, 174, 2, 3742, 2478, 414, 1908, 1940] combine_rows_per_src=[3168, 7358, 7950, 7848, 6582, 6962, 7978, 6782] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55390 payload_rows_remote=49690 dup_rows=10146 dup_rows_remote=7950 dedup_ratio=0.154816 router_tokens_per_expert=[643, 3841, 785, 9, 207, 1, 65, 0, 2, 4, 1, 0, 175, 235, 3757, 0, 0, 0, 0, 3811, 0, 4, 0, 1, 20, 451, 1, 1482, 0, 130, 0, 429, 405, 2504, 0, 2030, 1, 21, 420, 531, 99, 2, 1, 0, 3812, 108, 2, 0, 0, 0, 4028, 1, 637, 1, 8, 1, 0, 0, 391, 6, 1705, 0, 0, 0] payload_rows_per_dst=[6166, 7734, 7630, 5700, 5684, 7716, 8074, 6686] dup_rows_per_dst=[2066, 614, 2, 2196, 1132, 332, 1278, 2526] combine_rows_per_src=[6166, 7734, 7630, 5700, 5684, 7716, 8074, 6686] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=279523328 dispatch_meta_bytes=200224 combine_payload_bytes=279523328 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=279723552 ingress_bytes=830025728 link_bytes=830025728 token_bytes=559246880 weight_movement_bytes=550502400 total_nvlink_bytes=1109749280 weight_to_token_ratio=0.984364 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.443551257e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56336 payload_rows_remote=48748 dup_rows=9200 dup_rows_remote=9198 dedup_ratio=0.140381 router_tokens_per_expert=[527, 3799, 711, 33, 218, 0, 22, 0, 0, 34, 1, 6, 105, 43, 3966, 2, 0, 1, 0, 3792, 1, 1, 0, 0, 0, 515, 1, 985, 0, 49, 0, 336, 140, 3067, 0, 914, 0, 16, 1352, 1938, 212, 9, 0, 2, 3927, 35, 0, 0, 0, 0, 4072, 0, 541, 0, 22, 0, 0, 0, 60, 3, 1285, 5, 0, 20] payload_rows_per_dst=[7920, 7968, 7588, 3396, 6612, 7886, 8144, 6822] dup_rows_per_dst=[2700, 346, 2, 376, 2108, 484, 1126, 2058] combine_rows_per_src=[7920, 7968, 7588, 3396, 6612, 7886, 8144, 6822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54130 payload_rows_remote=45992 dup_rows=11406 dup_rows_remote=8848 dedup_ratio=0.174042 router_tokens_per_expert=[584, 3832, 504, 23, 120, 0, 31, 0, 0, 13, 0, 1, 121, 55, 3988, 0, 0, 2, 1, 3791, 3, 9, 0, 0, 3, 661, 1, 873, 0, 8, 1, 517, 303, 3225, 0, 1121, 0, 0, 1004, 744, 204, 2, 0, 0, 3966, 135, 0, 0, 0, 0, 4066, 0, 1210, 0, 72, 0, 0, 2, 140, 3, 1425, 0, 4, 0] payload_rows_per_dst=[2316, 8014, 7604, 7940, 5042, 7952, 8138, 7124] dup_rows_per_dst=[208, 342, 8, 3852, 1302, 662, 2558, 2474] combine_rows_per_src=[2316, 8014, 7604, 7940, 5042, 7952, 8138, 7124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54184 payload_rows_remote=47382 dup_rows=11352 dup_rows_remote=10154 dedup_ratio=0.173218 router_tokens_per_expert=[336, 3878, 706, 18, 28, 0, 14, 0, 11, 12, 5, 3, 65, 56, 3984, 0, 2, 11, 4, 3942, 3, 1, 1, 0, 0, 746, 1, 713, 0, 8, 2, 749, 74, 3253, 4, 1597, 1, 4, 840, 1847, 202, 4, 1, 3, 4003, 98, 1, 0, 3, 1, 4075, 3, 693, 0, 15, 0, 3, 7, 15, 1, 714, 1, 0, 6] payload_rows_per_dst=[2158, 6796, 7904, 7962, 6324, 8062, 8176, 6802] dup_rows_per_dst=[46, 194, 24, 4232, 2116, 856, 2686, 1198] combine_rows_per_src=[2158, 6796, 7904, 7962, 6324, 8062, 8176, 6802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344393728 dispatch_meta_bytes=225904 combine_payload_bytes=344393728 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=344619632 ingress_bytes=674695168 link_bytes=674695168 token_bytes=689013360 weight_movement_bytes=330301440 total_nvlink_bytes=1019314800 weight_to_token_ratio=0.479383 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.173405865e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50408 payload_rows_remote=42444 dup_rows=15128 dup_rows_remote=14814 dedup_ratio=0.230835 router_tokens_per_expert=[287, 3831, 832, 75, 180, 0, 78, 1, 13, 15, 0, 4, 89, 57, 3961, 0, 14, 12, 0, 3657, 5, 1, 0, 0, 7, 500, 0, 528, 1, 35, 3, 506, 55, 3312, 0, 1664, 33, 2, 1124, 1308, 89, 0, 0, 1, 3586, 64, 38, 0, 1, 0, 4063, 5, 1490, 0, 11, 1, 0, 0, 27, 4, 1168, 0, 0, 30] payload_rows_per_dst=[8150, 7964, 7358, 2900, 6232, 7304, 3606, 6894] dup_rows_per_dst=[9638, 314, 20, 260, 2496, 252, 316, 1832] combine_rows_per_src=[8150, 7964, 7358, 2900, 6232, 7304, 3606, 6894] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54494 payload_rows_remote=46630 dup_rows=11042 dup_rows_remote=9984 dedup_ratio=0.168488 router_tokens_per_expert=[760, 3755, 218, 63, 132, 0, 24, 1, 2, 2, 1, 0, 102, 39, 4056, 1, 0, 5, 0, 3945, 0, 0, 2, 2, 23, 464, 3, 862, 0, 33, 0, 357, 264, 2977, 0, 1126, 0, 0, 435, 1310, 222, 160, 2, 0, 3847, 222, 8, 0, 0, 0, 4055, 0, 995, 0, 1, 0, 0, 0, 29, 0, 2263, 0, 0, 0] payload_rows_per_dst=[2236, 8116, 7896, 7934, 5168, 7864, 8124, 7156] dup_rows_per_dst=[160, 290, 12, 3060, 1102, 1058, 1978, 3382] combine_rows_per_src=[2236, 8116, 7896, 7934, 5168, 7864, 8124, 7156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=356177920 dispatch_meta_bytes=230560 combine_payload_bytes=356177920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356408480 ingress_bytes=466278400 link_bytes=466278400 token_bytes=712586400 weight_movement_bytes=110100480 total_nvlink_bytes=822686880 weight_to_token_ratio=0.154508 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.109348272e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349425664 dispatch_meta_bytes=231784 combine_payload_bytes=349425664 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349657448 ingress_bytes=459526144 link_bytes=459526144 token_bytes=699083112 weight_movement_bytes=110100480 total_nvlink_bytes=809183592 weight_to_token_ratio=0.157493 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.991915435e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=329670656 dispatch_meta_bytes=219360 combine_payload_bytes=329670656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=329890016 ingress_bytes=549871616 link_bytes=549871616 token_bytes=659560672 weight_movement_bytes=220200960 total_nvlink_bytes=879761632 weight_to_token_ratio=0.333860 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.563171786e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=339634176 dispatch_meta_bytes=230144 combine_payload_bytes=339634176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339864320 ingress_bytes=449734656 link_bytes=449734656 token_bytes=679498496 weight_movement_bytes=110100480 total_nvlink_bytes=789598976 weight_to_token_ratio=0.162032 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.821625354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=304238592 dispatch_meta_bytes=229032 combine_payload_bytes=304238592 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=304467624 ingress_bytes=524439552 link_bytes=524439552 token_bytes=608706216 weight_movement_bytes=220200960 total_nvlink_bytes=828907176 weight_to_token_ratio=0.361752 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.120866364e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=334243840 dispatch_meta_bytes=226456 combine_payload_bytes=334243840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334470296 ingress_bytes=554444800 link_bytes=554444800 token_bytes=668714136 weight_movement_bytes=220200960 total_nvlink_bytes=888915096 weight_to_token_ratio=0.329290 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=30208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.642706977e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45848 payload_rows_remote=37656 dup_rows=19688 dup_rows_remote=9526 dedup_ratio=0.300415 router_tokens_per_expert=[392, 3895, 600, 20, 152, 0, 55, 0, 0, 14, 0, 0, 59, 183, 4037, 0, 2, 38, 0, 3815, 0, 4, 0, 0, 0, 885, 0, 1029, 0, 81, 1, 73, 205, 3020, 0, 2043, 0, 0, 1061, 1159, 140, 55, 0, 0, 3956, 78, 0, 0, 0, 3, 4063, 0, 692, 0, 28, 0, 0, 0, 189, 2, 739, 0, 0, 0] payload_rows_per_dst=[8192, 8082, 7958, 3660, 7388, 7314, 1440, 1814] dup_rows_per_dst=[10162, 1230, 3578, 478, 3770, 418, 6, 46] combine_rows_per_src=[8192, 8082, 7958, 3660, 7388, 7314, 1440, 1814] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55242 payload_rows_remote=52192 dup_rows=10294 dup_rows_remote=10290 dedup_ratio=0.157074 router_tokens_per_expert=[0, 7, 31, 1178, 2448, 0, 0, 0, 0, 0, 0, 0, 1, 4071, 0, 0, 2, 59, 1, 12, 0, 0, 11, 43, 4, 0, 11, 275, 0, 0, 2, 14, 0, 0, 1, 0, 0, 4096, 1522, 4, 7, 5, 1, 0, 1831, 0, 0, 0, 3713, 0, 4, 0, 0, 0, 2292, 3, 4093, 4070, 1437, 0, 0, 2, 1517, 0] payload_rows_per_dst=[6536, 8142, 8176, 8190, 3050, 8192, 7884, 5072] dup_rows_per_dst=[792, 2, 220, 608, 4, 3688, 4140, 840] combine_rows_per_src=[6536, 8142, 8176, 8190, 3050, 8192, 7884, 5072] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55930 payload_rows_remote=47790 dup_rows=9606 dup_rows_remote=8704 dedup_ratio=0.146576 router_tokens_per_expert=[0, 3, 27, 1692, 1837, 36, 91, 0, 0, 2, 1, 6, 0, 4087, 0, 0, 8, 23, 2, 9, 0, 0, 70, 34, 2, 11, 15, 345, 0, 0, 7, 89, 0, 0, 1, 1, 0, 4096, 1706, 3, 27, 7, 0, 0, 2623, 0, 9, 0, 3305, 0, 0, 3, 2, 0, 2269, 1, 4052, 4091, 1680, 0, 0, 7, 488, 0] payload_rows_per_dst=[6326, 8176, 8188, 8140, 7294, 5970, 7700, 4136] dup_rows_per_dst=[1046, 16, 286, 902, 2878, 804, 3460, 214] combine_rows_per_src=[6326, 8176, 8188, 8140, 7294, 5970, 7700, 4136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54844 payload_rows_remote=46738 dup_rows=10692 dup_rows_remote=10426 dedup_ratio=0.163147 router_tokens_per_expert=[0, 1, 1, 1858, 1770, 0, 9, 1, 24, 0, 0, 0, 0, 4082, 0, 0, 0, 29, 0, 1, 5, 0, 62, 42, 1, 0, 75, 579, 0, 0, 0, 275, 0, 0, 19, 7, 2, 4086, 1758, 14, 4, 0, 2, 0, 1284, 0, 35, 0, 3439, 0, 0, 0, 0, 0, 1929, 8, 3959, 4047, 2362, 0, 2, 3, 993, 0] payload_rows_per_dst=[6266, 8164, 8106, 8154, 8184, 2626, 7574, 5770] dup_rows_per_dst=[1014, 48, 266, 1624, 3588, 24, 3178, 950] combine_rows_per_src=[6266, 8164, 8106, 8154, 8184, 2626, 7574, 5770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269918208 dispatch_meta_bytes=188728 combine_payload_bytes=269918208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=270106936 ingress_bytes=490119168 link_bytes=490119168 token_bytes=540025144 weight_movement_bytes=220200960 total_nvlink_bytes=760226104 weight_to_token_ratio=0.407761 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.523978439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53738 payload_rows_remote=46178 dup_rows=11798 dup_rows_remote=8402 dedup_ratio=0.180023 router_tokens_per_expert=[0, 0, 5, 2171, 1786, 0, 2, 0, 0, 0, 0, 0, 1, 4096, 0, 1, 0, 64, 2, 0, 0, 1, 16, 70, 13, 11, 5, 445, 0, 0, 1, 34, 0, 0, 0, 0, 0, 4094, 1918, 12, 2, 4, 5, 0, 1764, 0, 0, 0, 3570, 0, 37, 0, 0, 0, 1870, 1, 4059, 4084, 1332, 0, 0, 3, 1289, 0] payload_rows_per_dst=[6510, 8192, 8174, 5630, 3850, 8190, 7560, 5632] dup_rows_per_dst=[1418, 4, 300, 774, 10, 3548, 3396, 2348] combine_rows_per_src=[6510, 8192, 8174, 5630, 3850, 8190, 7560, 5632] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47266 payload_rows_remote=39110 dup_rows=18270 dup_rows_remote=7074 dedup_ratio=0.278778 router_tokens_per_expert=[1, 0, 20, 2500, 1559, 2, 9, 0, 1, 0, 1, 0, 9, 4037, 0, 5, 12, 94, 0, 1, 0, 0, 13, 69, 9, 2, 0, 604, 2, 0, 0, 121, 0, 2, 7, 2, 0, 4095, 1896, 0, 5, 9, 2, 1, 1917, 1, 10, 0, 3774, 7, 2, 12, 2, 0, 1361, 10, 4055, 4086, 844, 1, 2, 3, 1590, 1] payload_rows_per_dst=[6612, 8098, 5928, 1470, 3800, 8190, 5012, 8156] dup_rows_per_dst=[1570, 8, 336, 6, 14, 3890, 1250, 11196] combine_rows_per_src=[6612, 8098, 5928, 1470, 3800, 8190, 5012, 8156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=374112256 dispatch_meta_bytes=249928 combine_payload_bytes=374112256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=374362184 ingress_bytes=594313216 link_bytes=594313216 token_bytes=748474440 weight_movement_bytes=220200960 total_nvlink_bytes=968675400 weight_to_token_ratio=0.294200 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.033608430e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51350 payload_rows_remote=43172 dup_rows=14186 dup_rows_remote=14174 dedup_ratio=0.216461 router_tokens_per_expert=[0, 2, 0, 2024, 1817, 0, 0, 0, 2, 0, 15, 1, 2, 4075, 0, 0, 5, 63, 15, 30, 0, 4, 91, 211, 51, 3, 0, 784, 1, 0, 7, 194, 2, 0, 0, 0, 1, 4090, 1156, 4, 2, 0, 0, 0, 1596, 0, 0, 3, 3482, 1, 0, 3, 0, 1, 1304, 0, 4043, 4025, 1467, 1, 0, 1, 2181, 8] payload_rows_per_dst=[6312, 8178, 8190, 8184, 8188, 3202, 3356, 5740] dup_rows_per_dst=[1374, 12, 698, 1982, 8082, 0, 462, 1576] combine_rows_per_src=[6312, 8178, 8190, 8184, 8188, 3202, 3356, 5740] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55094 payload_rows_remote=46902 dup_rows=10442 dup_rows_remote=6554 dedup_ratio=0.159332 router_tokens_per_expert=[2, 9, 1, 2326, 1354, 1, 14, 0, 2, 0, 0, 0, 2, 4093, 1, 0, 1, 97, 0, 2, 0, 0, 44, 26, 0, 9, 0, 716, 0, 0, 15, 289, 0, 0, 0, 7, 0, 4093, 1589, 6, 2, 1, 0, 0, 1932, 0, 12, 0, 3027, 0, 0, 0, 0, 0, 1094, 2, 4086, 4072, 1518, 0, 0, 1, 2322, 0] payload_rows_per_dst=[6214, 8190, 8154, 8186, 3200, 8192, 6862, 6096] dup_rows_per_dst=[1200, 6, 330, 2044, 4, 3888, 1384, 1586] combine_rows_per_src=[6214, 8190, 8154, 8186, 3200, 8192, 6862, 6096] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342558720 dispatch_meta_bytes=225976 combine_payload_bytes=342558720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342784696 ingress_bytes=452659200 link_bytes=452659200 token_bytes=685343416 weight_movement_bytes=110100480 total_nvlink_bytes=795443896 weight_to_token_ratio=0.160650 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.872487984e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=335017984 dispatch_meta_bytes=228656 combine_payload_bytes=335017984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335246640 ingress_bytes=445118464 link_bytes=445118464 token_bytes=670264624 weight_movement_bytes=110100480 total_nvlink_bytes=780365104 weight_to_token_ratio=0.164264 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.741342183e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331003904 dispatch_meta_bytes=218320 combine_payload_bytes=331003904 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=331222224 ingress_bytes=771405824 link_bytes=771405824 token_bytes=662226128 weight_movement_bytes=440401920 total_nvlink_bytes=1102628048 weight_to_token_ratio=0.665033 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.341601602e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=280340480 dispatch_meta_bytes=184736 combine_payload_bytes=280340480 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=280525216 ingress_bytes=610641920 link_bytes=610641920 token_bytes=560865696 weight_movement_bytes=330301440 total_nvlink_bytes=891167136 weight_to_token_ratio=0.588914 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062006732e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=309456896 dispatch_meta_bytes=229384 combine_payload_bytes=309456896 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=309686280 ingress_bytes=419557376 link_bytes=419557376 token_bytes=619143176 weight_movement_bytes=110100480 total_nvlink_bytes=729243656 weight_to_token_ratio=0.177827 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.296792822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=336193536 dispatch_meta_bytes=213824 combine_payload_bytes=336193536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336407360 ingress_bytes=446294016 link_bytes=446294016 token_bytes=672600896 weight_movement_bytes=110100480 total_nvlink_bytes=782701376 weight_to_token_ratio=0.163694 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.761786966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54388 payload_rows_remote=47354 dup_rows=11148 dup_rows_remote=9012 dedup_ratio=0.170105 router_tokens_per_expert=[0, 994, 296, 0, 0, 0, 4095, 0, 0, 0, 0, 9, 0, 841, 0, 0, 0, 0, 4096, 13, 0, 4096, 0, 4078, 2210, 0, 0, 0, 185, 11, 142, 0, 0, 0, 2135, 0, 0, 0, 2448, 2, 2, 0, 3342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 116, 0, 3634] payload_rows_per_dst=[2952, 8192, 8156, 4716, 7034, 7854, 8192, 7292] dup_rows_per_dst=[528, 1700, 26, 380, 2136, 6124, 0, 254] combine_rows_per_src=[2952, 8192, 8156, 4716, 7034, 7854, 8192, 7292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57474 payload_rows_remote=51512 dup_rows=8062 dup_rows_remote=7278 dedup_ratio=0.123016 router_tokens_per_expert=[0, 673, 1238, 0, 2, 0, 4072, 1, 0, 0, 0, 3, 0, 852, 0, 0, 0, 0, 4092, 19, 0, 4061, 1, 4093, 2483, 0, 0, 0, 317, 89, 203, 0, 1, 0, 1512, 3, 0, 0, 2550, 3, 32, 0, 2440, 2, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 36, 0, 1, 0, 1, 29, 0, 3955] payload_rows_per_dst=[8172, 8190, 8188, 5962, 6008, 4916, 8122, 7916] dup_rows_per_dst=[3800, 1704, 38, 784, 1568, 38, 2, 128] combine_rows_per_src=[8172, 8190, 8188, 5962, 6008, 4916, 8122, 7916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53054 payload_rows_remote=45408 dup_rows=12482 dup_rows_remote=7782 dedup_ratio=0.190460 router_tokens_per_expert=[0, 0, 8, 2479, 1922, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 8, 0, 3, 1, 0, 4, 3, 0, 21, 0, 512, 0, 0, 2, 191, 0, 0, 0, 2, 1, 4096, 1366, 3, 0, 0, 42, 0, 2509, 0, 15, 0, 3048, 0, 0, 0, 0, 0, 1159, 0, 4095, 4073, 1734, 0, 0, 2, 1376, 0] payload_rows_per_dst=[7646, 8186, 8150, 8190, 8192, 5048, 2318, 5324] dup_rows_per_dst=[4700, 114, 34, 1452, 5198, 84, 0, 900] combine_rows_per_src=[7646, 8186, 8150, 8190, 8192, 5048, 2318, 5324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57906 payload_rows_remote=49888 dup_rows=7630 dup_rows_remote=7618 dedup_ratio=0.116425 router_tokens_per_expert=[0, 366, 330, 2, 0, 0, 4095, 0, 0, 0, 0, 16, 0, 986, 0, 2, 0, 0, 4086, 10, 0, 4090, 0, 4005, 2688, 0, 0, 0, 259, 1, 279, 0, 2, 4, 2686, 0, 0, 0, 2472, 24, 0, 0, 2247, 0, 0, 1, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8, 0, 8, 0, 0, 164, 5, 3923] payload_rows_per_dst=[8190, 8178, 8018, 7580, 5374, 4504, 8182, 7880] dup_rows_per_dst=[1396, 2002, 12, 3818, 58, 2, 6, 336] combine_rows_per_src=[8190, 8178, 8018, 7580, 5374, 4504, 8182, 7880] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50594 payload_rows_remote=42428 dup_rows=14942 dup_rows_remote=14938 dedup_ratio=0.227997 router_tokens_per_expert=[0, 472, 511, 0, 3, 0, 4088, 0, 0, 0, 0, 45, 5, 1149, 0, 5, 0, 0, 4075, 7, 0, 4081, 0, 4055, 2749, 0, 0, 0, 134, 65, 480, 0, 0, 0, 2862, 0, 0, 0, 2227, 10, 7, 3, 1470, 4, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 22, 0, 1, 0, 0, 199, 1, 4032] payload_rows_per_dst=[1818, 3054, 8134, 5972, 7176, 8188, 8166, 8086] dup_rows_per_dst=[154, 112, 7382, 884, 3022, 2960, 4, 424] combine_rows_per_src=[1818, 3054, 8134, 5972, 7176, 8188, 8166, 8086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=339433472 dispatch_meta_bytes=225464 combine_payload_bytes=339433472 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=339658936 ingress_bytes=559634432 link_bytes=559634432 token_bytes=679092408 weight_movement_bytes=220200960 total_nvlink_bytes=899293368 weight_to_token_ratio=0.324258 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.732963213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54990 payload_rows_remote=46800 dup_rows=10546 dup_rows_remote=6224 dedup_ratio=0.160919 router_tokens_per_expert=[0, 1199, 419, 0, 3, 0, 4093, 0, 0, 3, 0, 50, 0, 1085, 0, 0, 0, 0, 4090, 101, 0, 4093, 0, 4077, 1825, 0, 2, 0, 29, 24, 155, 3, 0, 0, 3196, 6, 0, 0, 2148, 3, 12, 0, 2151, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 61, 1, 3934] payload_rows_per_dst=[2966, 8190, 8166, 3906, 7506, 8190, 8188, 7878] dup_rows_per_dst=[276, 2266, 190, 170, 3200, 4322, 6, 116] combine_rows_per_src=[2966, 8190, 8166, 3906, 7506, 8190, 8188, 7878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50734 payload_rows_remote=43486 dup_rows=14802 dup_rows_remote=12658 dedup_ratio=0.225861 router_tokens_per_expert=[0, 325, 1032, 4, 0, 0, 4073, 0, 0, 0, 4, 26, 17, 1176, 0, 0, 0, 0, 4069, 87, 6, 4077, 0, 3907, 3006, 0, 10, 13, 93, 0, 139, 0, 0, 0, 2839, 0, 0, 0, 3280, 5, 33, 6, 985, 8, 1, 0, 3, 0, 0, 3, 0, 17, 0, 0, 0, 0, 35, 0, 0, 15, 2, 93, 0, 3379] payload_rows_per_dst=[2528, 2438, 8170, 6142, 7860, 8170, 8178, 7248] dup_rows_per_dst=[194, 8, 4458, 380, 5456, 2048, 114, 2144] combine_rows_per_src=[2528, 2438, 8170, 6142, 7860, 8170, 8178, 7248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 2, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52578 payload_rows_remote=44502 dup_rows=12958 dup_rows_remote=10148 dedup_ratio=0.197723 router_tokens_per_expert=[0, 628, 410, 6, 3, 9, 4090, 4, 0, 0, 0, 9, 0, 1402, 0, 0, 0, 0, 4032, 48, 1, 4081, 0, 4039, 3010, 0, 2, 0, 141, 4, 114, 0, 0, 0, 2064, 1, 0, 0, 2641, 10, 2, 0, 1977, 2, 0, 0, 0, 8, 3, 0, 4, 3, 0, 0, 0, 0, 21, 1, 2, 0, 15, 16, 4, 3961] payload_rows_per_dst=[8192, 8076, 4530, 7554, 4132, 3974, 8178, 7942] dup_rows_per_dst=[5702, 2810, 52, 4270, 18, 4, 4, 98] combine_rows_per_src=[8192, 8076, 4530, 7554, 4132, 3974, 8178, 7942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=369238016 dispatch_meta_bytes=235160 combine_payload_bytes=369238016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369473176 ingress_bytes=479338496 link_bytes=479338496 token_bytes=738711192 weight_movement_bytes=110100480 total_nvlink_bytes=848811672 weight_to_token_ratio=0.149044 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.336484822e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=304123904 dispatch_meta_bytes=229464 combine_payload_bytes=304123904 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=304353368 ingress_bytes=524324864 link_bytes=524324864 token_bytes=608477272 weight_movement_bytes=220200960 total_nvlink_bytes=828678232 weight_to_token_ratio=0.361889 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.118871751e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=357597184 dispatch_meta_bytes=230024 combine_payload_bytes=357597184 prefetch_bytes=0 reduce_bytes=660602880 egress_bytes=357827208 ingress_bytes=1018200064 link_bytes=1018200064 token_bytes=715424392 weight_movement_bytes=660602880 total_nvlink_bytes=1376027272 weight_to_token_ratio=0.923372 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.770817376e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=335462400 dispatch_meta_bytes=212096 combine_payload_bytes=335462400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=335674496 ingress_bytes=445562880 link_bytes=445562880 token_bytes=671136896 weight_movement_bytes=110100480 total_nvlink_bytes=781237376 weight_to_token_ratio=0.164051 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.749071308e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=311707648 dispatch_meta_bytes=224576 combine_payload_bytes=311707648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=311932224 ingress_bytes=421808128 link_bytes=421808128 token_bytes=623639872 weight_movement_bytes=110100480 total_nvlink_bytes=733740352 weight_to_token_ratio=0.176545 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.335937101e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=318990336 dispatch_meta_bytes=218600 combine_payload_bytes=318990336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319208936 ingress_bytes=429090816 link_bytes=429090816 token_bytes=638199272 weight_movement_bytes=110100480 total_nvlink_bytes=748299752 weight_to_token_ratio=0.172517 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.462595023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57470 payload_rows_remote=49778 dup_rows=8066 dup_rows_remote=4628 dedup_ratio=0.123077 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4072, 0, 2211, 0, 0, 0, 3986, 188, 53, 0, 0, 0, 3725, 0, 0, 0, 0, 2846, 0, 0, 37, 0, 5, 2, 0, 0, 0, 2672, 0, 0, 3984, 0, 2893, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 4, 0, 1962, 0, 0, 0, 0, 0, 28, 0, 2, 0, 0, 0] payload_rows_per_dst=[8144, 3746, 7450, 6162, 7692, 8192, 8072, 8012] dup_rows_per_dst=[0, 86, 0, 690, 3438, 0, 3836, 16] combine_rows_per_src=[8144, 3746, 7450, 6162, 7692, 8192, 8072, 8012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=325484544 dispatch_meta_bytes=212760 combine_payload_bytes=325484544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325697304 ingress_bytes=435585024 link_bytes=435585024 token_bytes=651181848 weight_movement_bytes=110100480 total_nvlink_bytes=761282328 weight_to_token_ratio=0.169078 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.575539982e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56640 payload_rows_remote=51740 dup_rows=8896 dup_rows_remote=8812 dedup_ratio=0.135742 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 4027, 0, 3387, 0, 0, 0, 3716, 599, 35, 0, 0, 1, 3793, 0, 0, 0, 7, 2420, 0, 0, 49, 0, 1, 20, 2, 0, 0, 2624, 2, 0, 4090, 0, 1887, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 1985, 2, 0, 0, 0, 0, 12, 1, 4, 0, 0, 0] payload_rows_per_dst=[8056, 7576, 7590, 4900, 6598, 8190, 5540, 8190] dup_rows_per_dst=[0, 4888, 12, 84, 2428, 24, 1436, 24] combine_rows_per_src=[8056, 7576, 7590, 4900, 6598, 8190, 5540, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=356808704 dispatch_meta_bytes=217624 combine_payload_bytes=356808704 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=357026328 ingress_bytes=907311104 link_bytes=907311104 token_bytes=713835032 weight_movement_bytes=550502400 total_nvlink_bytes=1264337432 weight_to_token_ratio=0.771190 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.577963237e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54938 payload_rows_remote=47392 dup_rows=10598 dup_rows_remote=8084 dedup_ratio=0.161713 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3717, 0, 3472, 0, 3, 0, 3466, 556, 106, 1, 0, 13, 3772, 0, 0, 0, 0, 3580, 0, 3, 2, 0, 1, 29, 1, 0, 0, 1767, 11, 1, 4086, 0, 2507, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 1546, 1, 0, 0, 0, 2, 7, 7, 2, 0, 1, 0] payload_rows_per_dst=[7434, 1316, 7556, 8090, 6642, 8182, 7546, 8172] dup_rows_per_dst=[0, 14, 16, 6086, 1930, 0, 2514, 38] combine_rows_per_src=[7434, 1316, 7556, 8090, 6642, 8182, 7546, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56132 payload_rows_remote=47984 dup_rows=9404 dup_rows_remote=9196 dedup_ratio=0.143494 router_tokens_per_expert=[0, 0, 1, 0, 0, 1, 0, 3498, 0, 3445, 0, 2, 0, 3862, 1158, 24, 0, 0, 4, 3853, 0, 0, 0, 1, 1868, 0, 0, 23, 0, 17, 174, 0, 1, 0, 1410, 27, 0, 4065, 0, 3133, 0, 4092, 0, 0, 6, 0, 0, 2, 1, 0, 5, 0, 2055, 0, 5, 0, 0, 0, 1, 0, 22, 0, 12, 0] payload_rows_per_dst=[6996, 2356, 7708, 7476, 7258, 8190, 8000, 8148] dup_rows_per_dst=[4, 12, 8, 2998, 2308, 10, 3856, 208] combine_rows_per_src=[6996, 2356, 7708, 7476, 7258, 8190, 8000, 8148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56880 payload_rows_remote=48690 dup_rows=8656 dup_rows_remote=8656 dedup_ratio=0.132080 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 4000, 0, 2808, 5, 2, 0, 3983, 1618, 3, 0, 0, 6, 3900, 0, 0, 0, 1, 2297, 0, 0, 7, 1, 3, 18, 0, 0, 0, 2588, 0, 0, 3957, 0, 1532, 0, 4094, 0, 0, 0, 0, 1, 0, 12, 0, 2, 0, 1920, 1, 0, 3, 0, 0, 3, 0, 1, 0, 0, 1] payload_rows_per_dst=[8002, 3244, 7812, 7336, 6262, 8190, 8114, 7920] dup_rows_per_dst=[0, 12, 2, 2932, 1978, 0, 3728, 4] combine_rows_per_src=[8002, 3244, 7812, 7336, 6262, 8190, 8114, 7920] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56546 payload_rows_remote=48772 dup_rows=8990 dup_rows_remote=8930 dedup_ratio=0.137177 router_tokens_per_expert=[0, 4, 0, 0, 4, 4, 0, 4019, 0, 3230, 0, 0, 0, 3977, 341, 18, 0, 0, 28, 3885, 1, 0, 3, 0, 2804, 0, 1, 79, 0, 0, 7, 0, 0, 0, 1850, 0, 0, 4090, 0, 2414, 0, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 6, 0, 1885, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 18] payload_rows_per_dst=[8038, 8176, 7774, 5704, 6692, 8190, 3782, 8190] dup_rows_per_dst=[24, 6956, 60, 78, 1836, 4, 0, 32] combine_rows_per_src=[8038, 8176, 7774, 5704, 6692, 8190, 3782, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=370872320 dispatch_meta_bytes=242208 combine_payload_bytes=370872320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371114528 ingress_bytes=480972800 link_bytes=480972800 token_bytes=741986848 weight_movement_bytes=110100480 total_nvlink_bytes=852087328 weight_to_token_ratio=0.148386 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.364908056e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58356 payload_rows_remote=50270 dup_rows=7180 dup_rows_remote=2214 dedup_ratio=0.109558 router_tokens_per_expert=[0, 4, 0, 0, 2, 0, 2, 3982, 0, 2644, 0, 12, 0, 3477, 382, 11, 0, 16, 5, 3826, 1, 1, 0, 1, 3245, 0, 0, 27, 0, 0, 0, 0, 0, 0, 3037, 0, 7, 4064, 0, 1523, 0, 4081, 1, 1, 0, 1, 0, 2, 18, 5, 31, 1, 2289, 4, 2, 0, 1, 10, 13, 1, 5, 0, 30, 3] payload_rows_per_dst=[7974, 8086, 7666, 6522, 7110, 8170, 4666, 8162] dup_rows_per_dst=[6, 4966, 34, 22, 2024, 2, 34, 92] combine_rows_per_src=[7974, 8086, 7666, 6522, 7110, 8170, 4666, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=339705856 dispatch_meta_bytes=221904 combine_payload_bytes=339705856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339927760 ingress_bytes=449806336 link_bytes=449806336 token_bytes=679633616 weight_movement_bytes=110100480 total_nvlink_bytes=789734096 weight_to_token_ratio=0.162000 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.822871987e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=343949312 dispatch_meta_bytes=228720 combine_payload_bytes=343949312 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=344178032 ingress_bytes=564150272 link_bytes=564150272 token_bytes=688127344 weight_movement_bytes=220200960 total_nvlink_bytes=908328304 weight_to_token_ratio=0.320000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.811501098e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349009920 dispatch_meta_bytes=229384 combine_payload_bytes=349009920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349239304 ingress_bytes=459110400 link_bytes=459110400 token_bytes=698249224 weight_movement_bytes=110100480 total_nvlink_bytes=808349704 weight_to_token_ratio=0.157681 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.984684963e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=349597696 dispatch_meta_bytes=230808 combine_payload_bytes=349597696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349828504 ingress_bytes=459698176 link_bytes=459698176 token_bytes=699426200 weight_movement_bytes=110100480 total_nvlink_bytes=809526680 weight_to_token_ratio=0.157415 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.994907354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59844 payload_rows_remote=51876 dup_rows=5692 dup_rows_remote=5692 dedup_ratio=0.086853 router_tokens_per_expert=[4096, 0, 0, 0, 1720, 3590, 0, 0, 0, 32, 0, 0, 4096, 38, 0, 2146, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3984, 3950, 0, 1088, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3440, 7932, 8188, 7850, 7968, 8192, 8082, 8192] dup_rows_per_dst=[0, 3680, 0, 0, 0, 18, 1994, 0] combine_rows_per_src=[3440, 7932, 8188, 7850, 7968, 8192, 8082, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360335360 dispatch_meta_bytes=209936 combine_payload_bytes=360335360 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=360545296 ingress_bytes=800737280 link_bytes=800737280 token_bytes=720880656 weight_movement_bytes=440401920 total_nvlink_bytes=1161282576 weight_to_token_ratio=0.610922 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.392613829e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60394 payload_rows_remote=52740 dup_rows=5142 dup_rows_remote=5130 dedup_ratio=0.078461 router_tokens_per_expert=[4096, 0, 0, 0, 2230, 3700, 0, 0, 0, 7, 0, 0, 4094, 52, 0, 1703, 0, 5, 2, 4091, 0, 0, 0, 0, 0, 3827, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 3830, 1, 1061, 0, 5, 0, 0, 0, 0, 1, 2] payload_rows_per_dst=[6218, 5914, 8186, 7654, 8074, 8188, 7968, 8192] dup_rows_per_dst=[1170, 2082, 10, 12, 6, 30, 1816, 16] combine_rows_per_src=[6218, 5914, 8186, 7654, 8074, 8188, 7968, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=371847168 dispatch_meta_bytes=230272 combine_payload_bytes=371847168 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=372077440 ingress_bytes=592048128 link_bytes=592048128 token_bytes=743924608 weight_movement_bytes=220200960 total_nvlink_bytes=964125568 weight_to_token_ratio=0.295999 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029669070e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59856 payload_rows_remote=52174 dup_rows=5680 dup_rows_remote=4094 dedup_ratio=0.086670 router_tokens_per_expert=[4092, 0, 0, 0, 2028, 3212, 1, 0, 1, 0, 0, 6, 4095, 15, 0, 2760, 0, 0, 0, 4087, 0, 1, 0, 5, 0, 3758, 7, 0, 3, 9, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 1, 0, 0, 0, 0, 2, 4, 0, 4021, 3447, 3, 1178, 0, 5, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[4058, 7980, 8182, 7530, 8042, 8192, 7682, 8190] dup_rows_per_dst=[0, 4008, 4, 24, 4, 42, 1586, 12] combine_rows_per_src=[4058, 7980, 8182, 7530, 8042, 8192, 7682, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59648 payload_rows_remote=51458 dup_rows=5888 dup_rows_remote=5866 dedup_ratio=0.089844 router_tokens_per_expert=[4095, 0, 0, 0, 2161, 2486, 0, 0, 0, 3, 0, 0, 4094, 6, 0, 2626, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3834, 2, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 5, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 3851, 3451, 0, 2044, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[4322, 7584, 8190, 7670, 7706, 8190, 7794, 8192] dup_rows_per_dst=[0, 2658, 0, 6, 0, 22, 3200, 2] combine_rows_per_src=[4322, 7584, 8190, 7670, 7706, 8190, 7794, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61174 payload_rows_remote=53010 dup_rows=4362 dup_rows_remote=4328 dedup_ratio=0.066559 router_tokens_per_expert=[4079, 0, 0, 0, 3130, 3302, 0, 9, 3, 0, 0, 3, 4089, 14, 0, 1927, 0, 0, 2, 4078, 0, 0, 0, 0, 0, 3567, 27, 0, 0, 72, 0, 1, 0, 0, 1, 12, 0, 0, 1, 0, 2, 1, 7, 4, 0, 0, 0, 0, 0, 8, 0, 1, 3845, 3733, 3, 843, 2, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[6244, 7736, 8160, 7174, 7696, 8190, 7810, 8164] dup_rows_per_dst=[2, 2762, 0, 160, 22, 16, 1366, 34] combine_rows_per_src=[6244, 7736, 8160, 7174, 7696, 8190, 7810, 8164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60156 payload_rows_remote=51968 dup_rows=5380 dup_rows_remote=5376 dedup_ratio=0.082092 router_tokens_per_expert=[4087, 0, 0, 0, 2937, 3175, 1, 0, 0, 0, 0, 2, 4094, 10, 0, 2108, 0, 1, 0, 4092, 0, 0, 3, 0, 6, 3840, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 3967, 3724, 0, 699, 0, 7, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8006, 4236, 8188, 7686, 7934, 8192, 7736, 8178] dup_rows_per_dst=[4220, 4, 4, 20, 2, 10, 1110, 10] combine_rows_per_src=[8006, 4236, 8188, 7686, 7934, 8192, 7736, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378040320 dispatch_meta_bytes=231480 combine_payload_bytes=378040320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378271800 ingress_bytes=488140800 link_bytes=488140800 token_bytes=756312120 weight_movement_bytes=110100480 total_nvlink_bytes=866412600 weight_to_token_ratio=0.145575 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.489571366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49384 payload_rows_remote=41192 dup_rows=16152 dup_rows_remote=5676 dedup_ratio=0.246460 router_tokens_per_expert=[0, 507, 651, 0, 5, 0, 4096, 0, 0, 0, 0, 7, 0, 205, 0, 0, 1, 0, 4096, 7, 0, 4095, 0, 4075, 3433, 0, 0, 0, 93, 0, 99, 0, 0, 0, 2494, 0, 0, 0, 2884, 0, 5, 0, 2116, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 29, 0, 0, 1, 1, 20, 0, 3846] payload_rows_per_dst=[8192, 8192, 16, 7872, 4988, 4236, 8190, 7698] dup_rows_per_dst=[10476, 424, 0, 5146, 0, 6, 4, 96] combine_rows_per_src=[8192, 8192, 16, 7872, 4988, 4236, 8190, 7698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57246 payload_rows_remote=52398 dup_rows=8290 dup_rows_remote=8290 dedup_ratio=0.126495 router_tokens_per_expert=[4079, 0, 0, 0, 1507, 3312, 0, 0, 0, 7, 0, 0, 4092, 10, 3, 2404, 0, 6, 1, 4078, 0, 0, 4, 4, 0, 3378, 24, 0, 0, 1, 0, 1, 0, 0, 0, 17, 2, 1, 0, 0, 11, 1, 2, 0, 0, 0, 11, 1, 0, 8, 0, 0, 4052, 3912, 0, 1838, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[7602, 4848, 8164, 7936, 8140, 8190, 4206, 8160] dup_rows_per_dst=[2036, 0, 22, 4110, 1822, 46, 254, 0] combine_rows_per_src=[7602, 4848, 8164, 7936, 8140, 8190, 4206, 8160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=368850944 dispatch_meta_bytes=229296 combine_payload_bytes=368850944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369080240 ingress_bytes=478951424 link_bytes=478951424 token_bytes=737931184 weight_movement_bytes=110100480 total_nvlink_bytes=848031664 weight_to_token_ratio=0.149202 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.329753003e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=373983232 dispatch_meta_bytes=225072 combine_payload_bytes=373983232 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=374208304 ingress_bytes=924485632 link_bytes=924485632 token_bytes=748191536 weight_movement_bytes=550502400 total_nvlink_bytes=1298693936 weight_to_token_ratio=0.735777 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.607832566e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=379975680 dispatch_meta_bytes=229352 combine_payload_bytes=379975680 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=380205032 ingress_bytes=600176640 link_bytes=600176640 token_bytes=760180712 weight_movement_bytes=220200960 total_nvlink_bytes=980381672 weight_to_token_ratio=0.289669 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.043805889e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372506624 dispatch_meta_bytes=229376 combine_payload_bytes=372506624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372736000 ingress_bytes=482607104 link_bytes=482607104 token_bytes=745242624 weight_movement_bytes=110100480 total_nvlink_bytes=855343104 weight_to_token_ratio=0.147738 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.393331291e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295264256 dispatch_meta_bytes=187472 combine_payload_bytes=295264256 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=295451728 ingress_bytes=515465216 link_bytes=515465216 token_bytes=590715984 weight_movement_bytes=220200960 total_nvlink_bytes=810916944 weight_to_token_ratio=0.372770 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.964787901e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59818 payload_rows_remote=51626 dup_rows=5718 dup_rows_remote=5718 dedup_ratio=0.087250 router_tokens_per_expert=[0, 2901, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4019, 0, 0, 0, 0, 1412, 0, 0, 4096, 0, 0, 0, 0, 12, 0, 1454, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2657, 0, 0, 3788, 0, 0, 0, 0, 0, 0, 0, 0, 0, 141, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5802, 8038, 8192, 7896, 8192, 5314, 8192, 8192] dup_rows_per_dst=[0, 0, 2824, 2612, 0, 0, 0, 282] combine_rows_per_src=[5802, 8038, 8192, 7896, 8192, 5314, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=375588864 dispatch_meta_bytes=242752 combine_payload_bytes=375588864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375831616 ingress_bytes=595789824 link_bytes=595789824 token_bytes=751420480 weight_movement_bytes=220200960 total_nvlink_bytes=971621440 weight_to_token_ratio=0.293046 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036176495e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61870 payload_rows_remote=53788 dup_rows=3666 dup_rows_remote=1794 dedup_ratio=0.055939 router_tokens_per_expert=[0, 2914, 0, 0, 0, 1, 0, 0, 4069, 4, 0, 0, 0, 4051, 0, 0, 0, 0, 813, 0, 0, 4096, 0, 0, 0, 0, 99, 0, 902, 0, 0, 0, 3, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3567, 1, 2, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[5830, 8104, 8192, 8082, 8192, 7140, 8138, 8192] dup_rows_per_dst=[0, 6, 1626, 1872, 6, 0, 0, 156] combine_rows_per_src=[5830, 8104, 8192, 8082, 8192, 7140, 8138, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=370055168 dispatch_meta_bytes=229376 combine_payload_bytes=370055168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370284544 ingress_bytes=480155648 link_bytes=480155648 token_bytes=740339712 weight_movement_bytes=110100480 total_nvlink_bytes=850440192 weight_to_token_ratio=0.148716 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.350696439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60052 payload_rows_remote=52390 dup_rows=5484 dup_rows_remote=5484 dedup_ratio=0.083679 router_tokens_per_expert=[0, 1982, 1, 0, 1, 0, 0, 0, 4094, 3, 0, 0, 0, 4038, 0, 0, 0, 0, 1968, 0, 0, 4092, 1, 1, 1, 0, 33, 0, 782, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3829, 2, 0, 3649, 0, 0, 2, 0, 3, 0, 0, 0, 0, 95, 0, 0, 4094, 0, 2, 0, 0] payload_rows_per_dst=[3964, 8082, 8186, 7584, 8190, 7662, 8192, 8192] dup_rows_per_dst=[4, 0, 3938, 1346, 0, 0, 6, 190] combine_rows_per_src=[3964, 8082, 8186, 7584, 8190, 7662, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56254 payload_rows_remote=48074 dup_rows=9282 dup_rows_remote=9044 dedup_ratio=0.141632 router_tokens_per_expert=[0, 3225, 3, 17, 0, 0, 0, 0, 4074, 4, 0, 0, 4, 4088, 0, 6, 0, 0, 1303, 0, 0, 4095, 0, 0, 3, 0, 17, 0, 425, 0, 2, 0, 0, 4089, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3764, 0, 6, 3466, 0, 12, 0, 2, 0, 1, 1, 0, 0, 71, 0, 0, 4082, 0, 5, 0, 1] payload_rows_per_dst=[6468, 8088, 8090, 886, 8180, 8186, 8176, 8180] dup_rows_per_dst=[22, 16, 2592, 8, 16, 6286, 104, 238] combine_rows_per_src=[6468, 8088, 8090, 886, 8180, 8186, 8176, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61044 payload_rows_remote=52854 dup_rows=4492 dup_rows_remote=4492 dedup_ratio=0.068542 router_tokens_per_expert=[0, 2702, 1, 0, 0, 0, 0, 0, 4089, 0, 1, 0, 0, 4066, 0, 0, 0, 0, 1178, 0, 0, 4095, 2, 0, 0, 0, 64, 0, 599, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3843, 4, 0, 3869, 0, 6, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[5406, 8134, 8192, 6960, 8186, 7784, 8190, 8192] dup_rows_per_dst=[0, 0, 2358, 1060, 0, 954, 0, 120] combine_rows_per_src=[5406, 8134, 8192, 6960, 8186, 7784, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60178 payload_rows_remote=55562 dup_rows=5358 dup_rows_remote=4794 dedup_ratio=0.081757 router_tokens_per_expert=[0, 3068, 0, 0, 0, 0, 0, 0, 4073, 0, 1, 0, 1, 4090, 0, 0, 0, 0, 1117, 0, 0, 4096, 0, 0, 0, 0, 3, 0, 350, 2, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 2, 0, 3767, 0, 0, 0, 0, 0, 1, 1, 0, 0, 153, 0, 0, 4096, 0, 2, 0, 0] payload_rows_per_dst=[7522, 8180, 4616, 7630, 8192, 7696, 8150, 8192] dup_rows_per_dst=[3284, 580, 564, 614, 4, 2, 0, 310] combine_rows_per_src=[7522, 8180, 4616, 7630, 8192, 7696, 8150, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=385552384 dispatch_meta_bytes=222328 combine_payload_bytes=385552384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385774712 ingress_bytes=495652864 link_bytes=495652864 token_bytes=771327096 weight_movement_bytes=110100480 total_nvlink_bytes=881427576 weight_to_token_ratio=0.142742 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.620218514e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=344594432 dispatch_meta_bytes=228472 combine_payload_bytes=344594432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344822904 ingress_bytes=564795392 link_bytes=564795392 token_bytes=689417336 weight_movement_bytes=220200960 total_nvlink_bytes=909618296 weight_to_token_ratio=0.319402 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.822720796e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375531520 dispatch_meta_bytes=231496 combine_payload_bytes=375531520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375763016 ingress_bytes=595732480 link_bytes=595732480 token_bytes=751294536 weight_movement_bytes=220200960 total_nvlink_bytes=971495496 weight_to_token_ratio=0.293095 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.036076764e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58210 payload_rows_remote=50096 dup_rows=7326 dup_rows_remote=7324 dedup_ratio=0.111786 router_tokens_per_expert=[0, 2941, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 2, 1, 4055, 0, 0, 4, 0, 1713, 0, 0, 4077, 11, 5, 0, 2, 0, 0, 690, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 2, 0, 0, 3, 3119, 1, 0, 3762, 0, 0, 0, 0, 0, 0, 1, 2, 0, 80, 0, 8, 4095, 5, 0, 1, 0] payload_rows_per_dst=[8174, 8114, 3454, 7654, 8186, 6246, 8190, 8192] dup_rows_per_dst=[5866, 2, 12, 1254, 0, 0, 6, 186] combine_rows_per_src=[8174, 8114, 3454, 7654, 8186, 6246, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=378857472 dispatch_meta_bytes=229384 combine_payload_bytes=378857472 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=379086856 ingress_bytes=599058432 link_bytes=599058432 token_bytes=757944328 weight_movement_bytes=220200960 total_nvlink_bytes=978145288 weight_to_token_ratio=0.290524 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041861141e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398268416 dispatch_meta_bytes=241424 combine_payload_bytes=398268416 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=398509840 ingress_bytes=838670336 link_bytes=838670336 token_bytes=796778256 weight_movement_bytes=440401920 total_nvlink_bytes=1237180176 weight_to_token_ratio=0.552728 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.458585652e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52688 payload_rows_remote=44602 dup_rows=12848 dup_rows_remote=10076 dedup_ratio=0.196045 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4019, 0, 2616, 0, 0, 0, 3976, 433, 26, 0, 0, 7, 3983, 0, 0, 0, 0, 3120, 0, 0, 99, 0, 2, 7, 0, 0, 0, 2465, 0, 0, 4089, 0, 1549, 0, 4096, 0, 0, 0, 0, 0, 0, 42, 0, 1, 0, 2163, 0, 0, 0, 0, 0, 45, 0, 0, 0, 30, 0] payload_rows_per_dst=[8086, 8178, 8126, 6358, 1168, 8192, 4396, 8184] dup_rows_per_dst=[2772, 5924, 3722, 98, 172, 0, 16, 144] combine_rows_per_src=[8086, 8178, 8126, 6358, 1168, 8192, 4396, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58128 payload_rows_remote=50100 dup_rows=7408 dup_rows_remote=3634 dedup_ratio=0.113037 router_tokens_per_expert=[0, 0, 0, 3948, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1196, 0, 0, 0, 0, 215, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 711, 2487, 0, 1913, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4005, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 4096, 0, 2516, 0, 5, 1, 3, 0] payload_rows_per_dst=[7896, 8192, 8190, 5780, 8028, 8010, 6840, 5192] dup_rows_per_dst=[0, 2392, 430, 616, 3774, 12, 0, 184] combine_rows_per_src=[7896, 8192, 8190, 5780, 8028, 8010, 6840, 5192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359088128 dispatch_meta_bytes=229680 combine_payload_bytes=359088128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=359317808 ingress_bytes=689389568 link_bytes=689389568 token_bytes=718405936 weight_movement_bytes=330301440 total_nvlink_bytes=1048707376 weight_to_token_ratio=0.459770 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.198961844e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58796 payload_rows_remote=51550 dup_rows=6740 dup_rows_remote=5596 dedup_ratio=0.102844 router_tokens_per_expert=[0, 0, 1, 3932, 0, 0, 0, 0, 0, 2, 4095, 1, 0, 0, 0, 1034, 0, 0, 0, 0, 226, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 877, 3318, 0, 1362, 149, 0, 2, 0, 0, 0, 0, 0, 0, 0, 14, 0, 3900, 0, 0, 0, 0, 1, 0, 0, 2865, 2, 4095, 1, 2755, 0, 38, 0, 8, 1] payload_rows_per_dst=[7866, 8192, 8180, 7246, 8190, 7802, 5730, 5590] dup_rows_per_dst=[0, 2072, 450, 1144, 3026, 26, 6, 16] combine_rows_per_src=[7866, 8192, 8180, 7246, 8190, 7802, 5730, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=319707136 dispatch_meta_bytes=218712 combine_payload_bytes=319707136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319925848 ingress_bytes=429807616 link_bytes=429807616 token_bytes=639632984 weight_movement_bytes=110100480 total_nvlink_bytes=749733464 weight_to_token_ratio=0.172131 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.475061354e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359116800 dispatch_meta_bytes=214936 combine_payload_bytes=359116800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359331736 ingress_bytes=469217280 link_bytes=469217280 token_bytes=718448536 weight_movement_bytes=110100480 total_nvlink_bytes=828549016 weight_to_token_ratio=0.153248 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.160460229e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47850 payload_rows_remote=39664 dup_rows=17686 dup_rows_remote=7854 dedup_ratio=0.269867 router_tokens_per_expert=[0, 0, 21, 3954, 0, 2, 3, 0, 0, 0, 4077, 0, 0, 0, 0, 662, 0, 6, 0, 0, 92, 4087, 0, 1, 0, 0, 0, 1, 7, 1, 1667, 3927, 0, 1139, 314, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7, 0, 3903, 0, 0, 0, 1, 0, 0, 0, 3961, 0, 4087, 0, 822, 0, 17, 5, 1, 0] payload_rows_per_dst=[7950, 1324, 8180, 3348, 2866, 7808, 8188, 8186] dup_rows_per_dst=[10, 0, 192, 4, 46, 12, 7590, 9832] combine_rows_per_src=[7950, 1324, 8180, 3348, 2866, 7808, 8188, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57120 payload_rows_remote=49092 dup_rows=8416 dup_rows_remote=8344 dedup_ratio=0.128418 router_tokens_per_expert=[0, 3, 0, 3587, 0, 0, 1, 0, 2, 0, 4096, 0, 0, 0, 1, 821, 0, 2, 0, 0, 233, 4077, 1, 0, 0, 1, 0, 0, 0, 0, 1607, 3881, 0, 619, 508, 0, 1, 0, 0, 2, 0, 0, 0, 0, 37, 0, 4012, 1, 0, 0, 0, 0, 0, 0, 3592, 0, 4094, 0, 1588, 0, 0, 0, 0, 1] payload_rows_per_dst=[7174, 8192, 8160, 8014, 2178, 8028, 7184, 8190] dup_rows_per_dst=[8, 1648, 466, 2964, 82, 72, 0, 3176] combine_rows_per_src=[7174, 8192, 8160, 8014, 2178, 8028, 7184, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52898 payload_rows_remote=45014 dup_rows=12638 dup_rows_remote=11978 dedup_ratio=0.192841 router_tokens_per_expert=[0, 0, 0, 3838, 2, 0, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 507, 0, 0, 0, 1, 94, 4093, 1, 0, 0, 0, 0, 0, 0, 0, 1073, 3793, 0, 1748, 118, 3, 0, 0, 0, 2, 0, 0, 0, 0, 5, 0, 4067, 1, 0, 0, 0, 12, 0, 0, 3933, 0, 4076, 0, 1300, 0, 18, 5, 0, 1] payload_rows_per_dst=[7680, 1688, 8188, 7436, 3702, 8142, 7884, 8178] dup_rows_per_dst=[0, 166, 190, 1642, 40, 1782, 660, 8158] combine_rows_per_src=[7680, 1688, 8188, 7436, 3702, 8142, 7884, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57324 payload_rows_remote=49152 dup_rows=8212 dup_rows_remote=7676 dedup_ratio=0.125305 router_tokens_per_expert=[0, 20, 0, 3937, 0, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 900, 0, 1, 1, 0, 268, 4084, 0, 0, 0, 0, 0, 0, 5, 0, 1139, 3412, 0, 1774, 308, 0, 1, 0, 0, 49, 0, 2, 0, 0, 6, 0, 3969, 0, 0, 0, 0, 1, 0, 0, 3705, 1, 4095, 1, 957, 0, 34, 2, 0, 0] payload_rows_per_dst=[7906, 8190, 8172, 7532, 8190, 7944, 7410, 1980] dup_rows_per_dst=[10, 1800, 536, 1580, 4264, 10, 4, 8] combine_rows_per_src=[7906, 8190, 8172, 7532, 8190, 7944, 7410, 1980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=369510400 dispatch_meta_bytes=228584 combine_payload_bytes=369510400 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=369738984 ingress_bytes=589711360 link_bytes=589711360 token_bytes=739249384 weight_movement_bytes=220200960 total_nvlink_bytes=959450344 weight_to_token_ratio=0.297871 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.025605046e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=284311552 dispatch_meta_bytes=190072 combine_payload_bytes=284311552 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=284501624 ingress_bytes=504512512 link_bytes=504512512 token_bytes=568813176 weight_movement_bytes=220200960 total_nvlink_bytes=789014136 weight_to_token_ratio=0.387124 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=13440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.774302364e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=351891456 dispatch_meta_bytes=229744 combine_payload_bytes=351891456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352121200 ingress_bytes=461991936 link_bytes=461991936 token_bytes=704012656 weight_movement_bytes=110100480 total_nvlink_bytes=814113136 weight_to_token_ratio=0.156390 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.034799613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=322660352 dispatch_meta_bytes=227968 combine_payload_bytes=322660352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322888320 ingress_bytes=432760832 link_bytes=432760832 token_bytes=645548672 weight_movement_bytes=110100480 total_nvlink_bytes=755649152 weight_to_token_ratio=0.170553 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.526422638e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=352321536 dispatch_meta_bytes=227312 combine_payload_bytes=352321536 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=352548848 ingress_bytes=682622976 link_bytes=682622976 token_bytes=704870384 weight_movement_bytes=330301440 total_nvlink_bytes=1035171824 weight_to_token_ratio=0.468599 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.187193627e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58092 payload_rows_remote=49906 dup_rows=7444 dup_rows_remote=6376 dedup_ratio=0.113586 router_tokens_per_expert=[0, 1, 3, 3890, 2, 8, 0, 3, 3, 0, 4086, 0, 0, 4, 0, 534, 0, 0, 0, 0, 348, 4089, 0, 15, 0, 0, 0, 1, 0, 3, 1195, 3508, 0, 1464, 470, 0, 0, 3, 1, 12, 3, 0, 0, 8, 1, 0, 3802, 0, 0, 0, 0, 2, 0, 0, 3119, 0, 4065, 1, 2111, 0, 1, 10, 2, 0] payload_rows_per_dst=[7806, 8186, 8190, 7628, 8184, 7624, 6242, 4232] dup_rows_per_dst=[8, 1068, 714, 1786, 3846, 4, 0, 18] combine_rows_per_src=[7806, 8186, 8190, 7628, 8184, 7624, 6242, 4232] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56616 payload_rows_remote=52620 dup_rows=8920 dup_rows_remote=8846 dedup_ratio=0.136108 router_tokens_per_expert=[891, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 752, 0, 0, 0, 0, 0, 62, 0, 234, 0, 0, 0, 0, 0, 0, 3458, 1903, 4089, 0, 4096, 0, 132, 0, 0, 1664, 0, 220, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 2874] payload_rows_per_dst=[8192, 8192, 8192, 8182, 3996, 3678, 8192, 7992] dup_rows_per_dst=[1782, 0, 1504, 588, 74, 90, 210, 4672] combine_rows_per_src=[8192, 8192, 8192, 8182, 3996, 3678, 8192, 7992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54712 payload_rows_remote=46556 dup_rows=10824 dup_rows_remote=762 dedup_ratio=0.165161 router_tokens_per_expert=[4096, 0, 0, 0, 1932, 3563, 0, 0, 0, 0, 0, 0, 4096, 63, 0, 1903, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3483, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4060, 4006, 0, 1466, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8156, 4042, 8192, 7060, 8120, 8018, 2932, 8192] dup_rows_per_dst=[10062, 70, 46, 644, 0, 0, 0, 2] combine_rows_per_src=[8156, 4042, 8192, 7060, 8120, 8018, 2932, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58038 payload_rows_remote=49906 dup_rows=7498 dup_rows_remote=7146 dedup_ratio=0.114410 router_tokens_per_expert=[455, 0, 0, 2, 0, 4088, 0, 0, 0, 4084, 0, 0, 1, 0, 0, 0, 0, 0, 1967, 0, 0, 0, 1, 0, 31, 0, 154, 0, 0, 0, 0, 16, 0, 2004, 1662, 4041, 1, 4095, 0, 675, 2, 0, 1852, 0, 260, 4, 1, 4084, 0, 16, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 3191] payload_rows_per_dst=[8182, 8170, 8184, 8132, 6180, 4104, 8190, 6896] dup_rows_per_dst=[908, 0, 3920, 352, 1296, 134, 194, 694] combine_rows_per_src=[8182, 8170, 8184, 8132, 6180, 4104, 8190, 6896] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=357726208 dispatch_meta_bytes=225128 combine_payload_bytes=357726208 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=357951336 ingress_bytes=688027648 link_bytes=688027648 token_bytes=715677544 weight_movement_bytes=330301440 total_nvlink_bytes=1045978984 weight_to_token_ratio=0.461523 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.196593241e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377180160 dispatch_meta_bytes=245864 combine_payload_bytes=377180160 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=377426024 ingress_bytes=817582080 link_bytes=817582080 token_bytes=754606184 weight_movement_bytes=440401920 total_nvlink_bytes=1195008104 weight_to_token_ratio=0.583618 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.421909706e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46412 payload_rows_remote=39794 dup_rows=19124 dup_rows_remote=17664 dedup_ratio=0.291809 router_tokens_per_expert=[495, 0, 0, 0, 2, 4046, 0, 0, 0, 4076, 0, 0, 8, 0, 0, 0, 0, 0, 1148, 0, 0, 0, 13, 1, 173, 0, 283, 0, 0, 0, 0, 31, 4, 1676, 2372, 4090, 1, 4095, 0, 636, 1, 0, 1903, 0, 1134, 2, 1, 4094, 0, 9, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 2363] payload_rows_per_dst=[2014, 8168, 2304, 3054, 7874, 8190, 8190, 6618] dup_rows_per_dst=[140, 136, 20, 736, 7236, 9156, 240, 1460] combine_rows_per_src=[2014, 8168, 2304, 3054, 7874, 8190, 8190, 6618] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55342 payload_rows_remote=47992 dup_rows=10194 dup_rows_remote=5164 dedup_ratio=0.155548 router_tokens_per_expert=[291, 0, 0, 3, 4, 4095, 0, 0, 0, 4092, 0, 1, 0, 0, 0, 1, 0, 0, 1507, 0, 0, 0, 0, 0, 240, 0, 237, 0, 0, 0, 0, 9, 6, 3080, 1788, 4095, 0, 4096, 0, 114, 0, 1, 2392, 0, 451, 0, 0, 4095, 0, 0, 1, 0, 1, 0, 104, 0, 2, 0, 0, 0, 0, 0, 0, 2062] payload_rows_per_dst=[8190, 8186, 4046, 8190, 3728, 7350, 8192, 7460] dup_rows_per_dst=[596, 2, 466, 972, 88, 5030, 212, 2828] combine_rows_per_src=[8190, 8186, 4046, 8190, 3728, 7350, 8192, 7460] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56082 payload_rows_remote=47890 dup_rows=9454 dup_rows_remote=9216 dedup_ratio=0.144257 router_tokens_per_expert=[239, 1, 0, 0, 0, 4092, 1, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 1848, 0, 0, 0, 4, 11, 60, 0, 140, 0, 0, 0, 0, 0, 0, 3009, 1692, 4077, 0, 4096, 0, 313, 0, 0, 2346, 1, 406, 1, 1, 4093, 0, 0, 0, 0, 0, 0, 119, 0, 0, 0, 0, 0, 0, 0, 0, 2123] payload_rows_per_dst=[8190, 8188, 3714, 8188, 3912, 8190, 8192, 7508] dup_rows_per_dst=[476, 2, 12, 366, 98, 5506, 238, 2756] combine_rows_per_src=[8190, 8188, 3714, 8188, 3912, 8190, 8192, 7508] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=333713408 dispatch_meta_bytes=189272 combine_payload_bytes=333713408 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=333902680 ingress_bytes=774115328 link_bytes=774115328 token_bytes=667616088 weight_movement_bytes=440401920 total_nvlink_bytes=1108018008 weight_to_token_ratio=0.659663 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.346313875e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=357726208 dispatch_meta_bytes=228208 combine_payload_bytes=357726208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357954416 ingress_bytes=467826688 link_bytes=467826688 token_bytes=715680624 weight_movement_bytes=110100480 total_nvlink_bytes=825781104 weight_to_token_ratio=0.153840 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.136275547e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59506 payload_rows_remote=51318 dup_rows=6030 dup_rows_remote=3772 dedup_ratio=0.092010 router_tokens_per_expert=[230, 0, 0, 0, 0, 4066, 0, 0, 1, 4048, 0, 0, 0, 0, 2, 0, 0, 0, 1135, 0, 0, 0, 0, 0, 121, 0, 326, 0, 0, 0, 0, 34, 0, 1743, 1952, 4089, 0, 4095, 0, 295, 0, 0, 2483, 0, 1186, 0, 0, 4088, 0, 10, 0, 0, 0, 0, 111, 0, 0, 0, 0, 1, 0, 0, 0, 2752] payload_rows_per_dst=[8132, 8100, 8188, 8192, 6720, 6476, 8192, 5506] dup_rows_per_dst=[460, 2, 2258, 948, 1260, 862, 240, 0] combine_rows_per_src=[8132, 8100, 8188, 8192, 6720, 6476, 8192, 5506] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6984243496 prefetch_bytes=1101004800 reduce_bytes=1321205760 weight_movement_bytes=2422210560 egress_bytes=3493269800 ingress_bytes=5913184256 link_bytes=5913184256 total_nvlink_bytes=9406454056 weight_to_token_ratio=0.346811 prefetch_share_of_ingress=0.186195 dedup_ratio=0.157660 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=12 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=11176 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.028399997e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=285243392 dispatch_meta_bytes=229832 combine_payload_bytes=285243392 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285473224 ingress_bytes=395343872 link_bytes=395343872 token_bytes=570716616 weight_movement_bytes=110100480 total_nvlink_bytes=680817096 weight_to_token_ratio=0.192916 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=20992 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.875680163e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=344006656 dispatch_meta_bytes=212624 combine_payload_bytes=344006656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344219280 ingress_bytes=564207616 link_bytes=564207616 token_bytes=688225936 weight_movement_bytes=220200960 total_nvlink_bytes=908426896 weight_to_token_ratio=0.319954 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=15744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.812498404e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=343275520 dispatch_meta_bytes=228424 combine_payload_bytes=343275520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=343503944 ingress_bytes=453376000 link_bytes=453376000 token_bytes=686779464 weight_movement_bytes=110100480 total_nvlink_bytes=796879944 weight_to_token_ratio=0.160314 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.884954315e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7047445768 prefetch_bytes=990904320 reduce_bytes=330301440 weight_movement_bytes=1321205760 egress_bytes=3524875528 ingress_bytes=4843776000 link_bytes=4843776000 total_nvlink_bytes=8368651528 weight_to_token_ratio=0.187473 prefetch_share_of_ingress=0.204573 dedup_ratio=0.147452 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=3 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.424123128e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=367847424 dispatch_meta_bytes=220360 combine_payload_bytes=367847424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368067784 ingress_bytes=477947904 link_bytes=477947904 token_bytes=735915208 weight_movement_bytes=110100480 total_nvlink_bytes=846015688 weight_to_token_ratio=0.149610 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.312300140e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58026 payload_rows_remote=49836 dup_rows=7510 dup_rows_remote=7498 dedup_ratio=0.114594 router_tokens_per_expert=[534, 2, 1, 0, 0, 4086, 0, 0, 2, 4092, 2, 0, 2, 1, 1, 1, 0, 0, 1254, 0, 1, 0, 0, 0, 369, 0, 139, 0, 0, 0, 0, 1, 0, 1900, 2697, 4085, 1, 4086, 0, 428, 0, 0, 1757, 1, 127, 6, 0, 4092, 0, 1, 2, 1, 1, 0, 76, 0, 14, 0, 0, 0, 0, 1, 5, 2999] payload_rows_per_dst=[8176, 8190, 8186, 8188, 7342, 3734, 8184, 6026] dup_rows_per_dst=[1070, 12, 2508, 1000, 2710, 48, 150, 12] combine_rows_per_src=[8176, 8190, 8186, 8188, 7342, 3734, 8184, 6026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6256485888 prefetch_bytes=1321205760 reduce_bytes=660602880 weight_movement_bytes=1981808640 egress_bytes=3129331200 ingress_bytes=5108963328 link_bytes=5108963328 total_nvlink_bytes=8238294528 weight_to_token_ratio=0.316761 prefetch_share_of_ingress=0.258605 dedup_ratio=0.218817 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=6 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.036765 gemm_inner_pad_upper_bound=11684 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.885327507e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6748875536 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3375557392 ingress_bytes=5024825344 link_bytes=5024825344 total_nvlink_bytes=8400382736 weight_to_token_ratio=0.244708 prefetch_share_of_ingress=0.219113 dedup_ratio=0.163898 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037132 gemm_inner_pad_upper_bound=11684 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.738997715e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6651249080 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3326759352 ingress_bytes=5856800768 link_bytes=5856800768 total_nvlink_bytes=9183560120 weight_to_token_ratio=0.380727 prefetch_share_of_ingress=0.169189 dedup_ratio=0.180603 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.037868 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.018593981e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6932817872 prefetch_bytes=880803840 reduce_bytes=1321205760 weight_movement_bytes=2202009600 egress_bytes=3467548624 ingress_bytes=5667278848 link_bytes=5667278848 total_nvlink_bytes=9134827472 weight_to_token_ratio=0.317621 prefetch_share_of_ingress=0.155419 dedup_ratio=0.148880 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=12 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038971 gemm_inner_pad_upper_bound=10922 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.856330024e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56854 payload_rows_remote=48662 dup_rows=8682 dup_rows_remote=3680 dedup_ratio=0.132477 router_tokens_per_expert=[0, 2501, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1350, 0, 0, 4096, 0, 0, 0, 0, 5, 0, 1113, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3297, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 4095, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 2700, 7966, 8192, 6832, 8192, 6588] dup_rows_per_dst=[5002, 0, 0, 2112, 0, 1434, 0, 134] combine_rows_per_src=[8192, 8192, 2700, 7966, 8192, 6832, 8192, 6588] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=357224448 dispatch_meta_bytes=229336 combine_payload_bytes=357224448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357453784 ingress_bytes=467324928 link_bytes=467324928 token_bytes=714678232 weight_movement_bytes=110100480 total_nvlink_bytes=824778712 weight_to_token_ratio=0.154056 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.127549115e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=348809216 dispatch_meta_bytes=209368 combine_payload_bytes=348809216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349018584 ingress_bytes=458909696 link_bytes=458909696 token_bytes=697827800 weight_movement_bytes=110100480 total_nvlink_bytes=807928280 weight_to_token_ratio=0.157776 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.981194390e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6688565464 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3345424600 ingress_bytes=5435049984 link_bytes=5435049984 total_nvlink_bytes=8780474584 weight_to_token_ratio=0.312759 prefetch_share_of_ingress=0.182317 dedup_ratio=0.178473 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038235 gemm_inner_pad_upper_bound=11557 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.452445834e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55406 payload_rows_remote=47398 dup_rows=10130 dup_rows_remote=6916 dedup_ratio=0.154572 router_tokens_per_expert=[0, 1, 0, 3940, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1834, 0, 0, 0, 0, 438, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 842, 3635, 0, 438, 22, 0, 0, 0, 0, 4, 0, 0, 0, 0, 49, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 3899, 0, 4096, 0, 1280, 0, 18, 0, 0, 0] payload_rows_per_dst=[8008, 8192, 4874, 7584, 8192, 8162, 7798, 2596] dup_rows_per_dst=[3214, 4234, 424, 1232, 928, 98, 0, 0] combine_rows_per_src=[8008, 8192, 4874, 7584, 8192, 8162, 7798, 2596] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=339748864 dispatch_meta_bytes=217256 combine_payload_bytes=339748864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339966120 ingress_bytes=449849344 link_bytes=449849344 token_bytes=679714984 weight_movement_bytes=110100480 total_nvlink_bytes=789815464 weight_to_token_ratio=0.161980 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.823619967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55000 payload_rows_remote=46808 dup_rows=10536 dup_rows_remote=9700 dedup_ratio=0.160767 router_tokens_per_expert=[418, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 1921, 0, 0, 0, 0, 0, 67, 0, 93, 0, 0, 0, 0, 0, 0, 1957, 2893, 4064, 0, 4096, 0, 808, 0, 0, 1289, 0, 205, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 2554] payload_rows_per_dst=[8192, 8188, 8192, 8152, 7766, 2898, 6504, 5108] dup_rows_per_dst=[836, 0, 3842, 296, 5256, 90, 216, 0] combine_rows_per_src=[8192, 8188, 8192, 8152, 7766, 2898, 6504, 5108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=335519744 dispatch_meta_bytes=226032 combine_payload_bytes=335519744 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=335745776 ingress_bytes=665821184 link_bytes=665821184 token_bytes=671265520 weight_movement_bytes=330301440 total_nvlink_bytes=1001566960 weight_to_token_ratio=0.492058 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.157972548e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6252378280 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3127230632 ingress_bytes=5437257728 link_bytes=5437257728 total_nvlink_bytes=8564488360 weight_to_token_ratio=0.369797 prefetch_share_of_ingress=0.161994 dedup_ratio=0.215811 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=11049 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.456285464e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:57:46] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 372.4 | throughput per GPU (TFLOP/s/GPU): 331.6 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.110852E+01 | loss scale: 1.0 | grad norm: 107.525 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46816 payload_rows_remote=40488 dup_rows=18720 dup_rows_remote=16120 dedup_ratio=0.285645 router_tokens_per_expert=[263, 73, 104, 20, 40, 27, 997, 1000, 150, 158, 229, 515, 49, 164, 1, 150, 104, 561, 2435, 36, 515, 267, 14, 59, 1825, 270, 452, 889, 25, 1402, 710, 17, 26, 23, 119, 1745, 9, 7, 9, 345, 960, 945, 973, 24, 2319, 1357, 1268, 787, 251, 1309, 887, 105, 453, 615, 333, 708, 1197, 720, 203, 209, 300, 184, 857, 0] payload_rows_per_dst=[5410, 6604, 6328, 5720, 5944, 5262, 6174, 5374] dup_rows_per_dst=[2174, 2780, 2600, 1810, 2272, 1970, 3148, 1966] combine_rows_per_src=[5410, 6604, 6328, 5720, 5944, 5262, 6174, 5374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46618 payload_rows_remote=39684 dup_rows=18918 dup_rows_remote=15920 dedup_ratio=0.288666 router_tokens_per_expert=[276, 33, 121, 47, 244, 18, 891, 1994, 229, 163, 98, 797, 12, 120, 0, 243, 97, 797, 1711, 21, 143, 215, 60, 164, 1749, 444, 203, 1254, 75, 1763, 1002, 11, 27, 16, 113, 1855, 14, 9, 14, 199, 586, 941, 659, 70, 2645, 1066, 918, 719, 375, 1302, 569, 110, 403, 433, 452, 817, 939, 770, 152, 115, 521, 296, 664, 4] payload_rows_per_dst=[6104, 6934, 6126, 6448, 5878, 3868, 6136, 5124] dup_rows_per_dst=[2980, 2998, 2422, 3056, 2114, 764, 2786, 1798] combine_rows_per_src=[6104, 6934, 6126, 6448, 5878, 3868, 6136, 5124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47562 payload_rows_remote=41128 dup_rows=17974 dup_rows_remote=15232 dedup_ratio=0.274261 router_tokens_per_expert=[186, 88, 89, 15, 6, 10, 1357, 1065, 229, 79, 162, 371, 3, 145, 0, 178, 210, 307, 2529, 11, 344, 196, 25, 168, 1938, 350, 182, 1071, 20, 1794, 601, 6, 21, 7, 316, 2152, 3, 0, 13, 138, 471, 853, 948, 25, 2524, 1308, 835, 734, 107, 1279, 755, 55, 636, 564, 304, 1008, 1513, 696, 116, 69, 303, 429, 850, 1] payload_rows_per_dst=[5320, 6122, 5998, 5958, 6434, 5798, 6318, 5614] dup_rows_per_dst=[1982, 1260, 1582, 2090, 2742, 2880, 3098, 2340] combine_rows_per_src=[5320, 6122, 5998, 5958, 6434, 5798, 6318, 5614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47856 payload_rows_remote=42310 dup_rows=17680 dup_rows_remote=16048 dedup_ratio=0.269775 router_tokens_per_expert=[153, 77, 27, 10, 65, 20, 1445, 1748, 321, 44, 142, 594, 5, 33, 0, 267, 80, 480, 2617, 28, 154, 175, 21, 162, 1402, 182, 232, 666, 5, 1751, 732, 21, 37, 23, 93, 2330, 2, 0, 57, 391, 785, 663, 922, 13, 2319, 657, 2485, 1206, 205, 1096, 546, 26, 294, 460, 202, 597, 1419, 336, 116, 215, 263, 553, 795, 3] payload_rows_per_dst=[7286, 6088, 6146, 5546, 6182, 5922, 5448, 5238] dup_rows_per_dst=[4774, 1362, 1288, 1632, 2488, 2570, 1404, 2162] combine_rows_per_src=[7286, 6088, 6146, 5546, 6182, 5922, 5448, 5238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47614 payload_rows_remote=40424 dup_rows=17922 dup_rows_remote=12146 dedup_ratio=0.273468 router_tokens_per_expert=[252, 42, 67, 19, 15, 50, 1354, 1258, 146, 37, 135, 383, 7, 53, 2, 123, 176, 324, 2733, 5, 134, 294, 33, 125, 1446, 222, 182, 839, 8, 1635, 652, 6, 25, 16, 68, 2582, 11, 0, 4, 142, 880, 690, 865, 64, 2668, 1470, 2512, 500, 233, 1895, 233, 96, 338, 380, 449, 568, 1200, 635, 218, 98, 423, 177, 570, 1] payload_rows_per_dst=[5192, 6122, 6240, 5446, 6324, 7190, 6088, 5012] dup_rows_per_dst=[1918, 986, 1408, 1642, 2264, 5776, 2296, 1632] combine_rows_per_src=[5192, 6122, 6240, 5446, 6324, 7190, 6088, 5012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47510 payload_rows_remote=41578 dup_rows=18026 dup_rows_remote=14982 dedup_ratio=0.275055 router_tokens_per_expert=[208, 18, 375, 41, 10, 13, 1841, 877, 319, 97, 159, 260, 9, 244, 0, 182, 345, 198, 2996, 0, 230, 188, 45, 101, 1335, 836, 153, 802, 50, 1580, 789, 15, 20, 24, 385, 1887, 7, 9, 15, 88, 422, 654, 955, 51, 2347, 1148, 1456, 781, 217, 1626, 496, 99, 521, 658, 233, 519, 1465, 372, 269, 146, 332, 101, 1147, 2] payload_rows_per_dst=[5290, 5886, 6624, 5764, 5766, 6164, 6084, 5932] dup_rows_per_dst=[1476, 1348, 1582, 2108, 1774, 4040, 2654, 3044] combine_rows_per_src=[5290, 5886, 6624, 5764, 5766, 6164, 6084, 5932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46506 payload_rows_remote=41542 dup_rows=19030 dup_rows_remote=17724 dedup_ratio=0.290375 router_tokens_per_expert=[228, 54, 251, 57, 30, 17, 1219, 1413, 352, 72, 200, 438, 9, 118, 3, 264, 103, 741, 2016, 5, 325, 164, 39, 60, 1952, 354, 378, 1102, 9, 1938, 828, 32, 5, 26, 278, 1906, 22, 0, 17, 321, 503, 1091, 1007, 42, 2385, 1030, 660, 854, 71, 890, 681, 128, 444, 302, 212, 407, 1650, 690, 197, 147, 441, 623, 963, 4] payload_rows_per_dst=[4870, 6018, 5398, 6444, 6280, 5658, 4964, 6874] dup_rows_per_dst=[1668, 1664, 1508, 2838, 2774, 2756, 1306, 4516] combine_rows_per_src=[4870, 6018, 5398, 6444, 6280, 5658, 4964, 6874] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=290217984 dispatch_meta_bytes=226432 combine_payload_bytes=290217984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=290444416 ingress_bytes=400318464 link_bytes=400318464 token_bytes=580662400 weight_movement_bytes=110100480 total_nvlink_bytes=690762880 weight_to_token_ratio=0.189612 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.962196500e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284454912 dispatch_meta_bytes=222416 combine_payload_bytes=284454912 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=284677328 ingress_bytes=504655872 link_bytes=504655872 token_bytes=569132240 weight_movement_bytes=220200960 total_nvlink_bytes=789333200 weight_to_token_ratio=0.386906 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.776795630e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=294805504 dispatch_meta_bytes=225440 combine_payload_bytes=294805504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295030944 ingress_bytes=404905984 link_bytes=404905984 token_bytes=589836448 weight_movement_bytes=110100480 total_nvlink_bytes=699936928 weight_to_token_ratio=0.186663 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.041981018e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=303278080 dispatch_meta_bytes=233432 combine_payload_bytes=303278080 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=303511512 ingress_bytes=633579520 link_bytes=633579520 token_bytes=606789592 weight_movement_bytes=330301440 total_nvlink_bytes=937091032 weight_to_token_ratio=0.544343 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.101898991e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289759232 dispatch_meta_bytes=210280 combine_payload_bytes=289759232 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=289969512 ingress_bytes=950362112 link_bytes=950362112 token_bytes=579728744 weight_movement_bytes=660602880 total_nvlink_bytes=1240331624 weight_to_token_ratio=1.139503 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.652836020e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=298031104 dispatch_meta_bytes=226240 combine_payload_bytes=298031104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298257344 ingress_bytes=408131584 link_bytes=408131584 token_bytes=596288448 weight_movement_bytes=110100480 total_nvlink_bytes=706388928 weight_to_token_ratio=0.184643 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.098079507e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=297773056 dispatch_meta_bytes=237064 combine_payload_bytes=297773056 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=298010120 ingress_bytes=407873536 link_bytes=407873536 token_bytes=595783176 weight_movement_bytes=110100480 total_nvlink_bytes=705883656 weight_to_token_ratio=0.184800 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=10752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.093591628e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47064 payload_rows_remote=41032 dup_rows=18472 dup_rows_remote=15412 dedup_ratio=0.281860 router_tokens_per_expert=[196, 84, 106, 26, 68, 14, 1393, 1520, 339, 15, 95, 464, 23, 47, 4, 220, 249, 591, 2336, 10, 245, 246, 13, 140, 1567, 302, 273, 986, 25, 1266, 829, 34, 44, 14, 113, 1997, 9, 1, 8, 146, 725, 831, 836, 98, 2639, 1313, 1139, 805, 219, 1316, 324, 237, 573, 299, 232, 669, 1666, 1039, 247, 87, 303, 531, 574, 8] payload_rows_per_dst=[6032, 6808, 6938, 6334, 5042, 4032, 5754, 6124] dup_rows_per_dst=[3060, 2556, 3348, 2700, 1258, 886, 1878, 2786] combine_rows_per_src=[6032, 6808, 6938, 6334, 5042, 4032, 5754, 6124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52898 payload_rows_remote=44718 dup_rows=12638 dup_rows_remote=5810 dedup_ratio=0.192841 router_tokens_per_expert=[0, 4085, 2737, 362, 0, 129, 1104, 2066, 0, 744, 8, 0, 0, 0, 0, 0, 0, 1237, 2092, 17, 0, 92, 0, 0, 56, 4066, 117, 0, 0, 0, 2008, 0, 0, 100, 0, 0, 0, 0, 0, 0, 340, 0, 3123, 0, 1, 0, 0, 0, 0, 0, 0, 458, 0, 321, 0, 0, 1028, 0, 122, 0, 0, 0, 3342, 3013] payload_rows_per_dst=[5798, 5990, 8180, 4220, 8170, 6492, 7236, 6812] dup_rows_per_dst=[1524, 988, 6828, 142, 200, 436, 1006, 1514] combine_rows_per_src=[5798, 5990, 8180, 4220, 8170, 6492, 7236, 6812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52960 payload_rows_remote=44780 dup_rows=12576 dup_rows_remote=12462 dedup_ratio=0.191895 router_tokens_per_expert=[0, 4085, 2668, 366, 5, 347, 1419, 2277, 0, 782, 26, 0, 0, 0, 1, 1, 0, 1277, 824, 21, 0, 35, 0, 2, 27, 4049, 7, 0, 2, 0, 2895, 0, 0, 56, 0, 2, 0, 0, 0, 4, 240, 0, 2632, 3, 14, 0, 0, 0, 0, 0, 1, 92, 0, 187, 0, 0, 1349, 3, 117, 0, 0, 1, 3770, 3181] payload_rows_per_dst=[6518, 5872, 3766, 8152, 8180, 5468, 7774, 7230] dup_rows_per_dst=[2310, 1084, 552, 5808, 114, 310, 326, 2072] combine_rows_per_src=[6518, 5872, 3766, 8152, 8180, 5468, 7774, 7230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53582 payload_rows_remote=47460 dup_rows=11954 dup_rows_remote=10432 dedup_ratio=0.182404 router_tokens_per_expert=[0, 4081, 2700, 436, 3, 246, 1599, 2325, 0, 1103, 15, 1, 0, 1, 0, 2, 0, 544, 1788, 7, 0, 58, 9, 15, 133, 3791, 34, 0, 0, 4, 2087, 0, 0, 3, 0, 1, 1, 0, 0, 0, 521, 1, 2685, 4, 3, 0, 0, 0, 5, 0, 2, 85, 19, 168, 0, 0, 1542, 0, 74, 4, 0, 0, 3799, 2869] payload_rows_per_dst=[6602, 6122, 7938, 4390, 8168, 5688, 7756, 6918] dup_rows_per_dst=[2616, 1522, 4486, 126, 4, 740, 400, 2060] combine_rows_per_src=[6602, 6122, 7938, 4390, 8168, 5688, 7756, 6918] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54090 payload_rows_remote=47368 dup_rows=11446 dup_rows_remote=9958 dedup_ratio=0.174652 router_tokens_per_expert=[0, 4094, 2145, 269, 0, 468, 1489, 2228, 0, 829, 0, 1, 0, 0, 0, 42, 0, 1094, 820, 8, 0, 30, 4, 4, 39, 3943, 33, 0, 0, 0, 2873, 0, 15, 69, 0, 3, 0, 0, 0, 0, 402, 0, 2718, 0, 0, 2, 3, 0, 0, 22, 1, 239, 6, 353, 1, 0, 1653, 0, 132, 0, 0, 12, 3816, 2908] payload_rows_per_dst=[6532, 5112, 6796, 6722, 8188, 5752, 7894, 7094] dup_rows_per_dst=[2376, 922, 2690, 1488, 174, 498, 982, 2316] combine_rows_per_src=[6532, 5112, 6796, 6722, 8188, 5752, 7894, 7094] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52968 payload_rows_remote=46314 dup_rows=12568 dup_rows_remote=11656 dedup_ratio=0.191772 router_tokens_per_expert=[3, 4082, 2211, 476, 0, 124, 1826, 2165, 1, 1727, 23, 0, 0, 0, 7, 1, 0, 737, 551, 19, 0, 47, 1, 1, 61, 4059, 3, 3, 0, 1, 2354, 0, 1, 97, 1, 0, 0, 0, 0, 0, 665, 0, 3112, 2, 2, 0, 2, 0, 0, 0, 0, 146, 7, 107, 4, 6, 1038, 0, 60, 1, 3, 7, 3599, 3425] payload_rows_per_dst=[6330, 6472, 2440, 8166, 8178, 6654, 7330, 7398] dup_rows_per_dst=[2428, 1898, 272, 4796, 184, 912, 408, 1670] combine_rows_per_src=[6330, 6472, 2440, 8166, 8178, 6654, 7330, 7398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52542 payload_rows_remote=45306 dup_rows=12994 dup_rows_remote=10340 dedup_ratio=0.198273 router_tokens_per_expert=[0, 4094, 3356, 357, 0, 190, 1618, 1694, 0, 884, 1, 0, 3, 0, 0, 0, 0, 614, 1083, 4, 1, 51, 0, 4, 34, 4017, 4, 0, 11, 0, 2151, 1, 1, 57, 0, 0, 0, 0, 0, 0, 625, 0, 3342, 0, 1, 0, 0, 0, 1, 0, 0, 80, 2, 466, 0, 0, 1448, 0, 147, 0, 0, 0, 3391, 3035] payload_rows_per_dst=[3930, 7570, 3182, 7988, 8190, 7782, 6664, 7236] dup_rows_per_dst=[400, 4306, 332, 92, 114, 4456, 640, 2654] combine_rows_per_src=[3930, 7570, 3182, 7988, 8190, 7782, 6664, 7236] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52792 payload_rows_remote=44968 dup_rows=12744 dup_rows_remote=11692 dedup_ratio=0.194458 router_tokens_per_expert=[0, 4094, 2754, 350, 0, 187, 1421, 1473, 0, 1069, 29, 0, 0, 0, 0, 0, 0, 1567, 1014, 0, 0, 52, 0, 6, 66, 4032, 20, 0, 0, 0, 2743, 0, 0, 127, 0, 0, 0, 0, 0, 0, 620, 0, 3392, 0, 2, 0, 0, 0, 0, 0, 0, 136, 0, 580, 0, 0, 932, 0, 119, 0, 0, 0, 3722, 2261] payload_rows_per_dst=[3502, 7116, 4438, 8080, 8188, 7874, 7824, 5770] dup_rows_per_dst=[414, 3534, 840, 338, 254, 5458, 1052, 854] combine_rows_per_src=[3502, 7116, 4438, 8080, 8188, 7874, 7824, 5770] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=340193280 dispatch_meta_bytes=231568 combine_payload_bytes=340193280 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=340424848 ingress_bytes=560394240 link_bytes=560394240 token_bytes=680618128 weight_movement_bytes=220200960 total_nvlink_bytes=900819088 weight_to_token_ratio=0.323531 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.033088 gemm_active_groups=11 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.746177524e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=320538624 dispatch_meta_bytes=202112 combine_payload_bytes=320538624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=320740736 ingress_bytes=430639104 link_bytes=430639104 token_bytes=641279360 weight_movement_bytes=110100480 total_nvlink_bytes=751379840 weight_to_token_ratio=0.171689 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=14592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.489522298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=320983040 dispatch_meta_bytes=228968 combine_payload_bytes=320983040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=321212008 ingress_bytes=431083520 link_bytes=431083520 token_bytes=642195048 weight_movement_bytes=110100480 total_nvlink_bytes=752295528 weight_to_token_ratio=0.171444 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.497251423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=339533824 dispatch_meta_bytes=229304 combine_payload_bytes=339533824 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=339763128 ingress_bytes=669835264 link_bytes=669835264 token_bytes=679296952 weight_movement_bytes=330301440 total_nvlink_bytes=1009598392 weight_to_token_ratio=0.486240 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.164953693e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=331978752 dispatch_meta_bytes=231880 combine_payload_bytes=331978752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332210632 ingress_bytes=442079232 link_bytes=442079232 token_bytes=664189384 weight_movement_bytes=110100480 total_nvlink_bytes=774289864 weight_to_token_ratio=0.165767 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.688484940e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=324753408 dispatch_meta_bytes=222584 combine_payload_bytes=324753408 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=324975992 ingress_bytes=655054848 link_bytes=655054848 token_bytes=649729400 weight_movement_bytes=330301440 total_nvlink_bytes=980030840 weight_to_token_ratio=0.508368 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.139248119e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=322330624 dispatch_meta_bytes=226640 combine_payload_bytes=322330624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=322557264 ingress_bytes=432431104 link_bytes=432431104 token_bytes=644887888 weight_movement_bytes=110100480 total_nvlink_bytes=754988368 weight_to_token_ratio=0.170728 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.520688126e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=294117376 dispatch_meta_bytes=225776 combine_payload_bytes=294117376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=294343152 ingress_bytes=404217856 link_bytes=404217856 token_bytes=588460528 weight_movement_bytes=110100480 total_nvlink_bytes=698561008 weight_to_token_ratio=0.187099 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=10880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.030013340e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47900 payload_rows_remote=39714 dup_rows=17636 dup_rows_remote=13826 dedup_ratio=0.269104 router_tokens_per_expert=[2583, 3240, 2378, 3, 1652, 3, 0, 0, 0, 97, 0, 434, 340, 4, 1033, 0, 5, 0, 0, 1696, 1017, 2, 0, 3, 0, 4090, 0, 2483, 0, 17, 0, 1, 2672, 1028, 1, 699, 1, 0, 5, 1498, 0, 0, 2, 0, 4087, 0, 1077, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 3, 575, 0, 0, 6, 3, 27] payload_rows_per_dst=[8190, 8186, 7348, 4976, 5158, 2158, 6480, 5404] dup_rows_per_dst=[8466, 3810, 3442, 26, 1306, 0, 4, 582] combine_rows_per_src=[8190, 8186, 7348, 4976, 5158, 2158, 6480, 5404] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50304 payload_rows_remote=44108 dup_rows=15232 dup_rows_remote=13434 dedup_ratio=0.232422 router_tokens_per_expert=[2703, 3438, 2840, 0, 2207, 0, 0, 0, 0, 86, 0, 253, 101, 2, 1133, 0, 0, 0, 0, 823, 963, 0, 0, 0, 0, 4096, 0, 1912, 0, 25, 0, 0, 2211, 875, 0, 331, 0, 3, 56, 2220, 0, 0, 0, 0, 4096, 0, 1348, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 980, 0, 0, 0, 0, 62] payload_rows_per_dst=[7428, 8192, 6196, 3848, 5568, 5576, 6876, 6620] dup_rows_per_dst=[6172, 3150, 1798, 26, 1402, 1532, 8, 1144] combine_rows_per_src=[7428, 8192, 6196, 3848, 5568, 5576, 6876, 6620] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51338 payload_rows_remote=44728 dup_rows=14198 dup_rows_remote=11070 dedup_ratio=0.216644 router_tokens_per_expert=[2178, 3355, 2001, 8, 2245, 0, 0, 0, 0, 152, 0, 362, 41, 3, 1703, 0, 0, 0, 0, 1222, 1703, 0, 0, 0, 0, 4095, 0, 1389, 1, 1, 0, 0, 3168, 985, 1, 268, 0, 0, 7, 1338, 0, 0, 2, 0, 4095, 0, 873, 1, 0, 0, 3, 0, 9, 0, 0, 0, 0, 0, 1498, 2, 0, 0, 1, 58] payload_rows_per_dst=[6294, 4326, 5574, 7590, 6610, 8190, 6718, 6036] dup_rows_per_dst=[2568, 916, 2072, 2662, 3128, 1752, 16, 1084] combine_rows_per_src=[6294, 4326, 5574, 7590, 6610, 8190, 6718, 6036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48002 payload_rows_remote=45262 dup_rows=17534 dup_rows_remote=17534 dedup_ratio=0.267548 router_tokens_per_expert=[2343, 3720, 1720, 15, 1640, 36, 0, 0, 0, 132, 0, 1599, 20, 6, 1615, 0, 4, 0, 0, 1091, 1956, 0, 0, 0, 0, 4090, 0, 1368, 0, 2, 0, 0, 2465, 167, 0, 584, 0, 0, 16, 1391, 0, 8, 1, 0, 4079, 4, 2055, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 601, 0, 0, 3, 0, 18] payload_rows_per_dst=[5996, 8180, 7202, 2740, 3900, 8184, 7446, 4354] dup_rows_per_dst=[2072, 6744, 3830, 0, 416, 4110, 32, 330] combine_rows_per_src=[5996, 8180, 7202, 2740, 3900, 8184, 7446, 4354] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45848 payload_rows_remote=37656 dup_rows=19688 dup_rows_remote=15720 dedup_ratio=0.300415 router_tokens_per_expert=[2908, 3591, 2989, 0, 1382, 2, 0, 0, 7, 117, 0, 589, 38, 3, 792, 0, 0, 2, 0, 1645, 1400, 1, 0, 3, 8, 4080, 3, 2262, 0, 5, 10, 0, 2049, 429, 0, 439, 0, 1, 54, 1264, 3, 4, 9, 0, 4080, 11, 1973, 0, 0, 0, 0, 1, 4, 0, 2, 0, 3, 0, 513, 0, 0, 8, 0, 84] payload_rows_per_dst=[776, 2798, 4986, 8184, 6074, 8192, 7802, 7036] dup_rows_per_dst=[54, 294, 1116, 4552, 2398, 3968, 4924, 2382] combine_rows_per_src=[776, 2798, 4986, 8184, 6074, 8192, 7802, 7036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47248 payload_rows_remote=39666 dup_rows=18288 dup_rows_remote=13866 dedup_ratio=0.279053 router_tokens_per_expert=[2494, 2562, 3233, 0, 1874, 0, 0, 0, 0, 70, 1, 399, 115, 8, 1536, 0, 0, 0, 0, 1099, 1862, 0, 1, 0, 0, 4096, 1, 1269, 0, 22, 0, 1, 2592, 1386, 0, 484, 0, 1, 31, 1695, 0, 0, 3, 0, 4095, 0, 939, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 870, 0, 0, 0, 0, 25] payload_rows_per_dst=[0, 3850, 5106, 8192, 7124, 8190, 7204, 7582] dup_rows_per_dst=[0, 408, 818, 2586, 4876, 2262, 2916, 4422] combine_rows_per_src=[0, 3850, 5106, 8192, 7124, 8190, 7204, 7582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45274 payload_rows_remote=38294 dup_rows=20262 dup_rows_remote=18234 dedup_ratio=0.309174 router_tokens_per_expert=[1805, 2672, 3118, 0, 2719, 0, 0, 0, 0, 61, 0, 313, 12, 4, 1159, 0, 0, 0, 0, 847, 1539, 0, 0, 0, 0, 4096, 0, 2375, 0, 0, 0, 0, 2269, 1065, 0, 597, 0, 8, 26, 2284, 0, 0, 1, 0, 4096, 0, 899, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 770, 0, 0, 0, 0, 6] payload_rows_per_dst=[0, 2810, 4126, 8192, 7272, 8192, 6980, 7702] dup_rows_per_dst=[0, 288, 646, 4750, 5226, 1800, 2028, 5524] combine_rows_per_src=[0, 2810, 4126, 8192, 7272, 8192, 6980, 7702] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284669952 dispatch_meta_bytes=214160 combine_payload_bytes=284669952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284884112 ingress_bytes=394770432 link_bytes=394770432 token_bytes=569554064 weight_movement_bytes=110100480 total_nvlink_bytes=679654544 weight_to_token_ratio=0.193310 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.865707098e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=316166144 dispatch_meta_bytes=230168 combine_payload_bytes=316166144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316396312 ingress_bytes=426266624 link_bytes=426266624 token_bytes=632562456 weight_movement_bytes=110100480 total_nvlink_bytes=742662936 weight_to_token_ratio=0.174055 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.413477679e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=320610304 dispatch_meta_bytes=223192 combine_payload_bytes=320610304 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=320833496 ingress_bytes=650911744 link_bytes=650911744 token_bytes=641443800 weight_movement_bytes=330301440 total_nvlink_bytes=971745240 weight_to_token_ratio=0.514934 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=13056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.132042579e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=324438016 dispatch_meta_bytes=251184 combine_payload_bytes=324438016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=324689200 ingress_bytes=544638976 link_bytes=544638976 token_bytes=649127216 weight_movement_bytes=220200960 total_nvlink_bytes=869328176 weight_to_token_ratio=0.339226 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.472167570e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=269918208 dispatch_meta_bytes=213504 combine_payload_bytes=269918208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=270131712 ingress_bytes=490119168 link_bytes=490119168 token_bytes=540049920 weight_movement_bytes=220200960 total_nvlink_bytes=760250880 weight_to_token_ratio=0.407742 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.523978439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=284325888 dispatch_meta_bytes=214128 combine_payload_bytes=284325888 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=284540016 ingress_bytes=504526848 link_bytes=504526848 token_bytes=568865904 weight_movement_bytes=220200960 total_nvlink_bytes=789066864 weight_to_token_ratio=0.387088 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.029412 gemm_active_groups=11 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.774551691e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274491392 dispatch_meta_bytes=226112 combine_payload_bytes=274491392 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=274717504 ingress_bytes=494692352 link_bytes=494692352 token_bytes=549208896 weight_movement_bytes=220200960 total_nvlink_bytes=769409856 weight_to_token_ratio=0.400942 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.603513630e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55934 payload_rows_remote=47754 dup_rows=9602 dup_rows_remote=6362 dedup_ratio=0.146515 router_tokens_per_expert=[0, 0, 0, 287, 3793, 0, 0, 0, 7, 0, 0, 0, 0, 1619, 0, 0, 0, 1329, 13, 6, 0, 5, 18, 522, 4084, 0, 8, 1, 0, 0, 1, 692, 0, 112, 9, 0, 1, 4087, 3809, 0, 1, 0, 1, 0, 4082, 182, 0, 0, 1461, 0, 0, 2534, 0, 0, 3, 10, 7, 4082, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[7646, 8180, 8190, 3014, 5830, 8168, 6734, 8172] dup_rows_per_dst=[514, 3240, 3770, 266, 156, 364, 1282, 10] combine_rows_per_src=[7646, 8180, 8190, 3014, 5830, 8168, 6734, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54722 payload_rows_remote=46530 dup_rows=10814 dup_rows_remote=6460 dedup_ratio=0.165009 router_tokens_per_expert=[0, 0, 0, 253, 3610, 0, 0, 0, 3, 0, 0, 0, 0, 2492, 0, 0, 0, 1277, 0, 0, 0, 0, 0, 900, 4076, 0, 0, 1, 0, 0, 12, 1332, 0, 199, 0, 0, 0, 4096, 2835, 0, 0, 0, 0, 0, 4094, 115, 0, 0, 650, 0, 0, 2725, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7296, 8192, 8192, 2680, 5766, 8188, 6216, 8192] dup_rows_per_dst=[430, 4950, 4354, 10, 302, 230, 538, 0] combine_rows_per_src=[7296, 8192, 8192, 2680, 5766, 8188, 6216, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56442 payload_rows_remote=48938 dup_rows=9094 dup_rows_remote=5728 dedup_ratio=0.138763 router_tokens_per_expert=[0, 0, 0, 298, 3804, 0, 0, 0, 35, 0, 2, 0, 2, 2587, 0, 0, 0, 1266, 0, 0, 0, 3, 11, 469, 4089, 0, 1, 5, 0, 0, 0, 1124, 0, 113, 0, 0, 0, 4089, 3486, 0, 1, 0, 0, 0, 4093, 163, 0, 0, 575, 0, 0, 2304, 0, 0, 2, 157, 0, 4087, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7648, 5226, 5908, 8184, 7504, 8190, 5604, 8178] dup_rows_per_dst=[556, 26, 2096, 2254, 3366, 324, 472, 0] combine_rows_per_src=[7648, 5226, 5908, 8184, 7504, 8190, 5604, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50740 payload_rows_remote=43292 dup_rows=14796 dup_rows_remote=7988 dedup_ratio=0.225769 router_tokens_per_expert=[0, 4095, 2632, 266, 4, 318, 1958, 1922, 0, 861, 21, 0, 0, 0, 0, 0, 0, 1103, 1261, 53, 0, 63, 0, 10, 24, 3908, 0, 0, 0, 0, 2490, 0, 0, 109, 0, 0, 0, 0, 0, 0, 724, 0, 2783, 0, 0, 0, 0, 1, 0, 0, 0, 98, 1, 243, 0, 0, 948, 1, 112, 0, 0, 0, 3780, 2979] payload_rows_per_dst=[7448, 5930, 8046, 5000, 7040, 6146, 7744, 3386] dup_rows_per_dst=[6808, 1098, 4750, 28, 190, 870, 500, 552] combine_rows_per_src=[7448, 5930, 8046, 5000, 7040, 6146, 7744, 3386] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54458 payload_rows_remote=46280 dup_rows=11078 dup_rows_remote=10792 dedup_ratio=0.169037 router_tokens_per_expert=[0, 0, 1, 301, 3867, 0, 0, 1, 9, 0, 1, 3, 0, 2569, 0, 0, 6, 580, 0, 0, 0, 2, 2, 927, 4060, 0, 1, 0, 0, 1, 18, 1306, 2, 136, 1, 0, 5, 4096, 3401, 0, 0, 0, 0, 0, 4089, 143, 0, 0, 643, 0, 0, 2454, 0, 0, 0, 33, 9, 4094, 2, 0, 1, 0, 4, 0] payload_rows_per_dst=[7760, 5154, 2972, 8164, 8192, 8178, 5846, 8192] dup_rows_per_dst=[580, 10, 62, 2608, 7090, 286, 414, 28] combine_rows_per_src=[7760, 5154, 2972, 8164, 8192, 8178, 5846, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57162 payload_rows_remote=50098 dup_rows=8374 dup_rows_remote=6360 dedup_ratio=0.127777 router_tokens_per_expert=[0, 2, 0, 710, 3640, 0, 0, 0, 44, 0, 0, 0, 0, 2949, 0, 0, 0, 506, 0, 0, 0, 0, 7, 366, 4086, 0, 8, 0, 0, 0, 19, 2039, 0, 154, 0, 0, 0, 4096, 2628, 0, 8, 0, 1, 0, 4051, 188, 0, 0, 907, 0, 0, 2129, 0, 0, 0, 126, 10, 4088, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[7540, 7112, 8192, 7064, 5298, 8126, 5646, 8184] dup_rows_per_dst=[1164, 2100, 1758, 2014, 266, 370, 678, 24] combine_rows_per_src=[7540, 7112, 8192, 7064, 5298, 8126, 5646, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56278 payload_rows_remote=48086 dup_rows=9258 dup_rows_remote=9252 dedup_ratio=0.141266 router_tokens_per_expert=[0, 1, 0, 195, 3315, 1, 0, 0, 5, 0, 0, 0, 0, 2420, 0, 0, 0, 1200, 0, 0, 0, 0, 0, 900, 4087, 1, 6, 0, 0, 0, 0, 951, 1, 67, 0, 1, 0, 4094, 3523, 1, 0, 0, 0, 0, 4095, 120, 0, 0, 1136, 0, 0, 2546, 0, 0, 0, 3, 2, 4095, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[6666, 4850, 3908, 8180, 8192, 8190, 8100, 8192] dup_rows_per_dst=[358, 0, 292, 1910, 136, 240, 6316, 6] combine_rows_per_src=[6666, 4850, 3908, 8180, 8192, 8190, 8100, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=342300672 dispatch_meta_bytes=216464 combine_payload_bytes=342300672 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342517136 ingress_bytes=452401152 link_bytes=452401152 token_bytes=684817808 weight_movement_bytes=110100480 total_nvlink_bytes=794918288 weight_to_token_ratio=0.160773 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.868000105e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54952 payload_rows_remote=48372 dup_rows=10584 dup_rows_remote=8178 dedup_ratio=0.161499 router_tokens_per_expert=[0, 0, 0, 408, 3783, 0, 0, 0, 0, 0, 0, 0, 0, 2155, 0, 0, 0, 1064, 0, 0, 0, 0, 5, 905, 4093, 0, 1, 0, 0, 0, 1, 1428, 0, 175, 0, 0, 0, 4096, 3306, 0, 12, 0, 0, 0, 4096, 162, 0, 0, 634, 0, 0, 2336, 0, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7628, 4310, 3668, 8190, 8192, 8192, 6580, 8192] dup_rows_per_dst=[754, 0, 280, 2856, 3940, 348, 2406, 0] combine_rows_per_src=[7628, 4310, 3668, 8190, 8192, 8192, 6580, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=333527040 dispatch_meta_bytes=211960 combine_payload_bytes=333527040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=333739000 ingress_bytes=443627520 link_bytes=443627520 token_bytes=667266040 weight_movement_bytes=110100480 total_nvlink_bytes=777366520 weight_to_token_ratio=0.165002 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.715412215e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331735040 dispatch_meta_bytes=228288 combine_payload_bytes=331735040 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=331963328 ingress_bytes=662036480 link_bytes=662036480 token_bytes=663698368 weight_movement_bytes=330301440 total_nvlink_bytes=993999808 weight_to_token_ratio=0.497668 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.151390325e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=350787584 dispatch_meta_bytes=218664 combine_payload_bytes=350787584 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=351006248 ingress_bytes=791189504 link_bytes=791189504 token_bytes=701793832 weight_movement_bytes=440401920 total_nvlink_bytes=1142195752 weight_to_token_ratio=0.627537 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.376008676e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=359102464 dispatch_meta_bytes=225832 combine_payload_bytes=359102464 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=359328296 ingress_bytes=579303424 link_bytes=579303424 token_bytes=718430760 weight_movement_bytes=220200960 total_nvlink_bytes=938631720 weight_to_token_ratio=0.306503 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.007503933e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=344680448 dispatch_meta_bytes=229352 combine_payload_bytes=344680448 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=344909800 ingress_bytes=454780928 link_bytes=454780928 token_bytes=689590248 weight_movement_bytes=110100480 total_nvlink_bytes=799690728 weight_to_token_ratio=0.159661 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.909388324e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=310317056 dispatch_meta_bytes=205120 combine_payload_bytes=310317056 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=310522176 ingress_bytes=750718976 link_bytes=750718976 token_bytes=620839232 weight_movement_bytes=440401920 total_nvlink_bytes=1061241152 weight_to_token_ratio=0.709365 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.305623771e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=346730496 dispatch_meta_bytes=226200 combine_payload_bytes=346730496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346956696 ingress_bytes=456830976 link_bytes=456830976 token_bytes=693687192 weight_movement_bytes=110100480 total_nvlink_bytes=803787672 weight_to_token_ratio=0.158718 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.945042030e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65130 payload_rows_remote=56940 dup_rows=406 dup_rows_remote=396 dedup_ratio=0.006195 router_tokens_per_expert=[0, 0, 4092, 24, 0, 0, 4088, 0, 5, 0, 1, 0, 1, 0, 0, 0, 0, 0, 619, 3, 0, 4093, 4045, 3572, 0, 0, 0, 0, 4094, 0, 4051, 0, 0, 3, 4020, 0, 0, 0, 10, 0, 8, 2, 0, 15, 0, 0, 0, 1, 0, 4, 0, 12, 0, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 1] payload_rows_per_dst=[8184, 8190, 8116, 8102, 8044, 8120, 8192, 8182] dup_rows_per_dst=[48, 10, 272, 0, 22, 22, 28, 4] combine_rows_per_src=[8184, 8190, 8116, 8102, 8044, 8120, 8192, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65244 payload_rows_remote=57222 dup_rows=292 dup_rows_remote=170 dedup_ratio=0.004456 router_tokens_per_expert=[0, 0, 4096, 84, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 866, 0, 0, 4096, 4068, 3206, 0, 0, 0, 0, 4096, 0, 4073, 0, 0, 0, 4080, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8022, 8146, 8160, 8148, 8192, 8192] dup_rows_per_dst=[168, 0, 122, 0, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8022, 8146, 8160, 8148, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408145920 dispatch_meta_bytes=229344 combine_payload_bytes=408145920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408375264 ingress_bytes=518246400 link_bytes=518246400 token_bytes=816521184 weight_movement_bytes=110100480 total_nvlink_bytes=926621664 weight_to_token_ratio=0.134841 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.013157265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64960 payload_rows_remote=56846 dup_rows=576 dup_rows_remote=502 dedup_ratio=0.008789 router_tokens_per_expert=[0, 1, 4082, 141, 0, 0, 4084, 0, 0, 0, 0, 7, 1, 0, 0, 5, 0, 0, 999, 0, 0, 4095, 4056, 2998, 0, 2, 1, 0, 4088, 0, 4079, 0, 0, 0, 4060, 3, 0, 0, 4, 0, 29, 0, 1, 8, 0, 0, 0, 0, 5, 0, 1, 11, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8168, 8192, 7832, 8160, 8126, 8114, 8192, 8176] dup_rows_per_dst=[280, 24, 162, 4, 8, 74, 18, 6] combine_rows_per_src=[8168, 8192, 7832, 8160, 8126, 8114, 8192, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65050 payload_rows_remote=56918 dup_rows=486 dup_rows_remote=486 dedup_ratio=0.007416 router_tokens_per_expert=[0, 0, 4094, 155, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1703, 0, 0, 4096, 4036, 2314, 0, 0, 0, 0, 4096, 0, 4066, 0, 0, 0, 4084, 0, 0, 0, 1, 0, 16, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[8188, 8192, 7880, 8132, 8168, 8106, 8192, 8192] dup_rows_per_dst=[310, 0, 154, 0, 2, 18, 0, 2] combine_rows_per_src=[8188, 8192, 7880, 8132, 8168, 8106, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65278 payload_rows_remote=57160 dup_rows=258 dup_rows_remote=252 dedup_ratio=0.003937 router_tokens_per_expert=[0, 4, 4094, 22, 0, 0, 4094, 0, 2, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1251, 0, 0, 4092, 4060, 2911, 2, 0, 0, 0, 4093, 0, 4061, 0, 0, 0, 4058, 1, 0, 0, 3, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 2, 0, 0, 3, 5] payload_rows_per_dst=[8188, 8188, 8154, 8126, 8118, 8124, 8188, 8192] dup_rows_per_dst=[52, 4, 170, 0, 6, 4, 2, 20] combine_rows_per_src=[8188, 8188, 8154, 8126, 8118, 8124, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=410167296 dispatch_meta_bytes=229568 combine_payload_bytes=410167296 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=410396864 ingress_bytes=850569216 link_bytes=850569216 token_bytes=820564160 weight_movement_bytes=440401920 total_nvlink_bytes=1260966080 weight_to_token_ratio=0.536706 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.479279761e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63588 payload_rows_remote=55400 dup_rows=1948 dup_rows_remote=1948 dedup_ratio=0.029724 router_tokens_per_expert=[0, 0, 4095, 73, 0, 0, 4094, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1525, 1, 0, 4092, 4020, 2611, 1, 0, 0, 0, 4094, 0, 4039, 0, 0, 0, 4094, 2, 1, 1, 0, 0, 16, 0, 0, 4, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7034, 8162, 7694, 8084, 8190, 8046, 8190, 8188] dup_rows_per_dst=[92, 0, 540, 494, 384, 438, 0, 0] combine_rows_per_src=[7034, 8162, 7694, 8084, 8190, 8046, 8190, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65282 payload_rows_remote=57090 dup_rows=254 dup_rows_remote=254 dedup_ratio=0.003876 router_tokens_per_expert=[0, 0, 4096, 74, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 898, 0, 0, 4096, 4087, 3165, 0, 0, 0, 0, 4096, 0, 4094, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8032, 8188, 8120, 8174, 8192, 8192] dup_rows_per_dst=[148, 0, 94, 0, 0, 12, 0, 0] combine_rows_per_src=[8192, 8192, 8032, 8188, 8120, 8174, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46506 payload_rows_remote=38316 dup_rows=19030 dup_rows_remote=9700 dedup_ratio=0.290375 router_tokens_per_expert=[2613, 3027, 2226, 0, 2052, 0, 0, 0, 0, 83, 0, 173, 59, 0, 1382, 0, 0, 0, 0, 935, 1621, 0, 0, 0, 1, 4095, 0, 2408, 0, 16, 0, 0, 3082, 895, 0, 192, 0, 0, 1, 2632, 2, 0, 12, 0, 4095, 0, 701, 0, 0, 0, 0, 0, 22, 1, 0, 1, 0, 0, 327, 0, 1, 0, 0, 113] payload_rows_per_dst=[8190, 8190, 7598, 5698, 4360, 1428, 6066, 4976] dup_rows_per_dst=[9330, 3394, 3678, 1514, 718, 2, 36, 358] combine_rows_per_src=[8190, 8190, 7598, 5698, 4360, 1428, 6066, 4976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407988224 dispatch_meta_bytes=229616 combine_payload_bytes=407988224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408217840 ingress_bytes=628189184 link_bytes=628189184 token_bytes=816206064 weight_movement_bytes=220200960 total_nvlink_bytes=1036407024 weight_to_token_ratio=0.269786 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092524310e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407472128 dispatch_meta_bytes=229392 combine_payload_bytes=407472128 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=407701520 ingress_bytes=627673088 link_bytes=627673088 token_bytes=815173648 weight_movement_bytes=220200960 total_nvlink_bytes=1035374608 weight_to_token_ratio=0.270128 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091626735e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409722880 dispatch_meta_bytes=229648 combine_payload_bytes=409722880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409952528 ingress_bytes=519823360 link_bytes=519823360 token_bytes=819675408 weight_movement_bytes=110100480 total_nvlink_bytes=929775888 weight_to_token_ratio=0.134322 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.040583193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=397107200 dispatch_meta_bytes=229392 combine_payload_bytes=397107200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397336592 ingress_bytes=507207680 link_bytes=507207680 token_bytes=794443792 weight_movement_bytes=110100480 total_nvlink_bytes=904544272 weight_to_token_ratio=0.138588 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.821175768e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409221120 dispatch_meta_bytes=229376 combine_payload_bytes=409221120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409450496 ingress_bytes=519321600 link_bytes=519321600 token_bytes=818671616 weight_movement_bytes=110100480 total_nvlink_bytes=928772096 weight_to_token_ratio=0.134487 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.031856761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60422 payload_rows_remote=52230 dup_rows=5114 dup_rows_remote=4074 dedup_ratio=0.078033 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 4, 1, 0, 3, 0, 272, 0, 248, 0, 0, 0, 1, 3927, 1170, 0, 0, 0, 0, 742, 0, 0, 0, 0, 4095, 4067, 0, 0, 0, 4093, 0, 0, 260, 0, 3, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 3152, 0, 0, 0, 1, 0, 0, 4093, 5, 3, 2521, 0, 2, 10] payload_rows_per_dst=[8192, 8192, 8140, 8148, 8190, 8190, 6306, 5064] dup_rows_per_dst=[8, 1040, 2056, 1470, 522, 0, 0, 18] combine_rows_per_src=[8192, 8192, 8140, 8148, 8190, 8190, 6306, 5064] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274649088 dispatch_meta_bytes=192064 combine_payload_bytes=274649088 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=274841152 ingress_bytes=825151488 link_bytes=825151488 token_bytes=549490240 weight_movement_bytes=550502400 total_nvlink_bytes=1099992640 weight_to_token_ratio=1.001842 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=12672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.435074152e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61884 payload_rows_remote=53958 dup_rows=3652 dup_rows_remote=3176 dedup_ratio=0.055725 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 412, 0, 0, 0, 0, 3845, 356, 0, 0, 0, 0, 765, 0, 0, 0, 0, 4094, 4082, 0, 0, 0, 4096, 0, 0, 237, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3325, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3186, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 7926, 8172, 8192, 8192, 6650, 6372] dup_rows_per_dst=[0, 1180, 476, 1522, 474, 0, 0, 0] combine_rows_per_src=[8188, 8192, 7926, 8172, 8192, 8192, 6650, 6372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374384640 dispatch_meta_bytes=225216 combine_payload_bytes=374384640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374609856 ingress_bytes=484485120 link_bytes=484485120 token_bytes=748994496 weight_movement_bytes=110100480 total_nvlink_bytes=859094976 weight_to_token_ratio=0.146998 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.425993078e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=386770944 dispatch_meta_bytes=228536 combine_payload_bytes=386770944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=386999480 ingress_bytes=717072384 link_bytes=717072384 token_bytes=773770424 weight_movement_bytes=330301440 total_nvlink_bytes=1104071864 weight_to_token_ratio=0.426873 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.247106814e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62046 payload_rows_remote=54026 dup_rows=3490 dup_rows_remote=2920 dedup_ratio=0.053253 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 571, 0, 188, 0, 0, 0, 0, 3995, 682, 0, 0, 0, 0, 290, 0, 0, 0, 0, 4093, 4005, 0, 0, 0, 4096, 0, 0, 95, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 3482, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3076, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8146, 8020, 8192, 8192, 6966, 6152] dup_rows_per_dst=[4, 1518, 1208, 570, 190, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8146, 8020, 8192, 8192, 6966, 6152] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62930 payload_rows_remote=54738 dup_rows=2606 dup_rows_remote=2606 dedup_ratio=0.039764 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 0, 3, 0, 140, 0, 383, 0, 0, 0, 0, 3820, 489, 0, 3, 0, 0, 428, 0, 0, 0, 0, 4088, 4087, 0, 1, 0, 4090, 0, 0, 69, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 3448, 3, 2, 2, 0, 0, 0, 4091, 13, 0, 3500, 0, 1, 0] payload_rows_per_dst=[8186, 8192, 8058, 8178, 8188, 8192, 6908, 7028] dup_rows_per_dst=[6, 1042, 566, 852, 134, 0, 6, 0] combine_rows_per_src=[8186, 8192, 8058, 8178, 8188, 8192, 6908, 7028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60854 payload_rows_remote=52662 dup_rows=4682 dup_rows_remote=4438 dedup_ratio=0.071442 router_tokens_per_expert=[0, 2, 7, 2, 0, 1, 0, 0, 0, 0, 0, 861, 0, 244, 0, 0, 0, 0, 3559, 445, 0, 0, 0, 2, 888, 0, 0, 0, 0, 4084, 4035, 0, 0, 1, 4095, 0, 0, 122, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 1, 3004, 1, 0, 0, 0, 0, 0, 4087, 2, 0, 3224, 1, 0, 3] payload_rows_per_dst=[8176, 8186, 7548, 8090, 8192, 8192, 6012, 6458] dup_rows_per_dst=[16, 2198, 464, 1756, 244, 0, 2, 2] combine_rows_per_src=[8176, 8186, 7548, 8090, 8192, 8192, 6012, 6458] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48064 payload_rows_remote=39872 dup_rows=17472 dup_rows_remote=7686 dedup_ratio=0.266602 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 533, 0, 175, 0, 0, 0, 0, 3797, 12, 0, 0, 0, 0, 888, 0, 0, 0, 0, 4092, 4042, 0, 0, 0, 4095, 0, 0, 206, 0, 1, 0, 4095, 0, 0, 1, 2, 0, 1, 0, 1, 3421, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3307, 0, 0, 3] payload_rows_per_dst=[8188, 1410, 3800, 4816, 5278, 8192, 8188, 8192] dup_rows_per_dst=[0, 6, 8, 576, 350, 6, 6740, 9786] combine_rows_per_src=[8188, 1410, 3800, 4816, 5278, 8192, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56528 payload_rows_remote=48876 dup_rows=9008 dup_rows_remote=4882 dedup_ratio=0.137451 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 527, 0, 142, 0, 0, 0, 0, 3719, 452, 0, 0, 0, 0, 1674, 0, 0, 0, 0, 4086, 4046, 0, 0, 0, 4096, 0, 0, 249, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2151, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 3434, 0, 0, 0] payload_rows_per_dst=[8172, 5446, 7652, 3720, 8192, 8192, 7652, 7502] dup_rows_per_dst=[0, 838, 690, 244, 498, 0, 4126, 2612] combine_rows_per_src=[8172, 5446, 7652, 3720, 8192, 8192, 7652, 7502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387258368 dispatch_meta_bytes=227784 combine_payload_bytes=387258368 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=387486152 ingress_bytes=827660288 link_bytes=827660288 token_bytes=774744520 weight_movement_bytes=440401920 total_nvlink_bytes=1215146440 weight_to_token_ratio=0.568448 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.439437368e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=392361984 dispatch_meta_bytes=229376 combine_payload_bytes=392361984 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=392591360 ingress_bytes=502462464 link_bytes=502462464 token_bytes=784953344 weight_movement_bytes=110100480 total_nvlink_bytes=895053824 weight_to_token_ratio=0.140264 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.738648658e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=377481216 dispatch_meta_bytes=228400 combine_payload_bytes=377481216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=377709616 ingress_bytes=597682176 link_bytes=597682176 token_bytes=755190832 weight_movement_bytes=220200960 total_nvlink_bytes=975391792 weight_to_token_ratio=0.291583 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.039467606e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53638 payload_rows_remote=46056 dup_rows=11898 dup_rows_remote=10484 dedup_ratio=0.181549 router_tokens_per_expert=[0, 0, 1, 103, 3695, 0, 1, 0, 0, 1, 0, 0, 0, 2004, 0, 0, 0, 1376, 0, 0, 0, 0, 4, 471, 4088, 0, 0, 0, 0, 0, 0, 828, 0, 141, 0, 0, 0, 4096, 3243, 0, 0, 0, 0, 0, 4092, 177, 0, 0, 1434, 0, 0, 2907, 0, 0, 1, 10, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7582, 8192, 8192, 7262, 1282, 5922, 7040, 8166] dup_rows_per_dst=[1414, 3994, 3702, 880, 24, 220, 1664, 0] combine_rows_per_src=[7582, 8192, 8192, 7262, 1282, 5922, 7040, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=285802496 dispatch_meta_bytes=190232 combine_payload_bytes=285802496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=285992728 ingress_bytes=506003456 link_bytes=506003456 token_bytes=571795224 weight_movement_bytes=220200960 total_nvlink_bytes=791996184 weight_to_token_ratio=0.385105 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.800232333e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=350343168 dispatch_meta_bytes=215032 combine_payload_bytes=350343168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=350558200 ingress_bytes=460443648 link_bytes=460443648 token_bytes=700901368 weight_movement_bytes=110100480 total_nvlink_bytes=811001848 weight_to_token_ratio=0.157084 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.007872338e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63816 payload_rows_remote=55628 dup_rows=1720 dup_rows_remote=1720 dedup_ratio=0.026245 router_tokens_per_expert=[4096, 0, 725, 0, 3673, 2, 0, 3, 0, 0, 0, 1, 4093, 0, 0, 0, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 1, 1, 0, 0, 3497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 1, 0, 3, 0, 0, 0, 0, 220, 0, 0, 0, 4094, 4089, 0, 42, 0, 0, 0, 7, 0, 0, 3999, 0] payload_rows_per_dst=[7886, 8188, 8162, 6994, 8192, 8192, 8192, 8010] dup_rows_per_dst=[920, 0, 0, 4, 0, 284, 510, 2] combine_rows_per_src=[7886, 8188, 8162, 6994, 8192, 8192, 8192, 8010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63866 payload_rows_remote=55676 dup_rows=1670 dup_rows_remote=1670 dedup_ratio=0.025482 router_tokens_per_expert=[4096, 0, 563, 0, 3762, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3390, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 272, 0, 6, 0, 4096, 4096, 0, 35, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[7936, 8192, 8190, 6780, 8192, 8192, 8192, 8192] dup_rows_per_dst=[714, 0, 0, 0, 0, 330, 626, 0] combine_rows_per_src=[7936, 8192, 8190, 6780, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=398741504 dispatch_meta_bytes=229392 combine_payload_bytes=398741504 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=398970896 ingress_bytes=508841984 link_bytes=508841984 token_bytes=797712400 weight_movement_bytes=110100480 total_nvlink_bytes=907812880 weight_to_token_ratio=0.138020 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.849599003e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=330129408 dispatch_meta_bytes=226160 combine_payload_bytes=330129408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330355568 ingress_bytes=440229888 link_bytes=440229888 token_bytes=660484976 weight_movement_bytes=110100480 total_nvlink_bytes=770585456 weight_to_token_ratio=0.166696 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.656321806e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62838 payload_rows_remote=56746 dup_rows=2698 dup_rows_remote=2698 dedup_ratio=0.041168 router_tokens_per_expert=[4096, 0, 1010, 0, 3777, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 94, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 4090, 4096, 0, 135, 0, 0, 0, 0, 0, 0, 3911, 0] payload_rows_per_dst=[7976, 8192, 8192, 6092, 8192, 8180, 8192, 7822] dup_rows_per_dst=[1598, 0, 0, 0, 0, 188, 912, 0] combine_rows_per_src=[7976, 8192, 8192, 6092, 8192, 8180, 8192, 7822] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63430 payload_rows_remote=55244 dup_rows=2106 dup_rows_remote=2104 dedup_ratio=0.032135 router_tokens_per_expert=[4091, 0, 715, 0, 3811, 0, 0, 0, 0, 0, 0, 1, 4094, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 2, 3, 0, 0, 0, 0, 3243, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 336, 0, 0, 0, 0, 0, 0, 0, 131, 1, 0, 2, 4095, 4096, 0, 6, 0, 0, 0, 11, 0, 1, 4029, 0] payload_rows_per_dst=[7922, 8188, 8192, 6492, 8186, 8190, 8192, 8068] dup_rows_per_dst=[1130, 2, 6, 0, 2, 672, 280, 14] combine_rows_per_src=[7922, 8188, 8192, 6492, 8186, 8190, 8192, 8068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=399085568 dispatch_meta_bytes=229384 combine_payload_bytes=399085568 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399314952 ingress_bytes=509186048 link_bytes=509186048 token_bytes=798400520 weight_movement_bytes=110100480 total_nvlink_bytes=908501000 weight_to_token_ratio=0.137901 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.855582842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60766 payload_rows_remote=52574 dup_rows=4770 dup_rows_remote=4578 dedup_ratio=0.072784 router_tokens_per_expert=[4095, 0, 1274, 0, 4034, 0, 0, 1, 0, 0, 0, 2, 4096, 0, 4, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 1, 0, 1, 1, 0, 746, 0, 25, 0, 4094, 4088, 4, 97, 0, 1, 0, 11, 0, 0, 4060, 0] payload_rows_per_dst=[7634, 8192, 8150, 4082, 8190, 8192, 8192, 8134] dup_rows_per_dst=[2390, 12, 0, 438, 0, 192, 1728, 10] combine_rows_per_src=[7634, 8192, 8150, 4082, 8190, 8192, 8192, 8134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58268 payload_rows_remote=50076 dup_rows=7268 dup_rows_remote=6798 dedup_ratio=0.110901 router_tokens_per_expert=[4096, 0, 475, 0, 3772, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 301, 0, 0, 9, 0, 0, 0, 0, 219, 0, 0, 0, 4096, 4096, 0, 16, 0, 0, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[950, 8192, 8192, 8172, 8192, 8192, 8192, 8186] dup_rows_per_dst=[0, 0, 0, 6178, 0, 620, 470, 0] combine_rows_per_src=[950, 8192, 8192, 8172, 8192, 8192, 8192, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56940 payload_rows_remote=48762 dup_rows=8596 dup_rows_remote=7842 dedup_ratio=0.131165 router_tokens_per_expert=[4095, 0, 1187, 0, 3941, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 0, 0, 1, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2693, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 146, 0, 0, 2, 1, 0, 0, 1, 206, 0, 0, 1, 4094, 4096, 0, 27, 0, 0, 0, 0, 0, 0, 4083, 1] payload_rows_per_dst=[2374, 8188, 8190, 8178, 8192, 5448, 8192, 8178] dup_rows_per_dst=[0, 0, 2, 5088, 6, 180, 2566, 754] combine_rows_per_src=[2374, 8188, 8190, 8178, 8192, 5448, 8192, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406755328 dispatch_meta_bytes=237776 combine_payload_bytes=406755328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406993104 ingress_bytes=516855808 link_bytes=516855808 token_bytes=813748432 weight_movement_bytes=110100480 total_nvlink_bytes=923848912 weight_to_token_ratio=0.135300 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.988972583e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=395988992 dispatch_meta_bytes=229392 combine_payload_bytes=395988992 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=396218384 ingress_bytes=616189952 link_bytes=616189952 token_bytes=792207376 weight_movement_bytes=220200960 total_nvlink_bytes=1012408336 weight_to_token_ratio=0.277959 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071655672e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=376850432 dispatch_meta_bytes=228608 combine_payload_bytes=376850432 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377079040 ingress_bytes=486950912 link_bytes=486950912 token_bytes=753929472 weight_movement_bytes=110100480 total_nvlink_bytes=864029952 weight_to_token_ratio=0.146036 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.468877256e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64298 payload_rows_remote=56936 dup_rows=1238 dup_rows_remote=1238 dedup_ratio=0.018890 router_tokens_per_expert=[0, 4096, 4085, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 3676, 5, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 6, 0, 0, 0, 0, 274, 0, 3568, 142, 0, 145, 0, 0, 0, 0, 1, 2, 0, 0, 0, 1, 4094, 0, 0, 0, 384] payload_rows_per_dst=[8172, 7362, 8190, 8192, 8192, 7808, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 2, 12, 160, 294, 768] combine_rows_per_src=[8172, 7362, 8190, 8192, 8192, 7808, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=358944768 dispatch_meta_bytes=227496 combine_payload_bytes=358944768 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=359172264 ingress_bytes=909447168 link_bytes=909447168 token_bytes=718117032 weight_movement_bytes=550502400 total_nvlink_bytes=1268619432 weight_to_token_ratio=0.766591 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.581678204e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=349526016 dispatch_meta_bytes=226416 combine_payload_bytes=349526016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349752432 ingress_bytes=459626496 link_bytes=459626496 token_bytes=699278448 weight_movement_bytes=110100480 total_nvlink_bytes=809378928 weight_to_token_ratio=0.157449 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.993660721e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65178 payload_rows_remote=57004 dup_rows=358 dup_rows_remote=240 dedup_ratio=0.005463 router_tokens_per_expert=[0, 0, 4086, 59, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1037, 1, 0, 4095, 4057, 3108, 0, 0, 0, 0, 4095, 0, 4027, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8174, 8190, 8084, 8088, 8188, 8116, 8148, 8190] dup_rows_per_dst=[118, 0, 208, 12, 0, 18, 2, 0] combine_rows_per_src=[8174, 8190, 8084, 8088, 8188, 8116, 8148, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63504 payload_rows_remote=55312 dup_rows=2032 dup_rows_remote=1944 dedup_ratio=0.031006 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3290, 0, 0, 0, 0, 4096, 0, 0, 16, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 422, 0, 3557, 230, 0, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 350] payload_rows_per_dst=[8192, 6580, 8192, 8192, 8192, 7772, 8192, 8192] dup_rows_per_dst=[0, 0, 88, 0, 0, 646, 598, 700] combine_rows_per_src=[8192, 6580, 8192, 8192, 8192, 7772, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=408117248 dispatch_meta_bytes=232696 combine_payload_bytes=408117248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=408349944 ingress_bytes=628318208 link_bytes=628318208 token_bytes=816467192 weight_movement_bytes=220200960 total_nvlink_bytes=1036668152 weight_to_token_ratio=0.269700 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.092748704e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=408604672 dispatch_meta_bytes=228976 combine_payload_bytes=408604672 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=408833648 ingress_bytes=849006592 link_bytes=849006592 token_bytes=817438320 weight_movement_bytes=440401920 total_nvlink_bytes=1257840240 weight_to_token_ratio=0.538759 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.476562101e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64642 payload_rows_remote=56450 dup_rows=894 dup_rows_remote=894 dedup_ratio=0.013641 router_tokens_per_expert=[0, 4096, 4096, 0, 0, 0, 0, 0, 4096, 0, 7, 0, 0, 3786, 0, 0, 0, 0, 4096, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 271, 0, 3604, 226, 0, 194, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 93] payload_rows_per_dst=[8192, 7586, 8192, 8192, 8192, 7904, 8192, 8192] dup_rows_per_dst=[0, 0, 22, 0, 0, 298, 388, 186] combine_rows_per_src=[8192, 7586, 8192, 8192, 8192, 7904, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63318 payload_rows_remote=55126 dup_rows=2218 dup_rows_remote=2216 dedup_ratio=0.033844 router_tokens_per_expert=[1, 4094, 4092, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 3294, 1, 1, 0, 0, 4092, 0, 0, 1, 66, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 4095, 1, 0, 0, 0, 1, 0, 0, 0, 0, 410, 0, 3362, 259, 0, 241, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 4095, 0, 0, 0, 561] payload_rows_per_dst=[8186, 6588, 8184, 8190, 8192, 7596, 8190, 8192] dup_rows_per_dst=[2, 4, 134, 4, 2, 466, 482, 1124] combine_rows_per_src=[8186, 6588, 8184, 8190, 8192, 7596, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63438 payload_rows_remote=55246 dup_rows=2098 dup_rows_remote=1634 dedup_ratio=0.032013 router_tokens_per_expert=[0, 4096, 4093, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3602, 0, 0, 0, 0, 4096, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 433, 0, 3011, 224, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 676] payload_rows_per_dst=[8186, 7204, 8192, 8192, 8192, 7088, 8192, 8192] dup_rows_per_dst=[0, 0, 34, 0, 0, 248, 464, 1352] combine_rows_per_src=[8186, 7204, 8192, 8192, 8192, 7088, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396476416 dispatch_meta_bytes=229024 combine_payload_bytes=396476416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=396705440 ingress_bytes=616677376 link_bytes=616677376 token_bytes=793181856 weight_movement_bytes=220200960 total_nvlink_bytes=1013382816 weight_to_token_ratio=0.277617 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.072503383e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63286 payload_rows_remote=55634 dup_rows=2250 dup_rows_remote=1634 dedup_ratio=0.034332 router_tokens_per_expert=[0, 4096, 4094, 0, 0, 0, 0, 2, 4093, 0, 9, 0, 0, 3233, 3, 2, 0, 0, 4096, 0, 0, 2, 7, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 449, 0, 3472, 213, 0, 673, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 1, 126] payload_rows_per_dst=[8192, 6488, 8192, 8192, 8190, 7652, 8188, 8192] dup_rows_per_dst=[0, 6, 18, 10, 0, 616, 1346, 254] combine_rows_per_src=[8192, 6488, 8192, 8192, 8190, 7652, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62642 payload_rows_remote=54452 dup_rows=2894 dup_rows_remote=1204 dedup_ratio=0.044159 router_tokens_per_expert=[0, 4094, 4095, 0, 0, 0, 0, 0, 4095, 0, 2, 0, 0, 3198, 0, 0, 0, 1, 4096, 0, 1, 0, 4, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 5, 0, 0, 0, 1, 308, 0, 3438, 209, 0, 183, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 845] payload_rows_per_dst=[8190, 6400, 7682, 8190, 8192, 7606, 8192, 8190] dup_rows_per_dst=[0, 0, 14, 0, 8, 816, 366, 1690] combine_rows_per_src=[8190, 6400, 7682, 8190, 8192, 7606, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=404633600 dispatch_meta_bytes=229376 combine_payload_bytes=404633600 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404862976 ingress_bytes=514734080 link_bytes=514734080 token_bytes=809496576 weight_movement_bytes=110100480 total_nvlink_bytes=919597056 weight_to_token_ratio=0.136011 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.952072243e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396003328 dispatch_meta_bytes=227520 combine_payload_bytes=396003328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396230848 ingress_bytes=506103808 link_bytes=506103808 token_bytes=792234176 weight_movement_bytes=110100480 total_nvlink_bytes=902334656 weight_to_token_ratio=0.138975 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.801977619e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395143168 dispatch_meta_bytes=229368 combine_payload_bytes=395143168 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395372536 ingress_bytes=615344128 link_bytes=615344128 token_bytes=790515704 weight_movement_bytes=220200960 total_nvlink_bytes=1010716664 weight_to_token_ratio=0.278554 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070184645e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398784512 dispatch_meta_bytes=229072 combine_payload_bytes=398784512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399013584 ingress_bytes=508884992 link_bytes=508884992 token_bytes=797798096 weight_movement_bytes=110100480 total_nvlink_bytes=907898576 weight_to_token_ratio=0.138005 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.850346983e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61628 payload_rows_remote=53442 dup_rows=3908 dup_rows_remote=3236 dedup_ratio=0.059631 router_tokens_per_expert=[0, 4095, 1, 0, 0, 0, 6, 0, 335, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1290, 1, 0, 0, 0, 3866, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 2, 0, 8, 0, 2, 0, 0, 0, 0, 1619, 1829, 1, 0, 0, 4089, 4083, 4096, 0, 0, 0, 3334, 0] payload_rows_per_dst=[8192, 8186, 8186, 7806, 8192, 8192, 6206, 6668] dup_rows_per_dst=[12, 672, 2, 2508, 12, 10, 692, 0] combine_rows_per_src=[8192, 8186, 8186, 7806, 8192, 8192, 6206, 6668] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390311936 dispatch_meta_bytes=222624 combine_payload_bytes=390311936 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=390534560 ingress_bytes=720613376 link_bytes=720613376 token_bytes=780846496 weight_movement_bytes=330301440 total_nvlink_bytes=1111147936 weight_to_token_ratio=0.423004 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.253265181e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61226 payload_rows_remote=53056 dup_rows=4310 dup_rows_remote=4266 dedup_ratio=0.065765 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 523, 0, 334, 0, 0, 0, 0, 3915, 482, 0, 0, 0, 0, 658, 0, 1, 0, 0, 4082, 4065, 0, 0, 0, 4095, 0, 0, 255, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1, 0, 0, 2990, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 3174, 0, 0, 0] payload_rows_per_dst=[8170, 8190, 8064, 8094, 8192, 8188, 5980, 6348] dup_rows_per_dst=[44, 1714, 734, 1308, 508, 0, 0, 2] combine_rows_per_src=[8170, 8190, 8064, 8094, 8192, 8188, 5980, 6348] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60038 payload_rows_remote=51846 dup_rows=5498 dup_rows_remote=5498 dedup_ratio=0.083893 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1784, 0, 0, 0, 0, 4014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 519, 2910, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 2320, 0] payload_rows_per_dst=[8192, 8192, 8192, 8052, 8192, 8192, 6386, 4640] dup_rows_per_dst=[0, 1466, 0, 3544, 0, 16, 472, 0] combine_rows_per_src=[8192, 8192, 8192, 8052, 8192, 8192, 6386, 4640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=383072256 dispatch_meta_bytes=226712 combine_payload_bytes=383072256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383298968 ingress_bytes=493172736 link_bytes=493172736 token_bytes=766371224 weight_movement_bytes=110100480 total_nvlink_bytes=876471704 weight_to_token_ratio=0.143665 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.577085009e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62188 payload_rows_remote=54868 dup_rows=3348 dup_rows_remote=2120 dedup_ratio=0.051086 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 468, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 745, 0, 0, 0, 0, 3529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 11, 0, 0, 0, 0, 0, 3, 1413, 2742, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3369, 0] payload_rows_per_dst=[8192, 8192, 8192, 7320, 8192, 8192, 7170, 6738] dup_rows_per_dst=[0, 938, 0, 1228, 0, 36, 1146, 0] combine_rows_per_src=[8192, 8192, 8192, 7320, 8192, 8192, 7170, 6738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=380305408 dispatch_meta_bytes=229288 combine_payload_bytes=380305408 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=380534696 ingress_bytes=600506368 link_bytes=600506368 token_bytes=760840104 weight_movement_bytes=220200960 total_nvlink_bytes=981041064 weight_to_token_ratio=0.289418 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044379340e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57032 payload_rows_remote=48980 dup_rows=8504 dup_rows_remote=3502 dedup_ratio=0.129761 router_tokens_per_expert=[0, 4096, 0, 0, 0, 0, 0, 0, 624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 1418, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 245, 2572, 0, 0, 0, 4096, 4096, 4096, 0, 0, 0, 3453, 0] payload_rows_per_dst=[8192, 5722, 8192, 3166, 8192, 8192, 8052, 7324] dup_rows_per_dst=[0, 596, 0, 202, 0, 0, 5002, 2704] combine_rows_per_src=[8192, 5722, 8192, 3166, 8192, 8192, 8052, 7324] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62312 payload_rows_remote=54120 dup_rows=3224 dup_rows_remote=3222 dedup_ratio=0.049194 router_tokens_per_expert=[0, 4093, 0, 0, 0, 0, 0, 0, 728, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 840, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 2, 0, 2, 0, 13, 0, 0, 0, 14, 289, 2576, 0, 1, 0, 4088, 4094, 4096, 0, 1, 0, 3807, 0] payload_rows_per_dst=[8186, 8192, 8190, 8076, 8192, 8192, 5668, 7616] dup_rows_per_dst=[0, 1458, 0, 1650, 2, 22, 92, 0] combine_rows_per_src=[8186, 8192, 8190, 8076, 8192, 8192, 5668, 7616] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=371632128 dispatch_meta_bytes=229376 combine_payload_bytes=371632128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=371861504 ingress_bytes=481732608 link_bytes=481732608 token_bytes=743493632 weight_movement_bytes=110100480 total_nvlink_bytes=853594112 weight_to_token_ratio=0.148085 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.378122367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60094 payload_rows_remote=51902 dup_rows=5442 dup_rows_remote=5380 dedup_ratio=0.083038 router_tokens_per_expert=[1, 4095, 2, 0, 0, 0, 0, 0, 1400, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4093, 0, 0, 0, 0, 1251, 0, 0, 0, 0, 3852, 3, 0, 0, 2, 0, 1, 0, 0, 0, 0, 5, 8, 0, 7, 0, 13, 0, 0, 0, 3, 442, 1791, 0, 11, 0, 4094, 4083, 4092, 0, 1, 0, 3515, 0] payload_rows_per_dst=[8192, 8178, 8188, 7816, 8184, 8192, 4312, 7032] dup_rows_per_dst=[4, 2792, 0, 2396, 6, 62, 182, 0] combine_rows_per_src=[8192, 8178, 8188, 7816, 8184, 8192, 4312, 7032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53050 payload_rows_remote=44858 dup_rows=12486 dup_rows_remote=4434 dedup_ratio=0.190521 router_tokens_per_expert=[0, 4096, 0, 0, 0, 1, 0, 0, 667, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 1478, 0, 0, 0, 0, 3979, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 1, 0, 0, 1, 0, 1, 668, 1808, 0, 0, 0, 4094, 4095, 4095, 0, 0, 0, 3679, 0] payload_rows_per_dst=[8184, 1338, 8190, 2956, 8180, 8038, 7972, 8192] dup_rows_per_dst=[2, 4, 8, 0, 0, 8, 4412, 8052] combine_rows_per_src=[8184, 1338, 8190, 2956, 8180, 8038, 7972, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=393293824 dispatch_meta_bytes=227952 combine_payload_bytes=393293824 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=393521776 ingress_bytes=613494784 link_bytes=613494784 token_bytes=786815600 weight_movement_bytes=220200960 total_nvlink_bytes=1007016560 weight_to_token_ratio=0.279863 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.066968332e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=351088640 dispatch_meta_bytes=209928 combine_payload_bytes=351088640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351298568 ingress_bytes=461189120 link_bytes=461189120 token_bytes=702387208 weight_movement_bytes=110100480 total_nvlink_bytes=812487688 weight_to_token_ratio=0.156752 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.020837323e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=387932160 dispatch_meta_bytes=229368 combine_payload_bytes=387932160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388161528 ingress_bytes=498032640 link_bytes=498032640 token_bytes=776093688 weight_movement_bytes=110100480 total_nvlink_bytes=886194168 weight_to_token_ratio=0.141865 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.661606733e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=372033536 dispatch_meta_bytes=229128 combine_payload_bytes=372033536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372262664 ingress_bytes=482134016 link_bytes=482134016 token_bytes=744296200 weight_movement_bytes=110100480 total_nvlink_bytes=854396680 weight_to_token_ratio=0.147926 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.385103512e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=321542144 dispatch_meta_bytes=197168 combine_payload_bytes=321542144 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=321739312 ingress_bytes=761944064 link_bytes=761944064 token_bytes=643281456 weight_movement_bytes=440401920 total_nvlink_bytes=1083683376 weight_to_token_ratio=0.684618 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.325146046e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60520 payload_rows_remote=52328 dup_rows=5016 dup_rows_remote=5010 dedup_ratio=0.076538 router_tokens_per_expert=[0, 0, 40, 0, 4089, 0, 0, 4095, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 5, 1, 0, 8, 2507, 0, 56, 3, 0, 0, 0, 0, 0, 3980, 4094, 0, 9, 1707, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3938] payload_rows_per_dst=[8186, 8192, 8192, 8190, 8182, 8188, 3514, 7876] dup_rows_per_dst=[72, 6, 0, 0, 4932, 6, 0, 0] combine_rows_per_src=[8186, 8192, 8192, 8190, 8182, 8188, 3514, 7876] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57244 payload_rows_remote=49054 dup_rows=8292 dup_rows_remote=376 dedup_ratio=0.126526 router_tokens_per_expert=[4096, 0, 482, 0, 3930, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3519, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 1, 0, 0, 0, 0, 123, 1, 0, 0, 4096, 4095, 0, 7, 0, 1, 0, 0, 0, 0, 4082, 0] payload_rows_per_dst=[8190, 8190, 8190, 7038, 8192, 8192, 1086, 8166] dup_rows_per_dst=[7916, 0, 70, 0, 0, 292, 14, 0] combine_rows_per_src=[8190, 8190, 8190, 7038, 8192, 8192, 1086, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60266 payload_rows_remote=52074 dup_rows=5270 dup_rows_remote=5270 dedup_ratio=0.080414 router_tokens_per_expert=[0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 75, 0, 0, 0, 2247, 0, 51, 1, 0, 0, 0, 0, 0, 3891, 4096, 0, 0, 2024, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3906] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7262, 8192, 4232, 7812] dup_rows_per_dst=[2, 0, 0, 0, 3210, 2058, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7262, 8192, 4232, 7812] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375087104 dispatch_meta_bytes=229352 combine_payload_bytes=375087104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375316456 ingress_bytes=485187584 link_bytes=485187584 token_bytes=750403560 weight_movement_bytes=110100480 total_nvlink_bytes=860504040 weight_to_token_ratio=0.146722 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.438210082e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61312 payload_rows_remote=53120 dup_rows=4224 dup_rows_remote=4224 dedup_ratio=0.064453 router_tokens_per_expert=[1, 0, 83, 0, 4089, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 31, 0, 0, 1, 2622, 0, 61, 10, 0, 0, 0, 0, 0, 3937, 4096, 0, 24, 1419, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4083] payload_rows_per_dst=[8188, 8192, 8192, 8192, 5426, 8070, 6886, 8166] dup_rows_per_dst=[158, 0, 0, 0, 4, 1876, 2186, 0] combine_rows_per_src=[8188, 8192, 8192, 8192, 5426, 8070, 6886, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61288 payload_rows_remote=53096 dup_rows=4248 dup_rows_remote=12 dedup_ratio=0.064819 router_tokens_per_expert=[0, 0, 1, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 22, 0, 0, 0, 2465, 0, 45, 0, 0, 0, 0, 0, 0, 3615, 4096, 0, 0, 2052, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4022] payload_rows_per_dst=[8192, 8192, 8192, 8192, 5054, 7230, 8192, 8044] dup_rows_per_dst=[2, 0, 0, 0, 10, 0, 4236, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 5054, 7230, 8192, 8044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63518 payload_rows_remote=57192 dup_rows=2018 dup_rows_remote=2018 dedup_ratio=0.030792 router_tokens_per_expert=[0, 0, 12, 0, 4095, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 52, 0, 0, 0, 3061, 1, 49, 0, 0, 0, 0, 1, 0, 4057, 4095, 0, 9, 940, 0, 44, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4063] payload_rows_per_dst=[8190, 8192, 8192, 8182, 6326, 8116, 8192, 8128] dup_rows_per_dst=[26, 0, 6, 0, 0, 0, 1984, 2] combine_rows_per_src=[8190, 8192, 8192, 8182, 6326, 8116, 8192, 8128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=351619072 dispatch_meta_bytes=197720 combine_payload_bytes=351619072 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=351816792 ingress_bytes=792020992 link_bytes=792020992 token_bytes=703435864 weight_movement_bytes=440401920 total_nvlink_bytes=1143837784 weight_to_token_ratio=0.626073 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.377454770e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=373266432 dispatch_meta_bytes=229376 combine_payload_bytes=373266432 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373495808 ingress_bytes=593467392 link_bytes=593467392 token_bytes=746762240 weight_movement_bytes=220200960 total_nvlink_bytes=966963200 weight_to_token_ratio=0.294874 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032137403e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61846 payload_rows_remote=54216 dup_rows=3690 dup_rows_remote=3690 dedup_ratio=0.056305 router_tokens_per_expert=[2, 0, 32, 0, 4083, 0, 3, 4095, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 4095, 1, 18, 0, 0, 8, 2604, 0, 48, 0, 0, 0, 0, 1, 0, 3814, 4089, 0, 4, 1788, 0, 15, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3966] payload_rows_per_dst=[8186, 8190, 8188, 8190, 5338, 7630, 8192, 7932] dup_rows_per_dst=[54, 6, 0, 2, 20, 0, 3608, 0] combine_rows_per_src=[8186, 8190, 8188, 8190, 5338, 7630, 8192, 7932] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7200592296 prefetch_bytes=1211105280 reduce_bytes=220200960 weight_movement_bytes=1431306240 egress_bytes=3601424808 ingress_bytes=5030473728 link_bytes=5030473728 total_nvlink_bytes=8631898536 weight_to_token_ratio=0.198776 prefetch_share_of_ingress=0.240754 dedup_ratio=0.115222 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=2 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9906 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.748821184e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60256 payload_rows_remote=52404 dup_rows=5280 dup_rows_remote=3240 dedup_ratio=0.080566 router_tokens_per_expert=[0, 0, 7, 0, 4094, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 34, 0, 0, 0, 2843, 1, 31, 8, 0, 0, 0, 0, 0, 3891, 4093, 0, 0, 1592, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3879] payload_rows_per_dst=[7874, 8192, 8178, 8190, 5808, 5972, 8190, 7852] dup_rows_per_dst=[12, 0, 10, 14, 12, 6, 3186, 2040] combine_rows_per_src=[7874, 8192, 8178, 8190, 5808, 5972, 8190, 7852] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380764160 dispatch_meta_bytes=229376 combine_payload_bytes=380764160 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380993536 ingress_bytes=490864640 link_bytes=490864640 token_bytes=761757696 weight_movement_bytes=110100480 total_nvlink_bytes=871858176 weight_to_token_ratio=0.144535 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.536943423e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7197943088 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3600094512 ingress_bytes=5469556736 link_bytes=5469556736 total_nvlink_bytes=9069651248 weight_to_token_ratio=0.260034 prefetch_share_of_ingress=0.201297 dedup_ratio=0.115689 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040074 gemm_inner_pad_upper_bound=10160 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.512458751e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380592128 dispatch_meta_bytes=212432 combine_payload_bytes=380592128 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380804560 ingress_bytes=490692608 link_bytes=490692608 token_bytes=761396688 weight_movement_bytes=110100480 total_nvlink_bytes=871497168 weight_to_token_ratio=0.144603 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.533951504e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=409952256 dispatch_meta_bytes=236840 combine_payload_bytes=409952256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410189096 ingress_bytes=520052736 link_bytes=520052736 token_bytes=820141352 weight_movement_bytes=110100480 total_nvlink_bytes=930241832 weight_to_token_ratio=0.134246 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.044572419e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=388620288 dispatch_meta_bytes=231624 combine_payload_bytes=388620288 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=388851912 ingress_bytes=829022208 link_bytes=829022208 token_bytes=777472200 weight_movement_bytes=440401920 total_nvlink_bytes=1217874120 weight_to_token_ratio=0.566454 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.441805970e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=375631872 dispatch_meta_bytes=222576 combine_payload_bytes=375631872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375854448 ingress_bytes=485732352 link_bytes=485732352 token_bytes=751486320 weight_movement_bytes=110100480 total_nvlink_bytes=861586800 weight_to_token_ratio=0.146510 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.447684494e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7416413408 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3709367520 ingress_bytes=6019155968 link_bytes=6019155968 total_nvlink_bytes=9728523488 weight_to_token_ratio=0.311756 prefetch_share_of_ingress=0.146333 dedup_ratio=0.107077 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042647 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.046830221e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6977275240 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3489756520 ingress_bytes=5139025920 link_bytes=5139025920 total_nvlink_bytes=8628782440 weight_to_token_ratio=0.236698 prefetch_share_of_ingress=0.214244 dedup_ratio=0.143433 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=10541 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.937611299e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7329093488 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3665686384 ingress_bytes=5645215744 link_bytes=5645215744 total_nvlink_bytes=9310902128 weight_to_token_ratio=0.270403 prefetch_share_of_ingress=0.195033 dedup_ratio=0.104291 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039338 gemm_inner_pad_upper_bound=10287 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.817958657e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7121289376 prefetch_bytes=1101004800 reduce_bytes=1321205760 weight_movement_bytes=2422210560 egress_bytes=3561775264 ingress_bytes=5981724672 link_bytes=5981724672 total_nvlink_bytes=9543499936 weight_to_token_ratio=0.340137 prefetch_share_of_ingress=0.184061 dedup_ratio=0.122971 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=12 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9779 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.040320303e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6745605736 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3373893224 ingress_bytes=5463621632 link_bytes=5463621632 total_nvlink_bytes=8837514856 weight_to_token_ratio=0.310114 prefetch_share_of_ingress=0.181364 dedup_ratio=0.163638 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.038603 gemm_inner_pad_upper_bound=10541 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.502136629e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54704 payload_rows_remote=46512 dup_rows=10832 dup_rows_remote=10800 dedup_ratio=0.165283 router_tokens_per_expert=[0, 4095, 4094, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 3401, 1, 0, 0, 0, 4096, 1, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 501, 0, 3779, 150, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 318] payload_rows_per_dst=[8192, 7772, 8192, 8180, 8168, 8066, 4204, 1930] dup_rows_per_dst=[32, 9546, 208, 0, 4, 794, 56, 192] combine_rows_per_src=[8192, 7772, 8192, 8180, 8168, 8066, 4204, 1930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=333398016 dispatch_meta_bytes=229248 combine_payload_bytes=333398016 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=333627264 ingress_bytes=553598976 link_bytes=553598976 token_bytes=667025280 weight_movement_bytes=220200960 total_nvlink_bytes=887226240 weight_to_token_ratio=0.330124 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.627996706e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62162 payload_rows_remote=53974 dup_rows=3374 dup_rows_remote=3374 dedup_ratio=0.051483 router_tokens_per_expert=[0, 4094, 0, 0, 0, 0, 0, 0, 708, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 4095, 0, 0, 1, 0, 497, 0, 0, 0, 0, 3911, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1551, 2334, 0, 0, 0, 4095, 4095, 4095, 0, 0, 0, 3285, 0] payload_rows_per_dst=[8188, 8192, 8192, 7832, 8192, 8190, 6806, 6570] dup_rows_per_dst=[0, 1418, 4, 984, 0, 2, 966, 0] combine_rows_per_src=[8188, 8192, 8192, 7832, 8192, 8190, 6806, 6570] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386885632 dispatch_meta_bytes=229392 combine_payload_bytes=386885632 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=387115024 ingress_bytes=496986112 link_bytes=496986112 token_bytes=774000656 weight_movement_bytes=110100480 total_nvlink_bytes=884101136 weight_to_token_ratio=0.142249 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643405889e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59760 payload_rows_remote=51568 dup_rows=5776 dup_rows_remote=5766 dedup_ratio=0.088135 router_tokens_per_expert=[0, 0, 0, 0, 4096, 0, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 0, 2995, 0, 44, 0, 0, 0, 0, 0, 0, 3911, 4096, 0, 0, 1419, 0, 50, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3841] payload_rows_per_dst=[8192, 8182, 8192, 8192, 8190, 8192, 2938, 7682] dup_rows_per_dst=[10, 0, 2, 0, 5762, 0, 0, 2] combine_rows_per_src=[8192, 8182, 8192, 8192, 8190, 8192, 2938, 7682] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369639424 dispatch_meta_bytes=229336 combine_payload_bytes=369639424 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=369868760 ingress_bytes=699940864 link_bytes=699940864 token_bytes=739508184 weight_movement_bytes=330301440 total_nvlink_bytes=1069809624 weight_to_token_ratio=0.446650 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.217312283e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6881523384 prefetch_bytes=880803840 reduce_bytes=2091909120 weight_movement_bytes=2972712960 egress_bytes=3441858232 ingress_bytes=6412378112 link_bytes=6412378112 total_nvlink_bytes=9854236344 weight_to_token_ratio=0.431985 prefetch_share_of_ingress=0.137360 dedup_ratio=0.148221 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=19 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041912 gemm_inner_pad_upper_bound=9525 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.115218019e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:57:46] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 369.2 | throughput per GPU (TFLOP/s/GPU): 334.5 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 9.942751E+00 | loss scale: 1.0 | grad norm: 22.967 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47032 payload_rows_remote=41154 dup_rows=18504 dup_rows_remote=16426 dedup_ratio=0.282349 router_tokens_per_expert=[92, 75, 47, 35, 0, 11, 1146, 1679, 371, 103, 185, 721, 0, 142, 0, 422, 639, 536, 1816, 0, 169, 449, 7, 146, 915, 108, 531, 521, 6, 1873, 1029, 4, 11, 14, 89, 1673, 3, 0, 2, 313, 891, 303, 1279, 8, 2665, 642, 1787, 607, 149, 839, 1338, 171, 499, 317, 50, 128, 1285, 1184, 321, 843, 323, 214, 1042, 0] payload_rows_per_dst=[6426, 6818, 5516, 5068, 5878, 5516, 5244, 6566] dup_rows_per_dst=[3318, 2400, 2008, 1490, 2078, 1944, 1738, 3528] combine_rows_per_src=[6426, 6818, 5516, 5068, 5878, 5516, 5244, 6566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45288 payload_rows_remote=39842 dup_rows=20248 dup_rows_remote=18172 dedup_ratio=0.308960 router_tokens_per_expert=[130, 17, 32, 73, 30, 10, 876, 2446, 202, 29, 144, 1018, 2, 115, 1, 237, 134, 809, 1236, 0, 179, 364, 12, 241, 1411, 205, 303, 906, 3, 1884, 1333, 22, 31, 3, 54, 1739, 7, 3, 4, 387, 920, 677, 378, 8, 2401, 722, 716, 1059, 49, 789, 1248, 450, 675, 199, 71, 280, 920, 1662, 427, 421, 455, 480, 1127, 2] payload_rows_per_dst=[6226, 3412, 4662, 5986, 5990, 6842, 5446, 6724] dup_rows_per_dst=[2434, 468, 1288, 2380, 2234, 5104, 2076, 4264] combine_rows_per_src=[6226, 3412, 4662, 5986, 5990, 6842, 5446, 6724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46342 payload_rows_remote=41220 dup_rows=19194 dup_rows_remote=17206 dedup_ratio=0.292877 router_tokens_per_expert=[101, 53, 80, 72, 6, 34, 838, 1183, 294, 96, 307, 1774, 1, 89, 0, 309, 331, 494, 1995, 12, 599, 328, 3, 222, 744, 215, 544, 764, 1, 1246, 891, 18, 27, 13, 104, 1921, 5, 13, 3, 672, 977, 675, 637, 24, 2475, 504, 1094, 738, 35, 1381, 1117, 179, 666, 195, 70, 322, 1208, 1295, 416, 187, 334, 246, 1587, 4] payload_rows_per_dst=[5120, 7096, 5778, 5042, 5700, 5122, 5732, 6752] dup_rows_per_dst=[1802, 3594, 2190, 1312, 2308, 1988, 2198, 3802] combine_rows_per_src=[5120, 7096, 5778, 5042, 5700, 5122, 5732, 6752] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46936 payload_rows_remote=41834 dup_rows=18600 dup_rows_remote=16922 dedup_ratio=0.283813 router_tokens_per_expert=[119, 119, 46, 140, 3, 19, 1233, 1796, 320, 21, 155, 832, 0, 70, 1, 188, 289, 693, 1515, 0, 272, 247, 2, 372, 1088, 87, 400, 798, 6, 1615, 903, 0, 10, 4, 113, 2036, 10, 0, 6, 243, 1184, 437, 546, 6, 2830, 754, 1417, 928, 34, 1019, 1459, 412, 531, 160, 43, 166, 1326, 1258, 429, 504, 402, 127, 1025, 0] payload_rows_per_dst=[6264, 6876, 5102, 6176, 6018, 5510, 5512, 5478] dup_rows_per_dst=[3520, 1958, 1678, 2904, 2056, 2200, 2136, 2148] combine_rows_per_src=[6264, 6876, 5102, 6176, 6018, 5510, 5512, 5478] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45702 payload_rows_remote=39228 dup_rows=19834 dup_rows_remote=16582 dedup_ratio=0.302643 router_tokens_per_expert=[187, 39, 175, 57, 73, 6, 1016, 1299, 459, 76, 106, 1138, 1, 193, 0, 177, 214, 835, 1845, 4, 266, 574, 2, 524, 1483, 580, 376, 743, 9, 2113, 1221, 10, 6, 1, 115, 1497, 4, 5, 38, 431, 367, 684, 479, 51, 2044, 627, 1230, 1257, 14, 1758, 1421, 160, 1296, 163, 43, 252, 552, 441, 230, 267, 479, 221, 834, 0] payload_rows_per_dst=[5540, 6154, 5964, 6474, 6040, 4900, 6406, 4224] dup_rows_per_dst=[2624, 2234, 2564, 3252, 2380, 2030, 3808, 942] combine_rows_per_src=[5540, 6154, 5964, 6474, 6040, 4900, 6406, 4224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44814 payload_rows_remote=38498 dup_rows=20722 dup_rows_remote=17248 dedup_ratio=0.316193 router_tokens_per_expert=[120, 31, 55, 75, 10, 36, 1388, 1395, 408, 14, 151, 1280, 1, 91, 0, 218, 311, 531, 1634, 2, 533, 402, 8, 255, 997, 359, 306, 968, 3, 1077, 901, 0, 5, 7, 200, 2269, 0, 0, 24, 519, 1133, 394, 371, 4, 2395, 609, 1056, 1102, 72, 1434, 1039, 141, 1291, 443, 48, 285, 1136, 1062, 174, 180, 677, 143, 993, 2] payload_rows_per_dst=[5484, 3782, 5366, 4694, 6236, 7108, 5828, 6316] dup_rows_per_dst=[2308, 544, 1986, 1022, 3318, 5448, 2622, 3474] combine_rows_per_src=[5484, 3782, 5366, 4694, 6236, 7108, 5828, 6316] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45398 payload_rows_remote=38468 dup_rows=20138 dup_rows_remote=16066 dedup_ratio=0.307281 router_tokens_per_expert=[188, 60, 169, 101, 40, 11, 855, 1454, 1314, 235, 190, 1220, 0, 87, 1, 288, 175, 1488, 1627, 2, 317, 340, 8, 688, 1640, 252, 485, 661, 37, 1268, 793, 4, 44, 23, 176, 1496, 8, 4, 0, 423, 554, 751, 715, 32, 2166, 555, 890, 1300, 15, 947, 1106, 357, 1057, 193, 115, 36, 1106, 492, 172, 376, 519, 208, 933, 1] payload_rows_per_dst=[5386, 6930, 7176, 6068, 5124, 4500, 5584, 4630] dup_rows_per_dst=[2150, 4072, 4668, 2660, 1760, 1318, 2068, 1442] combine_rows_per_src=[5386, 6930, 7176, 6068, 5124, 4500, 5584, 4630] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=294991872 dispatch_meta_bytes=230320 combine_payload_bytes=294991872 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=295222192 ingress_bytes=515192832 link_bytes=515192832 token_bytes=590214064 weight_movement_bytes=220200960 total_nvlink_bytes=810415024 weight_to_token_ratio=0.373087 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=13952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.960050695e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285587456 dispatch_meta_bytes=232056 combine_payload_bytes=285587456 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=285819512 ingress_bytes=395687936 link_bytes=395687936 token_bytes=571406968 weight_movement_bytes=110100480 total_nvlink_bytes=681507448 weight_to_token_ratio=0.192683 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.881664002e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=295464960 dispatch_meta_bytes=233704 combine_payload_bytes=295464960 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=295698664 ingress_bytes=735866880 link_bytes=735866880 token_bytes=591163624 weight_movement_bytes=440401920 total_nvlink_bytes=1031565544 weight_to_token_ratio=0.744975 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279793534e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=299866112 dispatch_meta_bytes=235024 combine_payload_bytes=299866112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300101136 ingress_bytes=409966592 link_bytes=409966592 token_bytes=599967248 weight_movement_bytes=110100480 total_nvlink_bytes=710067728 weight_to_token_ratio=0.183511 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=13184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.129993314e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281186304 dispatch_meta_bytes=223240 combine_payload_bytes=281186304 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=281409544 ingress_bytes=611487744 link_bytes=611487744 token_bytes=562595848 weight_movement_bytes=330301440 total_nvlink_bytes=892897288 weight_to_token_ratio=0.587103 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=9088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.063477759e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275953664 dispatch_meta_bytes=222984 combine_payload_bytes=275953664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=276176648 ingress_bytes=606255104 link_bytes=606255104 token_bytes=552130312 weight_movement_bytes=330301440 total_nvlink_bytes=882431752 weight_to_token_ratio=0.598231 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=8448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.054377338e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275738624 dispatch_meta_bytes=218136 combine_payload_bytes=275738624 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275956760 ingress_bytes=385839104 link_bytes=385839104 token_bytes=551695384 weight_movement_bytes=110100480 total_nvlink_bytes=661795864 weight_to_token_ratio=0.199568 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.710376615e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45540 payload_rows_remote=40712 dup_rows=19996 dup_rows_remote=17890 dedup_ratio=0.305115 router_tokens_per_expert=[107, 39, 106, 73, 2, 38, 698, 1501, 154, 124, 192, 630, 1, 47, 39, 403, 245, 545, 1451, 60, 447, 715, 11, 380, 1420, 190, 381, 1162, 6, 1883, 1147, 13, 58, 19, 65, 1198, 22, 0, 11, 952, 584, 536, 522, 1, 3009, 535, 903, 941, 329, 1458, 1298, 302, 450, 293, 91, 348, 794, 1185, 439, 729, 417, 188, 825, 56] payload_rows_per_dst=[4828, 6894, 6286, 6772, 5914, 4472, 6160, 4214] dup_rows_per_dst=[2106, 2304, 3304, 4236, 2502, 1534, 2978, 1032] combine_rows_per_src=[4828, 6894, 6286, 6772, 5914, 4472, 6160, 4214] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58908 payload_rows_remote=50722 dup_rows=6628 dup_rows_remote=6128 dedup_ratio=0.101135 router_tokens_per_expert=[1, 3521, 0, 524, 0, 4091, 62, 37, 0, 4, 6, 0, 0, 0, 0, 1, 0, 2074, 3626, 118, 0, 1483, 0, 3108, 1, 3918, 211, 3, 0, 1, 0, 0, 2, 0, 0, 0, 1, 0, 0, 249, 462, 0, 5, 0, 134, 0, 0, 0, 0, 0, 0, 3884, 748, 3310, 0, 0, 131, 160, 12, 0, 0, 0, 873, 7] payload_rows_per_dst=[7324, 7254, 6106, 7868, 8186, 7912, 7160, 7098] dup_rows_per_dst=[966, 20, 1244, 400, 500, 1058, 956, 1484] combine_rows_per_src=[7324, 7254, 6106, 7868, 8186, 7912, 7160, 7098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=291823616 dispatch_meta_bytes=234408 combine_payload_bytes=291823616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=292058024 ingress_bytes=401924096 link_bytes=401924096 token_bytes=583881640 weight_movement_bytes=110100480 total_nvlink_bytes=693982120 weight_to_token_ratio=0.188566 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.990121081e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57534 payload_rows_remote=51894 dup_rows=8002 dup_rows_remote=7436 dedup_ratio=0.122101 router_tokens_per_expert=[0, 3418, 0, 733, 1, 4088, 60, 25, 0, 20, 14, 0, 0, 0, 2, 2, 0, 1586, 3397, 72, 0, 2754, 0, 3257, 36, 3871, 207, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 149, 963, 2, 11, 0, 189, 2, 2, 0, 0, 10, 1, 3800, 514, 2569, 9, 0, 191, 10, 13, 0, 0, 1, 778, 5] payload_rows_per_dst=[7188, 6830, 6914, 7780, 8180, 7988, 5640, 7014] dup_rows_per_dst=[1286, 40, 1910, 454, 300, 1950, 566, 1496] combine_rows_per_src=[7188, 6830, 6914, 7780, 8180, 7988, 5640, 7014] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58374 payload_rows_remote=50688 dup_rows=7162 dup_rows_remote=6310 dedup_ratio=0.109283 router_tokens_per_expert=[0, 3266, 3, 565, 0, 4029, 32, 13, 0, 2, 2, 0, 25, 0, 0, 0, 0, 1341, 3811, 54, 0, 2494, 0, 3058, 73, 3850, 502, 0, 0, 0, 7, 0, 0, 4, 0, 0, 0, 0, 0, 490, 347, 0, 3, 0, 173, 0, 0, 0, 0, 0, 0, 3746, 592, 3073, 0, 14, 136, 201, 48, 2, 12, 0, 798, 2] payload_rows_per_dst=[6884, 7634, 6428, 7866, 8104, 7686, 6704, 7068] dup_rows_per_dst=[874, 46, 1350, 998, 942, 852, 654, 1446] combine_rows_per_src=[6884, 7634, 6428, 7866, 8104, 7686, 6704, 7068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52414 payload_rows_remote=45354 dup_rows=13122 dup_rows_remote=11320 dedup_ratio=0.200226 router_tokens_per_expert=[0, 3035, 0, 662, 0, 4077, 12, 39, 6, 0, 2, 0, 0, 2, 0, 0, 0, 1782, 3627, 65, 1, 3011, 0, 2993, 9, 3972, 286, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 121, 172, 3, 2, 0, 179, 0, 0, 0, 0, 0, 0, 3871, 417, 2981, 0, 0, 853, 45, 7, 0, 0, 1, 527, 5] payload_rows_per_dst=[4316, 7266, 7880, 7960, 8176, 2154, 7602, 7060] dup_rows_per_dst=[476, 8, 4542, 574, 226, 180, 5314, 1802] combine_rows_per_src=[4316, 7266, 7880, 7960, 8176, 2154, 7602, 7060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58894 payload_rows_remote=52088 dup_rows=6642 dup_rows_remote=4982 dedup_ratio=0.101349 router_tokens_per_expert=[0, 3608, 5, 658, 0, 4091, 30, 44, 1, 0, 3, 0, 0, 0, 0, 0, 0, 1273, 3814, 115, 0, 2840, 5, 3019, 9, 3585, 251, 0, 2, 1, 0, 0, 2, 3, 0, 0, 1, 1, 0, 148, 218, 2, 0, 0, 194, 0, 0, 0, 0, 0, 4, 3901, 606, 3114, 0, 0, 189, 44, 27, 0, 1, 0, 944, 15] payload_rows_per_dst=[7474, 7630, 6806, 7242, 8190, 7918, 6732, 6902] dup_rows_per_dst=[1216, 6, 1660, 454, 302, 712, 716, 1576] combine_rows_per_src=[7474, 7630, 6806, 7242, 8190, 7918, 6732, 6902] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57422 payload_rows_remote=49548 dup_rows=8114 dup_rows_remote=7314 dedup_ratio=0.123810 router_tokens_per_expert=[0, 3187, 0, 512, 0, 4073, 52, 4, 0, 0, 28, 0, 0, 0, 0, 0, 0, 2100, 3450, 119, 0, 2440, 0, 3031, 32, 3887, 417, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 332, 644, 0, 1, 0, 616, 0, 0, 0, 0, 2, 0, 3405, 731, 3089, 0, 0, 101, 160, 12, 0, 0, 0, 342, 0] payload_rows_per_dst=[6662, 6916, 6988, 7874, 8158, 7412, 6810, 6602] dup_rows_per_dst=[848, 40, 2330, 800, 652, 1920, 834, 690] combine_rows_per_src=[6662, 6916, 6988, 7874, 8158, 7412, 6810, 6602] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363575296 dispatch_meta_bytes=227400 combine_payload_bytes=363575296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363802696 ingress_bytes=473675776 link_bytes=473675776 token_bytes=727377992 weight_movement_bytes=110100480 total_nvlink_bytes=837478472 weight_to_token_ratio=0.151366 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.238000807e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56526 payload_rows_remote=48842 dup_rows=9010 dup_rows_remote=6450 dedup_ratio=0.137482 router_tokens_per_expert=[0, 3512, 0, 385, 0, 4095, 61, 13, 0, 9, 4, 0, 0, 0, 0, 0, 0, 1451, 3617, 73, 0, 2624, 0, 2715, 6, 3979, 241, 0, 0, 0, 1, 0, 0, 14, 0, 0, 0, 0, 0, 440, 438, 0, 18, 1, 322, 0, 1, 0, 0, 0, 1, 3768, 924, 2949, 0, 0, 442, 160, 60, 0, 0, 0, 442, 2] payload_rows_per_dst=[7174, 7684, 4626, 7976, 8190, 7696, 6764, 6416] dup_rows_per_dst=[768, 2560, 686, 478, 908, 1400, 984, 1226] combine_rows_per_src=[7174, 7684, 4626, 7976, 8190, 7696, 6764, 6416] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=371976192 dispatch_meta_bytes=237320 combine_payload_bytes=371976192 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=372213512 ingress_bytes=592177152 link_bytes=592177152 token_bytes=744189704 weight_movement_bytes=220200960 total_nvlink_bytes=964390664 weight_to_token_ratio=0.295894 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029893464e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=363331584 dispatch_meta_bytes=227992 combine_payload_bytes=363331584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363559576 ingress_bytes=473432064 link_bytes=473432064 token_bytes=726891160 weight_movement_bytes=110100480 total_nvlink_bytes=836991640 weight_to_token_ratio=0.151468 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.233762255e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=325097472 dispatch_meta_bytes=226696 combine_payload_bytes=325097472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325324168 ingress_bytes=435197952 link_bytes=435197952 token_bytes=650421640 weight_movement_bytes=110100480 total_nvlink_bytes=760522120 weight_to_token_ratio=0.169276 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.568808163e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=373366784 dispatch_meta_bytes=228280 combine_payload_bytes=373366784 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=373595064 ingress_bytes=813768704 link_bytes=813768704 token_bytes=746961848 weight_movement_bytes=440401920 total_nvlink_bytes=1187363768 weight_to_token_ratio=0.589591 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.415277618e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=355160064 dispatch_meta_bytes=227448 combine_payload_bytes=355160064 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=355387512 ingress_bytes=575361024 link_bytes=575361024 token_bytes=710547576 weight_movement_bytes=220200960 total_nvlink_bytes=930748536 weight_to_token_ratio=0.309903 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.000647451e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=350099456 dispatch_meta_bytes=221168 combine_payload_bytes=350099456 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=350320624 ingress_bytes=570300416 link_bytes=570300416 token_bytes=700420080 weight_movement_bytes=220200960 total_nvlink_bytes=920621040 weight_to_token_ratio=0.314384 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=28800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.918462217e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51112 payload_rows_remote=43756 dup_rows=14424 dup_rows_remote=9700 dedup_ratio=0.220093 router_tokens_per_expert=[0, 2650, 0, 502, 18, 3790, 98, 136, 1, 4, 13, 0, 47, 0, 1, 2, 1, 2049, 3405, 124, 39, 2328, 0, 2819, 30, 3442, 288, 5, 5, 0, 12, 0, 0, 68, 0, 1, 0, 3, 0, 417, 372, 5, 8, 5, 661, 1, 0, 13, 2, 6, 2, 3284, 740, 3537, 0, 5, 831, 186, 5, 1, 0, 11, 656, 139] payload_rows_per_dst=[7356, 7826, 5060, 6342, 4196, 7230, 7034, 6068] dup_rows_per_dst=[4724, 3776, 1006, 470, 326, 1468, 1066, 1588] combine_rows_per_src=[7356, 7826, 5060, 6342, 4196, 7230, 7034, 6068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57556 payload_rows_remote=50200 dup_rows=7980 dup_rows_remote=7292 dedup_ratio=0.121765 router_tokens_per_expert=[0, 2, 0, 0, 133, 0, 0, 0, 1, 1, 0, 4094, 3850, 4019, 308, 0, 0, 0, 0, 1, 4032, 0, 0, 0, 0, 1058, 1, 0, 0, 0, 0, 0, 0, 435, 0, 3574, 0, 2, 9, 2, 0, 2, 0, 1, 3910, 0, 1625, 0, 0, 0, 0, 1, 3954, 0, 669, 0, 0, 0, 2, 0, 1080, 0, 0, 2] payload_rows_per_dst=[8192, 4044, 8066, 7990, 7356, 5844, 7930, 8134] dup_rows_per_dst=[266, 56, 0, 1948, 688, 1632, 1318, 2072] combine_rows_per_src=[8192, 4044, 8066, 7990, 7356, 5844, 7930, 8134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52762 payload_rows_remote=45182 dup_rows=12774 dup_rows_remote=11594 dedup_ratio=0.194916 router_tokens_per_expert=[1, 0, 0, 1, 254, 0, 0, 0, 13, 3, 0, 4092, 3875, 3967, 311, 0, 0, 1, 0, 0, 4030, 0, 1, 0, 0, 1557, 0, 0, 1, 0, 0, 0, 1, 648, 1, 3752, 0, 0, 18, 0, 0, 15, 7, 0, 3756, 0, 1590, 0, 0, 0, 0, 0, 3740, 0, 640, 0, 2, 0, 0, 0, 459, 3, 0, 29] payload_rows_per_dst=[8188, 654, 8064, 4384, 7728, 8180, 7580, 7984] dup_rows_per_dst=[508, 0, 0, 800, 1112, 8238, 1180, 936] combine_rows_per_src=[8188, 654, 8064, 4384, 7728, 8180, 7580, 7984] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56218 payload_rows_remote=48096 dup_rows=9318 dup_rows_remote=7552 dedup_ratio=0.142181 router_tokens_per_expert=[0, 0, 0, 13, 640, 0, 0, 0, 0, 0, 0, 4096, 3859, 3861, 380, 0, 0, 0, 0, 0, 4071, 0, 0, 0, 0, 1317, 0, 0, 0, 0, 0, 0, 0, 1062, 0, 3621, 0, 6, 128, 15, 0, 54, 0, 0, 3677, 0, 1031, 0, 0, 0, 0, 0, 3906, 0, 624, 0, 0, 0, 29, 0, 378, 0, 0, 0] payload_rows_per_dst=[8192, 760, 8142, 7590, 7732, 8122, 7878, 7802] dup_rows_per_dst=[1306, 0, 0, 2398, 1932, 1766, 1182, 734] combine_rows_per_src=[8192, 760, 8142, 7590, 7732, 8122, 7878, 7802] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46826 payload_rows_remote=38914 dup_rows=18710 dup_rows_remote=18110 dedup_ratio=0.285492 router_tokens_per_expert=[0, 1, 0, 0, 104, 4, 1, 0, 0, 3, 1, 4094, 3881, 3950, 246, 0, 0, 0, 0, 0, 4066, 1, 2, 0, 0, 1191, 0, 24, 1, 2, 0, 0, 0, 284, 0, 3930, 0, 0, 25, 6, 1, 0, 1, 0, 3995, 0, 1920, 0, 0, 0, 0, 0, 4053, 3, 672, 0, 0, 0, 0, 0, 305, 0, 0, 1] payload_rows_per_dst=[3464, 5020, 8136, 2400, 3548, 8190, 8156, 7912] dup_rows_per_dst=[122, 302, 2, 36, 78, 11406, 6164, 600] combine_rows_per_src=[3464, 5020, 8136, 2400, 3548, 8190, 8156, 7912] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58546 payload_rows_remote=50380 dup_rows=6990 dup_rows_remote=6988 dedup_ratio=0.106659 router_tokens_per_expert=[1, 1, 1, 0, 182, 0, 0, 1, 1, 0, 0, 4093, 3860, 4040, 304, 3, 0, 4, 0, 0, 4079, 1, 0, 0, 4, 1026, 0, 0, 1, 0, 0, 1, 0, 631, 0, 3708, 0, 0, 20, 0, 1, 10, 0, 1, 3935, 0, 1230, 0, 3, 1, 0, 0, 3979, 0, 876, 0, 1, 2, 0, 0, 748, 0, 0, 19] payload_rows_per_dst=[8190, 8060, 8166, 7956, 7576, 2476, 7992, 8130] dup_rows_per_dst=[368, 276, 2, 1978, 1142, 8, 1726, 1490] combine_rows_per_src=[8190, 8060, 8166, 7956, 7576, 2476, 7992, 8130] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=313643008 dispatch_meta_bytes=213824 combine_payload_bytes=313643008 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=313856832 ingress_bytes=643944448 link_bytes=643944448 token_bytes=627499840 weight_movement_bytes=330301440 total_nvlink_bytes=957801280 weight_to_token_ratio=0.526377 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.119925306e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57006 payload_rows_remote=49670 dup_rows=8530 dup_rows_remote=6642 dedup_ratio=0.130157 router_tokens_per_expert=[0, 0, 0, 1, 456, 0, 0, 0, 0, 0, 0, 4096, 3906, 3880, 585, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 0, 1188, 0, 0, 0, 0, 2, 0, 0, 1783, 0, 3737, 0, 0, 6, 0, 0, 0, 0, 0, 3422, 0, 791, 0, 0, 0, 0, 0, 4012, 0, 567, 0, 0, 0, 0, 0, 331, 0, 0, 4] payload_rows_per_dst=[8192, 8136, 8002, 7336, 7934, 1582, 8036, 7788] dup_rows_per_dst=[914, 846, 0, 1888, 3118, 0, 1122, 642] combine_rows_per_src=[8192, 8136, 8002, 7336, 7934, 1582, 8036, 7788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=359833600 dispatch_meta_bytes=229968 combine_payload_bytes=359833600 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=360063568 ingress_bytes=580034560 link_bytes=580034560 token_bytes=719897168 weight_movement_bytes=220200960 total_nvlink_bytes=940098128 weight_to_token_ratio=0.305878 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26368 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.008775499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53118 payload_rows_remote=44942 dup_rows=12418 dup_rows_remote=5886 dedup_ratio=0.189484 router_tokens_per_expert=[0, 0, 0, 1, 412, 0, 0, 0, 0, 0, 0, 4095, 3910, 3866, 302, 0, 0, 0, 0, 0, 3912, 0, 0, 0, 0, 1013, 0, 3, 0, 0, 20, 0, 1, 837, 0, 3703, 0, 0, 8, 0, 0, 1, 0, 0, 3637, 0, 1493, 0, 0, 0, 0, 0, 3980, 0, 772, 0, 0, 0, 0, 0, 796, 0, 0, 6] payload_rows_per_dst=[8192, 8176, 7824, 7542, 7656, 2986, 2816, 7926] dup_rows_per_dst=[824, 6532, 0, 1804, 1442, 2, 404, 1410] combine_rows_per_src=[8192, 8176, 7824, 7542, 7656, 2986, 2816, 7926] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=323864576 dispatch_meta_bytes=227104 combine_payload_bytes=323864576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=324091680 ingress_bytes=544065536 link_bytes=544065536 token_bytes=647956256 weight_movement_bytes=220200960 total_nvlink_bytes=868157216 weight_to_token_ratio=0.339839 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.462194505e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=344752128 dispatch_meta_bytes=222592 combine_payload_bytes=344752128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=344974720 ingress_bytes=564953088 link_bytes=564953088 token_bytes=689726848 weight_movement_bytes=220200960 total_nvlink_bytes=909927808 weight_to_token_ratio=0.319258 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.825463388e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=278935552 dispatch_meta_bytes=228096 combine_payload_bytes=278935552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279163648 ingress_bytes=389036032 link_bytes=389036032 token_bytes=558099200 weight_movement_bytes=110100480 total_nvlink_bytes=668199680 weight_to_token_ratio=0.197278 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765976451e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=361123840 dispatch_meta_bytes=229472 combine_payload_bytes=361123840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361353312 ingress_bytes=471224320 link_bytes=471224320 token_bytes=722477152 weight_movement_bytes=110100480 total_nvlink_bytes=832577632 weight_to_token_ratio=0.152393 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.195365956e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=356034560 dispatch_meta_bytes=225248 combine_payload_bytes=356034560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=356259808 ingress_bytes=466135040 link_bytes=466135040 token_bytes=712294368 weight_movement_bytes=110100480 total_nvlink_bytes=822394848 weight_to_token_ratio=0.154572 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=23296 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.106855006e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=322144256 dispatch_meta_bytes=203312 combine_payload_bytes=322144256 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=322347568 ingress_bytes=762546176 link_bytes=762546176 token_bytes=644491824 weight_movement_bytes=440401920 total_nvlink_bytes=1084893744 weight_to_token_ratio=0.683332 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.326193217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64554 payload_rows_remote=56362 dup_rows=982 dup_rows_remote=878 dedup_ratio=0.014984 router_tokens_per_expert=[0, 4, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 4092, 3611, 0, 4095, 1, 0, 0, 0, 4095, 0, 4095, 52, 0, 0, 1, 0, 0, 2, 0, 0, 0, 508, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8184, 7222, 8192, 8192, 8192, 8190] dup_rows_per_dst=[8, 6, 24, 2, 104, 838, 0, 0] combine_rows_per_src=[8192, 8190, 8184, 7222, 8192, 8192, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46842 payload_rows_remote=38680 dup_rows=18694 dup_rows_remote=18238 dedup_ratio=0.285248 router_tokens_per_expert=[0, 0, 0, 51, 174, 45, 1, 0, 172, 4, 0, 4038, 3722, 4016, 259, 0, 0, 0, 0, 0, 3695, 1, 1, 2, 0, 1070, 4, 4, 0, 0, 65, 0, 0, 522, 2, 3520, 0, 0, 13, 57, 3, 16, 1, 0, 3600, 0, 2532, 0, 0, 0, 18, 0, 3839, 0, 1008, 0, 0, 0, 4, 16, 266, 26, 1, 0] payload_rows_per_dst=[8162, 8192, 7934, 7782, 5824, 5078, 2040, 1830] dup_rows_per_dst=[456, 14482, 1198, 1704, 670, 26, 12, 146] combine_rows_per_src=[8162, 8192, 7934, 7782, 5824, 5078, 2040, 1830] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64142 payload_rows_remote=55956 dup_rows=1394 dup_rows_remote=1372 dedup_ratio=0.021271 router_tokens_per_expert=[0, 20, 0, 2, 4, 0, 0, 0, 25, 1, 0, 3, 0, 1, 0, 0, 0, 11, 0, 1, 0, 0, 1, 4091, 3430, 0, 4092, 0, 0, 1, 0, 4094, 0, 4091, 44, 1, 3, 0, 0, 1, 4, 0, 1, 0, 693, 3974, 0, 0, 4091, 0, 0, 0, 0, 1, 1, 0, 3, 4082, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 6862, 8192, 8164, 8184, 8170] dup_rows_per_dst=[48, 52, 22, 0, 88, 1180, 2, 2] combine_rows_per_src=[8192, 8192, 8186, 6862, 8192, 8164, 8184, 8170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63438 payload_rows_remote=55246 dup_rows=2098 dup_rows_remote=2094 dedup_ratio=0.032013 router_tokens_per_expert=[1, 3, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 1, 4084, 3065, 1, 4094, 1, 0, 3, 6, 4095, 0, 4095, 45, 0, 0, 0, 2, 2, 0, 0, 0, 0, 964, 4075, 1, 0, 4095, 0, 0, 0, 2, 0, 0, 1, 0, 4093, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8170, 6132, 8192, 8186, 8192, 8186] dup_rows_per_dst=[10, 2, 72, 20, 96, 1894, 4, 0] combine_rows_per_src=[8192, 8188, 8170, 6132, 8192, 8186, 8192, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62936 payload_rows_remote=54746 dup_rows=2600 dup_rows_remote=176 dedup_ratio=0.039673 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4070, 2845, 0, 4096, 0, 0, 0, 7, 4096, 0, 4076, 25, 0, 0, 7, 0, 0, 0, 0, 2, 0, 1224, 4081, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8140, 5692, 8164, 8190, 8192, 8174] dup_rows_per_dst=[6, 84, 22, 12, 52, 2424, 0, 0] combine_rows_per_src=[8192, 8192, 8140, 5692, 8164, 8190, 8192, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64298 payload_rows_remote=57324 dup_rows=1238 dup_rows_remote=1238 dedup_ratio=0.018890 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 4086, 3487, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 4005, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8172, 6974, 8192, 8192, 8192, 8192] dup_rows_per_dst=[12, 2, 16, 0, 52, 1156, 0, 0] combine_rows_per_src=[8192, 8192, 8172, 6974, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62640 payload_rows_remote=54480 dup_rows=2896 dup_rows_remote=2618 dedup_ratio=0.044189 router_tokens_per_expert=[0, 24, 0, 0, 0, 0, 0, 1, 49, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 4079, 3393, 0, 4096, 1, 1, 0, 0, 4094, 0, 4094, 49, 0, 0, 0, 2, 0, 1, 0, 0, 0, 674, 4029, 1, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 4076, 4, 0, 0, 0, 0, 1] payload_rows_per_dst=[7910, 7598, 8072, 6886, 7680, 8146, 8188, 8160] dup_rows_per_dst=[44, 98, 4, 782, 90, 1598, 2, 278] combine_rows_per_src=[7910, 7598, 8072, 6886, 7680, 8146, 8188, 8160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=277258240 dispatch_meta_bytes=227672 combine_payload_bytes=277258240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277485912 ingress_bytes=387358720 link_bytes=387358720 token_bytes=554744152 weight_movement_bytes=110100480 total_nvlink_bytes=664844632 weight_to_token_ratio=0.198471 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.736805236e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=404002816 dispatch_meta_bytes=228960 combine_payload_bytes=404002816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=404231776 ingress_bytes=734304256 link_bytes=734304256 token_bytes=808234592 weight_movement_bytes=330301440 total_nvlink_bytes=1138536032 weight_to_token_ratio=0.408670 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.277075873e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61080 payload_rows_remote=52888 dup_rows=4456 dup_rows_remote=4456 dedup_ratio=0.067993 router_tokens_per_expert=[0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 4, 4096, 3095, 0, 4096, 0, 0, 0, 0, 4096, 0, 4095, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 3992, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7278, 8190, 4654, 8192, 8190] dup_rows_per_dst=[12, 0, 16, 3518, 12, 896, 2, 0] combine_rows_per_src=[8192, 8192, 8192, 7278, 8190, 4654, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=401092608 dispatch_meta_bytes=229312 combine_payload_bytes=401092608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401321920 ingress_bytes=621293568 link_bytes=621293568 token_bytes=802414528 weight_movement_bytes=220200960 total_nvlink_bytes=1022615488 weight_to_token_ratio=0.274423 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080531700e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=396003328 dispatch_meta_bytes=229360 combine_payload_bytes=396003328 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=396232688 ingress_bytes=506103808 link_bytes=506103808 token_bytes=792236016 weight_movement_bytes=110100480 total_nvlink_bytes=902336496 weight_to_token_ratio=0.138974 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.801977619e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=392419328 dispatch_meta_bytes=219688 combine_payload_bytes=392419328 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=392639016 ingress_bytes=612620288 link_bytes=612620288 token_bytes=785058344 weight_movement_bytes=220200960 total_nvlink_bytes=1005259304 weight_to_token_ratio=0.280490 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=25984 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.065447440e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=410898432 dispatch_meta_bytes=234248 combine_payload_bytes=410898432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411132680 ingress_bytes=741199872 link_bytes=741199872 token_bytes=822031112 weight_movement_bytes=330301440 total_nvlink_bytes=1152332552 weight_to_token_ratio=0.401811 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289068484e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390512640 dispatch_meta_bytes=228392 combine_payload_bytes=390512640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=390741032 ingress_bytes=500613120 link_bytes=500613120 token_bytes=781253672 weight_movement_bytes=110100480 total_nvlink_bytes=891354152 weight_to_token_ratio=0.140928 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.706485524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=379101184 dispatch_meta_bytes=229376 combine_payload_bytes=379101184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379330560 ingress_bytes=489201664 link_bytes=489201664 token_bytes=758431744 weight_movement_bytes=110100480 total_nvlink_bytes=868532224 weight_to_token_ratio=0.145169 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.508021535e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64264 payload_rows_remote=56072 dup_rows=1272 dup_rows_remote=34 dedup_ratio=0.019409 router_tokens_per_expert=[0, 1, 0, 4096, 0, 0, 4094, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 1, 0, 1, 0, 3, 1, 0, 0, 4040, 0, 0, 4093, 0, 3539, 0, 0, 0, 619, 0, 0, 0, 4096, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8152, 8192, 7078, 8192, 8192, 8186, 8082] dup_rows_per_dst=[0, 0, 12, 0, 1238, 4, 0, 18] combine_rows_per_src=[8190, 8152, 8192, 7078, 8192, 8192, 8186, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58956 payload_rows_remote=50764 dup_rows=6580 dup_rows_remote=6438 dedup_ratio=0.100403 router_tokens_per_expert=[0, 68, 1, 2, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 5, 4094, 3176, 0, 4096, 15, 0, 0, 85, 4096, 0, 4096, 24, 0, 0, 0, 0, 0, 2, 0, 19, 1, 891, 3802, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 3, 1, 3996, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8188, 8192, 1824, 8184, 7994] dup_rows_per_dst=[142, 356, 54, 5968, 48, 2, 10, 0] combine_rows_per_src=[8192, 8192, 8190, 8188, 8192, 1824, 8184, 7994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63990 payload_rows_remote=55798 dup_rows=1546 dup_rows_remote=1528 dedup_ratio=0.023590 router_tokens_per_expert=[0, 0, 1, 4093, 5, 0, 4094, 0, 0, 0, 0, 4075, 0, 2, 0, 3, 0, 1, 7, 4, 0, 0, 0, 0, 0, 3960, 0, 3, 4090, 0, 3477, 0, 0, 1, 741, 0, 0, 0, 4091, 0, 13, 0, 2, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 5, 1, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8158, 8192, 6956, 8184, 8192, 8186, 7930] dup_rows_per_dst=[8, 2, 18, 4, 1482, 24, 4, 4] combine_rows_per_src=[8192, 8158, 8192, 6956, 8184, 8192, 8186, 7930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65026 payload_rows_remote=56834 dup_rows=510 dup_rows_remote=510 dedup_ratio=0.007782 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 4084, 0, 0, 0, 0, 0, 0, 2, 0, 0, 11, 0, 0, 0, 4028, 0, 0, 4095, 0, 3916, 0, 0, 0, 224, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8168, 8192, 7832, 8192, 8192, 8190, 8068] dup_rows_per_dst=[0, 0, 26, 0, 448, 0, 0, 36] combine_rows_per_src=[8192, 8168, 8192, 7832, 8192, 8192, 8190, 8068] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64106 payload_rows_remote=57164 dup_rows=1430 dup_rows_remote=1430 dedup_ratio=0.021820 router_tokens_per_expert=[0, 0, 2, 4096, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4008, 0, 0, 4096, 0, 3471, 0, 0, 0, 703, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 6942, 8192, 8190, 8192, 8018] dup_rows_per_dst=[4, 0, 0, 0, 1406, 0, 0, 20] combine_rows_per_src=[8192, 8188, 8192, 6942, 8192, 8190, 8192, 8018] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63652 payload_rows_remote=55506 dup_rows=1884 dup_rows_remote=1372 dedup_ratio=0.028748 router_tokens_per_expert=[0, 0, 0, 4094, 1, 0, 4095, 0, 0, 0, 2, 4092, 0, 0, 1, 0, 0, 7, 1, 2, 0, 0, 0, 1, 0, 4070, 0, 5, 4091, 0, 3742, 0, 0, 0, 338, 0, 0, 0, 4094, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 33, 0, 0, 0, 0, 1, 1, 1] payload_rows_per_dst=[7552, 8186, 8192, 7014, 8188, 8188, 8186, 8146] dup_rows_per_dst=[0, 630, 18, 4, 676, 14, 30, 512] combine_rows_per_src=[7552, 8186, 8192, 7014, 8188, 8188, 8186, 8146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64714 payload_rows_remote=56524 dup_rows=822 dup_rows_remote=822 dedup_ratio=0.012543 router_tokens_per_expert=[1, 2, 0, 4095, 0, 0, 4095, 0, 0, 0, 0, 4089, 0, 0, 1, 0, 0, 9, 3, 1, 0, 0, 4, 0, 0, 4080, 0, 3, 4095, 0, 3703, 0, 0, 0, 356, 0, 0, 0, 4094, 0, 2, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 1, 6, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8180, 8192, 7408, 8188, 8192, 8190, 8172] dup_rows_per_dst=[4, 0, 32, 4, 712, 6, 0, 64] combine_rows_per_src=[8192, 8180, 8192, 7408, 8188, 8192, 8190, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=363876352 dispatch_meta_bytes=228808 combine_payload_bytes=363876352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364105160 ingress_bytes=473976832 link_bytes=473976832 token_bytes=727981512 weight_movement_bytes=110100480 total_nvlink_bytes=838081992 weight_to_token_ratio=0.151241 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.243236666e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=401924096 dispatch_meta_bytes=224424 combine_payload_bytes=401924096 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402148520 ingress_bytes=622125056 link_bytes=622125056 token_bytes=804072616 weight_movement_bytes=220200960 total_nvlink_bytes=1024273576 weight_to_token_ratio=0.273857 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.081977794e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64534 payload_rows_remote=56348 dup_rows=1002 dup_rows_remote=1002 dedup_ratio=0.015289 router_tokens_per_expert=[0, 1, 31, 4096, 0, 0, 4096, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 4026, 0, 0, 4096, 0, 3655, 0, 0, 0, 388, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8186, 8192, 7310, 8190, 8192, 8192, 8080] dup_rows_per_dst=[64, 0, 10, 0, 776, 0, 0, 152] combine_rows_per_src=[8192, 8186, 8192, 7310, 8190, 8192, 8192, 8080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=399960064 dispatch_meta_bytes=229304 combine_payload_bytes=399960064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400189368 ingress_bytes=510060544 link_bytes=510060544 token_bytes=800149432 weight_movement_bytes=110100480 total_nvlink_bytes=910249912 weight_to_token_ratio=0.137600 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.870791766e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=407386112 dispatch_meta_bytes=229376 combine_payload_bytes=407386112 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407615488 ingress_bytes=627587072 link_bytes=627587072 token_bytes=815001600 weight_movement_bytes=220200960 total_nvlink_bytes=1035202560 weight_to_token_ratio=0.270185 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.091477139e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=409751552 dispatch_meta_bytes=234376 combine_payload_bytes=409751552 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=409985928 ingress_bytes=960253952 link_bytes=960253952 token_bytes=819737480 weight_movement_bytes=550502400 total_nvlink_bytes=1370239880 weight_to_token_ratio=0.671559 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.670039557e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=397867008 dispatch_meta_bytes=227512 combine_payload_bytes=397867008 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=398094520 ingress_bytes=618067968 link_bytes=618067968 token_bytes=795961528 weight_movement_bytes=220200960 total_nvlink_bytes=1016162488 weight_to_token_ratio=0.276648 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.074921851e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=405164032 dispatch_meta_bytes=229384 combine_payload_bytes=405164032 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=405393416 ingress_bytes=625364992 link_bytes=625364992 token_bytes=810557448 weight_movement_bytes=220200960 total_nvlink_bytes=1030758408 weight_to_token_ratio=0.271666 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.087612576e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=403902464 dispatch_meta_bytes=229400 combine_payload_bytes=403902464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404131864 ingress_bytes=514002944 link_bytes=514002944 token_bytes=808034328 weight_movement_bytes=110100480 total_nvlink_bytes=918134808 weight_to_token_ratio=0.136257 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.939356586e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60976 payload_rows_remote=52784 dup_rows=4560 dup_rows_remote=4302 dedup_ratio=0.069580 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1426, 0, 0, 0, 0, 3997, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 4080, 2662, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8190, 8124, 5252, 7994, 8192, 8190, 8192, 6842] dup_rows_per_dst=[0, 0, 1402, 0, 258, 0, 58, 2842] combine_rows_per_src=[8190, 8124, 5252, 7994, 8192, 8190, 8192, 6842] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57680 payload_rows_remote=49488 dup_rows=7856 dup_rows_remote=7294 dedup_ratio=0.119873 router_tokens_per_expert=[0, 60, 22, 4096, 0, 0, 4096, 0, 2, 0, 0, 3841, 0, 0, 0, 0, 0, 53, 0, 0, 0, 15, 73, 0, 0, 3881, 0, 10, 4090, 0, 3676, 0, 0, 0, 349, 0, 0, 0, 4096, 0, 100, 0, 0, 4072, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7686, 8002, 8150, 1354, 8126, 8180, 7990] dup_rows_per_dst=[562, 0, 74, 6834, 118, 24, 0, 244] combine_rows_per_src=[8192, 7686, 8002, 8150, 1354, 8126, 8180, 7990] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61686 payload_rows_remote=53500 dup_rows=3850 dup_rows_remote=756 dedup_ratio=0.058746 router_tokens_per_expert=[0, 0, 4091, 0, 0, 0, 0, 3, 0, 4023, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1553, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 5, 4093, 0, 0, 0, 4093, 0, 2, 352, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1, 0, 0, 0, 2, 0, 4086, 2429, 1, 0, 0, 1, 4096] payload_rows_per_dst=[8188, 8048, 8186, 7832, 8190, 8192, 8190, 4860] dup_rows_per_dst=[0, 2, 3094, 10, 704, 0, 38, 2] combine_rows_per_src=[8188, 8048, 8186, 7832, 8190, 8192, 8190, 4860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60212 payload_rows_remote=52020 dup_rows=5324 dup_rows_remote=5324 dedup_ratio=0.081238 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1656, 0, 0, 0, 0, 3961, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 508, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 4081, 2161, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8188, 8022, 3312, 7922, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 1016, 0, 16, 4292] combine_rows_per_src=[8188, 8022, 3312, 7922, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60542 payload_rows_remote=52686 dup_rows=4994 dup_rows_remote=4994 dedup_ratio=0.076202 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1385, 0, 0, 0, 0, 3928, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 0, 0, 0, 0, 0, 0, 4096, 1867, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7992, 8192, 7856, 8192, 8192, 8192, 3734] dup_rows_per_dst=[0, 0, 2770, 0, 2200, 0, 24, 0] combine_rows_per_src=[8192, 7992, 8192, 7856, 8192, 8192, 8192, 3734] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59604 payload_rows_remote=51418 dup_rows=5932 dup_rows_remote=426 dedup_ratio=0.090515 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 0, 1, 0, 1, 1, 0, 1, 2387, 0, 0, 0, 1, 3984, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 4094, 0, 0, 226, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 53, 0, 0, 1, 0, 0, 5, 0, 0, 4037, 1609, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8188, 8160, 4778, 7968, 5940, 8192, 8192, 8186] dup_rows_per_dst=[0, 2, 2, 0, 304, 2, 116, 5506] combine_rows_per_src=[8188, 8160, 4778, 7968, 5940, 8192, 8192, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60084 payload_rows_remote=51894 dup_rows=5452 dup_rows_remote=5422 dedup_ratio=0.083191 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 0, 1, 2, 4058, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1548, 1, 0, 0, 0, 3956, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 4095, 0, 0, 302, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 3, 0, 1, 4069, 2433, 0, 0, 1, 0, 4094] payload_rows_per_dst=[8190, 8120, 3098, 7914, 8190, 8190, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 604, 2, 30, 4816] combine_rows_per_src=[8190, 8120, 3098, 7914, 8190, 8190, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=378355712 dispatch_meta_bytes=228344 combine_payload_bytes=378355712 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=378584056 ingress_bytes=818757632 link_bytes=818757632 token_bytes=756939768 weight_movement_bytes=440401920 total_nvlink_bytes=1197341688 weight_to_token_ratio=0.581819 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.423954185e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59868 payload_rows_remote=51880 dup_rows=5668 dup_rows_remote=5668 dedup_ratio=0.086487 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2262, 0, 0, 0, 0, 3468, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 1, 0, 0, 4094, 1994, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 7988, 8188, 6936, 8192, 8192, 8192, 3988] dup_rows_per_dst=[0, 0, 4524, 0, 1072, 0, 72, 0] combine_rows_per_src=[8192, 7988, 8188, 6936, 8192, 8192, 8192, 3988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=354729984 dispatch_meta_bytes=227128 combine_payload_bytes=354729984 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=354957112 ingress_bytes=685031424 link_bytes=685031424 token_bytes=709687096 weight_movement_bytes=330301440 total_nvlink_bytes=1039988536 weight_to_token_ratio=0.465418 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.191382315e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=383488000 dispatch_meta_bytes=217024 combine_payload_bytes=383488000 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383705024 ingress_bytes=493588480 link_bytes=493588480 token_bytes=767193024 weight_movement_bytes=110100480 total_nvlink_bytes=877293504 weight_to_token_ratio=0.143511 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.584315481e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372879360 dispatch_meta_bytes=229376 combine_payload_bytes=372879360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=373108736 ingress_bytes=482979840 link_bytes=482979840 token_bytes=745988096 weight_movement_bytes=110100480 total_nvlink_bytes=856088576 weight_to_token_ratio=0.147590 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.399813783e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=377653248 dispatch_meta_bytes=230720 combine_payload_bytes=377653248 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=377883968 ingress_bytes=597854208 link_bytes=597854208 token_bytes=755537216 weight_movement_bytes=220200960 total_nvlink_bytes=975738176 weight_to_token_ratio=0.291450 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.039766798e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368564224 dispatch_meta_bytes=207376 combine_payload_bytes=368564224 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=368771600 ingress_bytes=698865664 link_bytes=698865664 token_bytes=737335824 weight_movement_bytes=330301440 total_nvlink_bytes=1067637264 weight_to_token_ratio=0.447966 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.215442333e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371976192 dispatch_meta_bytes=229264 combine_payload_bytes=371976192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372205456 ingress_bytes=482076672 link_bytes=482076672 token_bytes=744181648 weight_movement_bytes=110100480 total_nvlink_bytes=854282128 weight_to_token_ratio=0.147948 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.384106206e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=371875840 dispatch_meta_bytes=230192 combine_payload_bytes=371875840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372106032 ingress_bytes=481976320 link_bytes=481976320 token_bytes=743981872 weight_movement_bytes=110100480 total_nvlink_bytes=854082352 weight_to_token_ratio=0.147988 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.382360920e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59896 payload_rows_remote=51704 dup_rows=5640 dup_rows_remote=4646 dedup_ratio=0.086060 router_tokens_per_expert=[4028, 948, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4077, 0, 4096, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 497, 0, 0, 4096, 0, 0, 2264, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 297, 0, 1, 0, 0, 0, 4096, 0, 0, 5, 0] payload_rows_per_dst=[3128, 8192, 8154, 8192, 8192, 7654, 8192, 8192] dup_rows_per_dst=[140, 0, 0, 338, 994, 3558, 600, 10] combine_rows_per_src=[3128, 8192, 8154, 8192, 8192, 7654, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60056 payload_rows_remote=51904 dup_rows=5480 dup_rows_remote=5480 dedup_ratio=0.083618 router_tokens_per_expert=[4022, 1286, 4094, 0, 0, 4, 2, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4075, 0, 4094, 0, 370, 0, 0, 2, 1, 2, 0, 0, 0, 0, 0, 344, 0, 0, 4096, 0, 0, 1372, 0, 41, 1, 0, 7, 0, 0, 4093, 0, 756, 1, 0, 0, 0, 0, 4094, 0, 0, 0, 0] payload_rows_per_dst=[8136, 8192, 8152, 8192, 8192, 2812, 8192, 8188] dup_rows_per_dst=[2492, 16, 0, 746, 688, 16, 1522, 0] combine_rows_per_src=[8136, 8192, 8152, 8192, 8192, 2812, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56348 payload_rows_remote=48156 dup_rows=9188 dup_rows_remote=9172 dedup_ratio=0.140198 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 0, 0, 3536, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2421, 0, 0, 0, 0, 3566, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 213, 0, 0, 0, 0, 0, 15, 0, 0, 3948, 2042, 0, 0, 0, 0, 4096] payload_rows_per_dst=[8192, 8026, 8192, 7894, 4060, 8192, 7708, 4084] dup_rows_per_dst=[16, 1052, 4546, 3210, 230, 0, 134, 0] combine_rows_per_src=[8192, 8026, 8192, 7894, 4060, 8192, 7708, 4084] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59978 payload_rows_remote=51908 dup_rows=5558 dup_rows_remote=3512 dedup_ratio=0.084808 router_tokens_per_expert=[4009, 1429, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4045, 0, 4096, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 635, 0, 0, 4096, 0, 0, 1044, 0, 5, 0, 0, 13, 0, 0, 4096, 0, 998, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0] payload_rows_per_dst=[2858, 8192, 8090, 8192, 8192, 8070, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 186, 1270, 2046, 2022, 34] combine_rows_per_src=[2858, 8192, 8090, 8192, 8192, 8070, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51620 payload_rows_remote=43428 dup_rows=13916 dup_rows_remote=13914 dedup_ratio=0.212341 router_tokens_per_expert=[4043, 1479, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 4070, 0, 4096, 0, 162, 0, 0, 2, 8, 0, 0, 0, 0, 0, 0, 487, 0, 0, 4094, 0, 0, 1215, 2, 12, 1, 0, 12, 0, 0, 4094, 0, 797, 1, 0, 1, 0, 0, 4094, 0, 0, 1, 1] payload_rows_per_dst=[2958, 8188, 8142, 6212, 1552, 8184, 8192, 8192] dup_rows_per_dst=[0, 2, 0, 252, 52, 9920, 3688, 2] combine_rows_per_src=[2958, 8188, 8142, 6212, 1552, 8184, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=370614272 dispatch_meta_bytes=225400 combine_payload_bytes=370614272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370839672 ingress_bytes=480714752 link_bytes=480714752 token_bytes=741453944 weight_movement_bytes=110100480 total_nvlink_bytes=851554424 weight_to_token_ratio=0.148493 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.360420177e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61040 payload_rows_remote=52848 dup_rows=4496 dup_rows_remote=4116 dedup_ratio=0.068604 router_tokens_per_expert=[3950, 817, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 189, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 550, 0, 0, 4096, 0, 0, 1985, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 734, 0, 0, 0, 0, 0, 4096, 0, 0, 3, 0] payload_rows_per_dst=[7996, 8192, 8114, 8192, 8192, 3970, 8192, 8192] dup_rows_per_dst=[1538, 0, 0, 380, 1100, 4, 1468, 6] combine_rows_per_src=[7996, 8192, 8114, 8192, 8192, 3970, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60940 payload_rows_remote=52750 dup_rows=4596 dup_rows_remote=3426 dedup_ratio=0.070129 router_tokens_per_expert=[3969, 1928, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 4067, 1, 4095, 0, 222, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 561, 0, 0, 4095, 0, 0, 948, 0, 2, 0, 1, 4, 0, 0, 4095, 0, 581, 0, 0, 0, 0, 1, 4095, 1, 1, 0, 0] payload_rows_per_dst=[3856, 8190, 8136, 8190, 8190, 7996, 8190, 8192] dup_rows_per_dst=[0, 0, 4, 452, 1122, 1844, 1170, 4] combine_rows_per_src=[3856, 8190, 8136, 8190, 8190, 7996, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57824 payload_rows_remote=49632 dup_rows=7712 dup_rows_remote=7712 dedup_ratio=0.117676 router_tokens_per_expert=[3769, 543, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4011, 0, 4096, 0, 173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 4096, 0, 0, 2393, 0, 3, 0, 0, 1, 0, 0, 4096, 0, 1013, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0] payload_rows_per_dst=[8068, 8192, 8022, 8192, 8192, 4786, 4180, 8192] dup_rows_per_dst=[5774, 0, 0, 346, 764, 6, 822, 0] combine_rows_per_src=[8068, 8192, 8022, 8192, 8192, 4786, 4180, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372047872 dispatch_meta_bytes=229536 combine_payload_bytes=372047872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372277408 ingress_bytes=482148352 link_bytes=482148352 token_bytes=744325280 weight_movement_bytes=110100480 total_nvlink_bytes=854425760 weight_to_token_ratio=0.147920 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.385352839e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=345182208 dispatch_meta_bytes=229312 combine_payload_bytes=345182208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=345411520 ingress_bytes=455282688 link_bytes=455282688 token_bytes=690593728 weight_movement_bytes=110100480 total_nvlink_bytes=800694208 weight_to_token_ratio=0.159429 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.918114756e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=372076544 dispatch_meta_bytes=221680 combine_payload_bytes=372076544 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=372298224 ingress_bytes=592277504 link_bytes=592277504 token_bytes=744374768 weight_movement_bytes=220200960 total_nvlink_bytes=964575728 weight_to_token_ratio=0.295820 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.030067992e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=311291904 dispatch_meta_bytes=229368 combine_payload_bytes=311291904 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=311521272 ingress_bytes=421392384 link_bytes=421392384 token_bytes=622813176 weight_movement_bytes=110100480 total_nvlink_bytes=732913656 weight_to_token_ratio=0.176779 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.328706629e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378814464 dispatch_meta_bytes=227856 combine_payload_bytes=378814464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=379042320 ingress_bytes=709115904 link_bytes=709115904 token_bytes=757856784 weight_movement_bytes=330301440 total_nvlink_bytes=1088158224 weight_to_token_ratio=0.435836 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.233269187e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=378112000 dispatch_meta_bytes=224704 combine_payload_bytes=378112000 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=378336704 ingress_bytes=598312960 link_bytes=598312960 token_bytes=756448704 weight_movement_bytes=220200960 total_nvlink_bytes=976649664 weight_to_token_ratio=0.291098 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.040564643e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=355762176 dispatch_meta_bytes=229376 combine_payload_bytes=355762176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=355991552 ingress_bytes=465862656 link_bytes=465862656 token_bytes=711753728 weight_movement_bytes=110100480 total_nvlink_bytes=821854208 weight_to_token_ratio=0.154689 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.102117800e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61586 payload_rows_remote=53394 dup_rows=3950 dup_rows_remote=3946 dedup_ratio=0.060272 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3331, 4096, 0, 2886, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094, 0, 2, 4096, 0, 0, 0, 612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6662, 5772, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 2722, 0, 4, 0, 1224, 0] combine_rows_per_src=[6662, 5772, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63582 payload_rows_remote=55402 dup_rows=1954 dup_rows_remote=722 dedup_ratio=0.029816 router_tokens_per_expert=[0, 2, 4096, 0, 0, 0, 0, 3814, 4096, 0, 3412, 0, 0, 0, 2, 0, 1, 0, 4060, 0, 0, 0, 645, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 4089, 0, 1, 4094, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7628, 6828, 8180, 8190, 8192, 8180, 8192, 8192] dup_rows_per_dst=[4, 0, 1232, 0, 16, 0, 700, 2] combine_rows_per_src=[7628, 6828, 8180, 8190, 8192, 8180, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53540 payload_rows_remote=45348 dup_rows=11996 dup_rows_remote=1596 dedup_ratio=0.183044 router_tokens_per_expert=[3714, 1486, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3876, 0, 4096, 0, 80, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 326, 0, 0, 4096, 0, 0, 2248, 0, 87, 0, 0, 60, 0, 0, 4096, 0, 393, 0, 4, 0, 0, 0, 4096, 0, 0, 17, 0] payload_rows_per_dst=[8192, 8176, 8190, 7314, 8192, 4498, 838, 8140] dup_rows_per_dst=[10400, 4, 532, 160, 652, 172, 76, 0] combine_rows_per_src=[8192, 8176, 8190, 7314, 8192, 4498, 838, 8140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63312 payload_rows_remote=55122 dup_rows=2224 dup_rows_remote=2224 dedup_ratio=0.033936 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3592, 4096, 0, 3489, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 4096, 0, 0, 0, 525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7184, 6978, 8192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 1174, 0, 0, 0, 1050, 0] combine_rows_per_src=[7184, 6978, 8192, 8192, 8192, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=382728192 dispatch_meta_bytes=229360 combine_payload_bytes=382728192 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=382957552 ingress_bytes=602929152 link_bytes=602929152 token_bytes=765685744 weight_movement_bytes=220200960 total_nvlink_bytes=985886704 weight_to_token_ratio=0.287587 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048592960e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64334 payload_rows_remote=56142 dup_rows=1202 dup_rows_remote=558 dedup_ratio=0.018341 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 3834, 4095, 0, 3764, 1, 0, 0, 0, 0, 0, 0, 4057, 3, 1, 0, 308, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 4090, 0, 0, 4095, 0, 0, 0, 320, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7668, 7528, 8192, 8190, 8192, 8180, 8192, 8192] dup_rows_per_dst=[0, 2, 546, 0, 10, 0, 644, 0] combine_rows_per_src=[7668, 7528, 8192, 8190, 8192, 8180, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63080 payload_rows_remote=54888 dup_rows=2456 dup_rows_remote=2454 dedup_ratio=0.037476 router_tokens_per_expert=[0, 2, 4094, 0, 0, 0, 0, 3827, 4095, 0, 3218, 0, 0, 1, 0, 0, 0, 0, 4091, 2, 0, 0, 691, 0, 0, 2, 0, 0, 1, 0, 2, 0, 0, 0, 0, 1, 0, 0, 4094, 0, 0, 0, 0, 4091, 0, 1, 4094, 0, 0, 0, 458, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[7658, 6436, 8192, 8190, 8038, 8184, 8190, 8192] dup_rows_per_dst=[0, 2, 1376, 6, 0, 154, 916, 2] combine_rows_per_src=[7658, 6436, 8192, 8190, 8038, 8184, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63132 payload_rows_remote=54940 dup_rows=2404 dup_rows_remote=2404 dedup_ratio=0.036682 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 3241, 4096, 0, 3754, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 865, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4091, 0, 0, 4096, 0, 0, 0, 337, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6482, 7508, 8192, 8192, 8192, 8182, 8192, 8192] dup_rows_per_dst=[0, 0, 1730, 0, 0, 0, 674, 0] combine_rows_per_src=[6482, 7508, 8192, 8192, 8192, 8182, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59160 payload_rows_remote=51114 dup_rows=6376 dup_rows_remote=902 dedup_ratio=0.097290 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 2982, 4096, 0, 3901, 0, 0, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 1145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095, 0, 1, 4096, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5964, 8046, 4192, 8192, 8192, 8190, 8192, 8192] dup_rows_per_dst=[0, 5474, 560, 0, 2, 2, 338, 0] combine_rows_per_src=[5964, 8046, 4192, 8192, 8192, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=397121536 dispatch_meta_bytes=224496 combine_payload_bytes=397121536 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=397346032 ingress_bytes=727422976 link_bytes=727422976 token_bytes=794467568 weight_movement_bytes=330301440 total_nvlink_bytes=1124769008 weight_to_token_ratio=0.415752 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.265108196e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=395114496 dispatch_meta_bytes=229384 combine_payload_bytes=395114496 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395343880 ingress_bytes=615315456 link_bytes=615315456 token_bytes=790458376 weight_movement_bytes=220200960 total_nvlink_bytes=1010659336 weight_to_token_ratio=0.278574 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070134780e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=325054464 dispatch_meta_bytes=187776 combine_payload_bytes=325054464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=325242240 ingress_bytes=655355904 link_bytes=655355904 token_bytes=650296704 weight_movement_bytes=330301440 total_nvlink_bytes=980598144 weight_to_token_ratio=0.507924 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.139771705e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=402425856 dispatch_meta_bytes=226800 combine_payload_bytes=402425856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402652656 ingress_bytes=512526336 link_bytes=512526336 token_bytes=805078512 weight_movement_bytes=110100480 total_nvlink_bytes=915178992 weight_to_token_ratio=0.136757 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.913675944e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393437184 dispatch_meta_bytes=229368 combine_payload_bytes=393437184 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393666552 ingress_bytes=503537664 link_bytes=503537664 token_bytes=787103736 weight_movement_bytes=110100480 total_nvlink_bytes=897204216 weight_to_token_ratio=0.139881 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.757348154e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393809920 dispatch_meta_bytes=229376 combine_payload_bytes=393809920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394039296 ingress_bytes=503910400 link_bytes=503910400 token_bytes=787849216 weight_movement_bytes=110100480 total_nvlink_bytes=897949696 weight_to_token_ratio=0.139748 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.763830646e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=366385152 dispatch_meta_bytes=208064 combine_payload_bytes=366385152 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=366593216 ingress_bytes=586586112 link_bytes=586586112 token_bytes=732978368 weight_movement_bytes=220200960 total_nvlink_bytes=953179328 weight_to_token_ratio=0.300419 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.020169726e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64084 payload_rows_remote=57320 dup_rows=1452 dup_rows_remote=1432 dedup_ratio=0.022156 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 663, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 969, 2423, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 58, 0] payload_rows_per_dst=[8192, 8192, 8170, 8192, 6764, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 1312, 0, 20, 0, 4, 116] combine_rows_per_src=[8192, 8192, 8170, 8192, 6764, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64354 payload_rows_remote=56162 dup_rows=1182 dup_rows_remote=50 dedup_ratio=0.018036 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 565, 0, 0, 4093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 947, 2567, 4091, 4095, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 17, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 7028, 8186, 8190, 8184] dup_rows_per_dst=[0, 2, 1132, 0, 4, 0, 10, 34] combine_rows_per_src=[8192, 8190, 8192, 8192, 7028, 8186, 8190, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63610 payload_rows_remote=55418 dup_rows=1926 dup_rows_remote=1926 dedup_ratio=0.029388 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 940, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 835, 2301, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 22, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 6270, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 1878, 2, 2, 0, 0, 44] combine_rows_per_src=[8192, 8192, 8190, 8192, 6270, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410869760 dispatch_meta_bytes=235008 combine_payload_bytes=410869760 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411104768 ingress_bytes=741171200 link_bytes=741171200 token_bytes=821974528 weight_movement_bytes=330301440 total_nvlink_bytes=1152275968 weight_to_token_ratio=0.401839 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289018619e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53708 payload_rows_remote=45916 dup_rows=11828 dup_rows_remote=3334 dedup_ratio=0.180481 router_tokens_per_expert=[0, 13, 4096, 0, 0, 0, 0, 2758, 4096, 0, 3807, 0, 0, 0, 0, 0, 0, 0, 4087, 28, 0, 0, 1369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4037, 0, 4, 4096, 0, 0, 0, 182, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[7792, 7994, 2826, 8192, 8188, 8082, 2452, 8182] dup_rows_per_dst=[8494, 2968, 38, 0, 126, 0, 196, 6] combine_rows_per_src=[7792, 7994, 2826, 8192, 8188, 8082, 2452, 8182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61430 payload_rows_remote=53258 dup_rows=4106 dup_rows_remote=4106 dedup_ratio=0.062653 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 1, 4095, 4095, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 635, 0, 0, 4087, 1, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 4085, 0, 1093, 2297, 4094, 4095, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 85, 0] payload_rows_per_dst=[8192, 8190, 4862, 8192, 7442, 8190, 8172, 8190] dup_rows_per_dst=[4, 2, 672, 2, 3254, 0, 0, 172] combine_rows_per_src=[8192, 8190, 4862, 8192, 7442, 8190, 8172, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58316 payload_rows_remote=50124 dup_rows=7220 dup_rows_remote=7154 dedup_ratio=0.110168 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4086, 4094, 0, 0, 0, 1, 0, 1, 4, 2, 0, 0, 500, 0, 0, 4076, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 4094, 1, 763, 2818, 4094, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2, 0, 2, 2, 0, 0, 4094, 0, 0, 33, 0] payload_rows_per_dst=[8188, 8174, 1004, 8186, 8190, 8190, 8192, 8192] dup_rows_per_dst=[0, 2, 8, 0, 7122, 6, 16, 66] combine_rows_per_src=[8188, 8174, 1004, 8186, 8190, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64422 payload_rows_remote=56230 dup_rows=1114 dup_rows_remote=1114 dedup_ratio=0.016998 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 393, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 1455, 2097, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 160, 0] payload_rows_per_dst=[8192, 8192, 8184, 8192, 7092, 8192, 8186, 8192] dup_rows_per_dst=[0, 0, 782, 0, 12, 0, 0, 320] combine_rows_per_src=[8192, 8192, 8184, 8192, 7092, 8192, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64180 payload_rows_remote=55988 dup_rows=1356 dup_rows_remote=1356 dedup_ratio=0.020691 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4096, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 528, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 951, 2469, 4096, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 147, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 6838, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 1058, 2, 2, 0, 0, 294] combine_rows_per_src=[8192, 8192, 8190, 8192, 6838, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402569216 dispatch_meta_bytes=224848 combine_payload_bytes=402569216 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402794064 ingress_bytes=622770176 link_bytes=622770176 token_bytes=805363280 weight_movement_bytes=220200960 total_nvlink_bytes=1025564240 weight_to_token_ratio=0.273418 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083099764e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=397236224 dispatch_meta_bytes=229376 combine_payload_bytes=397236224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397465600 ingress_bytes=617437184 link_bytes=617437184 token_bytes=794701824 weight_movement_bytes=220200960 total_nvlink_bytes=1014902784 weight_to_token_ratio=0.277086 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073824814e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=329125888 dispatch_meta_bytes=197000 combine_payload_bytes=329125888 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=329322888 ingress_bytes=549326848 link_bytes=549326848 token_bytes=658448776 weight_movement_bytes=220200960 total_nvlink_bytes=878649736 weight_to_token_ratio=0.334424 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.553697374e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=381753344 dispatch_meta_bytes=229456 combine_payload_bytes=381753344 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=381982800 ingress_bytes=601954304 link_bytes=601954304 token_bytes=763736144 weight_movement_bytes=220200960 total_nvlink_bytes=983937104 weight_to_token_ratio=0.288321 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046897539e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=403056640 dispatch_meta_bytes=229376 combine_payload_bytes=403056640 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403286016 ingress_bytes=513157120 link_bytes=513157120 token_bytes=806342656 weight_movement_bytes=110100480 total_nvlink_bytes=916443136 weight_to_token_ratio=0.136543 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.924646315e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=359288832 dispatch_meta_bytes=229112 combine_payload_bytes=359288832 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=359517944 ingress_bytes=689590272 link_bytes=689590272 token_bytes=718806776 weight_movement_bytes=330301440 total_nvlink_bytes=1049108216 weight_to_token_ratio=0.459514 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.199310901e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=401321984 dispatch_meta_bytes=229376 combine_payload_bytes=401321984 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=401551360 ingress_bytes=621522944 link_bytes=621522944 token_bytes=802873344 weight_movement_bytes=220200960 total_nvlink_bytes=1023074304 weight_to_token_ratio=0.274266 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080930623e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60584 payload_rows_remote=55886 dup_rows=4952 dup_rows_remote=3834 dedup_ratio=0.075562 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1071, 3804, 0, 636, 0, 0, 384, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4090, 0, 2301, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 4698, 8192, 6742, 8184] dup_rows_per_dst=[0, 0, 0, 0, 1118, 768, 3066, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 4698, 8192, 6742, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62460 payload_rows_remote=54268 dup_rows=3076 dup_rows_remote=3068 dedup_ratio=0.046936 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 1, 0, 0, 0, 4093, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1488, 3699, 0, 205, 0, 0, 218, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 4096, 0, 2575, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8162, 8192, 5150, 8192] dup_rows_per_dst=[0, 4, 8, 0, 2622, 438, 0, 4] combine_rows_per_src=[8190, 8192, 8192, 8190, 8162, 8192, 5150, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400590848 dispatch_meta_bytes=238880 combine_payload_bytes=400590848 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=400829728 ingress_bytes=840992768 link_bytes=840992768 token_bytes=801420576 weight_movement_bytes=440401920 total_nvlink_bytes=1241822496 weight_to_token_ratio=0.549527 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.462624743e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60500 payload_rows_remote=52308 dup_rows=5036 dup_rows_remote=4460 dedup_ratio=0.076843 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1450, 3925, 0, 244, 0, 0, 288, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 2295, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 3324, 8192, 8044, 8172] dup_rows_per_dst=[0, 0, 0, 0, 64, 576, 4396, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 3324, 8192, 8044, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61504 payload_rows_remote=53312 dup_rows=4032 dup_rows_remote=4032 dedup_ratio=0.061523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1040, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1119, 1629, 4088, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 315, 0] payload_rows_per_dst=[8192, 8190, 8190, 8192, 5494, 8176, 8192, 6878] dup_rows_per_dst=[0, 18, 3712, 0, 2, 22, 0, 278] combine_rows_per_src=[8192, 8190, 8190, 8192, 5494, 8176, 8192, 6878] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61092 payload_rows_remote=53098 dup_rows=4444 dup_rows_remote=752 dedup_ratio=0.067810 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4095, 0, 0, 1666, 3944, 0, 347, 0, 0, 322, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4095, 1, 1898, 0, 0, 11, 1, 0, 0, 1, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8192, 3950, 8190, 7994, 8192] dup_rows_per_dst=[0, 0, 6, 2, 76, 644, 3692, 24] combine_rows_per_src=[8190, 8192, 8192, 8192, 3950, 8190, 7994, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=388993024 dispatch_meta_bytes=229344 combine_payload_bytes=388993024 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=389222368 ingress_bytes=609193984 link_bytes=609193984 token_bytes=778215392 weight_movement_bytes=220200960 total_nvlink_bytes=998416352 weight_to_token_ratio=0.282956 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.059488533e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7537270992 prefetch_bytes=880803840 reduce_bytes=1761607680 weight_movement_bytes=2642411520 egress_bytes=3769784528 ingress_bytes=6409897984 link_bytes=6409897984 total_nvlink_bytes=10179682512 weight_to_token_ratio=0.350579 prefetch_share_of_ingress=0.137413 dedup_ratio=0.085327 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=16 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=8890 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.114786684e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58442 payload_rows_remote=50250 dup_rows=7094 dup_rows_remote=6974 dedup_ratio=0.108246 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 1, 4095, 0, 1, 0, 1, 1, 0, 0, 0, 4094, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 385, 3985, 0, 264, 0, 0, 755, 0, 0, 1, 0, 4094, 0, 0, 0, 0, 4096, 0, 2793, 0, 0, 5, 0, 2, 0, 1, 0, 0] payload_rows_per_dst=[8192, 8192, 8190, 8188, 1284, 8190, 8014, 8192] dup_rows_per_dst=[0, 6, 2, 0, 16, 1512, 5438, 120] combine_rows_per_src=[8192, 8192, 8190, 8188, 1284, 8190, 8014, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61594 payload_rows_remote=53402 dup_rows=3942 dup_rows_remote=3942 dedup_ratio=0.060150 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2055, 3603, 0, 125, 0, 0, 257, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 2145, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8152, 8192, 4290, 8192] dup_rows_per_dst=[0, 0, 14, 0, 3414, 514, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8152, 8192, 4290, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374943744 dispatch_meta_bytes=227072 combine_payload_bytes=374943744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=375170816 ingress_bytes=595144704 link_bytes=595144704 token_bytes=750114560 weight_movement_bytes=220200960 total_nvlink_bytes=970315520 weight_to_token_ratio=0.293556 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=29696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.035054525e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61388 payload_rows_remote=53196 dup_rows=4148 dup_rows_remote=4148 dedup_ratio=0.063293 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1912, 3606, 0, 252, 0, 0, 390, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086, 0, 2040, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8176, 8192, 4080, 8172] dup_rows_per_dst=[0, 0, 4, 0, 3364, 780, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8176, 8192, 4080, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=382140416 dispatch_meta_bytes=229376 combine_payload_bytes=382140416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382369792 ingress_bytes=492240896 link_bytes=492240896 token_bytes=764510208 weight_movement_bytes=110100480 total_nvlink_bytes=874610688 weight_to_token_ratio=0.144014 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.560878779e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7561534752 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3781905696 ingress_bytes=5761437696 link_bytes=5761437696 total_nvlink_bytes=9543343392 weight_to_token_ratio=0.262091 prefetch_share_of_ingress=0.171989 dedup_ratio=0.077383 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.040809 gemm_inner_pad_upper_bound=9652 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.002008775e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380606464 dispatch_meta_bytes=215400 combine_payload_bytes=380606464 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=380821864 ingress_bytes=710907904 link_bytes=710907904 token_bytes=761428328 weight_movement_bytes=330301440 total_nvlink_bytes=1091729768 weight_to_token_ratio=0.433792 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.236385769e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=360192000 dispatch_meta_bytes=228896 combine_payload_bytes=360192000 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=360420896 ingress_bytes=580392960 link_bytes=580392960 token_bytes=720612896 weight_movement_bytes=220200960 total_nvlink_bytes=940813856 weight_to_token_ratio=0.305575 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.009398816e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382785536 dispatch_meta_bytes=229376 combine_payload_bytes=382785536 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=383014912 ingress_bytes=602986496 link_bytes=602986496 token_bytes=765800448 weight_movement_bytes=220200960 total_nvlink_bytes=986001408 weight_to_token_ratio=0.287544 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048692691e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7433479200 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3717874720 ingress_bytes=5917614080 link_bytes=5917614080 total_nvlink_bytes=9635488800 weight_to_token_ratio=0.296229 prefetch_share_of_ingress=0.186056 dedup_ratio=0.089801 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041544 gemm_inner_pad_upper_bound=9398 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.029170417e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=381308928 dispatch_meta_bytes=229376 combine_payload_bytes=381308928 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=381538304 ingress_bytes=491409408 link_bytes=491409408 token_bytes=762847232 weight_movement_bytes=110100480 total_nvlink_bytes=872947712 weight_to_token_ratio=0.144328 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.546417835e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7397219728 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3699750288 ingress_bytes=5569177600 link_bytes=5569177600 total_nvlink_bytes=9268927888 weight_to_token_ratio=0.253029 prefetch_share_of_ingress=0.177927 dedup_ratio=0.097266 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.685715819e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6924538760 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3463398280 ingress_bytes=5442949120 link_bytes=5442949120 total_nvlink_bytes=8906347400 weight_to_token_ratio=0.286201 prefetch_share_of_ingress=0.202281 dedup_ratio=0.143359 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.039706 gemm_inner_pad_upper_bound=10033 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.466183731e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7500592704 prefetch_bytes=1211105280 reduce_bytes=1321205760 weight_movement_bytes=2532311040 egress_bytes=3751441984 ingress_bytes=6281461760 link_bytes=6281461760 total_nvlink_bytes=10032903744 weight_to_token_ratio=0.337615 prefetch_share_of_ingress=0.192806 dedup_ratio=0.085596 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=12 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043015 gemm_inner_pad_upper_bound=9144 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.092449512e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7217507904 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3609867840 ingress_bytes=5369247744 link_bytes=5369247744 total_nvlink_bytes=8979115584 weight_to_token_ratio=0.244074 prefetch_share_of_ingress=0.225563 dedup_ratio=0.110297 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.041176 gemm_inner_pad_upper_bound=9398 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.338004916e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56260 payload_rows_remote=48106 dup_rows=9276 dup_rows_remote=9234 dedup_ratio=0.141541 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1425, 3757, 0, 203, 0, 0, 561, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 3995, 0, 2315, 0, 0, 51, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8154, 8182, 8160, 8192, 8192, 2680, 4630, 8070] dup_rows_per_dst=[42, 0, 0, 4, 8924, 284, 0, 22] combine_rows_per_src=[8154, 8182, 8160, 8192, 8192, 2680, 4630, 8070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=344823808 dispatch_meta_bytes=229360 combine_payload_bytes=344823808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=345053168 ingress_bytes=565024768 link_bytes=565024768 token_bytes=689876976 weight_movement_bytes=220200960 total_nvlink_bytes=910077936 weight_to_token_ratio=0.319189 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.826710022e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6657520632 prefetch_bytes=990904320 reduce_bytes=990904320 weight_movement_bytes=1981808640 egress_bytes=3329862648 ingress_bytes=5309466624 link_bytes=5309466624 total_nvlink_bytes=8639329272 weight_to_token_ratio=0.297680 prefetch_share_of_ingress=0.186630 dedup_ratio=0.173752 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=9 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043382 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.234035716e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none - [2026-08-26 16:57:46] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 366.1 | throughput per GPU (TFLOP/s/GPU): 337.3 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.046572E+01 | loss scale: 1.0 | grad norm: 8.608 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45552 payload_rows_remote=39088 dup_rows=19984 dup_rows_remote=17170 dedup_ratio=0.304932 router_tokens_per_expert=[353, 155, 428, 503, 11, 49, 416, 250, 1142, 771, 355, 726, 1, 345, 10, 778, 792, 357, 371, 13, 1027, 1709, 18, 352, 399, 731, 894, 419, 36, 472, 408, 0, 70, 23, 525, 175, 17, 0, 185, 877, 1022, 717, 1578, 91, 100, 376, 102, 945, 24, 234, 265, 1177, 1267, 437, 86, 695, 479, 292, 879, 2057, 2164, 1255, 357, 6] payload_rows_per_dst=[5838, 5474, 6464, 5162, 5774, 5002, 5800, 6038] dup_rows_per_dst=[2606, 2782, 2814, 1556, 2298, 1704, 2570, 3654] combine_rows_per_src=[5838, 5474, 6464, 5162, 5774, 5002, 5800, 6038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45910 payload_rows_remote=40000 dup_rows=19626 dup_rows_remote=17808 dedup_ratio=0.299469 router_tokens_per_expert=[381, 105, 314, 596, 33, 40, 434, 146, 583, 481, 965, 1586, 0, 379, 0, 700, 1347, 209, 345, 0, 953, 1942, 8, 529, 258, 1149, 948, 615, 19, 518, 258, 1, 32, 14, 375, 198, 14, 0, 110, 704, 770, 647, 1559, 44, 84, 480, 106, 888, 14, 117, 141, 664, 2073, 540, 27, 507, 279, 343, 969, 2294, 2417, 371, 145, 0] payload_rows_per_dst=[6138, 6056, 6786, 5430, 5910, 4556, 5804, 5230] dup_rows_per_dst=[2548, 3332, 3880, 2102, 1818, 1482, 2362, 2102] combine_rows_per_src=[6138, 6056, 6786, 5430, 5910, 4556, 5804, 5230] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45346 payload_rows_remote=40024 dup_rows=20190 dup_rows_remote=18490 dedup_ratio=0.308075 router_tokens_per_expert=[554, 157, 925, 865, 12, 15, 168, 261, 950, 458, 443, 1038, 0, 369, 0, 783, 1145, 238, 316, 1, 793, 1591, 20, 255, 383, 571, 878, 585, 6, 651, 432, 5, 66, 23, 360, 136, 24, 1, 87, 1098, 976, 996, 867, 148, 158, 737, 147, 658, 6, 207, 247, 1119, 2400, 71, 18, 686, 295, 238, 1080, 1775, 1734, 1406, 136, 0] payload_rows_per_dst=[6068, 5400, 6146, 5322, 5156, 5348, 6508, 5398] dup_rows_per_dst=[3396, 2682, 2572, 1700, 1902, 2292, 3000, 2646] combine_rows_per_src=[6068, 5400, 6146, 5322, 5156, 5348, 6508, 5398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45998 payload_rows_remote=40390 dup_rows=19538 dup_rows_remote=16984 dedup_ratio=0.298126 router_tokens_per_expert=[588, 95, 326, 667, 12, 26, 179, 116, 617, 826, 873, 1779, 0, 209, 2, 722, 969, 462, 226, 2, 941, 1518, 21, 704, 309, 626, 1101, 489, 11, 417, 402, 0, 41, 7, 265, 138, 14, 0, 58, 795, 778, 979, 1201, 105, 67, 411, 296, 1087, 2, 232, 270, 788, 1558, 417, 5, 809, 275, 346, 1050, 1653, 2366, 1314, 206, 0] payload_rows_per_dst=[5420, 4870, 5106, 6742, 5770, 6778, 5608, 5704] dup_rows_per_dst=[1904, 1628, 1544, 3526, 1598, 3704, 2554, 3080] combine_rows_per_src=[5420, 4870, 5106, 6742, 5770, 6778, 5608, 5704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44870 payload_rows_remote=38096 dup_rows=20666 dup_rows_remote=16080 dedup_ratio=0.315338 router_tokens_per_expert=[396, 165, 513, 656, 7, 33, 351, 309, 732, 601, 325, 1319, 0, 376, 4, 835, 595, 397, 196, 1, 921, 1841, 8, 492, 279, 804, 797, 495, 46, 462, 607, 0, 35, 15, 207, 328, 28, 0, 121, 1476, 899, 616, 1387, 124, 513, 273, 111, 837, 15, 247, 580, 1357, 1969, 234, 4, 426, 210, 376, 776, 1712, 1565, 1479, 284, 1] payload_rows_per_dst=[4558, 4428, 4186, 6236, 5432, 6842, 6414, 6774] dup_rows_per_dst=[1392, 1318, 1034, 3382, 2118, 3586, 3250, 4586] combine_rows_per_src=[4558, 4428, 4186, 6236, 5432, 6842, 6414, 6774] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=225032 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=280407816 ingress_bytes=500383744 link_bytes=500383744 token_bytes=560590600 weight_movement_bytes=220200960 total_nvlink_bytes=780791560 weight_to_token_ratio=0.392802 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.702496298e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286720000 dispatch_meta_bytes=231232 combine_payload_bytes=286720000 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=286951232 ingress_bytes=506920960 link_bytes=506920960 token_bytes=573671232 weight_movement_bytes=220200960 total_nvlink_bytes=793872192 weight_to_token_ratio=0.383845 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.036765 gemm_active_groups=11 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.816189236e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286892032 dispatch_meta_bytes=234056 combine_payload_bytes=286892032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=287126088 ingress_bytes=396992512 link_bytes=396992512 token_bytes=574018120 weight_movement_bytes=110100480 total_nvlink_bytes=684118600 weight_to_token_ratio=0.191807 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.904352724e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45548 payload_rows_remote=39252 dup_rows=19988 dup_rows_remote=17074 dedup_ratio=0.304993 router_tokens_per_expert=[349, 261, 862, 597, 20, 96, 115, 305, 1036, 720, 449, 1468, 11, 142, 6, 960, 672, 505, 108, 3, 1065, 1398, 10, 454, 257, 589, 1080, 643, 18, 336, 336, 1, 365, 105, 234, 205, 14, 0, 21, 733, 1168, 623, 870, 80, 185, 312, 248, 669, 50, 86, 445, 1299, 1198, 95, 43, 593, 258, 497, 1457, 2236, 2380, 1010, 417, 0] payload_rows_per_dst=[6220, 5302, 4640, 5890, 5930, 6296, 5474, 5796] dup_rows_per_dst=[3462, 2352, 1150, 2560, 2184, 2914, 2144, 3222] combine_rows_per_src=[6220, 5302, 4640, 5890, 5930, 6296, 5474, 5796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44392 payload_rows_remote=38576 dup_rows=21144 dup_rows_remote=18592 dedup_ratio=0.322632 router_tokens_per_expert=[557, 110, 474, 675, 11, 42, 290, 264, 816, 660, 447, 1248, 0, 261, 0, 942, 1098, 349, 371, 13, 1038, 1034, 24, 267, 401, 695, 1055, 714, 49, 477, 486, 7, 153, 12, 272, 169, 48, 1, 95, 1139, 758, 838, 1066, 141, 314, 667, 70, 1019, 25, 138, 527, 750, 1592, 531, 11, 801, 181, 467, 1090, 1761, 1784, 1352, 121, 0] payload_rows_per_dst=[5816, 6378, 6226, 5648, 6420, 5430, 4066, 4408] dup_rows_per_dst=[2552, 4866, 2850, 2120, 3630, 2390, 1500, 1236] combine_rows_per_src=[5816, 6378, 6226, 5648, 6420, 5430, 4066, 4408] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289515520 dispatch_meta_bytes=229496 combine_payload_bytes=289515520 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=289745016 ingress_bytes=509716480 link_bytes=509716480 token_bytes=579260536 weight_movement_bytes=220200960 total_nvlink_bytes=799461496 weight_to_token_ratio=0.380141 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.864807927e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273072128 dispatch_meta_bytes=216704 combine_payload_bytes=273072128 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=273288832 ingress_bytes=713474048 link_bytes=713474048 token_bytes=546360960 weight_movement_bytes=440401920 total_nvlink_bytes=986762880 weight_to_token_ratio=0.806064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=9344 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.240848716e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281358336 dispatch_meta_bytes=225304 combine_payload_bytes=281358336 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=281583640 ingress_bytes=501559296 link_bytes=501559296 token_bytes=562941976 weight_movement_bytes=220200960 total_nvlink_bytes=783142936 weight_to_token_ratio=0.391161 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.722941081e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57500 payload_rows_remote=49308 dup_rows=8036 dup_rows_remote=8032 dedup_ratio=0.122620 router_tokens_per_expert=[0, 2, 2, 549, 0, 0, 3, 18, 0, 0, 39, 0, 2447, 0, 0, 0, 3, 0, 0, 4094, 0, 1, 0, 0, 4075, 0, 0, 0, 0, 30, 0, 0, 0, 3288, 0, 0, 0, 0, 0, 5, 4041, 0, 0, 0, 4093, 1, 0, 0, 0, 0, 1, 0, 1125, 7, 0, 0, 2, 2715, 4090, 0, 2134, 0, 3, 0] payload_rows_per_dst=[8188, 7080, 8192, 8172, 7272, 7312, 5124, 6160] dup_rows_per_dst=[1140, 2772, 4, 38, 1884, 0, 1220, 978] combine_rows_per_src=[8188, 7080, 8192, 8172, 7272, 7312, 5124, 6160] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60592 payload_rows_remote=54084 dup_rows=4944 dup_rows_remote=4944 dedup_ratio=0.075439 router_tokens_per_expert=[0, 0, 0, 624, 0, 0, 0, 33, 0, 0, 0, 0, 2954, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 3250, 0, 0, 0, 0, 0, 4, 4085, 0, 0, 0, 4095, 0, 3, 0, 0, 0, 0, 0, 851, 0, 0, 0, 0, 2518, 4096, 0, 2067, 0, 0, 0] payload_rows_per_dst=[8192, 5908, 8190, 8186, 6508, 8170, 8190, 7248] dup_rows_per_dst=[1314, 0, 0, 0, 0, 6, 1702, 1922] combine_rows_per_src=[8192, 5908, 8190, 8186, 6508, 8170, 8190, 7248] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59470 payload_rows_remote=51310 dup_rows=6066 dup_rows_remote=6064 dedup_ratio=0.092560 router_tokens_per_expert=[0, 0, 0, 314, 0, 1, 4, 58, 1, 0, 3, 0, 2746, 1, 0, 0, 0, 0, 0, 4095, 0, 1, 0, 1, 4080, 0, 0, 0, 0, 1, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 26, 4060, 2, 0, 0, 4095, 0, 0, 0, 0, 2, 1, 0, 1370, 0, 0, 0, 0, 2994, 4094, 0, 1940, 0, 0, 1] payload_rows_per_dst=[8192, 5494, 8192, 8160, 5790, 8124, 8192, 7326] dup_rows_per_dst=[750, 8, 2, 2, 16, 0, 2744, 2544] combine_rows_per_src=[8192, 5494, 8192, 8160, 5790, 8124, 8192, 7326] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276512768 dispatch_meta_bytes=228672 combine_payload_bytes=276512768 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276741440 ingress_bytes=386613248 link_bytes=386613248 token_bytes=553254208 weight_movement_bytes=110100480 total_nvlink_bytes=663354688 weight_to_token_ratio=0.199005 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723840252e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45224 payload_rows_remote=39472 dup_rows=20312 dup_rows_remote=17396 dedup_ratio=0.309937 router_tokens_per_expert=[576, 107, 299, 436, 43, 49, 232, 186, 696, 829, 206, 1351, 0, 358, 0, 894, 673, 342, 398, 0, 579, 1743, 15, 673, 630, 237, 1028, 324, 19, 1213, 629, 0, 22, 31, 147, 183, 16, 2, 104, 1561, 1150, 570, 881, 108, 361, 400, 129, 888, 20, 310, 310, 1302, 2065, 416, 6, 458, 230, 283, 1042, 1881, 1389, 1475, 246, 17] payload_rows_per_dst=[5684, 5752, 6542, 5812, 6294, 4960, 5898, 4282] dup_rows_per_dst=[1934, 2916, 3596, 2348, 3302, 2252, 2584, 1380] combine_rows_per_src=[5684, 5752, 6542, 5812, 6294, 4960, 5898, 4282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59110 payload_rows_remote=50918 dup_rows=6426 dup_rows_remote=3982 dedup_ratio=0.098053 router_tokens_per_expert=[0, 0, 0, 545, 2, 0, 0, 1, 2, 0, 1, 0, 2855, 0, 0, 0, 0, 0, 0, 4093, 3, 3, 0, 0, 4091, 0, 0, 0, 0, 1, 0, 0, 0, 3374, 0, 0, 0, 3, 0, 0, 4058, 0, 0, 1, 4096, 0, 0, 0, 2, 1, 0, 0, 1212, 5, 0, 2, 0, 2219, 4095, 0, 2103, 0, 0, 0] payload_rows_per_dst=[6734, 5716, 8190, 8182, 6754, 8118, 8192, 7224] dup_rows_per_dst=[834, 0, 8, 2, 0, 0, 2444, 3138] combine_rows_per_src=[6734, 5716, 8190, 8182, 6754, 8118, 8192, 7224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54440 payload_rows_remote=46252 dup_rows=11096 dup_rows_remote=2844 dedup_ratio=0.169312 router_tokens_per_expert=[0, 1, 0, 599, 0, 0, 0, 13, 0, 0, 2, 1, 2538, 0, 0, 0, 0, 0, 0, 4091, 7, 0, 1, 6, 4084, 0, 0, 0, 0, 5, 2, 1, 0, 3640, 1, 0, 0, 4, 0, 25, 4064, 0, 5, 0, 4094, 1, 0, 1, 0, 0, 0, 0, 1389, 2, 0, 2, 0, 1886, 4091, 0, 2208, 0, 4, 0] payload_rows_per_dst=[1224, 5080, 8130, 8178, 7312, 8138, 8190, 8188] dup_rows_per_dst=[2, 2, 18, 6, 28, 4, 2784, 8252] combine_rows_per_src=[1224, 5080, 8130, 8178, 7312, 8138, 8190, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=353439744 dispatch_meta_bytes=229360 combine_payload_bytes=353439744 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=353669104 ingress_bytes=573640704 link_bytes=573640704 token_bytes=707108848 weight_movement_bytes=220200960 total_nvlink_bytes=927309808 weight_to_token_ratio=0.311410 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.976555319e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=387674112 dispatch_meta_bytes=236112 combine_payload_bytes=387674112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387910224 ingress_bytes=497774592 link_bytes=497774592 token_bytes=775584336 weight_movement_bytes=110100480 total_nvlink_bytes=885684816 weight_to_token_ratio=0.141958 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.657118853e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=367790080 dispatch_meta_bytes=229496 combine_payload_bytes=367790080 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=368019576 ingress_bytes=477890560 link_bytes=477890560 token_bytes=735809656 weight_movement_bytes=110100480 total_nvlink_bytes=845910136 weight_to_token_ratio=0.149632 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.311302833e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=364980224 dispatch_meta_bytes=219600 combine_payload_bytes=364980224 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365199824 ingress_bytes=475080704 link_bytes=475080704 token_bytes=730180048 weight_movement_bytes=110100480 total_nvlink_bytes=840280528 weight_to_token_ratio=0.150785 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=22528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.262434816e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=331534336 dispatch_meta_bytes=196384 combine_payload_bytes=331534336 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=331730720 ingress_bytes=771936256 link_bytes=771936256 token_bytes=663265056 weight_movement_bytes=440401920 total_nvlink_bytes=1103666976 weight_to_token_ratio=0.663991 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.342524111e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282935296 dispatch_meta_bytes=227472 combine_payload_bytes=282935296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=283162768 ingress_bytes=503136256 link_bytes=503136256 token_bytes=566098064 weight_movement_bytes=220200960 total_nvlink_bytes=786299024 weight_to_token_ratio=0.388980 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.750367009e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60898 payload_rows_remote=52950 dup_rows=4638 dup_rows_remote=4632 dedup_ratio=0.070770 router_tokens_per_expert=[0, 0, 0, 763, 2, 0, 0, 32, 0, 0, 32, 0, 3068, 0, 0, 0, 0, 1, 1, 4094, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 20, 2, 0, 3, 3884, 1, 0, 0, 0, 0, 41, 3973, 0, 4, 0, 4093, 0, 0, 0, 0, 1, 0, 2, 1173, 0, 0, 0, 0, 1528, 4092, 0, 1868, 0, 2, 0] payload_rows_per_dst=[8192, 6160, 8190, 8190, 7788, 7948, 8190, 6240] dup_rows_per_dst=[1586, 40, 2, 30, 70, 6, 2348, 556] combine_rows_per_src=[8192, 6160, 8190, 8190, 7788, 7948, 8190, 6240] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53494 payload_rows_remote=45308 dup_rows=12042 dup_rows_remote=10180 dedup_ratio=0.183746 router_tokens_per_expert=[0, 0, 0, 789, 0, 0, 0, 147, 0, 0, 1, 0, 2611, 0, 0, 6, 0, 3, 1, 4095, 0, 0, 1, 0, 4083, 4, 0, 0, 1, 13, 0, 2, 0, 3105, 0, 0, 1, 0, 0, 22, 4032, 0, 3, 0, 4096, 2, 2, 0, 1, 0, 0, 2, 1629, 3, 0, 0, 6, 2509, 4088, 0, 1509, 1, 0, 0] payload_rows_per_dst=[8186, 8160, 8192, 8152, 7620, 8192, 3268, 1724] dup_rows_per_dst=[1862, 5140, 34, 28, 3654, 1320, 2, 2] combine_rows_per_src=[8186, 8160, 8192, 8152, 7620, 8192, 3268, 1724] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62388 payload_rows_remote=54236 dup_rows=3148 dup_rows_remote=3148 dedup_ratio=0.048035 router_tokens_per_expert=[0, 0, 0, 147, 4095, 0, 0, 0, 4019, 0, 0, 0, 0, 0, 1429, 0, 0, 4075, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4093, 0, 1, 4095, 0, 1, 0, 1, 1, 3, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 29, 2, 2582, 0, 0, 0] payload_rows_per_dst=[8192, 8078, 8152, 8192, 8192, 8188, 8186, 5208] dup_rows_per_dst=[292, 2818, 0, 8, 12, 0, 0, 18] combine_rows_per_src=[8192, 8078, 8152, 8192, 8192, 8188, 8186, 5208] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63432 payload_rows_remote=55240 dup_rows=2104 dup_rows_remote=2082 dedup_ratio=0.032104 router_tokens_per_expert=[0, 0, 0, 61, 4095, 2, 0, 1, 4091, 0, 0, 0, 0, 0, 980, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 1, 0, 4092, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3057, 0, 0, 5] payload_rows_per_dst=[8192, 8190, 8166, 8192, 8192, 8186, 8190, 6124] dup_rows_per_dst=[126, 1952, 0, 22, 4, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8166, 8192, 8192, 8186, 8190, 6124] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=324767744 dispatch_meta_bytes=221952 combine_payload_bytes=324767744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324989696 ingress_bytes=434868224 link_bytes=434868224 token_bytes=649757440 weight_movement_bytes=110100480 total_nvlink_bytes=759857920 weight_to_token_ratio=0.169449 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.563073651e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=379545600 dispatch_meta_bytes=230328 combine_payload_bytes=379545600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=379775928 ingress_bytes=709847040 link_bytes=709847040 token_bytes=759321528 weight_movement_bytes=330301440 total_nvlink_bytes=1089622968 weight_to_token_ratio=0.434995 gemm_rows_defined=33792 gemm_rows_charged=34816 gemm_tail_pad_rows=1024 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.029412 gemm_active_groups=10 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.234540752e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62698 payload_rows_remote=54506 dup_rows=2838 dup_rows_remote=2836 dedup_ratio=0.043304 router_tokens_per_expert=[0, 0, 0, 84, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 1333, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 1, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4, 0, 2683, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8182, 8192, 8192, 8184, 8192, 5374] dup_rows_per_dst=[168, 2660, 0, 8, 2, 0, 0, 0] combine_rows_per_src=[8192, 8190, 8182, 8192, 8192, 8184, 8192, 5374] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59340 payload_rows_remote=51152 dup_rows=6196 dup_rows_remote=6196 dedup_ratio=0.094543 router_tokens_per_expert=[0, 0, 0, 65, 4092, 2, 0, 0, 4090, 0, 0, 2, 0, 0, 1062, 0, 0, 4064, 1, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 5, 4094, 2, 0, 4091, 0, 0, 0, 4, 1, 2, 1, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 3021, 0, 0, 0] payload_rows_per_dst=[8188, 2128, 8128, 8192, 8186, 8144, 8188, 8186] dup_rows_per_dst=[130, 0, 4, 16, 10, 0, 0, 6036] combine_rows_per_src=[8188, 2128, 8128, 8192, 8186, 8144, 8188, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58572 payload_rows_remote=50400 dup_rows=6964 dup_rows_remote=1240 dedup_ratio=0.106262 router_tokens_per_expert=[0, 0, 0, 106, 4092, 6, 0, 0, 4066, 1, 0, 0, 0, 0, 1199, 0, 0, 4016, 0, 0, 0, 2, 0, 1, 0, 0, 6, 14, 0, 0, 4090, 0, 3, 4090, 0, 0, 0, 2, 0, 1, 1, 4090, 4, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 4092, 0, 0, 0, 4, 2, 2876, 0, 0, 0] payload_rows_per_dst=[6598, 3112, 8046, 8080, 8188, 8190, 8186, 8172] dup_rows_per_dst=[194, 396, 500, 28, 4, 86, 32, 5724] combine_rows_per_src=[6598, 3112, 8046, 8080, 8188, 8190, 8186, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=388763648 dispatch_meta_bytes=229536 combine_payload_bytes=388763648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388993184 ingress_bytes=498864128 link_bytes=498864128 token_bytes=777756832 weight_movement_bytes=110100480 total_nvlink_bytes=887857312 weight_to_token_ratio=0.141562 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.676067676e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=395960320 dispatch_meta_bytes=229288 combine_payload_bytes=395960320 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=396189608 ingress_bytes=616161280 link_bytes=616161280 token_bytes=792149928 weight_movement_bytes=220200960 total_nvlink_bytes=1012350888 weight_to_token_ratio=0.277979 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071605807e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=366657536 dispatch_meta_bytes=229392 combine_payload_bytes=366657536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366886928 ingress_bytes=476758016 link_bytes=476758016 token_bytes=733544464 weight_movement_bytes=110100480 total_nvlink_bytes=843644944 weight_to_token_ratio=0.150094 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.291606030e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=390699008 dispatch_meta_bytes=229368 combine_payload_bytes=390699008 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=390928376 ingress_bytes=500799488 link_bytes=500799488 token_bytes=781627384 weight_movement_bytes=110100480 total_nvlink_bytes=891727864 weight_to_token_ratio=0.140861 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.709726770e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55630 payload_rows_remote=47456 dup_rows=9906 dup_rows_remote=4104 dedup_ratio=0.151154 router_tokens_per_expert=[0, 0, 0, 476, 0, 0, 0, 12, 0, 0, 5, 0, 2944, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4089, 0, 3, 0, 0, 13, 0, 0, 0, 3193, 0, 0, 0, 0, 0, 19, 4039, 0, 3, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 0, 0, 2208, 4082, 0, 2017, 0, 0, 0] payload_rows_per_dst=[8182, 8174, 8192, 8178, 7732, 8192, 2946, 4034] dup_rows_per_dst=[958, 5802, 0, 32, 3108, 6, 0, 0] combine_rows_per_src=[8182, 8174, 8192, 8178, 7732, 8192, 2946, 4034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=361267200 dispatch_meta_bytes=206560 combine_payload_bytes=361267200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361473760 ingress_bytes=471367680 link_bytes=471367680 token_bytes=722740960 weight_movement_bytes=110100480 total_nvlink_bytes=832841440 weight_to_token_ratio=0.152337 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.197859222e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62750 payload_rows_remote=54558 dup_rows=2786 dup_rows_remote=2462 dedup_ratio=0.042511 router_tokens_per_expert=[0, 0, 0, 153, 4095, 0, 0, 0, 4061, 0, 0, 0, 5, 0, 1248, 0, 3, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4094, 0, 0, 4095, 0, 0, 0, 0, 0, 2, 0, 4086, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 4089, 0, 0, 0, 6, 4, 2826, 0, 0, 0] payload_rows_per_dst=[8192, 8178, 7994, 8190, 8174, 8176, 8182, 5664] dup_rows_per_dst=[324, 2450, 0, 2, 0, 0, 2, 8] combine_rows_per_src=[8192, 8178, 7994, 8190, 8174, 8176, 8182, 5664] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61786 payload_rows_remote=53594 dup_rows=3750 dup_rows_remote=3748 dedup_ratio=0.057220 router_tokens_per_expert=[0, 0, 0, 128, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 952, 0, 2, 4087, 0, 0, 0, 0, 0, 1, 2, 0, 0, 8, 0, 3, 4095, 0, 0, 4092, 0, 0, 0, 1, 1, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 7, 0, 3030, 0, 0, 0] payload_rows_per_dst=[8192, 5842, 8174, 8192, 8188, 8192, 8186, 6820] dup_rows_per_dst=[256, 1174, 6, 24, 0, 2, 0, 2288] combine_rows_per_src=[8192, 5842, 8174, 8192, 8188, 8192, 8186, 6820] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=340164608 dispatch_meta_bytes=206240 combine_payload_bytes=340164608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=340370848 ingress_bytes=450265088 link_bytes=450265088 token_bytes=680535456 weight_movement_bytes=110100480 total_nvlink_bytes=790635936 weight_to_token_ratio=0.161785 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=22272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.830850439e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63740 payload_rows_remote=55630 dup_rows=1796 dup_rows_remote=1790 dedup_ratio=0.027405 router_tokens_per_expert=[0, 1282, 1, 3274, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 17, 0, 1, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4094, 0, 0, 0, 0, 1, 4094, 0, 41, 0, 0, 0, 0, 0, 3444, 0, 0, 0, 0, 0, 0, 329, 1, 2, 3955, 0, 0, 0, 1, 0] payload_rows_per_dst=[7810, 8190, 8110, 8190, 8188, 8190, 7152, 7910] dup_rows_per_dst=[1304, 0, 6, 2, 2, 80, 394, 8] combine_rows_per_src=[7810, 8190, 8110, 8190, 8188, 8190, 7152, 7910] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64370 payload_rows_remote=56178 dup_rows=1166 dup_rows_remote=1164 dedup_ratio=0.017792 router_tokens_per_expert=[0, 1476, 0, 3087, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 6, 1, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 0, 100, 0, 0, 4059, 0, 0, 0, 1, 0] payload_rows_per_dst=[8080, 8192, 8186, 8192, 8192, 8192, 7218, 8118] dup_rows_per_dst=[1046, 0, 0, 2, 0, 14, 102, 2] combine_rows_per_src=[8080, 8192, 8186, 8192, 8192, 8192, 7218, 8118] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63582 payload_rows_remote=56150 dup_rows=1954 dup_rows_remote=978 dedup_ratio=0.029816 router_tokens_per_expert=[0, 1009, 0, 3466, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4078, 0, 0, 0, 0, 17, 0, 0, 0, 0, 1, 0, 1, 4093, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4096, 0, 2, 1, 0, 0, 1, 0, 3602, 0, 0, 0, 0, 0, 1, 153, 0, 0, 4058, 0, 0, 2, 0, 0] payload_rows_per_dst=[7092, 8188, 8186, 8190, 8182, 8192, 7432, 8120] dup_rows_per_dst=[808, 0, 4, 0, 62, 104, 976, 0] combine_rows_per_src=[7092, 8188, 8186, 8190, 8182, 8192, 7432, 8120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63698 payload_rows_remote=55508 dup_rows=1838 dup_rows_remote=1838 dedup_ratio=0.028046 router_tokens_per_expert=[0, 1693, 0, 3084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 3215, 0, 0, 0, 0, 0, 0, 244, 0, 0, 4054, 0, 0, 0, 0, 0] payload_rows_per_dst=[7972, 8192, 8174, 8192, 8190, 8192, 6678, 8108] dup_rows_per_dst=[1582, 0, 10, 0, 0, 6, 240, 0] combine_rows_per_src=[7972, 8192, 8174, 8192, 8190, 8192, 6678, 8108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59450 payload_rows_remote=51412 dup_rows=6086 dup_rows_remote=5084 dedup_ratio=0.092865 router_tokens_per_expert=[0, 1262, 0, 3209, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 1, 1, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 0, 0, 0, 0, 1, 4096, 0, 39, 0, 0, 0, 0, 0, 3687, 0, 0, 0, 0, 0, 0, 131, 4, 0, 4012, 0, 0, 0, 0, 0] payload_rows_per_dst=[2528, 8192, 8076, 8192, 8170, 8192, 8062, 8038] dup_rows_per_dst=[14, 0, 0, 8, 0, 78, 4984, 1002] combine_rows_per_src=[2528, 8192, 8076, 8192, 8170, 8192, 8062, 8038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=391071744 dispatch_meta_bytes=228080 combine_payload_bytes=391071744 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=391299824 ingress_bytes=721373184 link_bytes=721373184 token_bytes=782371568 weight_movement_bytes=330301440 total_nvlink_bytes=1112673008 weight_to_token_ratio=0.422180 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.254586612e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=384161792 dispatch_meta_bytes=229368 combine_payload_bytes=384161792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384391160 ingress_bytes=494262272 link_bytes=494262272 token_bytes=768552952 weight_movement_bytes=110100480 total_nvlink_bytes=878653432 weight_to_token_ratio=0.143257 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.596033832e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=398755840 dispatch_meta_bytes=229680 combine_payload_bytes=398755840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398985520 ingress_bytes=508856320 link_bytes=508856320 token_bytes=797741360 weight_movement_bytes=110100480 total_nvlink_bytes=907841840 weight_to_token_ratio=0.138015 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.849848330e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=402683904 dispatch_meta_bytes=229368 combine_payload_bytes=402683904 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=402913272 ingress_bytes=622884864 link_bytes=622884864 token_bytes=805597176 weight_movement_bytes=220200960 total_nvlink_bytes=1025798136 weight_to_token_ratio=0.273339 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083299225e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=402483200 dispatch_meta_bytes=228512 combine_payload_bytes=402483200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402711712 ingress_bytes=512583680 link_bytes=512583680 token_bytes=805194912 weight_movement_bytes=110100480 total_nvlink_bytes=915295392 weight_to_token_ratio=0.136738 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.914673250e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=397881344 dispatch_meta_bytes=229384 combine_payload_bytes=397881344 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=398110728 ingress_bytes=507981824 link_bytes=507981824 token_bytes=795992072 weight_movement_bytes=110100480 total_nvlink_bytes=906092552 weight_to_token_ratio=0.138319 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.834639406e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=368521216 dispatch_meta_bytes=225984 combine_payload_bytes=368521216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=368747200 ingress_bytes=478621696 link_bytes=478621696 token_bytes=737268416 weight_movement_bytes=110100480 total_nvlink_bytes=847368896 weight_to_token_ratio=0.149336 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.324018491e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63242 payload_rows_remote=55054 dup_rows=2294 dup_rows_remote=114 dedup_ratio=0.035004 router_tokens_per_expert=[0, 0, 0, 54, 4096, 0, 0, 0, 4081, 0, 0, 0, 0, 0, 1103, 0, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4096, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 1, 0, 2956, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8184, 8192, 8192, 8188, 8192, 5914] dup_rows_per_dst=[108, 2180, 0, 6, 0, 0, 0, 0] combine_rows_per_src=[8192, 8188, 8184, 8192, 8192, 8188, 8192, 5914] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63974 payload_rows_remote=55868 dup_rows=1562 dup_rows_remote=384 dedup_ratio=0.023834 router_tokens_per_expert=[0, 1146, 0, 3485, 0, 6, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 3, 0, 1, 0, 0, 1, 0, 0, 4096, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 3502, 0, 0, 0, 0, 0, 0, 133, 5, 0, 4062, 0, 0, 1, 0, 0] payload_rows_per_dst=[8106, 8188, 8170, 8180, 8176, 7904, 7116, 8134] dup_rows_per_dst=[1178, 6, 222, 0, 0, 0, 154, 2] combine_rows_per_src=[8106, 8188, 8170, 8180, 8176, 7904, 7116, 8134] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=394627072 dispatch_meta_bytes=220672 combine_payload_bytes=394627072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394847744 ingress_bytes=614828032 link_bytes=614828032 token_bytes=789474816 weight_movement_bytes=220200960 total_nvlink_bytes=1009675776 weight_to_token_ratio=0.278921 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069287069e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64664 payload_rows_remote=56472 dup_rows=872 dup_rows_remote=872 dedup_ratio=0.013306 router_tokens_per_expert=[0, 679, 0, 3658, 0, 0, 0, 0, 4096, 0, 0, 1, 1, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 15, 0, 1, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3707, 0, 0, 0, 0, 0, 0, 166, 1, 0, 4065, 0, 0, 0, 4, 0] payload_rows_per_dst=[8010, 8192, 8186, 8192, 8184, 8192, 7572, 8136] dup_rows_per_dst=[664, 4, 22, 4, 0, 0, 174, 4] combine_rows_per_src=[8010, 8192, 8186, 8192, 8184, 8192, 7572, 8136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54410 payload_rows_remote=47290 dup_rows=11126 dup_rows_remote=10448 dedup_ratio=0.169769 router_tokens_per_expert=[0, 7, 0, 2571, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2354, 0, 3766, 507, 3392, 0, 0, 0, 0, 718, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3451, 29, 0, 0, 4096, 2656, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 3, 0, 952, 4096, 0, 39, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 7734, 7120, 7730, 5342, 8192, 1910, 8192] dup_rows_per_dst=[5094, 3876, 678, 1244, 28, 128, 0, 78] combine_rows_per_src=[8190, 7734, 7120, 7730, 5342, 8192, 1910, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57404 payload_rows_remote=49444 dup_rows=8132 dup_rows_remote=6384 dedup_ratio=0.124084 router_tokens_per_expert=[0, 2, 0, 2307, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2351, 0, 3954, 598, 3332, 0, 0, 0, 0, 900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3258, 8, 0, 0, 4096, 2852, 46, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4081, 0, 1, 0, 883, 4096, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[5182, 7680, 7048, 7960, 5710, 8192, 7440, 8192] dup_rows_per_dst=[290, 3538, 812, 1748, 10, 94, 1636, 4] combine_rows_per_src=[5182, 7680, 7048, 7960, 5710, 8192, 7440, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57898 payload_rows_remote=50126 dup_rows=7638 dup_rows_remote=4102 dedup_ratio=0.116547 router_tokens_per_expert=[0, 3, 0, 2593, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2619, 0, 3955, 428, 3330, 0, 0, 0, 0, 1023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3146, 20, 0, 0, 4096, 2644, 25, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 690, 4096, 0, 7, 0, 0, 0, 0, 0] payload_rows_per_dst=[5192, 5440, 6938, 7990, 7772, 8192, 8182, 8192] dup_rows_per_dst=[0, 110, 578, 1966, 3536, 54, 1380, 14] combine_rows_per_src=[5192, 5440, 6938, 7990, 7772, 8192, 8182, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52166 payload_rows_remote=43974 dup_rows=13370 dup_rows_remote=5646 dedup_ratio=0.204010 router_tokens_per_expert=[0, 5, 0, 2731, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2319, 0, 3958, 797, 3087, 0, 0, 0, 0, 743, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2872, 5, 0, 0, 4095, 3124, 48, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4086, 0, 0, 0, 784, 4096, 0, 11, 0, 0, 0, 2, 1] payload_rows_per_dst=[5472, 4638, 1594, 7972, 7916, 8190, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 1436, 4086, 96, 7724, 28] combine_rows_per_src=[5472, 4638, 1594, 7972, 7916, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51272 payload_rows_remote=43080 dup_rows=14264 dup_rows_remote=14200 dedup_ratio=0.217651 router_tokens_per_expert=[0, 16, 0, 1983, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2639, 0, 3769, 881, 3356, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3590, 9, 0, 0, 4096, 2823, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3959, 0, 0, 0, 784, 4096, 0, 29, 0, 0, 0, 3, 0] payload_rows_per_dst=[3968, 5278, 1762, 7626, 8068, 8192, 8186, 8192] dup_rows_per_dst=[30, 0, 0, 938, 4776, 444, 8012, 64] combine_rows_per_src=[3968, 5278, 1762, 7626, 8068, 8192, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=400461824 dispatch_meta_bytes=225008 combine_payload_bytes=400461824 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=400686832 ingress_bytes=950964224 link_bytes=950964224 token_bytes=801148656 weight_movement_bytes=550502400 total_nvlink_bytes=1351651056 weight_to_token_ratio=0.687141 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.653883192e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=404791296 dispatch_meta_bytes=229376 combine_payload_bytes=404791296 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=405020672 ingress_bytes=624992256 link_bytes=624992256 token_bytes=809811968 weight_movement_bytes=220200960 total_nvlink_bytes=1030012928 weight_to_token_ratio=0.271916 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.086964327e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338974720 dispatch_meta_bytes=230952 combine_payload_bytes=338974720 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=339205672 ingress_bytes=669276160 link_bytes=669276160 token_bytes=678180392 weight_movement_bytes=330301440 total_nvlink_bytes=1008481832 weight_to_token_ratio=0.487041 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.163981319e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=354414592 dispatch_meta_bytes=223312 combine_payload_bytes=354414592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354637904 ingress_bytes=464515072 link_bytes=464515072 token_bytes=709052496 weight_movement_bytes=110100480 total_nvlink_bytes=819152976 weight_to_token_ratio=0.155278 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.078681098e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=359303168 dispatch_meta_bytes=216912 combine_payload_bytes=359303168 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=359520080 ingress_bytes=799705088 link_bytes=799705088 token_bytes=718823248 weight_movement_bytes=440401920 total_nvlink_bytes=1159225168 weight_to_token_ratio=0.612671 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.390818677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=315205632 dispatch_meta_bytes=198480 combine_payload_bytes=315205632 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=315404112 ingress_bytes=535406592 link_bytes=535406592 token_bytes=630609744 weight_movement_bytes=220200960 total_nvlink_bytes=850810704 weight_to_token_ratio=0.349187 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.311601227e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=308797440 dispatch_meta_bytes=229120 combine_payload_bytes=308797440 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=309026560 ingress_bytes=418897920 link_bytes=418897920 token_bytes=617824000 weight_movement_bytes=110100480 total_nvlink_bytes=727924480 weight_to_token_ratio=0.178207 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.285323797e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64050 payload_rows_remote=55858 dup_rows=1486 dup_rows_remote=1486 dedup_ratio=0.022675 router_tokens_per_expert=[0, 1322, 0, 3349, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4096, 0, 4, 0, 0, 0, 0, 0, 3481, 0, 0, 0, 0, 0, 0, 160, 0, 0, 3977, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 8192, 8184, 8192, 8190, 8192, 7092, 7954] dup_rows_per_dst=[1288, 0, 0, 0, 0, 8, 190, 0] combine_rows_per_src=[8054, 8192, 8184, 8192, 8190, 8192, 7092, 7954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43718 payload_rows_remote=35528 dup_rows=21818 dup_rows_remote=16440 dedup_ratio=0.332916 router_tokens_per_expert=[0, 22, 0, 2732, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2222, 0, 3693, 574, 3424, 0, 0, 0, 0, 749, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3628, 24, 0, 0, 4096, 2485, 189, 0, 0, 8, 0, 1, 0, 0, 0, 0, 0, 4030, 0, 1, 0, 747, 4096, 0, 45, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 7972, 8178, 1498, 1678, 6836, 1496, 7870] dup_rows_per_dst=[5378, 3730, 12174, 0, 410, 114, 0, 12] combine_rows_per_src=[8190, 7972, 8178, 1498, 1678, 6836, 1496, 7870] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65002 payload_rows_remote=56812 dup_rows=534 dup_rows_remote=534 dedup_ratio=0.008148 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 4011, 0, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 28, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4004, 0, 0, 0, 238, 0, 0, 0, 0, 1] payload_rows_per_dst=[8192, 8022, 8190, 8192, 8016, 8190, 8008, 8192] dup_rows_per_dst=[6, 0, 0, 0, 50, 0, 0, 478] combine_rows_per_src=[8192, 8022, 8190, 8192, 8016, 8190, 8008, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=400390144 dispatch_meta_bytes=229376 combine_payload_bytes=400390144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400619520 ingress_bytes=510490624 link_bytes=510490624 token_bytes=801009664 weight_movement_bytes=110100480 total_nvlink_bytes=911110144 weight_to_token_ratio=0.137452 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.878271564e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64972 payload_rows_remote=56780 dup_rows=564 dup_rows_remote=564 dedup_ratio=0.008606 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4065, 0, 4095, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3998, 0, 79, 4096, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3934, 0, 0, 0, 212, 0, 0, 0, 0, 2] payload_rows_per_dst=[8190, 8130, 8192, 8192, 8018, 8190, 7868, 8192] dup_rows_per_dst=[0, 0, 0, 0, 136, 0, 0, 428] combine_rows_per_src=[8190, 8130, 8192, 8192, 8018, 8190, 7868, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57248 payload_rows_remote=49056 dup_rows=8288 dup_rows_remote=7996 dedup_ratio=0.126465 router_tokens_per_expert=[0, 4, 0, 2906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2287, 0, 3894, 454, 3420, 0, 0, 0, 0, 933, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3095, 23, 0, 0, 4096, 2600, 141, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4053, 0, 3, 0, 718, 4096, 0, 39, 0, 0, 0, 0, 0] payload_rows_per_dst=[5814, 4574, 6684, 7924, 7752, 8192, 8116, 8192] dup_rows_per_dst=[6, 0, 614, 1730, 3686, 292, 1882, 78] combine_rows_per_src=[5814, 4574, 6684, 7924, 7752, 8192, 8116, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65224 payload_rows_remote=57054 dup_rows=312 dup_rows_remote=268 dedup_ratio=0.004761 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4048, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 22, 4096, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3999, 0, 0, 0, 131, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 8096, 8192, 8192, 8170, 8192, 7998, 8192] dup_rows_per_dst=[0, 0, 0, 0, 44, 0, 0, 268] combine_rows_per_src=[8192, 8096, 8192, 8192, 8170, 8192, 7998, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407228416 dispatch_meta_bytes=229384 combine_payload_bytes=407228416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407457800 ingress_bytes=517328896 link_bytes=517328896 token_bytes=814686216 weight_movement_bytes=110100480 total_nvlink_bytes=924786696 weight_to_token_ratio=0.135145 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.997200361e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=254664704 dispatch_meta_bytes=207872 combine_payload_bytes=254664704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=254872576 ingress_bytes=364765184 link_bytes=364765184 token_bytes=509537280 weight_movement_bytes=110100480 total_nvlink_bytes=619637760 weight_to_token_ratio=0.216079 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.343866485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65286 payload_rows_remote=57248 dup_rows=250 dup_rows_remote=250 dedup_ratio=0.003815 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 4086, 0, 4096, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 38, 4096, 0, 4092, 0, 0, 0, 0, 2, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4019, 0, 0, 0, 88, 0, 0, 0, 0, 3] payload_rows_per_dst=[8192, 8174, 8190, 8192, 8120, 8188, 8038, 8192] dup_rows_per_dst=[2, 0, 0, 0, 66, 0, 0, 182] combine_rows_per_src=[8192, 8174, 8190, 8192, 8120, 8188, 8038, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64902 payload_rows_remote=56710 dup_rows=634 dup_rows_remote=204 dedup_ratio=0.009674 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 4057, 0, 4096, 0, 4095, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3933, 0, 141, 4096, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3947, 0, 0, 0, 208, 0, 0, 0, 0, 7] payload_rows_per_dst=[8180, 8114, 8192, 8190, 7962, 8178, 7894, 8192] dup_rows_per_dst=[2, 12, 0, 2, 160, 28, 0, 430] combine_rows_per_src=[8180, 8114, 8192, 8190, 7962, 8178, 7894, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=351633408 dispatch_meta_bytes=228208 combine_payload_bytes=351633408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351861616 ingress_bytes=461733888 link_bytes=461733888 token_bytes=703495024 weight_movement_bytes=110100480 total_nvlink_bytes=813595504 weight_to_token_ratio=0.156505 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.030311734e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406999040 dispatch_meta_bytes=229376 combine_payload_bytes=406999040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407228416 ingress_bytes=517099520 link_bytes=517099520 token_bytes=814227456 weight_movement_bytes=110100480 total_nvlink_bytes=924327936 weight_to_token_ratio=0.135221 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.993211135e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=408963072 dispatch_meta_bytes=229288 combine_payload_bytes=408963072 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=409192360 ingress_bytes=849364992 link_bytes=849364992 token_bytes=818155432 weight_movement_bytes=440401920 total_nvlink_bytes=1258557352 weight_to_token_ratio=0.538286 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.477185418e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=410353664 dispatch_meta_bytes=229992 combine_payload_bytes=410353664 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=410583656 ingress_bytes=740655104 link_bytes=740655104 token_bytes=820937320 weight_movement_bytes=330301440 total_nvlink_bytes=1151238760 weight_to_token_ratio=0.402347 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.288121043e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406497280 dispatch_meta_bytes=227656 combine_payload_bytes=406497280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406724936 ingress_bytes=516597760 link_bytes=516597760 token_bytes=813222216 weight_movement_bytes=110100480 total_nvlink_bytes=923322696 weight_to_token_ratio=0.135388 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.984484704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52778 payload_rows_remote=45068 dup_rows=12758 dup_rows_remote=9502 dedup_ratio=0.194672 router_tokens_per_expert=[0, 8, 0, 2321, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2545, 0, 3990, 775, 3493, 0, 0, 0, 0, 497, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2938, 13, 0, 0, 4096, 3180, 62, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 4036, 0, 0, 0, 690, 4096, 0, 22, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 7710, 7438, 6922, 4764, 8192, 1380, 8192] dup_rows_per_dst=[4550, 3256, 3918, 852, 2, 136, 0, 44] combine_rows_per_src=[8180, 7710, 7438, 6922, 4764, 8192, 1380, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63676 payload_rows_remote=55484 dup_rows=1860 dup_rows_remote=1538 dedup_ratio=0.028381 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 736, 4096, 0, 4096, 0, 2710, 0, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4029, 17, 0, 0, 0, 0, 0, 0, 678, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6744, 8192, 7914, 8192, 8058, 8192, 8192] dup_rows_per_dst=[0, 148, 322, 0, 0, 34, 1356, 0] combine_rows_per_src=[8192, 6744, 8192, 7914, 8192, 8058, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62238 payload_rows_remote=54046 dup_rows=3298 dup_rows_remote=3294 dedup_ratio=0.050323 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 4021, 0, 4096, 0, 4095, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3975, 0, 52, 4096, 0, 4092, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3990, 0, 0, 0, 236, 0, 0, 0, 0, 7] payload_rows_per_dst=[8192, 8140, 8182, 8192, 7974, 8192, 8016, 5350] dup_rows_per_dst=[4, 776, 2, 0, 634, 0, 1734, 148] combine_rows_per_src=[8192, 8140, 8182, 8192, 7974, 8192, 8016, 5350] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=323047424 dispatch_meta_bytes=218280 combine_payload_bytes=323047424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323265704 ingress_bytes=433147904 link_bytes=433147904 token_bytes=646313128 weight_movement_bytes=110100480 total_nvlink_bytes=756413608 weight_to_token_ratio=0.170352 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.533154456e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64908 payload_rows_remote=56720 dup_rows=628 dup_rows_remote=626 dedup_ratio=0.009583 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4002, 0, 4096, 0, 4095, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3980, 0, 66, 4096, 0, 4094, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 259, 0, 0, 0, 0, 4] payload_rows_per_dst=[8192, 8004, 8192, 8192, 7998, 8188, 7950, 8192] dup_rows_per_dst=[0, 0, 6, 0, 94, 2, 0, 526] combine_rows_per_src=[8192, 8004, 8192, 8192, 7998, 8188, 7950, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63348 payload_rows_remote=55396 dup_rows=2188 dup_rows_remote=2188 dedup_ratio=0.033386 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 839, 4096, 0, 4096, 0, 2414, 0, 0, 0, 323, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3976, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4027, 63, 0, 0, 0, 0, 0, 0, 645, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6380, 8192, 7952, 8192, 8056, 8192, 8192] dup_rows_per_dst=[0, 126, 646, 0, 0, 124, 1292, 0] combine_rows_per_src=[8192, 6380, 8192, 7952, 8192, 8056, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397709312 dispatch_meta_bytes=228088 combine_payload_bytes=397709312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=397937400 ingress_bytes=507809792 link_bytes=507809792 token_bytes=795646712 weight_movement_bytes=110100480 total_nvlink_bytes=905747192 weight_to_token_ratio=0.138379 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.831647486e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387401728 dispatch_meta_bytes=229360 combine_payload_bytes=387401728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=387631088 ingress_bytes=497502208 link_bytes=497502208 token_bytes=775032816 weight_movement_bytes=110100480 total_nvlink_bytes=885133296 weight_to_token_ratio=0.142059 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.652381648e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64416 payload_rows_remote=56224 dup_rows=1120 dup_rows_remote=1120 dedup_ratio=0.017090 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 703, 4096, 0, 4096, 0, 3227, 0, 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 3875, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4040, 6, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7626, 8192, 7750, 8192, 8080, 8192, 8192] dup_rows_per_dst=[0, 234, 288, 0, 0, 12, 586, 0] combine_rows_per_src=[8192, 7626, 8192, 7750, 8192, 8080, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62998 payload_rows_remote=54806 dup_rows=2538 dup_rows_remote=680 dedup_ratio=0.038727 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 776, 4096, 0, 4096, 0, 2325, 0, 0, 0, 234, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3979, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 4009, 37, 0, 0, 0, 0, 0, 0, 927, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6068, 8192, 7958, 8192, 8018, 8192, 8186] dup_rows_per_dst=[0, 134, 468, 2, 2, 74, 1858, 0] combine_rows_per_src=[8192, 6068, 8192, 7958, 8192, 8018, 8192, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63316 payload_rows_remote=55128 dup_rows=2220 dup_rows_remote=2204 dedup_ratio=0.033875 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 800, 4096, 0, 4096, 0, 2384, 0, 0, 0, 363, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3981, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4066, 93, 0, 0, 0, 0, 0, 0, 600, 0, 0, 0, 0, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8190, 6286, 8192, 7946, 8190, 8132, 8192, 8188] dup_rows_per_dst=[2, 84, 726, 0, 0, 186, 1206, 16] combine_rows_per_src=[8190, 6286, 8192, 7946, 8190, 8132, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406568960 dispatch_meta_bytes=229384 combine_payload_bytes=406568960 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=406798344 ingress_bytes=626769920 link_bytes=626769920 token_bytes=813367304 weight_movement_bytes=220200960 total_nvlink_bytes=1033568264 weight_to_token_ratio=0.270728 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090055977e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397078528 dispatch_meta_bytes=230336 combine_payload_bytes=397078528 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=397308864 ingress_bytes=727379968 link_bytes=727379968 token_bytes=794387392 weight_movement_bytes=330301440 total_nvlink_bytes=1124688832 weight_to_token_ratio=0.415794 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.265033398e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=392849408 dispatch_meta_bytes=221944 combine_payload_bytes=392849408 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=393071352 ingress_bytes=833251328 link_bytes=833251328 token_bytes=785920760 weight_movement_bytes=440401920 total_nvlink_bytes=1226322680 weight_to_token_ratio=0.560364 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.449161106e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=403013632 dispatch_meta_bytes=229376 combine_payload_bytes=403013632 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=403243008 ingress_bytes=513114112 link_bytes=513114112 token_bytes=806256640 weight_movement_bytes=110100480 total_nvlink_bytes=916357120 weight_to_token_ratio=0.136558 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.923898335e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=395157504 dispatch_meta_bytes=229328 combine_payload_bytes=395157504 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=395386832 ingress_bytes=615358464 link_bytes=615358464 token_bytes=790544336 weight_movement_bytes=220200960 total_nvlink_bytes=1010745296 weight_to_token_ratio=0.278543 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070209578e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65070 payload_rows_remote=57002 dup_rows=466 dup_rows_remote=466 dedup_ratio=0.007111 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4034, 0, 4096, 0, 4096, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4005, 0, 42, 4096, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4015, 0, 0, 0, 176, 0, 0, 0, 0, 15] payload_rows_per_dst=[8192, 8068, 8192, 8192, 8016, 8188, 8030, 8192] dup_rows_per_dst=[0, 0, 6, 0, 78, 0, 0, 382] combine_rows_per_src=[8192, 8068, 8192, 8192, 8016, 8188, 8030, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63454 payload_rows_remote=55264 dup_rows=2082 dup_rows_remote=20 dedup_ratio=0.031769 router_tokens_per_expert=[0, 4093, 1, 0, 4050, 0, 0, 0, 0, 0, 3, 0, 0, 4, 0, 0, 0, 0, 39, 994, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3113, 3, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4086] payload_rows_per_dst=[8102, 8190, 8190, 8190, 8192, 6226, 8192, 8172] dup_rows_per_dst=[0, 10, 2062, 0, 0, 8, 2, 0] combine_rows_per_src=[8102, 8190, 8190, 8190, 8192, 6226, 8192, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55178 payload_rows_remote=46986 dup_rows=10358 dup_rows_remote=10358 dedup_ratio=0.158051 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 729, 4096, 0, 4096, 0, 2381, 0, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 3956, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4054, 117, 0, 0, 0, 0, 0, 0, 850, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 4696, 7912, 8192, 8108, 1700, 8186] dup_rows_per_dst=[0, 9892, 226, 4, 0, 234, 2, 0] combine_rows_per_src=[8192, 8192, 4696, 7912, 8192, 8108, 1700, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=408590336 dispatch_meta_bytes=229872 combine_payload_bytes=408590336 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=408820208 ingress_bytes=738891776 link_bytes=738891776 token_bytes=817410544 weight_movement_bytes=330301440 total_nvlink_bytes=1147711984 weight_to_token_ratio=0.404083 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.285054325e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62906 payload_rows_remote=54716 dup_rows=2630 dup_rows_remote=2628 dedup_ratio=0.040131 router_tokens_per_expert=[0, 4081, 1, 0, 4041, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 45, 1267, 0, 4095, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2855, 1, 1, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4090] payload_rows_per_dst=[8084, 8164, 8192, 8190, 8192, 5712, 8192, 8180] dup_rows_per_dst=[0, 2, 2622, 2, 0, 2, 2, 0] combine_rows_per_src=[8084, 8164, 8192, 8190, 8192, 5712, 8192, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62888 payload_rows_remote=54878 dup_rows=2648 dup_rows_remote=2538 dedup_ratio=0.040405 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 848, 4096, 0, 4096, 0, 2263, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 3939, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4004, 56, 0, 0, 0, 0, 0, 0, 951, 0, 0, 0, 0, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[8192, 6040, 8192, 7878, 8192, 8010, 8192, 8192] dup_rows_per_dst=[0, 182, 452, 0, 0, 110, 1904, 0] combine_rows_per_src=[8192, 6040, 8192, 7878, 8192, 8010, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=396132352 dispatch_meta_bytes=221136 combine_payload_bytes=396132352 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=396353488 ingress_bytes=946634752 link_bytes=946634752 token_bytes=792485840 weight_movement_bytes=550502400 total_nvlink_bytes=1342988240 weight_to_token_ratio=0.694653 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=22400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.646353528e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60160 payload_rows_remote=51968 dup_rows=5376 dup_rows_remote=5372 dedup_ratio=0.082031 router_tokens_per_expert=[0, 4093, 0, 0, 4075, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 42, 1398, 0, 4095, 4096, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2678, 3, 1, 0, 2, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4088] payload_rows_per_dst=[8150, 8186, 2880, 8192, 8192, 8192, 8192, 8176] dup_rows_per_dst=[0, 4, 0, 6, 0, 5362, 4, 0] combine_rows_per_src=[8150, 8186, 2880, 8192, 8192, 8192, 8192, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=336795648 dispatch_meta_bytes=229376 combine_payload_bytes=336795648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337025024 ingress_bytes=446896128 link_bytes=446896128 token_bytes=673820672 weight_movement_bytes=110100480 total_nvlink_bytes=783921152 weight_to_token_ratio=0.163397 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.772258684e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64074 payload_rows_remote=55882 dup_rows=1462 dup_rows_remote=1462 dedup_ratio=0.022308 router_tokens_per_expert=[0, 4095, 0, 0, 4087, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 43, 686, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3376, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4095] payload_rows_per_dst=[8174, 8190, 8192, 8192, 8192, 6752, 8192, 8190] dup_rows_per_dst=[0, 2, 1458, 2, 0, 0, 0, 0] combine_rows_per_src=[8174, 8190, 8192, 8192, 8192, 6752, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60714 payload_rows_remote=52542 dup_rows=4822 dup_rows_remote=4704 dedup_ratio=0.073578 router_tokens_per_expert=[0, 4031, 0, 0, 4025, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 44, 1896, 0, 4094, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2282, 3, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4086] payload_rows_per_dst=[7852, 8086, 3876, 8192, 8192, 8152, 8192, 8172] dup_rows_per_dst=[0, 200, 4, 0, 0, 4500, 0, 118] combine_rows_per_src=[7852, 8086, 3876, 8192, 8192, 8152, 8192, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=392204288 dispatch_meta_bytes=229376 combine_payload_bytes=392204288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392433664 ingress_bytes=502304768 link_bytes=502304768 token_bytes=784637952 weight_movement_bytes=110100480 total_nvlink_bytes=894738432 weight_to_token_ratio=0.140320 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.735906065e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=393365504 dispatch_meta_bytes=229664 combine_payload_bytes=393365504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=393595168 ingress_bytes=503465984 link_bytes=503465984 token_bytes=786960672 weight_movement_bytes=110100480 total_nvlink_bytes=897061152 weight_to_token_ratio=0.139906 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.756101521e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=372506624 dispatch_meta_bytes=229360 combine_payload_bytes=372506624 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=372735984 ingress_bytes=482607104 link_bytes=482607104 token_bytes=745242608 weight_movement_bytes=110100480 total_nvlink_bytes=855343088 weight_to_token_ratio=0.147738 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.393331291e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=400562176 dispatch_meta_bytes=229376 combine_payload_bytes=400562176 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400791552 ingress_bytes=510662656 link_bytes=510662656 token_bytes=801353728 weight_movement_bytes=110100480 total_nvlink_bytes=911454208 weight_to_token_ratio=0.137393 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.881263484e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=376621056 dispatch_meta_bytes=228984 combine_payload_bytes=376621056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=376850040 ingress_bytes=596822016 link_bytes=596822016 token_bytes=753471096 weight_movement_bytes=220200960 total_nvlink_bytes=973672056 weight_to_token_ratio=0.292249 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.037971646e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61024 payload_rows_remote=54422 dup_rows=4512 dup_rows_remote=3714 dedup_ratio=0.068848 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1103, 4096, 0, 4096, 0, 2157, 0, 0, 0, 184, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 4041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4036, 74, 0, 0, 0, 0, 0, 0, 693, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6602, 8192, 8108, 8192, 8072, 5474, 8192] dup_rows_per_dst=[0, 798, 368, 2112, 0, 228, 1006, 0] combine_rows_per_src=[8192, 6602, 8192, 8108, 8192, 8072, 5474, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65520 payload_rows_remote=57328 dup_rows=16 dup_rows_remote=12 dedup_ratio=0.000244 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4095, 3, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 1, 0, 4093, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8186, 8192, 8192, 8190, 8186] dup_rows_per_dst=[2, 0, 4, 0, 8, 0, 0, 2] combine_rows_per_src=[8190, 8192, 8192, 8186, 8192, 8192, 8190, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=390096896 dispatch_meta_bytes=232544 combine_payload_bytes=390096896 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=390329440 ingress_bytes=720398336 link_bytes=720398336 token_bytes=780426336 weight_movement_bytes=330301440 total_nvlink_bytes=1110727776 weight_to_token_ratio=0.423232 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252891191e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62184 payload_rows_remote=54004 dup_rows=3352 dup_rows_remote=2558 dedup_ratio=0.051147 router_tokens_per_expert=[0, 4071, 0, 0, 4068, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 43, 1477, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2622, 3, 4, 0, 7, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 0, 4087] payload_rows_per_dst=[8180, 8146, 7868, 8188, 8192, 5252, 8182, 8176] dup_rows_per_dst=[794, 4, 2548, 0, 0, 6, 0, 0] combine_rows_per_src=[8180, 8146, 7868, 8188, 8192, 5252, 8182, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61552 payload_rows_remote=54746 dup_rows=3984 dup_rows_remote=1472 dedup_ratio=0.060791 router_tokens_per_expert=[0, 4089, 3, 0, 4061, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 35, 1322, 0, 4093, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2788, 3, 3, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4082] payload_rows_per_dst=[8128, 8180, 5702, 8188, 8192, 6806, 8192, 8164] dup_rows_per_dst=[0, 2, 1468, 0, 0, 2512, 2, 0] combine_rows_per_src=[8128, 8180, 5702, 8188, 8192, 6806, 8192, 8164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65530 payload_rows_remote=57340 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 1, 1, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8190, 8192, 8192, 8190, 8192, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 2, 0, 4, 0, 0, 0] combine_rows_per_src=[8190, 8192, 8192, 8190, 8192, 8192, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410927104 dispatch_meta_bytes=229360 combine_payload_bytes=410927104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411156464 ingress_bytes=521027584 link_bytes=521027584 token_bytes=822083568 weight_movement_bytes=110100480 total_nvlink_bytes=932184048 weight_to_token_ratio=0.133929 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.061526629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65526 payload_rows_remote=57336 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 2, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 8192, 8192, 8190, 8190] dup_rows_per_dst=[2, 0, 0, 0, 8, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 8192, 8192, 8190, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65536 payload_rows_remote=57344 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65496 payload_rows_remote=57306 dup_rows=40 dup_rows_remote=32 dedup_ratio=0.000610 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4092, 2, 6, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4092, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8178, 8188, 8180, 8192, 8184, 8190] dup_rows_per_dst=[4, 0, 2, 0, 8, 0, 18, 8] combine_rows_per_src=[8192, 8192, 8178, 8188, 8180, 8192, 8184, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=387100672 dispatch_meta_bytes=226248 combine_payload_bytes=387100672 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=387326920 ingress_bytes=717402112 link_bytes=717402112 token_bytes=774427592 weight_movement_bytes=330301440 total_nvlink_bytes=1104729032 weight_to_token_ratio=0.426510 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.247680265e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=392419328 dispatch_meta_bytes=224872 combine_payload_bytes=392419328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=392644200 ingress_bytes=502519808 link_bytes=502519808 token_bytes=785063528 weight_movement_bytes=110100480 total_nvlink_bytes=895164008 weight_to_token_ratio=0.140244 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.739645964e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411013120 dispatch_meta_bytes=229384 combine_payload_bytes=411013120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411242504 ingress_bytes=631214080 link_bytes=631214080 token_bytes=822255624 weight_movement_bytes=220200960 total_nvlink_bytes=1042456584 weight_to_token_ratio=0.267801 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097785102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410984448 dispatch_meta_bytes=229384 combine_payload_bytes=410984448 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411213832 ingress_bytes=741285888 link_bytes=741285888 token_bytes=822198280 weight_movement_bytes=330301440 total_nvlink_bytes=1152499720 weight_to_token_ratio=0.401730 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289218080e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=229376 combine_payload_bytes=411041792 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=411271168 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822312960 weight_movement_bytes=330301440 total_nvlink_bytes=1152614400 weight_to_token_ratio=0.401674 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410769408 dispatch_meta_bytes=229352 combine_payload_bytes=410769408 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410998760 ingress_bytes=520869888 link_bytes=520869888 token_bytes=821768168 weight_movement_bytes=110100480 total_nvlink_bytes=931868648 weight_to_token_ratio=0.133980 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.058784036e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63252 payload_rows_remote=55082 dup_rows=2284 dup_rows_remote=2284 dedup_ratio=0.034851 router_tokens_per_expert=[0, 4085, 0, 0, 4036, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 1096, 0, 4096, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3027, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094] payload_rows_per_dst=[8072, 8170, 8192, 8192, 8192, 6054, 8192, 8188] dup_rows_per_dst=[0, 0, 2282, 2, 0, 0, 0, 0] combine_rows_per_src=[8072, 8170, 8192, 8192, 8192, 6054, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64042 payload_rows_remote=55850 dup_rows=1494 dup_rows_remote=1488 dedup_ratio=0.022797 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 17, 0, 0, 0, 0, 0, 0, 3352, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 418, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 6704, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 34, 0, 836, 618, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 6704, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394827776 dispatch_meta_bytes=229464 combine_payload_bytes=394827776 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=395057240 ingress_bytes=615028736 link_bytes=615028736 token_bytes=789885016 weight_movement_bytes=220200960 total_nvlink_bytes=1010085976 weight_to_token_ratio=0.278776 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.069636127e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400332800 dispatch_meta_bytes=229352 combine_payload_bytes=400332800 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400562152 ingress_bytes=510433280 link_bytes=510433280 token_bytes=800894952 weight_movement_bytes=110100480 total_nvlink_bytes=910995432 weight_to_token_ratio=0.137472 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.877274258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65506 payload_rows_remote=57314 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000458 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 8166, 8192, 8192, 8190] dup_rows_per_dst=[0, 0, 4, 26, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 8166, 8192, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65516 payload_rows_remote=57324 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 2, 3, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4093, 0, 0, 0, 2, 0, 4094, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8186, 8192, 8192, 8186, 8188] dup_rows_per_dst=[4, 0, 2, 0, 10, 0, 0, 4] combine_rows_per_src=[8188, 8192, 8192, 8186, 8192, 8192, 8186, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63834 payload_rows_remote=55644 dup_rows=1702 dup_rows_remote=1654 dedup_ratio=0.025970 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 0, 24, 0, 0, 0, 0, 0, 0, 3246, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8190, 6492, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 2, 48, 0, 528, 1122, 0] combine_rows_per_src=[8192, 8192, 8192, 8190, 6492, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64230 payload_rows_remote=56038 dup_rows=1306 dup_rows_remote=698 dedup_ratio=0.019928 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4094, 0, 31, 0, 0, 0, 0, 0, 0, 3445, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 315, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 303, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 6890, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 62, 0, 630, 608, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 6890, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64552 payload_rows_remote=57344 dup_rows=984 dup_rows_remote=984 dedup_ratio=0.015015 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 40, 0, 0, 0, 0, 0, 0, 3604, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7208, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 80, 0, 472, 432, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7208, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63434 payload_rows_remote=55242 dup_rows=2102 dup_rows_remote=2102 dedup_ratio=0.032074 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4090, 0, 24, 0, 0, 0, 0, 0, 0, 3240, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 217, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8188, 8158, 7840, 6480, 8192, 8192, 8192] dup_rows_per_dst=[0, 2, 10, 46, 0, 814, 1230, 0] combine_rows_per_src=[8192, 8188, 8158, 7840, 6480, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7547175320 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3774728600 ingress_bytes=5754255360 link_bytes=5754255360 total_nvlink_bytes=9528983960 weight_to_token_ratio=0.262589 prefetch_share_of_ingress=0.191337 dedup_ratio=0.076410 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046691 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.000759648e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410826752 dispatch_meta_bytes=229376 combine_payload_bytes=410826752 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411056128 ingress_bytes=520927232 link_bytes=520927232 token_bytes=821882880 weight_movement_bytes=110100480 total_nvlink_bytes=931983360 weight_to_token_ratio=0.133961 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.059781343e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=410898432 dispatch_meta_bytes=229376 combine_payload_bytes=410898432 prefetch_bytes=0 reduce_bytes=0 egress_bytes=411127808 ingress_bytes=410898432 link_bytes=411127808 token_bytes=822026240 weight_movement_bytes=0 total_nvlink_bytes=822026240 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.150188770e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=398856192 dispatch_meta_bytes=229192 combine_payload_bytes=398856192 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=399085384 ingress_bytes=619057152 link_bytes=619057152 token_bytes=797941576 weight_movement_bytes=220200960 total_nvlink_bytes=1018142536 weight_to_token_ratio=0.275961 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076642205e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401680384 dispatch_meta_bytes=226944 combine_payload_bytes=401680384 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=401907328 ingress_bytes=511780864 link_bytes=511780864 token_bytes=803587712 weight_movement_bytes=110100480 total_nvlink_bytes=913688192 weight_to_token_ratio=0.137011 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.900710960e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=411041792 dispatch_meta_bytes=233312 combine_payload_bytes=411041792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=411275104 ingress_bytes=741343232 link_bytes=741343232 token_bytes=822316896 weight_movement_bytes=330301440 total_nvlink_bytes=1152618336 weight_to_token_ratio=0.401672 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.289317810e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=395974656 dispatch_meta_bytes=229376 combine_payload_bytes=395974656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=396204032 ingress_bytes=506075136 link_bytes=506075136 token_bytes=792178688 weight_movement_bytes=110100480 total_nvlink_bytes=902279168 weight_to_token_ratio=0.138984 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.801478966e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65532 payload_rows_remote=57340 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 2, 0, 4096, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[8188, 8192, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7630077376 prefetch_bytes=770703360 reduce_bytes=990904320 weight_movement_bytes=1761607680 egress_bytes=3816185280 ingress_bytes=5575499776 link_bytes=5575499776 total_nvlink_bytes=9391685056 weight_to_token_ratio=0.230877 prefetch_share_of_ingress=0.138230 dedup_ratio=0.068280 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=9 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.696711123e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7456676384 prefetch_bytes=770703360 reduce_bytes=1321205760 weight_movement_bytes=2091909120 egress_bytes=3729459744 ingress_bytes=5819125760 link_bytes=5819125760 total_nvlink_bytes=9548585504 weight_to_token_ratio=0.280542 prefetch_share_of_ingress=0.132443 dedup_ratio=0.086920 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=12 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.012041678e-04 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7716093928 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3859193832 ingress_bytes=6169010176 link_bytes=6169010176 total_nvlink_bytes=10028204008 weight_to_token_ratio=0.299648 prefetch_share_of_ingress=0.142779 dedup_ratio=0.062198 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.072892332e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7258643896 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3630431672 ingress_bytes=5610020864 link_bytes=5610020864 total_nvlink_bytes=9240452536 weight_to_token_ratio=0.273027 prefetch_share_of_ingress=0.196257 dedup_ratio=0.105124 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.756748972e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=411013120 dispatch_meta_bytes=229376 combine_payload_bytes=411013120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=411242496 ingress_bytes=521113600 link_bytes=521113600 token_bytes=822255616 weight_movement_bytes=110100480 total_nvlink_bytes=932356096 weight_to_token_ratio=0.133901 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.063022589e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64136 payload_rows_remote=55944 dup_rows=1400 dup_rows_remote=802 dedup_ratio=0.021362 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 4093, 0, 16, 0, 0, 0, 0, 0, 0, 3399, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 382, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8186, 6798, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 6, 32, 0, 598, 764, 0] combine_rows_per_src=[8192, 8192, 8192, 8186, 6798, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54678 payload_rows_remote=46486 dup_rows=10858 dup_rows_remote=10858 dedup_ratio=0.165680 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 34, 0, 0, 0, 0, 0, 0, 3380, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 491, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 7746, 3792, 2184, 8192] dup_rows_per_dst=[0, 0, 4, 68, 10244, 250, 292, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 7746, 3792, 2184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401006592 dispatch_meta_bytes=226984 combine_payload_bytes=401006592 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=401233576 ingress_bytes=731308032 link_bytes=731308032 token_bytes=802240168 weight_movement_bytes=330301440 total_nvlink_bytes=1132541608 weight_to_token_ratio=0.411724 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.271864947e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=333211648 dispatch_meta_bytes=229376 combine_payload_bytes=333211648 prefetch_bytes=0 reduce_bytes=0 egress_bytes=333441024 ingress_bytes=333211648 link_bytes=333441024 token_bytes=666652672 weight_movement_bytes=0 total_nvlink_bytes=666652672 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=5.799087824e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64318 payload_rows_remote=56126 dup_rows=1218 dup_rows_remote=1218 dedup_ratio=0.018585 router_tokens_per_expert=[4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3487, 0, 0, 4096, 0, 0, 4096, 4096, 0, 4096, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 471, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6974, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 4, 0, 272, 942, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6974, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7613781360 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3808032112 ingress_bytes=5567356928 link_bytes=5567586304 total_nvlink_bytes=9375389040 weight_to_token_ratio=0.231371 prefetch_share_of_ingress=0.197761 dedup_ratio=0.070520 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047059 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.682948293e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7007885784 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3505070552 ingress_bytes=5374523392 link_bytes=5374752768 total_nvlink_bytes=8879593944 weight_to_token_ratio=0.267086 prefetch_share_of_ingress=0.184371 dedup_ratio=0.133130 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.347579058e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402311168 dispatch_meta_bytes=229376 combine_payload_bytes=402311168 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=402540544 ingress_bytes=512411648 link_bytes=512411648 token_bytes=804851712 weight_movement_bytes=110100480 total_nvlink_bytes=914952192 weight_to_token_ratio=0.136796 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.911681331e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7498260352 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3750256512 ingress_bytes=5619712000 link_bytes=5619712000 total_nvlink_bytes=9369968512 weight_to_token_ratio=0.249619 prefetch_share_of_ingress=0.195918 dedup_ratio=0.084290 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047426 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.773603452e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:57:47] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 369.4 | throughput per GPU (TFLOP/s/GPU): 334.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 9.389159E+00 | loss scale: 1.0 | grad norm: 4.969 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44746 payload_rows_remote=39026 dup_rows=20790 dup_rows_remote=18074 dedup_ratio=0.317230 router_tokens_per_expert=[2201, 763, 1630, 1731, 20, 198, 192, 45, 338, 2080, 1531, 486, 1, 743, 3, 1453, 1149, 854, 48, 2, 597, 818, 18, 1399, 90, 1004, 521, 562, 84, 94, 196, 1, 567, 92, 655, 5, 189, 0, 658, 112, 153, 640, 554, 587, 3, 895, 116, 369, 24, 104, 153, 861, 232, 759, 68, 1068, 362, 461, 448, 369, 675, 561, 176, 0] payload_rows_per_dst=[6710, 6144, 5720, 5108, 6176, 5378, 4838, 4672] dup_rows_per_dst=[4786, 2758, 2716, 2060, 2540, 2590, 1908, 1432] combine_rows_per_src=[6710, 6144, 5720, 5108, 6176, 5378, 4838, 4672] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45780 payload_rows_remote=39910 dup_rows=19756 dup_rows_remote=16896 dedup_ratio=0.301453 router_tokens_per_expert=[1966, 1527, 1462, 1348, 9, 97, 126, 37, 1025, 1688, 1284, 408, 0, 660, 0, 2004, 812, 611, 29, 5, 1398, 594, 33, 935, 200, 615, 432, 694, 30, 149, 278, 1, 390, 199, 919, 20, 99, 0, 896, 268, 247, 604, 429, 892, 13, 523, 52, 569, 65, 74, 238, 656, 199, 530, 37, 1030, 262, 666, 553, 572, 547, 499, 263, 0] payload_rows_per_dst=[5810, 6032, 5174, 5870, 6122, 5542, 5688, 5542] dup_rows_per_dst=[2096, 2162, 2036, 2860, 2836, 2740, 2538, 2488] combine_rows_per_src=[5810, 6032, 5174, 5870, 6122, 5542, 5688, 5542] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45028 payload_rows_remote=38214 dup_rows=20508 dup_rows_remote=15920 dedup_ratio=0.312927 router_tokens_per_expert=[1481, 627, 1273, 1680, 77, 333, 192, 36, 835, 2212, 1651, 936, 1, 703, 0, 1600, 821, 494, 7, 1, 1129, 529, 110, 753, 229, 954, 561, 327, 95, 82, 279, 1, 411, 147, 846, 7, 101, 8, 764, 225, 599, 545, 292, 351, 3, 683, 57, 653, 164, 208, 44, 1269, 344, 1032, 23, 966, 282, 538, 345, 463, 430, 706, 251, 2] payload_rows_per_dst=[3886, 5896, 5228, 5488, 6500, 5412, 6814, 5804] dup_rows_per_dst=[874, 2254, 1948, 2530, 2942, 2596, 4588, 2776] combine_rows_per_src=[3886, 5896, 5228, 5488, 6500, 5412, 6814, 5804] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44958 payload_rows_remote=39698 dup_rows=20578 dup_rows_remote=17916 dedup_ratio=0.313995 router_tokens_per_expert=[1762, 802, 1891, 1261, 95, 88, 401, 124, 352, 1586, 1190, 477, 0, 1039, 1, 1648, 860, 783, 9, 67, 909, 647, 196, 679, 307, 1095, 525, 430, 42, 197, 447, 1, 588, 390, 746, 9, 111, 0, 431, 174, 115, 476, 274, 650, 2, 1027, 47, 510, 29, 59, 191, 667, 519, 848, 34, 1079, 307, 220, 592, 1024, 814, 870, 54, 0] payload_rows_per_dst=[4634, 5304, 4910, 6074, 5792, 5260, 6022, 6962] dup_rows_per_dst=[908, 1730, 1670, 3538, 2278, 2662, 3210, 4582] combine_rows_per_src=[4634, 5304, 4910, 6074, 5792, 5260, 6022, 6962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44830 payload_rows_remote=37732 dup_rows=20706 dup_rows_remote=16602 dedup_ratio=0.315948 router_tokens_per_expert=[2400, 939, 2257, 1585, 164, 319, 137, 31, 208, 1846, 1512, 713, 0, 890, 9, 1233, 1282, 628, 41, 2, 715, 513, 104, 1057, 144, 1104, 430, 517, 64, 223, 286, 0, 411, 71, 410, 19, 86, 0, 594, 239, 326, 682, 366, 476, 0, 884, 132, 410, 34, 138, 469, 530, 366, 708, 35, 685, 368, 388, 490, 411, 452, 1126, 109, 0] payload_rows_per_dst=[2870, 4630, 6244, 6548, 5644, 5890, 5906, 7098] dup_rows_per_dst=[310, 1476, 3046, 3788, 1708, 3226, 3048, 4104] combine_rows_per_src=[2870, 4630, 6244, 6548, 5644, 5890, 5906, 7098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45720 payload_rows_remote=39842 dup_rows=19816 dup_rows_remote=17822 dedup_ratio=0.302368 router_tokens_per_expert=[1884, 668, 1407, 1357, 11, 201, 330, 70, 845, 1961, 1342, 533, 5, 660, 0, 1192, 1664, 1317, 18, 0, 831, 625, 59, 1120, 114, 931, 610, 533, 82, 189, 509, 1, 176, 165, 887, 15, 113, 0, 529, 90, 529, 644, 197, 396, 2, 1033, 250, 402, 79, 349, 83, 1041, 421, 529, 25, 953, 233, 365, 298, 469, 448, 905, 73, 0] payload_rows_per_dst=[4314, 5140, 5502, 6350, 5878, 6420, 6180, 5936] dup_rows_per_dst=[960, 1330, 2438, 3356, 1994, 3814, 3464, 2460] combine_rows_per_src=[4314, 5140, 5502, 6350, 5878, 6420, 6180, 5936] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279738368 dispatch_meta_bytes=228400 combine_payload_bytes=279738368 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=279966768 ingress_bytes=499939328 link_bytes=499939328 token_bytes=559705136 weight_movement_bytes=220200960 total_nvlink_bytes=779906096 weight_to_token_ratio=0.393423 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.694767173e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286074880 dispatch_meta_bytes=227224 combine_payload_bytes=286074880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286302104 ingress_bytes=396175360 link_bytes=396175360 token_bytes=572376984 weight_movement_bytes=110100480 total_nvlink_bytes=682477464 weight_to_token_ratio=0.192357 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.890141107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273917952 dispatch_meta_bytes=216536 combine_payload_bytes=273917952 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274134488 ingress_bytes=384018432 link_bytes=384018432 token_bytes=548052440 weight_movement_bytes=110100480 total_nvlink_bytes=658152920 weight_to_token_ratio=0.200894 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.678712134e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=284555264 dispatch_meta_bytes=230456 combine_payload_bytes=284555264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=284785720 ingress_bytes=394655744 link_bytes=394655744 token_bytes=569340984 weight_movement_bytes=110100480 total_nvlink_bytes=679441464 weight_to_token_ratio=0.193382 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.863712485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43914 payload_rows_remote=36796 dup_rows=21622 dup_rows_remote=15470 dedup_ratio=0.329926 router_tokens_per_expert=[1660, 566, 1644, 1503, 41, 272, 184, 42, 395, 1599, 1754, 519, 14, 393, 10, 1433, 1544, 629, 31, 1, 977, 706, 39, 943, 64, 1169, 576, 439, 185, 246, 450, 0, 581, 305, 1106, 14, 347, 1, 444, 86, 348, 835, 672, 477, 1, 954, 61, 216, 93, 72, 271, 317, 333, 1001, 55, 833, 199, 424, 725, 776, 668, 388, 133, 4] payload_rows_per_dst=[7118, 6600, 6266, 4666, 4388, 5228, 4626, 5022] dup_rows_per_dst=[6152, 4126, 3474, 1592, 1442, 1900, 1324, 1612] combine_rows_per_src=[7118, 6600, 6266, 4666, 4388, 5228, 4626, 5022] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270462976 dispatch_meta_bytes=217336 combine_payload_bytes=270462976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=270680312 ingress_bytes=380563456 link_bytes=380563456 token_bytes=541143288 weight_movement_bytes=110100480 total_nvlink_bytes=651243768 weight_to_token_ratio=0.203459 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.618624419e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=285587456 dispatch_meta_bytes=230656 combine_payload_bytes=285587456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=285818112 ingress_bytes=395687936 link_bytes=395687936 token_bytes=571405568 weight_movement_bytes=110100480 total_nvlink_bytes=681506048 weight_to_token_ratio=0.192684 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=13696 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.881664002e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56786 payload_rows_remote=48646 dup_rows=8750 dup_rows_remote=8720 dedup_ratio=0.133514 router_tokens_per_expert=[0, 0, 0, 4095, 1272, 0, 3, 4094, 2532, 0, 2715, 0, 1, 0, 1, 0, 0, 0, 0, 4070, 15, 0, 0, 0, 0, 1, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3086, 1, 0, 794, 0, 3926, 0, 0, 0, 0, 0, 1320, 0, 159, 0, 0, 0, 2, 0, 1, 0, 456, 0, 2, 125] payload_rows_per_dst=[4854, 7158, 8140, 8188, 6174, 7822, 6258, 8192] dup_rows_per_dst=[1066, 3340, 30, 6, 0, 1514, 1624, 1170] combine_rows_per_src=[4854, 7158, 8140, 8188, 6174, 7822, 6258, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56512 payload_rows_remote=48324 dup_rows=9024 dup_rows_remote=9020 dedup_ratio=0.137695 router_tokens_per_expert=[0, 0, 0, 4096, 1088, 0, 36, 4096, 2190, 0, 2598, 0, 0, 0, 0, 0, 0, 0, 0, 4062, 14, 7, 0, 0, 0, 0, 2, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3378, 0, 0, 726, 0, 3928, 0, 0, 0, 0, 0, 1456, 0, 138, 0, 0, 0, 0, 0, 2, 0, 646, 0, 1, 210] payload_rows_per_dst=[2246, 6930, 8136, 8188, 6756, 7872, 8192, 8192] dup_rows_per_dst=[2, 2646, 30, 4, 0, 1436, 3188, 1718] combine_rows_per_src=[2246, 6930, 8136, 8188, 6756, 7872, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44248 payload_rows_remote=37814 dup_rows=21288 dup_rows_remote=17768 dedup_ratio=0.324829 router_tokens_per_expert=[2099, 889, 1589, 1909, 19, 282, 251, 16, 489, 2174, 1620, 871, 7, 735, 19, 1236, 928, 625, 45, 3, 1211, 523, 68, 1179, 104, 1221, 505, 451, 118, 91, 364, 0, 184, 121, 652, 10, 344, 0, 917, 246, 120, 632, 727, 317, 0, 1083, 126, 434, 31, 94, 79, 328, 320, 870, 26, 943, 217, 439, 357, 634, 609, 152, 115, 0] payload_rows_per_dst=[7308, 6434, 6050, 4460, 6360, 5176, 4408, 4052] dup_rows_per_dst=[6800, 3520, 3114, 1248, 2936, 1702, 974, 994] combine_rows_per_src=[7308, 6434, 6050, 4460, 6360, 5176, 4408, 4052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=348694528 dispatch_meta_bytes=229464 combine_payload_bytes=348694528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348923992 ingress_bytes=458795008 link_bytes=458795008 token_bytes=697618520 weight_movement_bytes=110100480 total_nvlink_bytes=807719000 weight_to_token_ratio=0.157823 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.979199777e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=346386432 dispatch_meta_bytes=229376 combine_payload_bytes=346386432 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=346615808 ingress_bytes=566587392 link_bytes=566587392 token_bytes=693002240 weight_movement_bytes=220200960 total_nvlink_bytes=913203200 weight_to_token_ratio=0.317749 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.853886623e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=263753728 dispatch_meta_bytes=209064 combine_payload_bytes=263753728 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=263962792 ingress_bytes=704155648 link_bytes=704155648 token_bytes=527716520 weight_movement_bytes=440401920 total_nvlink_bytes=968118440 weight_to_token_ratio=0.834543 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=10240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.224642486e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55140 payload_rows_remote=47146 dup_rows=10396 dup_rows_remote=9108 dedup_ratio=0.158630 router_tokens_per_expert=[0, 0, 0, 4096, 745, 0, 1, 4094, 2709, 0, 3245, 0, 0, 0, 0, 0, 0, 0, 0, 4009, 22, 0, 0, 0, 0, 0, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2838, 0, 0, 660, 0, 3981, 0, 0, 0, 0, 0, 1116, 0, 86, 0, 0, 0, 4, 0, 0, 0, 960, 0, 0, 102] payload_rows_per_dst=[1492, 6306, 8024, 8192, 6748, 7994, 8192, 8192] dup_rows_per_dst=[0, 2132, 38, 8, 2398, 1288, 2400, 2132] combine_rows_per_src=[1492, 6306, 8024, 8192, 6748, 7994, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271050752 dispatch_meta_bytes=222328 combine_payload_bytes=271050752 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=271273080 ingress_bytes=601352192 link_bytes=601352192 token_bytes=542323832 weight_movement_bytes=330301440 total_nvlink_bytes=872625272 weight_to_token_ratio=0.609048 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=11136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045850367e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53982 payload_rows_remote=45792 dup_rows=11554 dup_rows_remote=9164 dedup_ratio=0.176300 router_tokens_per_expert=[0, 0, 0, 4096, 1334, 0, 3, 4095, 1984, 0, 2877, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 80, 1, 0, 0, 0, 0, 74, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2835, 0, 0, 912, 0, 3978, 0, 0, 0, 0, 0, 1120, 0, 75, 0, 0, 0, 22, 0, 0, 0, 870, 0, 0, 223] payload_rows_per_dst=[2672, 2978, 8188, 8190, 7538, 8034, 8190, 8192] dup_rows_per_dst=[2, 0, 162, 148, 3886, 2736, 2390, 2230] combine_rows_per_src=[2672, 2978, 8188, 8190, 7538, 8034, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50222 payload_rows_remote=42030 dup_rows=15314 dup_rows_remote=13370 dedup_ratio=0.233673 router_tokens_per_expert=[0, 0, 2, 4096, 693, 0, 4, 4096, 2600, 1, 3383, 0, 1, 0, 0, 1, 0, 0, 0, 4078, 6, 0, 0, 0, 0, 0, 28, 0, 1, 4095, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3098, 0, 0, 618, 1, 3981, 0, 0, 0, 0, 0, 895, 0, 116, 0, 0, 0, 3, 0, 0, 0, 933, 0, 1, 35] payload_rows_per_dst=[1394, 6, 8156, 8190, 7910, 8182, 8192, 8192] dup_rows_per_dst=[4, 0, 12, 62, 5052, 6218, 2022, 1944] combine_rows_per_src=[1394, 6, 8156, 8190, 7910, 8182, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56462 payload_rows_remote=49808 dup_rows=9074 dup_rows_remote=9074 dedup_ratio=0.138458 router_tokens_per_expert=[0, 0, 0, 4096, 779, 0, 10, 4096, 2453, 0, 3259, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 13, 2, 0, 0, 0, 0, 15, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 514, 0, 3949, 0, 0, 0, 0, 1, 938, 0, 97, 0, 0, 0, 1, 0, 0, 0, 877, 0, 0, 155] payload_rows_per_dst=[1576, 7564, 8182, 8190, 6654, 7912, 8192, 8192] dup_rows_per_dst=[2, 3860, 30, 30, 0, 1014, 2072, 2066] combine_rows_per_src=[1576, 7564, 8182, 8190, 6654, 7912, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64826 payload_rows_remote=56634 dup_rows=710 dup_rows_remote=702 dedup_ratio=0.010834 router_tokens_per_expert=[0, 0, 0, 4096, 3, 3747, 4096, 0, 0, 4095, 0, 0, 0, 0, 338, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 7, 4095, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 4095, 1, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4089] payload_rows_per_dst=[7500, 8190, 8192, 8190, 8192, 8192, 8192, 8178] dup_rows_per_dst=[0, 676, 8, 20, 2, 4, 0, 0] combine_rows_per_src=[7500, 8190, 8192, 8190, 8192, 8192, 8192, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=337942528 dispatch_meta_bytes=225016 combine_payload_bytes=337942528 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338167544 ingress_bytes=558143488 link_bytes=558143488 token_bytes=676110072 weight_movement_bytes=220200960 total_nvlink_bytes=896311032 weight_to_token_ratio=0.325688 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=23808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.707033245e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64710 payload_rows_remote=56518 dup_rows=826 dup_rows_remote=816 dedup_ratio=0.012604 router_tokens_per_expert=[0, 0, 0, 4096, 1, 3682, 4096, 0, 0, 4096, 0, 0, 0, 0, 376, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 3, 4096, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7366, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 752, 64, 10, 0, 0, 0, 0] combine_rows_per_src=[7366, 8192, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=301271040 dispatch_meta_bytes=221600 combine_payload_bytes=301271040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301492640 ingress_bytes=411371520 link_bytes=411371520 token_bytes=602763680 weight_movement_bytes=110100480 total_nvlink_bytes=712864160 weight_to_token_ratio=0.182659 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.154427323e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=328237056 dispatch_meta_bytes=219824 combine_payload_bytes=328237056 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=328456880 ingress_bytes=438337536 link_bytes=438337536 token_bytes=656693936 weight_movement_bytes=110100480 total_nvlink_bytes=766794416 weight_to_token_ratio=0.167659 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.623410692e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=357023744 dispatch_meta_bytes=235528 combine_payload_bytes=357023744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357259272 ingress_bytes=467124224 link_bytes=467124224 token_bytes=714283016 weight_movement_bytes=110100480 total_nvlink_bytes=824383496 weight_to_token_ratio=0.154141 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24832 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.124058543e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43514 payload_rows_remote=35324 dup_rows=22022 dup_rows_remote=4866 dedup_ratio=0.336029 router_tokens_per_expert=[0, 0, 0, 4088, 629, 1, 2, 4090, 2729, 0, 3204, 0, 1, 0, 0, 0, 0, 1, 0, 4045, 26, 1, 0, 0, 0, 0, 26, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 1, 0, 3153, 4, 0, 1050, 0, 3863, 2, 4, 1, 0, 0, 976, 0, 161, 0, 0, 0, 0, 2, 11, 0, 548, 0, 0, 55] payload_rows_per_dst=[8190, 7504, 8094, 7830, 6310, 2110, 2274, 1202] dup_rows_per_dst=[17156, 4680, 92, 52, 0, 12, 0, 30] combine_rows_per_src=[8190, 7504, 8094, 7830, 6310, 2110, 2274, 1202] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405952512 dispatch_meta_bytes=229344 combine_payload_bytes=405952512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406181856 ingress_bytes=516052992 link_bytes=516052992 token_bytes=812134368 weight_movement_bytes=110100480 total_nvlink_bytes=922234848 weight_to_token_ratio=0.135569 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.975010292e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=405121024 dispatch_meta_bytes=229336 combine_payload_bytes=405121024 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=405350360 ingress_bytes=735422464 link_bytes=735422464 token_bytes=810471384 weight_movement_bytes=330301440 total_nvlink_bytes=1140772824 weight_to_token_ratio=0.407542 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279020621e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45070 payload_rows_remote=37906 dup_rows=20466 dup_rows_remote=17198 dedup_ratio=0.312286 router_tokens_per_expert=[0, 0, 0, 4095, 1074, 0, 8, 4089, 2381, 0, 2832, 0, 0, 0, 0, 3, 0, 0, 1, 4078, 32, 0, 0, 0, 0, 0, 33, 0, 0, 4095, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3055, 0, 0, 696, 0, 3953, 0, 0, 0, 0, 1, 1068, 0, 106, 0, 0, 0, 4, 0, 1, 0, 1098, 0, 0, 64] payload_rows_per_dst=[8190, 7164, 8156, 8192, 6110, 1392, 2346, 3520] dup_rows_per_dst=[16690, 3268, 66, 66, 0, 0, 4, 372] combine_rows_per_src=[8190, 7164, 8156, 8192, 6110, 1392, 2346, 3520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=253202432 dispatch_meta_bytes=160760 combine_payload_bytes=253202432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=253363192 ingress_bytes=583503872 link_bytes=583503872 token_bytes=506565624 weight_movement_bytes=330301440 total_nvlink_bytes=836867064 weight_to_token_ratio=0.652041 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=9856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.014809203e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64684 payload_rows_remote=56492 dup_rows=852 dup_rows_remote=850 dedup_ratio=0.013000 router_tokens_per_expert=[0, 0, 0, 4096, 3, 3672, 4096, 0, 0, 4096, 0, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 10, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4092] payload_rows_per_dst=[7348, 8192, 8192, 8192, 8192, 8192, 8192, 8184] dup_rows_per_dst=[2, 624, 204, 20, 0, 2, 0, 0] combine_rows_per_src=[7348, 8192, 8192, 8192, 8192, 8192, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271710208 dispatch_meta_bytes=220416 combine_payload_bytes=271710208 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=271930624 ingress_bytes=602011648 link_bytes=602011648 token_bytes=543640832 weight_movement_bytes=330301440 total_nvlink_bytes=873942272 weight_to_token_ratio=0.607573 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046997270e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64300 payload_rows_remote=56154 dup_rows=1236 dup_rows_remote=936 dedup_ratio=0.018860 router_tokens_per_expert=[0, 0, 0, 4096, 3, 3659, 4095, 0, 0, 4094, 0, 0, 1, 0, 410, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 33, 4096, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 0, 4071] payload_rows_per_dst=[7322, 8190, 8192, 7980, 8188, 8090, 8192, 8146] dup_rows_per_dst=[0, 820, 24, 70, 14, 6, 2, 300] combine_rows_per_src=[7322, 8190, 8192, 7980, 8188, 8090, 8192, 8146] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64598 payload_rows_remote=56406 dup_rows=938 dup_rows_remote=938 dedup_ratio=0.014313 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3627, 4096, 0, 0, 4096, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 13, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096] payload_rows_per_dst=[7254, 8192, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 800, 94, 28, 0, 16, 0, 0] combine_rows_per_src=[7254, 8192, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64300 payload_rows_remote=56108 dup_rows=1236 dup_rows_remote=1236 dedup_ratio=0.018860 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3480, 4096, 0, 0, 4096, 0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 84, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4094] payload_rows_per_dst=[6960, 8192, 8192, 8192, 8192, 8192, 8192, 8188] dup_rows_per_dst=[0, 924, 72, 168, 0, 72, 0, 0] combine_rows_per_src=[6960, 8192, 8192, 8192, 8192, 8192, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59034 payload_rows_remote=50842 dup_rows=6502 dup_rows_remote=5058 dedup_ratio=0.099213 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 19, 0, 0, 2675, 0, 0, 0, 0, 0, 0, 722, 0, 0, 0, 4096, 0, 0, 865, 0, 3, 0, 1, 3892, 0, 0, 0, 0, 0, 0, 29, 0, 4095, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8178, 8192, 1738, 8190, 8162, 8190, 8192] dup_rows_per_dst=[6, 4994, 1444, 0, 58, 0, 0, 0] combine_rows_per_src=[8192, 8178, 8192, 1738, 8190, 8162, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=404934656 dispatch_meta_bytes=229368 combine_payload_bytes=404934656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405164024 ingress_bytes=515035136 link_bytes=515035136 token_bytes=810098680 weight_movement_bytes=110100480 total_nvlink_bytes=920199160 weight_to_token_ratio=0.135910 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.957308102e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60720 payload_rows_remote=55870 dup_rows=4816 dup_rows_remote=3784 dedup_ratio=0.073486 router_tokens_per_expert=[0, 0, 7, 0, 0, 4096, 0, 0, 5, 0, 0, 2204, 0, 0, 0, 0, 0, 0, 789, 0, 0, 0, 4096, 0, 0, 1320, 0, 20, 0, 0, 3882, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4062, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 6788, 8192, 4850, 8192, 8124, 8190, 8192] dup_rows_per_dst=[14, 2192, 1578, 1032, 0, 0, 0, 0] combine_rows_per_src=[8192, 6788, 8192, 4850, 8192, 8124, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=402511872 dispatch_meta_bytes=228360 combine_payload_bytes=402511872 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402740232 ingress_bytes=622712832 link_bytes=622712832 token_bytes=805252104 weight_movement_bytes=220200960 total_nvlink_bytes=1025453064 weight_to_token_ratio=0.273456 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.083000034e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58926 payload_rows_remote=50866 dup_rows=6610 dup_rows_remote=268 dedup_ratio=0.100861 router_tokens_per_expert=[0, 0, 0, 4095, 13, 3875, 4096, 0, 0, 4090, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 27, 4095, 0, 2, 2, 0, 1, 0, 0, 0, 0, 0, 4087, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 1, 0, 4063] payload_rows_per_dst=[8060, 2686, 7638, 7856, 8178, 8190, 8190, 8128] dup_rows_per_dst=[6342, 182, 32, 48, 2, 4, 0, 0] combine_rows_per_src=[8060, 2686, 7638, 7856, 8178, 8190, 8190, 8128] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=404318208 dispatch_meta_bytes=229376 combine_payload_bytes=404318208 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=404547584 ingress_bytes=624519168 link_bytes=624519168 token_bytes=808865792 weight_movement_bytes=220200960 total_nvlink_bytes=1029066752 weight_to_token_ratio=0.272234 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.086141549e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=364435456 dispatch_meta_bytes=223600 combine_payload_bytes=364435456 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=364659056 ingress_bytes=694736896 link_bytes=694736896 token_bytes=729094512 weight_movement_bytes=330301440 total_nvlink_bytes=1059395952 weight_to_token_ratio=0.453030 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.208261727e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=402182144 dispatch_meta_bytes=229376 combine_payload_bytes=402182144 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402411520 ingress_bytes=512282624 link_bytes=512282624 token_bytes=804593664 weight_movement_bytes=110100480 total_nvlink_bytes=914694144 weight_to_token_ratio=0.136840 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.909437391e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64548 payload_rows_remote=56362 dup_rows=988 dup_rows_remote=250 dedup_ratio=0.015076 router_tokens_per_expert=[0, 0, 0, 4096, 2, 3689, 4095, 0, 0, 4093, 0, 0, 0, 0, 369, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 84, 4096, 0, 7, 2, 0, 1, 0, 0, 0, 0, 0, 4094, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4014] payload_rows_per_dst=[7380, 8186, 8190, 8192, 8188, 8192, 8192, 8028] dup_rows_per_dst=[2, 738, 58, 186, 4, 0, 0, 0] combine_rows_per_src=[7380, 8186, 8190, 8192, 8188, 8192, 8192, 8028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=400476160 dispatch_meta_bytes=238616 combine_payload_bytes=400476160 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=400714776 ingress_bytes=620677120 link_bytes=620677120 token_bytes=801190936 weight_movement_bytes=220200960 total_nvlink_bytes=1021391896 weight_to_token_ratio=0.274842 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.079459596e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=364607488 dispatch_meta_bytes=204536 combine_payload_bytes=364607488 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=364812024 ingress_bytes=805009408 link_bytes=805009408 token_bytes=729419512 weight_movement_bytes=440401920 total_nvlink_bytes=1169821432 weight_to_token_ratio=0.603770 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.400043762e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=404002816 dispatch_meta_bytes=226448 combine_payload_bytes=404002816 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=404229264 ingress_bytes=624203776 link_bytes=624203776 token_bytes=808232080 weight_movement_bytes=220200960 total_nvlink_bytes=1028433040 weight_to_token_ratio=0.272448 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.085593030e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61512 payload_rows_remote=53402 dup_rows=4024 dup_rows_remote=4024 dedup_ratio=0.061401 router_tokens_per_expert=[0, 0, 0, 0, 0, 4094, 0, 0, 2, 0, 0, 2126, 0, 0, 0, 0, 0, 0, 336, 0, 0, 0, 4096, 0, 0, 1784, 0, 7, 0, 0, 3967, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 4055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8188, 4256, 8192, 8190, 8192, 8110, 8192, 8192] dup_rows_per_dst=[0, 0, 672, 3326, 26, 0, 0, 0] combine_rows_per_src=[8188, 4256, 8192, 8190, 8192, 8110, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55216 payload_rows_remote=47026 dup_rows=10320 dup_rows_remote=10316 dedup_ratio=0.157471 router_tokens_per_expert=[0, 0, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 2422, 0, 0, 0, 0, 0, 0, 663, 0, 0, 0, 4096, 0, 0, 1181, 0, 3, 0, 0, 3943, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 1, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 1, 0, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8142, 4852, 1326, 8192, 8192, 8132, 8190, 8190] dup_rows_per_dst=[4, 0, 0, 9858, 32, 396, 26, 4] combine_rows_per_src=[8142, 4852, 1326, 8192, 8192, 8132, 8190, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59578 payload_rows_remote=51440 dup_rows=5958 dup_rows_remote=5958 dedup_ratio=0.090912 router_tokens_per_expert=[0, 3198, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 0, 0, 0, 0, 2719, 0, 4095, 0, 0, 0, 0, 4096, 0, 13, 0, 0, 5, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3705, 1222, 0, 4096, 0, 0, 0, 1440, 0] payload_rows_per_dst=[6396, 8192, 8138, 8192, 8192, 8192, 7416, 4860] dup_rows_per_dst=[0, 0, 0, 20, 5438, 36, 0, 464] combine_rows_per_src=[6396, 8192, 8138, 8192, 8192, 8192, 7416, 4860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62530 payload_rows_remote=54338 dup_rows=3006 dup_rows_remote=2986 dedup_ratio=0.045868 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 2616, 0, 0, 0, 0, 0, 0, 451, 0, 0, 0, 4096, 0, 0, 1232, 0, 10, 0, 0, 3879, 0, 0, 0, 0, 0, 0, 10, 0, 4096, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 5234, 8192, 8158, 8192, 8178, 8192, 8192] dup_rows_per_dst=[0, 0, 902, 2084, 20, 0, 0, 0] combine_rows_per_src=[8192, 5234, 8192, 8158, 8192, 8178, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59252 payload_rows_remote=51062 dup_rows=6284 dup_rows_remote=6284 dedup_ratio=0.095886 router_tokens_per_expert=[0, 0, 3, 0, 0, 4096, 0, 0, 13, 0, 0, 2397, 0, 0, 0, 0, 0, 0, 929, 0, 0, 0, 4096, 0, 0, 847, 0, 9, 0, 0, 4010, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 4820, 5354, 8164, 8192, 8148, 8190, 8192] dup_rows_per_dst=[6, 0, 1238, 5026, 14, 0, 0, 0] combine_rows_per_src=[8192, 4820, 5354, 8164, 8192, 8148, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=382785536 dispatch_meta_bytes=229704 combine_payload_bytes=382785536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383015240 ingress_bytes=492886016 link_bytes=492886016 token_bytes=765800776 weight_movement_bytes=110100480 total_nvlink_bytes=875901256 weight_to_token_ratio=0.143772 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.572098476e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=337082368 dispatch_meta_bytes=229368 combine_payload_bytes=337082368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337311736 ingress_bytes=447182848 link_bytes=447182848 token_bytes=674394104 weight_movement_bytes=110100480 total_nvlink_bytes=784494584 weight_to_token_ratio=0.163258 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.777245216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58882 payload_rows_remote=50690 dup_rows=6654 dup_rows_remote=6642 dedup_ratio=0.101532 router_tokens_per_expert=[0, 3192, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 3152, 0, 4096, 0, 0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 2, 0, 3380, 1776, 0, 4096, 0, 0, 0, 776, 0] payload_rows_per_dst=[6384, 8190, 8178, 8192, 8192, 8192, 6764, 4790] dup_rows_per_dst=[0, 0, 0, 12, 6304, 24, 0, 314] combine_rows_per_src=[6384, 8190, 8178, 8192, 8192, 8192, 6764, 4790] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59484 payload_rows_remote=51292 dup_rows=6052 dup_rows_remote=6042 dedup_ratio=0.092346 router_tokens_per_expert=[0, 0, 5, 0, 0, 4096, 0, 0, 6, 0, 0, 1994, 0, 0, 0, 0, 0, 0, 1046, 0, 0, 0, 4095, 0, 0, 1158, 0, 43, 0, 0, 3965, 0, 0, 0, 0, 0, 0, 2, 0, 4092, 1, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 2, 0, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8138, 8190, 2400, 8034, 8150, 8188, 8192] dup_rows_per_dst=[10, 3792, 2246, 2, 0, 0, 0, 2] combine_rows_per_src=[8192, 8138, 8190, 2400, 8034, 8150, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=368721920 dispatch_meta_bytes=229592 combine_payload_bytes=368721920 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=368951512 ingress_bytes=588922880 link_bytes=588922880 token_bytes=737673432 weight_movement_bytes=220200960 total_nvlink_bytes=957874392 weight_to_token_ratio=0.298507 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.024233749e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=389494784 dispatch_meta_bytes=229296 combine_payload_bytes=389494784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=389724080 ingress_bytes=609695744 link_bytes=609695744 token_bytes=779218864 weight_movement_bytes=220200960 total_nvlink_bytes=999419824 weight_to_token_ratio=0.282592 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.060361177e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366012416 dispatch_meta_bytes=229384 combine_payload_bytes=366012416 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366241800 ingress_bytes=476112896 link_bytes=476112896 token_bytes=732254216 weight_movement_bytes=110100480 total_nvlink_bytes=842354696 weight_to_token_ratio=0.150358 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.280386333e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59716 payload_rows_remote=51594 dup_rows=5820 dup_rows_remote=1650 dedup_ratio=0.088806 router_tokens_per_expert=[0, 0, 7, 0, 0, 4096, 0, 0, 17, 0, 0, 2378, 0, 0, 0, 0, 0, 0, 813, 0, 0, 0, 4096, 0, 0, 1231, 0, 27, 0, 0, 3751, 0, 0, 0, 0, 0, 0, 5, 0, 4096, 3, 0, 4065, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4089, 1, 0, 0, 0, 0, 0, 4093, 0] payload_rows_per_dst=[8192, 8122, 8192, 2516, 8192, 8136, 8178, 8188] dup_rows_per_dst=[14, 4170, 1626, 0, 10, 0, 0, 0] combine_rows_per_src=[8192, 8122, 8192, 2516, 8192, 8136, 8178, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=363345920 dispatch_meta_bytes=229328 combine_payload_bytes=363345920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=363575248 ingress_bytes=473446400 link_bytes=473446400 token_bytes=726921168 weight_movement_bytes=110100480 total_nvlink_bytes=837021648 weight_to_token_ratio=0.151461 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.234011582e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=367661056 dispatch_meta_bytes=229336 combine_payload_bytes=367661056 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=367890392 ingress_bytes=697962496 link_bytes=697962496 token_bytes=735551448 weight_movement_bytes=330301440 total_nvlink_bytes=1065852888 weight_to_token_ratio=0.449053 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.213871576e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=369825792 dispatch_meta_bytes=212976 combine_payload_bytes=369825792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370038768 ingress_bytes=479926272 link_bytes=479926272 token_bytes=739864560 weight_movement_bytes=110100480 total_nvlink_bytes=849965040 weight_to_token_ratio=0.148812 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.346707213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58384 payload_rows_remote=50192 dup_rows=7152 dup_rows_remote=6750 dedup_ratio=0.109131 router_tokens_per_expert=[0, 2877, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 2217, 0, 4096, 0, 0, 0, 0, 4096, 0, 201, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3716, 1871, 0, 4096, 0, 0, 0, 1417, 0] payload_rows_per_dst=[5754, 8190, 8158, 8192, 4434, 8192, 7604, 7860] dup_rows_per_dst=[0, 0, 0, 14, 0, 402, 1044, 5692] combine_rows_per_src=[5754, 8190, 8158, 8192, 4434, 8192, 7604, 7860] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59338 payload_rows_remote=51146 dup_rows=6198 dup_rows_remote=500 dedup_ratio=0.094574 router_tokens_per_expert=[0, 2826, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 3350, 0, 4094, 0, 0, 0, 0, 4096, 0, 26, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 3, 0, 3790, 1264, 0, 4096, 0, 0, 0, 1042, 0] payload_rows_per_dst=[5302, 8192, 8150, 8090, 6716, 7110, 7586, 8192] dup_rows_per_dst=[0, 0, 350, 14, 82, 54, 0, 5698] combine_rows_per_src=[5302, 8192, 8150, 8090, 6716, 7110, 7586, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65108 payload_rows_remote=56916 dup_rows=428 dup_rows_remote=428 dedup_ratio=0.006531 router_tokens_per_expert=[0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 29, 0, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 163, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 4096, 0, 3936, 0, 4095, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8190, 8192, 8192, 8192, 8192, 7872, 8086] dup_rows_per_dst=[2, 60, 0, 326, 0, 40, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8192, 8192, 8192, 7872, 8086] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58148 payload_rows_remote=51506 dup_rows=7388 dup_rows_remote=5656 dedup_ratio=0.112732 router_tokens_per_expert=[0, 3573, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 3154, 0, 4096, 0, 0, 0, 0, 4096, 0, 32, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3758, 932, 0, 4096, 0, 0, 0, 839, 0] payload_rows_per_dst=[7146, 8190, 8164, 8192, 6642, 8192, 8178, 3444] dup_rows_per_dst=[0, 0, 0, 30, 1732, 64, 5464, 98] combine_rows_per_src=[7146, 8190, 8164, 8192, 6642, 8192, 8178, 3444] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59496 payload_rows_remote=52016 dup_rows=6040 dup_rows_remote=6040 dedup_ratio=0.092163 router_tokens_per_expert=[0, 2411, 0, 0, 4044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3170, 0, 4096, 0, 0, 0, 0, 4096, 0, 117, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 3740, 1844, 0, 4096, 0, 0, 0, 1058, 0] payload_rows_per_dst=[4822, 8190, 8088, 8192, 6340, 8192, 7480, 8192] dup_rows_per_dst=[0, 0, 0, 2, 0, 234, 0, 5804] combine_rows_per_src=[4822, 8190, 8088, 8192, 6340, 8192, 7480, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=359776256 dispatch_meta_bytes=227768 combine_payload_bytes=359776256 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=360004024 ingress_bytes=469876736 link_bytes=469876736 token_bytes=719780280 weight_movement_bytes=110100480 total_nvlink_bytes=829880760 weight_to_token_ratio=0.152964 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.171929253e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=366614528 dispatch_meta_bytes=206584 combine_payload_bytes=366614528 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=366821112 ingress_bytes=807016448 link_bytes=807016448 token_bytes=733435640 weight_movement_bytes=440401920 total_nvlink_bytes=1173837560 weight_to_token_ratio=0.600464 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=11904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.403534334e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65030 payload_rows_remote=56838 dup_rows=506 dup_rows_remote=100 dedup_ratio=0.007721 router_tokens_per_expert=[0, 0, 0, 4096, 0, 3, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 4096, 0, 3940, 0, 4096, 0, 0, 0, 0, 0, 4000, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8190, 8192, 8192, 8192, 7880, 8000] dup_rows_per_dst=[6, 68, 0, 406, 0, 26, 0, 0] combine_rows_per_src=[8192, 8192, 8190, 8192, 8192, 8192, 7880, 8000] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58216 payload_rows_remote=50024 dup_rows=7320 dup_rows_remote=490 dedup_ratio=0.111694 router_tokens_per_expert=[0, 3415, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2770, 0, 4094, 0, 0, 0, 0, 4096, 0, 50, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3545, 1373, 0, 4096, 0, 0, 0, 1152, 0] payload_rows_per_dst=[8192, 8192, 8146, 8188, 5540, 8192, 7082, 4684] dup_rows_per_dst=[6830, 8, 2, 6, 0, 108, 0, 366] combine_rows_per_src=[8192, 8192, 8146, 8188, 5540, 8192, 7082, 4684] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407973888 dispatch_meta_bytes=229376 combine_payload_bytes=407973888 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408203264 ingress_bytes=518074368 link_bytes=518074368 token_bytes=816177152 weight_movement_bytes=110100480 total_nvlink_bytes=926277632 weight_to_token_ratio=0.134898 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.010165345e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=369195008 dispatch_meta_bytes=228648 combine_payload_bytes=369195008 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369423656 ingress_bytes=589395968 link_bytes=589395968 token_bytes=738618664 weight_movement_bytes=220200960 total_nvlink_bytes=958819624 weight_to_token_ratio=0.298125 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.025056527e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58452 payload_rows_remote=50260 dup_rows=7084 dup_rows_remote=7084 dedup_ratio=0.108093 router_tokens_per_expert=[0, 3266, 0, 0, 4081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1, 0, 0, 0, 2955, 0, 4095, 0, 0, 0, 0, 4096, 0, 79, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 9, 0, 3450, 1517, 0, 4096, 0, 0, 0, 1022, 0] payload_rows_per_dst=[8186, 8192, 8162, 8192, 5914, 8192, 6918, 4696] dup_rows_per_dst=[6524, 0, 0, 8, 10, 160, 0, 382] combine_rows_per_src=[8186, 8192, 8162, 8192, 5914, 8192, 6918, 4696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=372850688 dispatch_meta_bytes=232224 combine_payload_bytes=372850688 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=373082912 ingress_bytes=703152128 link_bytes=703152128 token_bytes=745933600 weight_movement_bytes=330301440 total_nvlink_bytes=1076235040 weight_to_token_ratio=0.442803 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.222897199e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=407414784 dispatch_meta_bytes=227752 combine_payload_bytes=407414784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407642536 ingress_bytes=517515264 link_bytes=517515264 token_bytes=815057320 weight_movement_bytes=110100480 total_nvlink_bytes=925157800 weight_to_token_ratio=0.135083 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.000441607e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=358572032 dispatch_meta_bytes=202056 combine_payload_bytes=358572032 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=358774088 ingress_bytes=688873472 link_bytes=688873472 token_bytes=717346120 weight_movement_bytes=330301440 total_nvlink_bytes=1047647560 weight_to_token_ratio=0.460449 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.198064268e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=360263680 dispatch_meta_bytes=229376 combine_payload_bytes=360263680 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=360493056 ingress_bytes=580464640 link_bytes=580464640 token_bytes=720756736 weight_movement_bytes=220200960 total_nvlink_bytes=940957696 weight_to_token_ratio=0.305514 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.009523479e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64430 payload_rows_remote=56238 dup_rows=1106 dup_rows_remote=1102 dedup_ratio=0.016876 router_tokens_per_expert=[0, 0, 0, 4096, 0, 31, 0, 0, 0, 0, 0, 0, 0, 178, 0, 12, 0, 0, 4074, 0, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 3870, 0, 4096, 0, 0, 0, 0, 0, 3791, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8148, 8192, 8192, 8192, 7740, 7582] dup_rows_per_dst=[62, 380, 0, 660, 0, 4, 0, 0] combine_rows_per_src=[8192, 8192, 8148, 8192, 8192, 8192, 7740, 7582] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59520 payload_rows_remote=51482 dup_rows=6016 dup_rows_remote=2342 dedup_ratio=0.091797 router_tokens_per_expert=[0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 3954, 0, 4095, 0, 0, 0, 0, 0, 3913, 0, 0, 0, 4095, 0] payload_rows_per_dst=[7984, 6388, 8192, 4714, 8192, 8048, 7964, 8038] dup_rows_per_dst=[4, 128, 0, 242, 22, 0, 1946, 3674] combine_rows_per_src=[7984, 6388, 8192, 4714, 8192, 8048, 7964, 8038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65318 payload_rows_remote=57128 dup_rows=218 dup_rows_remote=218 dedup_ratio=0.003326 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 4095, 3996, 0, 3, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 105, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7992, 8190, 8192, 8182, 8186, 8192, 8192] dup_rows_per_dst=[0, 6, 0, 0, 0, 0, 210, 2] combine_rows_per_src=[8192, 7992, 8190, 8192, 8182, 8186, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64946 payload_rows_remote=56754 dup_rows=590 dup_rows_remote=590 dedup_ratio=0.009003 router_tokens_per_expert=[0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 4096, 0, 3946, 0, 4096, 0, 0, 0, 0, 0, 3954, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8186, 8192, 8192, 8192, 7892, 7908] dup_rows_per_dst=[0, 72, 0, 498, 0, 20, 0, 0] combine_rows_per_src=[8192, 8192, 8186, 8192, 8192, 8192, 7892, 7908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=403113984 dispatch_meta_bytes=229360 combine_payload_bytes=403113984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403343344 ingress_bytes=513214464 link_bytes=513214464 token_bytes=806457328 weight_movement_bytes=110100480 total_nvlink_bytes=916557808 weight_to_token_ratio=0.136524 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.925643622e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64436 payload_rows_remote=56344 dup_rows=1100 dup_rows_remote=1100 dedup_ratio=0.016785 router_tokens_per_expert=[0, 0, 0, 4096, 0, 12, 0, 0, 0, 0, 0, 0, 0, 349, 0, 9, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4096, 0, 4046, 0, 4096, 0, 0, 0, 0, 0, 3596, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 8192, 8092, 7192] dup_rows_per_dst=[24, 716, 0, 358, 0, 2, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 8192, 8092, 7192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=369022976 dispatch_meta_bytes=215296 combine_payload_bytes=369022976 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=369238272 ingress_bytes=699324416 link_bytes=699324416 token_bytes=738261248 weight_movement_bytes=330301440 total_nvlink_bytes=1068562688 weight_to_token_ratio=0.447405 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.216240179e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64726 payload_rows_remote=56534 dup_rows=810 dup_rows_remote=810 dedup_ratio=0.012360 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3693, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 405, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7386, 8192, 8192, 8188, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 810, 0] combine_rows_per_src=[8192, 7386, 8192, 8192, 8188, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=409493504 dispatch_meta_bytes=229384 combine_payload_bytes=409493504 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409722888 ingress_bytes=519593984 link_bytes=519593984 token_bytes=819216392 weight_movement_bytes=110100480 total_nvlink_bytes=929316872 weight_to_token_ratio=0.134397 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.036593967e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63984 payload_rows_remote=55792 dup_rows=1552 dup_rows_remote=1526 dedup_ratio=0.023682 router_tokens_per_expert=[0, 0, 0, 4096, 0, 13, 0, 0, 0, 0, 0, 0, 0, 138, 0, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 321, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 14, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 4096, 0, 3813, 0, 4091, 0, 0, 0, 0, 0, 3880, 0, 0, 0, 4093, 0] payload_rows_per_dst=[8192, 8182, 8192, 8192, 8168, 8192, 7872, 6994] dup_rows_per_dst=[26, 282, 52, 636, 0, 36, 520, 0] combine_rows_per_src=[8192, 8182, 8192, 8192, 8168, 8192, 7872, 6994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64968 payload_rows_remote=56778 dup_rows=568 dup_rows_remote=456 dedup_ratio=0.008667 router_tokens_per_expert=[0, 0, 0, 4096, 0, 42, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 177, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 1, 7, 0, 0, 4096, 0, 3879, 0, 4095, 0, 0, 0, 0, 0, 4031, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8192, 8190, 8192, 8190, 8192, 8192, 7758, 8062] dup_rows_per_dst=[84, 112, 0, 354, 2, 16, 0, 0] combine_rows_per_src=[8192, 8190, 8192, 8190, 8192, 8192, 7758, 8062] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406812672 dispatch_meta_bytes=229376 combine_payload_bytes=406812672 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407042048 ingress_bytes=627013632 link_bytes=627013632 token_bytes=813854720 weight_movement_bytes=220200960 total_nvlink_bytes=1034055680 weight_to_token_ratio=0.270565 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090479832e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=403873792 dispatch_meta_bytes=229776 combine_payload_bytes=403873792 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=404103568 ingress_bytes=734175232 link_bytes=734175232 token_bytes=807977360 weight_movement_bytes=330301440 total_nvlink_bytes=1138278800 weight_to_token_ratio=0.408800 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.276851480e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=399917056 dispatch_meta_bytes=229272 combine_payload_bytes=399917056 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=400146328 ingress_bytes=620118016 link_bytes=620118016 token_bytes=800063384 weight_movement_bytes=220200960 total_nvlink_bytes=1020264344 weight_to_token_ratio=0.275229 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.078487222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=405235712 dispatch_meta_bytes=229376 combine_payload_bytes=405235712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405465088 ingress_bytes=515336192 link_bytes=515336192 token_bytes=810700800 weight_movement_bytes=110100480 total_nvlink_bytes=920801280 weight_to_token_ratio=0.135809 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.962543961e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64514 payload_rows_remote=56322 dup_rows=1022 dup_rows_remote=1022 dedup_ratio=0.015594 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3653, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4028, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 511, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7306, 8192, 8192, 8056, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 1022, 0] combine_rows_per_src=[8192, 7306, 8192, 8192, 8056, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=406984704 dispatch_meta_bytes=228936 combine_payload_bytes=406984704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407213640 ingress_bytes=517085184 link_bytes=517085184 token_bytes=814198344 weight_movement_bytes=110100480 total_nvlink_bytes=924298824 weight_to_token_ratio=0.135226 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.992961809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65044 payload_rows_remote=56852 dup_rows=492 dup_rows_remote=492 dedup_ratio=0.007507 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 4096, 3860, 0, 2, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 236, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8178, 7726, 8192, 8188, 8188, 8188, 8192, 8192] dup_rows_per_dst=[4, 12, 0, 0, 0, 4, 472, 0] combine_rows_per_src=[8178, 7726, 8192, 8188, 8188, 8188, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62660 payload_rows_remote=54540 dup_rows=2876 dup_rows_remote=2438 dedup_ratio=0.043884 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3736, 0, 0, 2, 0, 0, 1215, 4096, 0, 0, 0, 44, 277, 0, 0, 3958, 0, 4096, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3327, 0, 0, 3823, 0, 0] payload_rows_per_dst=[7472, 8192, 8120, 8192, 8192, 7646, 8192, 6654] dup_rows_per_dst=[0, 2434, 438, 4, 0, 0, 0, 0] combine_rows_per_src=[7472, 8192, 8120, 8192, 8192, 7646, 8192, 6654] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=403716096 dispatch_meta_bytes=229376 combine_payload_bytes=403716096 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=403945472 ingress_bytes=623917056 link_bytes=623917056 token_bytes=807661568 weight_movement_bytes=220200960 total_nvlink_bytes=1027862528 weight_to_token_ratio=0.272640 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.085094377e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65362 payload_rows_remote=57170 dup_rows=174 dup_rows_remote=174 dedup_ratio=0.002655 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 4009, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 87, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8018, 8192, 8192, 8192, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 174, 0] combine_rows_per_src=[8192, 8018, 8192, 8192, 8192, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65038 payload_rows_remote=56846 dup_rows=498 dup_rows_remote=0 dedup_ratio=0.007599 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 4096, 3901, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 249, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7802, 8192, 8192, 8084, 8192, 8192, 8192] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 498, 0] combine_rows_per_src=[8192, 7802, 8192, 8192, 8084, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=407515136 dispatch_meta_bytes=229376 combine_payload_bytes=407515136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407744512 ingress_bytes=517615616 link_bytes=517615616 token_bytes=815259648 weight_movement_bytes=110100480 total_nvlink_bytes=925360128 weight_to_token_ratio=0.135050 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.002186894e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61932 payload_rows_remote=53740 dup_rows=3604 dup_rows_remote=3596 dedup_ratio=0.054993 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 4095, 3998, 0, 6, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4087, 0, 1, 0, 0, 0, 4089, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 103, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8130, 8188, 8192, 8176, 8178, 4686, 8190] dup_rows_per_dst=[8, 3194, 8, 4, 300, 28, 62, 0] combine_rows_per_src=[8192, 8130, 8188, 8192, 8176, 8178, 4686, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=390942720 dispatch_meta_bytes=227912 combine_payload_bytes=390942720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391170632 ingress_bytes=611143680 link_bytes=611143680 token_bytes=782113352 weight_movement_bytes=220200960 total_nvlink_bytes=1002314312 weight_to_token_ratio=0.281546 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062879375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63172 payload_rows_remote=54980 dup_rows=2364 dup_rows_remote=2350 dedup_ratio=0.036072 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3623, 0, 0, 0, 0, 5, 922, 4096, 0, 0, 0, 37, 306, 0, 0, 3940, 0, 4096, 0, 0, 0, 4096, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 0, 0, 3838, 0, 0] payload_rows_per_dst=[7246, 8192, 8082, 8192, 8192, 7676, 8192, 7400] dup_rows_per_dst=[0, 1854, 484, 14, 0, 12, 0, 0] combine_rows_per_src=[7246, 8192, 8082, 8192, 8192, 7676, 8192, 7400] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65276 payload_rows_remote=57312 dup_rows=260 dup_rows_remote=260 dedup_ratio=0.003967 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 4096, 3980, 0, 2, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 4092, 0, 0, 0, 0, 4096, 0, 4096, 0, 0, 0, 0, 0, 4096, 129, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7964, 8190, 8192, 8170, 8184, 8192, 8192] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 258, 0] combine_rows_per_src=[8192, 7964, 8190, 8192, 8170, 8184, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=409794560 dispatch_meta_bytes=229376 combine_payload_bytes=409794560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410023936 ingress_bytes=519895040 link_bytes=519895040 token_bytes=819818496 weight_movement_bytes=110100480 total_nvlink_bytes=929918976 weight_to_token_ratio=0.134299 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.041829826e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=407472128 dispatch_meta_bytes=227384 combine_payload_bytes=407472128 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=407699512 ingress_bytes=957974528 link_bytes=957974528 token_bytes=815171640 weight_movement_bytes=550502400 total_nvlink_bytes=1365674040 weight_to_token_ratio=0.675321 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.666075264e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=385208320 dispatch_meta_bytes=229344 combine_payload_bytes=385208320 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385437664 ingress_bytes=495308800 link_bytes=495308800 token_bytes=770645984 weight_movement_bytes=110100480 total_nvlink_bytes=880746464 weight_to_token_ratio=0.142868 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.614234675e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63350 payload_rows_remote=55648 dup_rows=2186 dup_rows_remote=2164 dedup_ratio=0.033356 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3591, 0, 0, 0, 0, 0, 765, 4096, 0, 0, 0, 66, 329, 0, 0, 3931, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3840, 0, 0, 3841, 0, 0] payload_rows_per_dst=[7182, 8192, 8018, 8192, 8192, 7702, 8192, 7680] dup_rows_per_dst=[0, 1530, 634, 0, 0, 22, 0, 0] combine_rows_per_src=[7182, 8192, 8018, 8192, 8192, 7702, 8192, 7680] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394096640 dispatch_meta_bytes=229320 combine_payload_bytes=394096640 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=394325960 ingress_bytes=614297600 link_bytes=614297600 token_bytes=788422600 weight_movement_bytes=220200960 total_nvlink_bytes=1008623560 weight_to_token_ratio=0.279293 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.068364561e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410812416 dispatch_meta_bytes=230288 combine_payload_bytes=410812416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=411042704 ingress_bytes=631013376 link_bytes=631013376 token_bytes=821855120 weight_movement_bytes=220200960 total_nvlink_bytes=1042056080 weight_to_token_ratio=0.267932 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.097436045e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57988 payload_rows_remote=49796 dup_rows=7548 dup_rows_remote=532 dedup_ratio=0.115173 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3027, 0, 0, 1, 0, 0, 1533, 4096, 0, 0, 0, 51, 336, 0, 0, 3936, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3508, 0, 0, 3990, 0, 0] payload_rows_per_dst=[6054, 3066, 8120, 8192, 8192, 7980, 8192, 8192] dup_rows_per_dst=[0, 2, 526, 2, 0, 2, 0, 7016] combine_rows_per_src=[6054, 3066, 8120, 8192, 8192, 7980, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63398 payload_rows_remote=55206 dup_rows=2138 dup_rows_remote=2136 dedup_ratio=0.032623 router_tokens_per_expert=[0, 0, 200, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3913, 0, 2, 4095, 0, 0, 0, 1, 0, 3211, 0, 0, 0, 0, 66, 4096, 177, 0, 0, 0, 624, 4094, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8188, 7830, 6422, 8192, 8190] dup_rows_per_dst=[400, 0, 2, 0, 0, 2, 1734, 0] combine_rows_per_src=[8192, 8192, 8192, 8188, 7830, 6422, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=398884864 dispatch_meta_bytes=231248 combine_payload_bytes=398884864 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=399116112 ingress_bytes=619085824 link_bytes=619085824 token_bytes=798000976 weight_movement_bytes=220200960 total_nvlink_bytes=1018201936 weight_to_token_ratio=0.275941 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.076692070e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61158 payload_rows_remote=52966 dup_rows=4378 dup_rows_remote=4378 dedup_ratio=0.066803 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3074, 0, 0, 0, 0, 0, 1752, 4096, 0, 0, 0, 31, 558, 0, 0, 3926, 0, 4096, 0, 0, 0, 4096, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3320, 0, 0, 3719, 0, 0] payload_rows_per_dst=[6148, 8192, 8164, 8192, 8192, 7438, 8192, 6640] dup_rows_per_dst=[0, 3504, 866, 2, 0, 6, 0, 0] combine_rows_per_src=[6148, 8192, 8164, 8192, 8192, 7438, 8192, 6640] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60014 payload_rows_remote=51822 dup_rows=5522 dup_rows_remote=5522 dedup_ratio=0.084259 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3688, 0, 0, 0, 0, 0, 840, 4096, 0, 0, 0, 83, 185, 0, 0, 3951, 0, 4096, 0, 0, 0, 4096, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3610, 0, 0, 3980, 0, 0] payload_rows_per_dst=[7376, 4436, 8092, 8192, 8192, 7986, 8192, 7548] dup_rows_per_dst=[0, 1024, 346, 30, 0, 38, 0, 4084] combine_rows_per_src=[7376, 4436, 8092, 8192, 8192, 7986, 8192, 7548] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=356937728 dispatch_meta_bytes=201312 combine_payload_bytes=356937728 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=357139040 ingress_bytes=687239168 link_bytes=687239168 token_bytes=714076768 weight_movement_bytes=330301440 total_nvlink_bytes=1044378208 weight_to_token_ratio=0.462557 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.195221945e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395716608 dispatch_meta_bytes=229368 combine_payload_bytes=395716608 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=395945976 ingress_bytes=505817088 link_bytes=505817088 token_bytes=791662584 weight_movement_bytes=110100480 total_nvlink_bytes=901763064 weight_to_token_ratio=0.139075 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.796991086e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54122 payload_rows_remote=45930 dup_rows=11414 dup_rows_remote=4392 dedup_ratio=0.174164 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3511, 0, 0, 1, 0, 0, 1047, 4096, 0, 0, 0, 43, 314, 0, 0, 3955, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3637, 0, 0, 3870, 0, 0] payload_rows_per_dst=[8192, 4492, 5074, 8140, 8192, 8014, 8192, 3826] dup_rows_per_dst=[7022, 1006, 200, 0, 0, 3186, 0, 0] combine_rows_per_src=[8192, 4492, 5074, 8140, 8192, 8014, 8192, 3826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63306 payload_rows_remote=55114 dup_rows=2230 dup_rows_remote=2228 dedup_ratio=0.034027 router_tokens_per_expert=[0, 0, 165, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4046, 0, 2, 4096, 0, 0, 0, 0, 0, 3029, 0, 0, 0, 0, 109, 4096, 408, 0, 0, 0, 432, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8096, 6058, 8192, 8192] dup_rows_per_dst=[330, 0, 0, 2, 0, 0, 1898, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8096, 6058, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61122 payload_rows_remote=54064 dup_rows=4414 dup_rows_remote=2052 dedup_ratio=0.067352 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 3215, 0, 0, 2, 0, 0, 1819, 4096, 0, 0, 0, 24, 248, 0, 0, 3914, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3248, 0, 0, 3910, 0, 0] payload_rows_per_dst=[7122, 7058, 8044, 8192, 8192, 7826, 8192, 6496] dup_rows_per_dst=[1722, 2362, 328, 0, 0, 2, 0, 0] combine_rows_per_src=[7122, 7058, 8044, 8192, 8192, 7826, 8192, 6496] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=379660288 dispatch_meta_bytes=229376 combine_payload_bytes=379660288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=379889664 ingress_bytes=489760768 link_bytes=489760768 token_bytes=759549952 weight_movement_bytes=110100480 total_nvlink_bytes=869650432 weight_to_token_ratio=0.144955 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.517745274e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371460096 dispatch_meta_bytes=229376 combine_payload_bytes=371460096 prefetch_bytes=0 reduce_bytes=0 egress_bytes=371689472 ingress_bytes=371460096 link_bytes=371689472 token_bytes=743149568 weight_movement_bytes=0 total_nvlink_bytes=743149568 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.464291242e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=329226240 dispatch_meta_bytes=201288 combine_payload_bytes=329226240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329427528 ingress_bytes=439326720 link_bytes=439326720 token_bytes=658653768 weight_movement_bytes=110100480 total_nvlink_bytes=768754248 weight_to_token_ratio=0.167160 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=27520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.640614229e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63772 payload_rows_remote=56780 dup_rows=1764 dup_rows_remote=1764 dedup_ratio=0.026917 router_tokens_per_expert=[0, 0, 131, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 0, 4096, 0, 0, 0, 0, 0, 3496, 0, 0, 0, 0, 65, 4096, 172, 0, 0, 0, 514, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7628, 6992, 8192, 8192] dup_rows_per_dst=[262, 0, 0, 0, 0, 0, 1502, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7628, 6992, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395057152 dispatch_meta_bytes=229368 combine_payload_bytes=395057152 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=395286520 ingress_bytes=615258112 link_bytes=615258112 token_bytes=790343672 weight_movement_bytes=220200960 total_nvlink_bytes=1010544632 weight_to_token_ratio=0.278614 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070035049e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=387530752 dispatch_meta_bytes=224464 combine_payload_bytes=387530752 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=387755216 ingress_bytes=717832192 link_bytes=717832192 token_bytes=775285968 weight_movement_bytes=330301440 total_nvlink_bytes=1105587408 weight_to_token_ratio=0.426038 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.248428245e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61064 payload_rows_remote=52876 dup_rows=4472 dup_rows_remote=4450 dedup_ratio=0.068237 router_tokens_per_expert=[0, 0, 116, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3946, 0, 1, 4094, 0, 0, 0, 2, 0, 3421, 0, 0, 0, 0, 123, 4096, 163, 0, 0, 0, 424, 4094, 0, 0, 0, 0, 4094, 0, 0] payload_rows_per_dst=[5328, 8192, 8190, 8188, 7962, 6842, 8174, 8188] dup_rows_per_dst=[148, 0, 0, 0, 2880, 4, 1418, 22] combine_rows_per_src=[5328, 8192, 8190, 8188, 7962, 6842, 8174, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64144 payload_rows_remote=55954 dup_rows=1392 dup_rows_remote=1392 dedup_ratio=0.021240 router_tokens_per_expert=[547, 0, 0, 0, 0, 4095, 0, 0, 0, 3423, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 4095, 1, 0, 0, 4096, 0, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6848, 8190, 8192, 8192, 8192, 8146, 8192] dup_rows_per_dst=[1092, 0, 0, 0, 0, 2, 298, 0] combine_rows_per_src=[8192, 6848, 8190, 8192, 8192, 8192, 8146, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=406999040 dispatch_meta_bytes=234176 combine_payload_bytes=406999040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407233216 ingress_bytes=517099520 link_bytes=517099520 token_bytes=814232256 weight_movement_bytes=110100480 total_nvlink_bytes=924332736 weight_to_token_ratio=0.135220 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26880 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.993211135e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64062 payload_rows_remote=56196 dup_rows=1474 dup_rows_remote=1474 dedup_ratio=0.022491 router_tokens_per_expert=[0, 0, 185, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3933, 0, 0, 4096, 0, 0, 0, 0, 0, 3522, 0, 0, 0, 0, 163, 4096, 46, 0, 0, 0, 343, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7866, 7044, 8192, 8192] dup_rows_per_dst=[370, 0, 0, 0, 0, 0, 1104, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7866, 7044, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63584 payload_rows_remote=55392 dup_rows=1952 dup_rows_remote=372 dedup_ratio=0.029785 router_tokens_per_expert=[0, 0, 185, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3940, 0, 0, 4096, 0, 0, 0, 0, 0, 3276, 0, 0, 0, 0, 96, 4096, 108, 0, 0, 0, 586, 4096, 0, 0, 0, 0, 4096, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7880, 6552, 8192, 8192] dup_rows_per_dst=[370, 0, 0, 2, 0, 0, 1580, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7880, 6552, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=379015168 dispatch_meta_bytes=229304 combine_payload_bytes=379015168 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=379244472 ingress_bytes=709316608 link_bytes=709316608 token_bytes=758259640 weight_movement_bytes=330301440 total_nvlink_bytes=1088561080 weight_to_token_ratio=0.435605 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.233618244e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=401078272 dispatch_meta_bytes=229384 combine_payload_bytes=401078272 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=401307656 ingress_bytes=621279232 link_bytes=621279232 token_bytes=802385928 weight_movement_bytes=220200960 total_nvlink_bytes=1022586888 weight_to_token_ratio=0.274433 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080506767e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58504 payload_rows_remote=50312 dup_rows=7032 dup_rows_remote=6346 dedup_ratio=0.107300 router_tokens_per_expert=[0, 0, 159, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3757, 0, 4, 4092, 0, 0, 0, 5, 0, 3339, 0, 0, 0, 0, 80, 4096, 223, 0, 0, 0, 632, 4094, 0, 0, 0, 0, 4093, 0, 0] payload_rows_per_dst=[8192, 8188, 8192, 8188, 7518, 8180, 1868, 8178] dup_rows_per_dst=[686, 4, 0, 8, 0, 6332, 2, 0] combine_rows_per_src=[8192, 8188, 8192, 8188, 7518, 8180, 1868, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61026 payload_rows_remote=52834 dup_rows=4510 dup_rows_remote=4510 dedup_ratio=0.068817 router_tokens_per_expert=[0, 0, 149, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 2, 4096, 0, 0, 0, 4, 0, 3559, 0, 0, 0, 0, 121, 4096, 45, 0, 0, 0, 377, 4096, 0, 0, 0, 0, 4095, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 7876, 7514, 4678, 8190] dup_rows_per_dst=[298, 0, 0, 0, 0, 3594, 618, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 7876, 7514, 4678, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64298 payload_rows_remote=56106 dup_rows=1238 dup_rows_remote=1238 dedup_ratio=0.018890 router_tokens_per_expert=[410, 0, 0, 0, 0, 4096, 0, 0, 0, 3518, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 5, 0, 0, 4054, 0, 0, 0, 0, 205, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7036, 8192, 8192, 8192, 8192, 8110, 8192] dup_rows_per_dst=[820, 0, 0, 0, 0, 10, 408, 0] combine_rows_per_src=[8192, 7036, 8192, 8192, 8192, 8192, 8110, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=402812928 dispatch_meta_bytes=230680 combine_payload_bytes=402812928 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=403043608 ingress_bytes=733114368 link_bytes=733114368 token_bytes=805856536 weight_movement_bytes=330301440 total_nvlink_bytes=1136157976 weight_to_token_ratio=0.409876 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.275006463e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7547781376 prefetch_bytes=1211105280 reduce_bytes=550502400 weight_movement_bytes=1761607680 egress_bytes=3775033600 ingress_bytes=5534355456 link_bytes=5534355456 total_nvlink_bytes=9309389056 weight_to_token_ratio=0.233394 prefetch_share_of_ingress=0.218834 dedup_ratio=0.075931 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=5 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=5969 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.625154383e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=397049856 dispatch_meta_bytes=223056 combine_payload_bytes=397049856 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=397272912 ingress_bytes=837451776 link_bytes=837451776 token_bytes=794322768 weight_movement_bytes=440401920 total_nvlink_bytes=1234724688 weight_to_token_ratio=0.554437 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.456466376e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62580 payload_rows_remote=54388 dup_rows=2956 dup_rows_remote=2942 dedup_ratio=0.045105 router_tokens_per_expert=[1224, 0, 0, 0, 0, 4096, 0, 0, 0, 2905, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 7, 0, 0, 3906, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7982, 5826, 8192, 8192, 8192, 8192, 7812, 8192] dup_rows_per_dst=[2436, 206, 0, 0, 0, 14, 300, 0] combine_rows_per_src=[7982, 5826, 8192, 8192, 8192, 8192, 7812, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=360636416 dispatch_meta_bytes=226632 combine_payload_bytes=360636416 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=360863048 ingress_bytes=580837376 link_bytes=580837376 token_bytes=721499464 weight_movement_bytes=220200960 total_nvlink_bytes=941700424 weight_to_token_ratio=0.305199 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.010171728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=378714112 dispatch_meta_bytes=229376 combine_payload_bytes=378714112 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=378943488 ingress_bytes=598915072 link_bytes=598915072 token_bytes=757657600 weight_movement_bytes=220200960 total_nvlink_bytes=977858560 weight_to_token_ratio=0.290634 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041611815e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=402167808 dispatch_meta_bytes=229376 combine_payload_bytes=402167808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=402397184 ingress_bytes=622368768 link_bytes=622368768 token_bytes=804564992 weight_movement_bytes=220200960 total_nvlink_bytes=1024765952 weight_to_token_ratio=0.273689 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.082401650e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58796 payload_rows_remote=50604 dup_rows=6740 dup_rows_remote=6740 dedup_ratio=0.102844 router_tokens_per_expert=[822, 0, 0, 0, 0, 4096, 0, 0, 0, 3187, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 4096, 0, 0, 4, 0, 0, 4032, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1644, 8192, 8188, 8188, 8192, 8136, 8064, 8192] dup_rows_per_dst=[0, 6372, 6, 0, 0, 10, 352, 0] combine_rows_per_src=[1644, 8192, 8188, 8188, 8192, 8136, 8064, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=389853184 dispatch_meta_bytes=229320 combine_payload_bytes=389853184 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=390082504 ingress_bytes=720154624 link_bytes=720154624 token_bytes=779935688 weight_movement_bytes=330301440 total_nvlink_bytes=1110237128 weight_to_token_ratio=0.423498 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.252467336e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63284 payload_rows_remote=55092 dup_rows=2252 dup_rows_remote=2252 dedup_ratio=0.034363 router_tokens_per_expert=[891, 0, 0, 0, 0, 4096, 0, 0, 0, 2996, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 4064, 0, 0, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5992, 8192, 8192, 8192, 8192, 8140, 8192] dup_rows_per_dst=[1782, 0, 0, 0, 0, 2, 468, 0] combine_rows_per_src=[8192, 5992, 8192, 8192, 8192, 8192, 8140, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7613052096 prefetch_bytes=880803840 reduce_bytes=990904320 weight_movement_bytes=1871708160 egress_bytes=3807675584 ingress_bytes=5677084672 link_bytes=5677084672 total_nvlink_bytes=9484760256 weight_to_token_ratio=0.245855 prefetch_share_of_ingress=0.155151 dedup_ratio=0.073584 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=9 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.873383965e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58494 payload_rows_remote=50346 dup_rows=7042 dup_rows_remote=6682 dedup_ratio=0.107452 router_tokens_per_expert=[597, 0, 0, 0, 0, 4096, 0, 0, 0, 3338, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 3, 0, 0, 4064, 0, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1194, 8192, 8192, 8192, 8192, 8192, 8148, 8192] dup_rows_per_dst=[0, 6676, 0, 0, 0, 6, 360, 0] combine_rows_per_src=[1194, 8192, 8192, 8192, 8192, 8192, 8148, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=362729472 dispatch_meta_bytes=229376 combine_payload_bytes=362729472 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362958848 ingress_bytes=472829952 link_bytes=472829952 token_bytes=725688320 weight_movement_bytes=110100480 total_nvlink_bytes=835788800 weight_to_token_ratio=0.151719 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.223290537e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7547418608 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3774857200 ingress_bytes=5424068608 link_bytes=5424068608 total_nvlink_bytes=9198925808 weight_to_token_ratio=0.218817 prefetch_share_of_ingress=0.182687 dedup_ratio=0.079401 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.433347415e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394899456 dispatch_meta_bytes=229376 combine_payload_bytes=394899456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395128832 ingress_bytes=504999936 link_bytes=504999936 token_bytes=790028288 weight_movement_bytes=110100480 total_nvlink_bytes=900128768 weight_to_token_ratio=0.139363 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.782779469e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62696 payload_rows_remote=57216 dup_rows=2840 dup_rows_remote=2840 dedup_ratio=0.043335 router_tokens_per_expert=[1251, 0, 0, 0, 0, 4094, 0, 0, 0, 2739, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 2, 0, 0, 4096, 0, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 169, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5480, 8192, 8192, 8190, 8192, 8066, 8192] dup_rows_per_dst=[2498, 0, 0, 0, 0, 4, 338, 0] combine_rows_per_src=[8192, 5480, 8192, 8192, 8190, 8192, 8066, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60082 payload_rows_remote=51890 dup_rows=5454 dup_rows_remote=326 dedup_ratio=0.083221 router_tokens_per_expert=[1612, 0, 0, 0, 0, 4089, 0, 0, 0, 2357, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 1, 0, 0, 4019, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 4716, 8192, 8192, 8192, 8188, 6218, 8192] dup_rows_per_dst=[5128, 0, 0, 4, 2, 12, 308, 0] combine_rows_per_src=[8192, 4716, 8192, 8192, 8192, 8188, 6218, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7108534440 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3555371176 ingress_bytes=5755172864 link_bytes=5755172864 total_nvlink_bytes=9310544040 weight_to_token_ratio=0.309770 prefetch_share_of_ingress=0.191307 dedup_ratio=0.120609 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.000919217e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=360880128 dispatch_meta_bytes=228112 combine_payload_bytes=360880128 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=361108240 ingress_bytes=581081088 link_bytes=581081088 token_bytes=721988368 weight_movement_bytes=220200960 total_nvlink_bytes=942189328 weight_to_token_ratio=0.304992 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.010595583e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7597227768 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3799764728 ingress_bytes=5448970240 link_bytes=5448970240 total_nvlink_bytes=9248734968 weight_to_token_ratio=0.217383 prefetch_share_of_ingress=0.202057 dedup_ratio=0.075360 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6096 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.476655449e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=410124288 dispatch_meta_bytes=240224 combine_payload_bytes=410124288 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=410364512 ingress_bytes=740425728 link_bytes=740425728 token_bytes=820488800 weight_movement_bytes=330301440 total_nvlink_bytes=1150790240 weight_to_token_ratio=0.402567 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.287722120e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7374409688 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3688337368 ingress_bytes=6108282880 link_bytes=6108512256 total_nvlink_bytes=9796620248 weight_to_token_ratio=0.328462 prefetch_share_of_ingress=0.162223 dedup_ratio=0.093747 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050735 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.062370749e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=371947520 dispatch_meta_bytes=208864 combine_payload_bytes=371947520 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=372156384 ingress_bytes=592148480 link_bytes=592148480 token_bytes=744103904 weight_movement_bytes=220200960 total_nvlink_bytes=964304864 weight_to_token_ratio=0.295928 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.029843598e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7344303872 prefetch_bytes=1211105280 reduce_bytes=1211105280 weight_movement_bytes=2422210560 egress_bytes=3673284352 ingress_bytes=6093230080 link_bytes=6093230080 total_nvlink_bytes=9766514432 weight_to_token_ratio=0.329808 prefetch_share_of_ingress=0.198762 dedup_ratio=0.104123 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=11 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.059712927e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6911565728 prefetch_bytes=880803840 reduce_bytes=1871708160 weight_movement_bytes=2752512000 egress_bytes=3456833440 ingress_bytes=6207244288 link_bytes=6207244288 total_nvlink_bytes=9664077728 weight_to_token_ratio=0.398247 prefetch_share_of_ingress=0.141899 dedup_ratio=0.141422 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=17 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051838 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.079541873e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:57:47] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 371.2 | throughput per GPU (TFLOP/s/GPU): 332.7 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.894554E+00 | loss scale: 1.0 | grad norm: 4.611 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44180 payload_rows_remote=37146 dup_rows=21356 dup_rows_remote=17452 dedup_ratio=0.325867 router_tokens_per_expert=[1587, 1450, 948, 663, 272, 809, 401, 55, 227, 1331, 1129, 394, 4, 907, 12, 1528, 838, 376, 12, 43, 730, 506, 562, 481, 173, 1107, 335, 625, 342, 232, 589, 34, 840, 350, 1013, 2, 575, 2, 1068, 80, 173, 643, 349, 604, 0, 947, 92, 382, 212, 199, 334, 709, 263, 983, 80, 1239, 354, 889, 398, 164, 495, 366, 260, 1] payload_rows_per_dst=[4436, 3206, 5390, 6246, 5798, 6222, 7034, 5848] dup_rows_per_dst=[1740, 326, 1826, 3290, 3876, 3214, 3904, 3180] combine_rows_per_src=[4436, 3206, 5390, 6246, 5798, 6222, 7034, 5848] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45300 payload_rows_remote=40328 dup_rows=20236 dup_rows_remote=17854 dedup_ratio=0.308777 router_tokens_per_expert=[1345, 1342, 829, 944, 115, 1120, 448, 88, 284, 1202, 1245, 334, 3, 1260, 26, 1335, 805, 720, 32, 24, 862, 309, 633, 684, 339, 773, 356, 926, 383, 170, 410, 1, 534, 537, 756, 4, 519, 0, 1145, 182, 455, 501, 295, 849, 0, 725, 73, 541, 156, 107, 292, 272, 171, 996, 129, 778, 450, 691, 544, 441, 593, 435, 249, 1] payload_rows_per_dst=[5070, 6234, 5794, 6054, 4972, 4962, 6184, 6030] dup_rows_per_dst=[2018, 2740, 2344, 3066, 2382, 1916, 2302, 3468] combine_rows_per_src=[5070, 6234, 5794, 6054, 4972, 4962, 6184, 6030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45148 payload_rows_remote=39190 dup_rows=20388 dup_rows_remote=17382 dedup_ratio=0.311096 router_tokens_per_expert=[1614, 1041, 1263, 1001, 310, 361, 461, 29, 265, 1486, 910, 181, 3, 1297, 10, 1031, 1063, 757, 15, 30, 594, 415, 305, 836, 136, 1085, 332, 622, 318, 107, 522, 7, 640, 294, 1138, 8, 726, 0, 970, 209, 286, 723, 404, 1100, 1, 768, 70, 639, 118, 219, 156, 374, 323, 978, 79, 604, 775, 1022, 398, 200, 329, 556, 254, 0] payload_rows_per_dst=[4888, 5608, 5736, 5958, 5334, 5554, 5736, 6334] dup_rows_per_dst=[1962, 2052, 2294, 3006, 2636, 2428, 2048, 3962] combine_rows_per_src=[4888, 5608, 5736, 5958, 5334, 5554, 5736, 6334] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45016 payload_rows_remote=39970 dup_rows=20520 dup_rows_remote=18450 dedup_ratio=0.313110 router_tokens_per_expert=[1082, 1354, 1355, 970, 97, 1102, 374, 44, 383, 1195, 1237, 327, 5, 1269, 33, 1040, 1004, 558, 32, 67, 1088, 410, 125, 546, 219, 681, 477, 843, 539, 129, 385, 1, 561, 413, 894, 35, 477, 1, 1241, 155, 542, 475, 358, 1201, 6, 405, 87, 430, 226, 197, 177, 713, 243, 727, 39, 1196, 495, 493, 655, 264, 166, 593, 310, 22] payload_rows_per_dst=[5250, 5648, 5604, 5972, 5512, 5046, 6462, 5522] dup_rows_per_dst=[2634, 1982, 2056, 2966, 2892, 2070, 3282, 2638] combine_rows_per_src=[5250, 5648, 5604, 5972, 5512, 5046, 6462, 5522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44112 payload_rows_remote=37944 dup_rows=21424 dup_rows_remote=18252 dedup_ratio=0.326904 router_tokens_per_expert=[1354, 1341, 1252, 576, 31, 1125, 336, 82, 469, 1752, 934, 356, 2, 716, 52, 1404, 881, 855, 11, 46, 1182, 287, 711, 645, 302, 613, 566, 700, 417, 224, 335, 9, 683, 538, 713, 7, 668, 0, 840, 103, 216, 1006, 479, 645, 1, 567, 54, 306, 157, 229, 241, 355, 142, 1109, 155, 672, 427, 735, 1077, 362, 263, 175, 275, 2] payload_rows_per_dst=[3390, 3222, 7094, 6340, 5422, 6118, 6358, 6168] dup_rows_per_dst=[910, 404, 4646, 3496, 3114, 3238, 2444, 3172] combine_rows_per_src=[3390, 3222, 7094, 6340, 5422, 6118, 6358, 6168] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44206 payload_rows_remote=38906 dup_rows=21330 dup_rows_remote=19684 dedup_ratio=0.325470 router_tokens_per_expert=[1556, 893, 1096, 830, 180, 1013, 497, 152, 239, 1374, 771, 351, 4, 1189, 30, 956, 681, 972, 9, 38, 632, 309, 402, 430, 421, 1016, 268, 584, 161, 217, 653, 3, 720, 537, 675, 16, 687, 2, 986, 207, 261, 830, 264, 1334, 2, 920, 38, 507, 90, 118, 551, 427, 397, 992, 109, 578, 394, 619, 861, 177, 313, 988, 241, 0] payload_rows_per_dst=[6270, 6184, 5300, 5038, 5006, 5676, 4996, 5736] dup_rows_per_dst=[4470, 3644, 1646, 1608, 2654, 2636, 1806, 2866] combine_rows_per_src=[6270, 6184, 5300, 5038, 5006, 5676, 4996, 5736] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43712 payload_rows_remote=36690 dup_rows=21824 dup_rows_remote=15972 dedup_ratio=0.333008 router_tokens_per_expert=[1447, 1247, 938, 692, 48, 1022, 214, 90, 722, 1494, 958, 323, 7, 1033, 30, 1870, 1118, 871, 31, 42, 803, 308, 241, 792, 341, 624, 372, 558, 288, 113, 701, 1, 519, 479, 1424, 7, 427, 8, 825, 166, 313, 758, 564, 1127, 10, 784, 36, 505, 314, 222, 228, 328, 135, 852, 99, 699, 367, 602, 534, 297, 252, 411, 137, 0] payload_rows_per_dst=[6664, 7022, 5932, 4614, 5274, 5620, 4478, 4108] dup_rows_per_dst=[4732, 5852, 2480, 1382, 2436, 2574, 1276, 1092] combine_rows_per_src=[6664, 7022, 5932, 4614, 5274, 5620, 4478, 4108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266262528 dispatch_meta_bytes=218392 combine_payload_bytes=266262528 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=266480920 ingress_bytes=376363008 link_bytes=376363008 token_bytes=532743448 weight_movement_bytes=110100480 total_nvlink_bytes=642843928 weight_to_token_ratio=0.206667 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.545571720e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=289071104 dispatch_meta_bytes=232728 combine_payload_bytes=289071104 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=289303832 ingress_bytes=399171584 link_bytes=399171584 token_bytes=578374936 weight_movement_bytes=110100480 total_nvlink_bytes=688475416 weight_to_token_ratio=0.190362 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.942250370e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280913920 dispatch_meta_bytes=226288 combine_payload_bytes=280913920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281140208 ingress_bytes=391014400 link_bytes=391014400 token_bytes=562054128 weight_movement_bytes=110100480 total_nvlink_bytes=672154608 weight_to_token_ratio=0.195889 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6912 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.800383524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=286504960 dispatch_meta_bytes=233680 combine_payload_bytes=286504960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=286738640 ingress_bytes=396605440 link_bytes=396605440 token_bytes=573243600 weight_movement_bytes=110100480 total_nvlink_bytes=683344080 weight_to_token_ratio=0.192066 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8192 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.897620905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=271982592 dispatch_meta_bytes=224784 combine_payload_bytes=271982592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=272207376 ingress_bytes=382083072 link_bytes=382083072 token_bytes=544189968 weight_movement_bytes=110100480 total_nvlink_bytes=654290448 weight_to_token_ratio=0.202320 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=7808 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.645053041e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278878208 dispatch_meta_bytes=234360 combine_payload_bytes=278878208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279112568 ingress_bytes=388978688 link_bytes=388978688 token_bytes=557990776 weight_movement_bytes=110100480 total_nvlink_bytes=668091256 weight_to_token_ratio=0.197316 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.764979144e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44352 payload_rows_remote=37704 dup_rows=21184 dup_rows_remote=16730 dedup_ratio=0.323242 router_tokens_per_expert=[1424, 1166, 1102, 543, 114, 654, 247, 91, 321, 1009, 1041, 369, 16, 1662, 15, 1483, 1163, 306, 17, 68, 868, 328, 504, 443, 397, 846, 273, 740, 239, 234, 511, 10, 754, 392, 1031, 21, 837, 0, 739, 340, 570, 979, 324, 1243, 11, 712, 11, 281, 61, 196, 308, 350, 349, 1352, 78, 919, 318, 543, 645, 387, 237, 443, 132, 1] payload_rows_per_dst=[6648, 6792, 5478, 4862, 5264, 5678, 5472, 4158] dup_rows_per_dst=[4454, 4620, 1916, 1638, 2964, 2584, 1754, 1254] combine_rows_per_src=[6648, 6792, 5478, 4862, 5264, 5678, 5472, 4158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=262993920 dispatch_meta_bytes=210648 combine_payload_bytes=262993920 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=263204568 ingress_bytes=703395840 link_bytes=703395840 token_bytes=526198488 weight_movement_bytes=440401920 total_nvlink_bytes=966600408 weight_to_token_ratio=0.836950 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=8064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.223321054e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49142 payload_rows_remote=41944 dup_rows=16394 dup_rows_remote=12666 dedup_ratio=0.250153 router_tokens_per_expert=[0, 0, 0, 1216, 4004, 0, 48, 3452, 1902, 2, 61, 0, 0, 0, 0, 52, 0, 0, 0, 1, 3115, 2549, 0, 0, 0, 0, 2070, 1, 4, 2349, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3740, 0, 0, 0, 0, 0, 0, 217, 2325, 0, 572, 0, 0, 0, 1551, 0, 2, 0, 13, 0, 1, 3521] payload_rows_per_dst=[2526, 3932, 8098, 4146, 8008, 7480, 7198, 7754] dup_rows_per_dst=[2, 102, 10136, 4, 0, 0, 3728, 2422] combine_rows_per_src=[2526, 3932, 8098, 4146, 8008, 7480, 7198, 7754] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53132 payload_rows_remote=45954 dup_rows=12404 dup_rows_remote=12404 dedup_ratio=0.189270 router_tokens_per_expert=[0, 0, 0, 1648, 3897, 0, 36, 3460, 1512, 0, 41, 0, 0, 1, 0, 193, 0, 6, 0, 9, 3205, 2163, 0, 2, 0, 0, 3028, 1, 19, 2786, 0, 2, 0, 0, 0, 0, 3, 1, 0, 23, 0, 2, 3587, 0, 0, 0, 0, 0, 0, 224, 1970, 0, 426, 0, 0, 0, 1659, 1, 15, 0, 6, 2, 13, 2827] payload_rows_per_dst=[6670, 3384, 7682, 6072, 7804, 7178, 7260, 7082] dup_rows_per_dst=[2258, 110, 4448, 28, 44, 0, 3552, 1964] combine_rows_per_src=[6670, 3384, 7682, 6072, 7804, 7178, 7260, 7082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54596 payload_rows_remote=46584 dup_rows=10940 dup_rows_remote=10940 dedup_ratio=0.166931 router_tokens_per_expert=[0, 0, 0, 1705, 4005, 0, 55, 3444, 1818, 0, 16, 0, 0, 1, 0, 7, 0, 1, 0, 33, 3290, 1874, 1, 0, 0, 0, 2654, 0, 4, 2774, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 3817, 0, 0, 0, 9, 0, 0, 220, 2261, 0, 436, 0, 0, 0, 1294, 0, 0, 0, 3, 1, 0, 3043] payload_rows_per_dst=[7566, 5902, 5496, 7454, 8012, 7652, 5422, 7092] dup_rows_per_dst=[2842, 1308, 1376, 2850, 0, 2, 972, 1590] combine_rows_per_src=[7566, 5902, 5496, 7454, 8012, 7652, 5422, 7092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54586 payload_rows_remote=46938 dup_rows=10950 dup_rows_remote=8190 dedup_ratio=0.167084 router_tokens_per_expert=[0, 0, 0, 1534, 3860, 0, 66, 2886, 2268, 3, 28, 0, 1, 0, 0, 16, 0, 0, 0, 18, 3270, 2300, 0, 0, 0, 0, 2361, 0, 0, 2843, 0, 0, 0, 3, 0, 0, 0, 6, 0, 5, 0, 0, 3647, 0, 0, 0, 6, 0, 0, 215, 2410, 2, 616, 0, 0, 0, 1441, 0, 0, 0, 20, 0, 2, 2941] payload_rows_per_dst=[6864, 7724, 4602, 7648, 7732, 7300, 5690, 7026] dup_rows_per_dst=[2108, 3448, 34, 2760, 16, 6, 796, 1782] combine_rows_per_src=[6864, 7724, 4602, 7648, 7732, 7300, 5690, 7026] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48262 payload_rows_remote=41918 dup_rows=17274 dup_rows_remote=13576 dedup_ratio=0.263580 router_tokens_per_expert=[0, 0, 0, 2142, 3695, 0, 60, 2971, 1746, 4, 372, 0, 0, 16, 0, 243, 0, 1, 0, 44, 3342, 2455, 0, 0, 0, 0, 2491, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 3914, 0, 0, 0, 11, 0, 0, 256, 1860, 0, 490, 0, 0, 0, 912, 0, 0, 1, 96, 4, 4, 2393] payload_rows_per_dst=[4348, 4160, 7808, 2710, 7500, 7828, 7564, 6344] dup_rows_per_dst=[56, 602, 4980, 586, 3198, 22, 4132, 3698] combine_rows_per_src=[4348, 4160, 7808, 2710, 7500, 7828, 7564, 6344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270262272 dispatch_meta_bytes=217736 combine_payload_bytes=270262272 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=270480008 ingress_bytes=710664192 link_bytes=710664192 token_bytes=540742280 weight_movement_bytes=440401920 total_nvlink_bytes=981144200 weight_to_token_ratio=0.814440 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.235961914e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54508 payload_rows_remote=50176 dup_rows=11028 dup_rows_remote=10976 dedup_ratio=0.168274 router_tokens_per_expert=[0, 0, 0, 1333, 3857, 0, 58, 3116, 2760, 0, 195, 0, 0, 0, 0, 78, 2, 1, 0, 37, 3049, 2152, 0, 0, 0, 0, 2240, 0, 2, 2906, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 3851, 0, 0, 1, 0, 0, 0, 379, 2542, 0, 743, 0, 1, 0, 544, 0, 0, 3, 70, 1, 22, 2823] payload_rows_per_dst=[7020, 7936, 4332, 7618, 7716, 7704, 5982, 6200] dup_rows_per_dst=[1994, 4228, 52, 2678, 2, 0, 1348, 726] combine_rows_per_src=[7020, 7936, 4332, 7618, 7716, 7704, 5982, 6200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=300654592 dispatch_meta_bytes=218440 combine_payload_bytes=300654592 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=300873032 ingress_bytes=410755072 link_bytes=410755072 token_bytes=601527624 weight_movement_bytes=110100480 total_nvlink_bytes=711628104 weight_to_token_ratio=0.183035 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17920 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.143706278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=329398272 dispatch_meta_bytes=233432 combine_payload_bytes=329398272 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329631704 ingress_bytes=439498752 link_bytes=439498752 token_bytes=659029976 weight_movement_bytes=110100480 total_nvlink_bytes=769130456 weight_to_token_ratio=0.167064 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.643606149e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=333914112 dispatch_meta_bytes=230096 combine_payload_bytes=333914112 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=334144208 ingress_bytes=554115072 link_bytes=554115072 token_bytes=668058320 weight_movement_bytes=220200960 total_nvlink_bytes=888259280 weight_to_token_ratio=0.329613 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.636972465e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=336451584 dispatch_meta_bytes=220512 combine_payload_bytes=336451584 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=336672096 ingress_bytes=666753024 link_bytes=666753024 token_bytes=673123680 weight_movement_bytes=330301440 total_nvlink_bytes=1003425120 weight_to_token_ratio=0.490699 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.047794 gemm_active_groups=7 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.159593171e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=300468224 dispatch_meta_bytes=221976 combine_payload_bytes=300468224 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=300690200 ingress_bytes=520669184 link_bytes=520669184 token_bytes=601158424 weight_movement_bytes=220200960 total_nvlink_bytes=821359384 weight_to_token_ratio=0.366294 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.055293463e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54294 payload_rows_remote=46584 dup_rows=11242 dup_rows_remote=8020 dedup_ratio=0.171539 router_tokens_per_expert=[0, 0, 0, 1854, 3976, 0, 82, 3349, 1846, 2, 96, 0, 0, 0, 0, 69, 0, 0, 0, 28, 3453, 2023, 0, 0, 0, 0, 2306, 0, 0, 2944, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 3790, 0, 0, 0, 2, 0, 0, 236, 1857, 0, 531, 0, 0, 0, 1053, 0, 0, 0, 19, 0, 12, 3235] payload_rows_per_dst=[7492, 7710, 4068, 7760, 7962, 7580, 4552, 7170] dup_rows_per_dst=[3078, 3222, 34, 2740, 0, 4, 696, 1468] combine_rows_per_src=[7492, 7710, 4068, 7760, 7962, 7580, 4552, 7170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=359661568 dispatch_meta_bytes=244608 combine_payload_bytes=359661568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=359906176 ingress_bytes=579862528 link_bytes=579862528 token_bytes=719567744 weight_movement_bytes=220200960 total_nvlink_bytes=939768704 weight_to_token_ratio=0.306018 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.008476307e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=333914112 dispatch_meta_bytes=218416 combine_payload_bytes=333914112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334132528 ingress_bytes=444014592 link_bytes=444014592 token_bytes=668046640 weight_movement_bytes=110100480 total_nvlink_bytes=778147120 weight_to_token_ratio=0.164810 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.722144033e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59090 payload_rows_remote=50900 dup_rows=6446 dup_rows_remote=6446 dedup_ratio=0.098358 router_tokens_per_expert=[0, 0, 0, 3850, 2, 0, 4095, 0, 0, 3806, 0, 0, 0, 0, 1145, 0, 38, 0, 0, 0, 0, 4096, 22, 0, 0, 0, 411, 4096, 0, 66, 0, 0, 0, 0, 0, 0, 0, 1, 2915, 0, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 37] payload_rows_per_dst=[7700, 2290, 8192, 8192, 8150, 8184, 8190, 8192] dup_rows_per_dst=[4, 0, 120, 954, 5294, 0, 0, 74] combine_rows_per_src=[7700, 2290, 8192, 8192, 8150, 8184, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49726 payload_rows_remote=42176 dup_rows=15810 dup_rows_remote=12648 dedup_ratio=0.241241 router_tokens_per_expert=[0, 0, 0, 1882, 3771, 0, 48, 3426, 2369, 1, 48, 0, 0, 1, 0, 30, 0, 0, 0, 25, 3435, 1170, 0, 0, 0, 0, 2607, 0, 1, 2633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3661, 0, 2, 0, 1, 0, 1, 232, 2651, 0, 438, 0, 0, 0, 932, 0, 5, 2, 4, 0, 6, 3385] payload_rows_per_dst=[7550, 7916, 2390, 7624, 7542, 7896, 5904, 2904] dup_rows_per_dst=[3162, 3852, 0, 2858, 0, 4884, 740, 314] combine_rows_per_src=[7550, 7916, 2390, 7624, 7542, 7896, 5904, 2904] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59148 payload_rows_remote=51048 dup_rows=6388 dup_rows_remote=6382 dedup_ratio=0.097473 router_tokens_per_expert=[0, 0, 0, 3836, 16, 0, 4091, 0, 0, 3765, 0, 0, 0, 0, 1284, 0, 33, 0, 0, 0, 0, 4083, 34, 0, 0, 0, 493, 4096, 0, 110, 0, 0, 0, 0, 0, 0, 0, 7, 2741, 8, 4035, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 31] payload_rows_per_dst=[7674, 2568, 8172, 8192, 8086, 8100, 8182, 8174] dup_rows_per_dst=[30, 0, 128, 1206, 4956, 6, 0, 62] combine_rows_per_src=[7674, 2568, 8172, 8192, 8086, 8100, 8182, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62154 payload_rows_remote=56264 dup_rows=3382 dup_rows_remote=3012 dedup_ratio=0.051605 router_tokens_per_expert=[0, 0, 0, 3780, 1, 0, 4096, 0, 0, 3828, 0, 0, 0, 0, 1277, 0, 25, 0, 0, 0, 0, 4091, 31, 0, 0, 0, 416, 4096, 0, 52, 0, 0, 0, 1, 0, 0, 0, 0, 2867, 1, 4085, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 26] payload_rows_per_dst=[7562, 7770, 8188, 8192, 5890, 8170, 8192, 8190] dup_rows_per_dst=[0, 1918, 106, 936, 370, 0, 0, 52] combine_rows_per_src=[7562, 7770, 8188, 8192, 5890, 8170, 8192, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63538 payload_rows_remote=55346 dup_rows=1998 dup_rows_remote=1192 dedup_ratio=0.030487 router_tokens_per_expert=[0, 0, 0, 3941, 2, 0, 4095, 0, 0, 3741, 0, 0, 0, 0, 846, 0, 8, 0, 0, 0, 0, 4094, 38, 0, 0, 0, 347, 4096, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 3288, 2, 4087, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 27] payload_rows_per_dst=[7882, 8136, 8188, 8192, 6580, 8178, 8190, 8192] dup_rows_per_dst=[4, 1038, 92, 806, 0, 2, 0, 56] combine_rows_per_src=[7882, 8136, 8188, 8192, 6580, 8178, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58946 payload_rows_remote=50754 dup_rows=6590 dup_rows_remote=6586 dedup_ratio=0.100555 router_tokens_per_expert=[0, 0, 0, 3793, 0, 0, 4092, 0, 0, 3582, 0, 0, 0, 0, 1817, 0, 23, 0, 0, 0, 0, 4094, 91, 0, 0, 0, 561, 4096, 0, 157, 0, 0, 0, 0, 0, 0, 0, 85, 2195, 0, 4083, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 2] payload_rows_per_dst=[7560, 3634, 7166, 8192, 7852, 8166, 8184, 8192] dup_rows_per_dst=[0, 0, 206, 2480, 3872, 2, 26, 4] combine_rows_per_src=[7560, 3634, 7166, 8192, 7852, 8166, 8184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61336 payload_rows_remote=53150 dup_rows=4200 dup_rows_remote=3980 dedup_ratio=0.064087 router_tokens_per_expert=[0, 0, 0, 3818, 3, 0, 4094, 0, 0, 3817, 0, 0, 0, 0, 1437, 0, 36, 0, 0, 0, 0, 4093, 74, 0, 0, 0, 580, 4096, 0, 196, 1, 0, 0, 2, 0, 0, 0, 33, 2285, 0, 4071, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 35] payload_rows_per_dst=[7638, 8156, 8186, 8192, 4640, 8142, 8190, 8192] dup_rows_per_dst=[4, 2352, 220, 1554, 0, 0, 0, 70] combine_rows_per_src=[7638, 8156, 8186, 8192, 4640, 8142, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=364851200 dispatch_meta_bytes=229384 combine_payload_bytes=364851200 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=365080584 ingress_bytes=585052160 link_bytes=585052160 token_bytes=729931784 weight_movement_bytes=220200960 total_nvlink_bytes=950132744 weight_to_token_ratio=0.301673 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.017501931e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=302317568 dispatch_meta_bytes=219296 combine_payload_bytes=302317568 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=302536864 ingress_bytes=742719488 link_bytes=742719488 token_bytes=604854432 weight_movement_bytes=440401920 total_nvlink_bytes=1045256352 weight_to_token_ratio=0.728112 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.291711346e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=365912064 dispatch_meta_bytes=229720 combine_payload_bytes=365912064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366141784 ingress_bytes=476012544 link_bytes=476012544 token_bytes=732053848 weight_movement_bytes=110100480 total_nvlink_bytes=842154328 weight_to_token_ratio=0.150399 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.278641046e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=403300352 dispatch_meta_bytes=237104 combine_payload_bytes=403300352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403537456 ingress_bytes=513400832 link_bytes=513400832 token_bytes=806837808 weight_movement_bytes=110100480 total_nvlink_bytes=916938288 weight_to_token_ratio=0.136459 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=21248 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.928884868e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=396720128 dispatch_meta_bytes=226152 combine_payload_bytes=396720128 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=396946280 ingress_bytes=727021568 link_bytes=727021568 token_bytes=793666408 weight_movement_bytes=330301440 total_nvlink_bytes=1123967848 weight_to_token_ratio=0.416172 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.264410081e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60280 payload_rows_remote=52094 dup_rows=5256 dup_rows_remote=738 dedup_ratio=0.080200 router_tokens_per_expert=[0, 0, 0, 3863, 0, 0, 4095, 0, 0, 3730, 0, 0, 0, 0, 1734, 0, 3, 0, 0, 0, 0, 4096, 26, 0, 0, 0, 491, 4096, 0, 70, 0, 0, 0, 0, 0, 0, 0, 45, 2323, 0, 4074, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 24] payload_rows_per_dst=[7726, 8186, 8192, 6908, 4736, 8150, 8190, 8192] dup_rows_per_dst=[0, 4518, 58, 630, 0, 2, 0, 48] combine_rows_per_src=[7726, 8186, 8192, 6908, 4736, 8150, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=380979200 dispatch_meta_bytes=228520 combine_payload_bytes=380979200 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=381207720 ingress_bytes=601180160 link_bytes=601180160 token_bytes=762186920 weight_movement_bytes=220200960 total_nvlink_bytes=982387880 weight_to_token_ratio=0.288907 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045551175e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=363804672 dispatch_meta_bytes=229360 combine_payload_bytes=363804672 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=364034032 ingress_bytes=584005632 link_bytes=584005632 token_bytes=727838704 weight_movement_bytes=220200960 total_nvlink_bytes=948039664 weight_to_token_ratio=0.302541 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.015681846e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=373409792 dispatch_meta_bytes=211328 combine_payload_bytes=373409792 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=373621120 ingress_bytes=593610752 link_bytes=593610752 token_bytes=747030912 weight_movement_bytes=220200960 total_nvlink_bytes=967231872 weight_to_token_ratio=0.294768 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.032386730e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57078 payload_rows_remote=48904 dup_rows=8458 dup_rows_remote=8458 dedup_ratio=0.129059 router_tokens_per_expert=[0, 0, 1148, 0, 0, 4095, 0, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 4093, 0, 0, 4034, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 40, 850, 0, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 1486, 4096, 0] payload_rows_per_dst=[2296, 8190, 8092, 8068, 5880, 8186, 8174, 8192] dup_rows_per_dst=[0, 1372, 0, 0, 1268, 0, 0, 5818] combine_rows_per_src=[2296, 8190, 8092, 8068, 5880, 8186, 8174, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60816 payload_rows_remote=52624 dup_rows=4720 dup_rows_remote=4700 dedup_ratio=0.072021 router_tokens_per_expert=[0, 0, 1941, 0, 0, 4096, 0, 0, 1, 0, 0, 430, 0, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 4096, 0, 0, 4043, 0, 4066, 0, 1, 8, 0, 0, 0, 0, 0, 42, 656, 0, 4087, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 1146, 4088, 0] payload_rows_per_dst=[3882, 8192, 7956, 8094, 8178, 8192, 8132, 8190] dup_rows_per_dst=[0, 862, 0, 10, 1392, 20, 0, 2436] combine_rows_per_src=[3882, 8192, 7956, 8094, 8178, 8192, 8132, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54460 payload_rows_remote=46448 dup_rows=11076 dup_rows_remote=7336 dedup_ratio=0.169006 router_tokens_per_expert=[0, 0, 0, 3985, 1, 0, 4096, 0, 0, 3863, 0, 0, 0, 0, 1416, 0, 4, 0, 0, 0, 0, 4091, 10, 0, 0, 0, 350, 4096, 0, 105, 0, 0, 0, 0, 0, 0, 0, 0, 2553, 2, 4069, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 32] payload_rows_per_dst=[8012, 8158, 8190, 866, 5110, 8146, 8192, 7786] dup_rows_per_dst=[3740, 6812, 132, 44, 0, 288, 0, 60] combine_rows_per_src=[8012, 8158, 8190, 866, 5110, 8146, 8192, 7786] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60616 payload_rows_remote=52440 dup_rows=4920 dup_rows_remote=2572 dedup_ratio=0.075073 router_tokens_per_expert=[0, 0, 1799, 0, 0, 4094, 0, 0, 1, 0, 0, 509, 1, 0, 0, 0, 0, 0, 4025, 0, 0, 0, 4095, 0, 0, 4035, 0, 4075, 0, 0, 0, 0, 0, 0, 0, 0, 64, 1116, 0, 4082, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 734, 4095, 0] payload_rows_per_dst=[3598, 8188, 8050, 8070, 8176, 8192, 8150, 8192] dup_rows_per_dst=[0, 1022, 0, 0, 2348, 2, 0, 1548] combine_rows_per_src=[3598, 8188, 8050, 8070, 8176, 8192, 8150, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59498 payload_rows_remote=51450 dup_rows=6038 dup_rows_remote=6036 dedup_ratio=0.092133 router_tokens_per_expert=[0, 0, 1354, 0, 0, 4096, 0, 0, 2, 0, 0, 635, 0, 0, 0, 0, 0, 0, 3993, 0, 0, 0, 4095, 0, 0, 4024, 0, 4084, 0, 0, 1, 0, 0, 0, 0, 0, 58, 822, 0, 4090, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 1368, 4094, 0] payload_rows_per_dst=[4604, 8192, 7986, 8048, 8182, 8192, 8168, 6126] dup_rows_per_dst=[1290, 1274, 0, 2, 1758, 6, 0, 1708] combine_rows_per_src=[4604, 8192, 7986, 8048, 8182, 8192, 8168, 6126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58574 payload_rows_remote=50948 dup_rows=6962 dup_rows_remote=6962 dedup_ratio=0.106232 router_tokens_per_expert=[0, 0, 1874, 0, 0, 4096, 0, 0, 3, 0, 0, 698, 2, 0, 0, 0, 0, 0, 3813, 0, 0, 0, 4095, 0, 0, 3995, 0, 4089, 0, 0, 2, 0, 0, 0, 0, 0, 69, 840, 0, 4080, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 966, 4093, 0] payload_rows_per_dst=[8192, 8192, 7626, 7994, 8168, 8192, 8178, 2032] dup_rows_per_dst=[3742, 1406, 0, 0, 1810, 2, 0, 2] combine_rows_per_src=[8192, 8192, 7626, 7994, 8168, 8192, 8178, 2032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54840 payload_rows_remote=46648 dup_rows=10696 dup_rows_remote=1522 dedup_ratio=0.163208 router_tokens_per_expert=[0, 0, 2784, 0, 0, 4079, 0, 0, 0, 0, 0, 575, 0, 0, 0, 0, 0, 0, 3875, 0, 0, 0, 4096, 0, 0, 4003, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 57, 547, 0, 4002, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 540, 4096, 0] payload_rows_per_dst=[5568, 8158, 7750, 7660, 1186, 8192, 8134, 8192] dup_rows_per_dst=[0, 1150, 0, 0, 22, 4, 346, 9174] combine_rows_per_src=[5568, 8158, 7750, 7660, 1186, 8192, 8134, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=350543872 dispatch_meta_bytes=229448 combine_payload_bytes=350543872 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=350773320 ingress_bytes=570744832 link_bytes=570744832 token_bytes=701317192 weight_movement_bytes=220200960 total_nvlink_bytes=921518152 weight_to_token_ratio=0.313982 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.926191342e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=377208832 dispatch_meta_bytes=229296 combine_payload_bytes=377208832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377438128 ingress_bytes=487309312 link_bytes=487309312 token_bytes=754646960 weight_movement_bytes=110100480 total_nvlink_bytes=864747440 weight_to_token_ratio=0.145897 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.475110422e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=332939264 dispatch_meta_bytes=215136 combine_payload_bytes=332939264 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=333154400 ingress_bytes=663240704 link_bytes=663240704 token_bytes=666093664 weight_movement_bytes=330301440 total_nvlink_bytes=996395104 weight_to_token_ratio=0.495878 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.153484669e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=375889920 dispatch_meta_bytes=220048 combine_payload_bytes=375889920 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=376109968 ingress_bytes=706191360 link_bytes=706191360 token_bytes=751999888 weight_movement_bytes=330301440 total_nvlink_bytes=1082301328 weight_to_token_ratio=0.439231 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.228182924e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=368793600 dispatch_meta_bytes=229944 combine_payload_bytes=368793600 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=369023544 ingress_bytes=699095040 link_bytes=699095040 token_bytes=737817144 weight_movement_bytes=330301440 total_nvlink_bytes=1068118584 weight_to_token_ratio=0.447674 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.215841256e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=334372864 dispatch_meta_bytes=192680 combine_payload_bytes=334372864 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=334565544 ingress_bytes=554573824 link_bytes=554573824 token_bytes=668938408 weight_movement_bytes=220200960 total_nvlink_bytes=889139368 weight_to_token_ratio=0.329180 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.644950917e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57522 payload_rows_remote=50584 dup_rows=8014 dup_rows_remote=7418 dedup_ratio=0.122284 router_tokens_per_expert=[0, 0, 1974, 0, 0, 4088, 0, 0, 0, 0, 0, 316, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 0, 4096, 0, 0, 4031, 0, 4048, 0, 0, 0, 0, 0, 0, 0, 0, 61, 1053, 0, 4077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 40, 0, 0, 0, 0, 904, 4096, 0] payload_rows_per_dst=[8192, 6938, 7966, 8062, 8186, 8192, 8098, 1888] dup_rows_per_dst=[5222, 596, 0, 0, 2196, 0, 0, 0] combine_rows_per_src=[8192, 6938, 7966, 8062, 8186, 8192, 8098, 1888] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=365195264 dispatch_meta_bytes=231640 combine_payload_bytes=365195264 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=365426904 ingress_bytes=585396224 link_bytes=585396224 token_bytes=730622168 weight_movement_bytes=220200960 total_nvlink_bytes=950823128 weight_to_token_ratio=0.301388 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.018100315e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=362586112 dispatch_meta_bytes=232008 combine_payload_bytes=362586112 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362818120 ingress_bytes=472686592 link_bytes=472686592 token_bytes=725404232 weight_movement_bytes=110100480 total_nvlink_bytes=835504712 weight_to_token_ratio=0.151778 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28032 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.220797271e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62428 payload_rows_remote=55022 dup_rows=3108 dup_rows_remote=2648 dedup_ratio=0.047424 router_tokens_per_expert=[0, 2, 0, 0, 441, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 4026, 0, 0, 0, 0, 0, 4095, 576, 0, 4096, 0, 0, 0, 0, 27, 0, 4094, 0, 0, 4051, 0, 0, 0, 195, 0, 0, 0, 3691, 47, 0, 0, 0, 3179, 0, 0, 0, 4038, 0] payload_rows_per_dst=[8098, 8192, 8188, 8054, 8028, 8104, 7406, 6358] dup_rows_per_dst=[864, 0, 0, 418, 1126, 240, 460, 0] combine_rows_per_src=[8098, 8192, 8188, 8054, 8028, 8104, 7406, 6358] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63266 payload_rows_remote=55206 dup_rows=2270 dup_rows_remote=2214 dedup_ratio=0.034637 router_tokens_per_expert=[0, 38, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 3919, 11, 0, 0, 0, 0, 4087, 392, 0, 4085, 0, 0, 0, 0, 28, 1, 4082, 0, 0, 4029, 0, 0, 0, 331, 0, 0, 0, 3876, 53, 0, 0, 0, 3527, 0, 0, 0, 3932, 0] payload_rows_per_dst=[7994, 8170, 8164, 7868, 8186, 8060, 7770, 7054] dup_rows_per_dst=[432, 0, 0, 260, 772, 56, 750, 0] combine_rows_per_src=[7994, 8170, 8164, 7868, 8186, 8060, 7770, 7054] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49962 payload_rows_remote=41770 dup_rows=15574 dup_rows_remote=4816 dedup_ratio=0.237640 router_tokens_per_expert=[0, 0, 1289, 0, 0, 4092, 0, 0, 0, 0, 0, 902, 0, 0, 0, 0, 0, 0, 4001, 0, 0, 0, 4096, 0, 0, 3993, 0, 4078, 0, 0, 3, 0, 0, 0, 0, 0, 65, 1057, 0, 4077, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 0, 0, 0, 0, 965, 4094, 0] payload_rows_per_dst=[8192, 1804, 8052, 8062, 5494, 8164, 8156, 2038] dup_rows_per_dst=[10758, 0, 2058, 1466, 1292, 0, 0, 0] combine_rows_per_src=[8192, 1804, 8052, 8062, 5494, 8164, 8156, 2038] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62798 payload_rows_remote=54606 dup_rows=2738 dup_rows_remote=1650 dedup_ratio=0.041779 router_tokens_per_expert=[0, 1, 0, 0, 392, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 264, 0, 4087, 1, 0, 0, 0, 0, 4095, 545, 0, 4095, 0, 0, 0, 0, 33, 0, 4095, 0, 0, 4052, 0, 0, 0, 138, 0, 0, 0, 3749, 39, 0, 0, 0, 3261, 0, 0, 0, 3921, 0] payload_rows_per_dst=[7918, 8190, 8190, 8176, 8192, 8104, 7506, 6522] dup_rows_per_dst=[710, 0, 0, 528, 1088, 66, 346, 0] combine_rows_per_src=[7918, 8190, 8190, 8176, 8192, 8104, 7506, 6522] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62838 payload_rows_remote=54708 dup_rows=2698 dup_rows_remote=2342 dedup_ratio=0.041168 router_tokens_per_expert=[0, 19, 0, 0, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 179, 0, 4061, 3, 0, 0, 0, 0, 4094, 653, 0, 4094, 0, 0, 0, 0, 8, 0, 4093, 0, 0, 4044, 0, 0, 0, 24, 0, 0, 0, 3738, 77, 0, 0, 0, 3226, 0, 0, 0, 4033, 0] payload_rows_per_dst=[8122, 8188, 8186, 8130, 8190, 8088, 7482, 6452] dup_rows_per_dst=[826, 0, 0, 356, 1304, 16, 196, 0] combine_rows_per_src=[8122, 8188, 8186, 8130, 8190, 8088, 7482, 6452] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=394397696 dispatch_meta_bytes=230680 combine_payload_bytes=394397696 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394628376 ingress_bytes=504498176 link_bytes=504498176 token_bytes=789026072 weight_movement_bytes=110100480 total_nvlink_bytes=899126552 weight_to_token_ratio=0.139540 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.774053038e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63398 payload_rows_remote=55206 dup_rows=2138 dup_rows_remote=2134 dedup_ratio=0.032623 router_tokens_per_expert=[0, 4, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 194, 0, 4040, 4, 0, 0, 0, 0, 4080, 369, 0, 4094, 0, 0, 0, 0, 31, 0, 4096, 0, 0, 3918, 0, 0, 0, 151, 0, 0, 0, 3664, 44, 0, 0, 0, 3755, 0, 0, 0, 3963, 0] payload_rows_per_dst=[8028, 8188, 8192, 8086, 8168, 7836, 7390, 7510] dup_rows_per_dst=[624, 0, 4, 390, 730, 62, 328, 0] combine_rows_per_src=[8028, 8188, 8192, 8086, 8168, 7836, 7390, 7510] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52012 payload_rows_remote=43838 dup_rows=13524 dup_rows_remote=3296 dedup_ratio=0.206360 router_tokens_per_expert=[0, 0, 0, 0, 284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 187, 0, 3905, 1, 0, 0, 0, 0, 4096, 439, 0, 4096, 0, 0, 0, 0, 75, 3, 4096, 0, 0, 3973, 0, 0, 0, 239, 0, 0, 0, 3726, 45, 0, 0, 0, 3725, 0, 0, 0, 3878, 0] payload_rows_per_dst=[4634, 8192, 8192, 6110, 1090, 7956, 7664, 8174] dup_rows_per_dst=[266, 0, 0, 250, 18, 810, 1952, 10228] combine_rows_per_src=[4634, 8192, 8192, 6110, 1090, 7956, 7664, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=395716608 dispatch_meta_bytes=229680 combine_payload_bytes=395716608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=395946288 ingress_bytes=726018048 link_bytes=726018048 token_bytes=791662896 weight_movement_bytes=330301440 total_nvlink_bytes=1121964336 weight_to_token_ratio=0.417225 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.262664795e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62618 payload_rows_remote=54426 dup_rows=2918 dup_rows_remote=2918 dedup_ratio=0.044525 router_tokens_per_expert=[0, 1, 0, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 210, 0, 4001, 2, 0, 0, 0, 0, 4094, 669, 0, 4096, 0, 0, 0, 0, 55, 0, 4096, 0, 0, 4044, 0, 0, 0, 217, 0, 0, 0, 3796, 56, 0, 0, 0, 3135, 0, 0, 0, 4016, 0] payload_rows_per_dst=[8072, 8192, 8192, 8012, 8188, 8088, 7604, 6270] dup_rows_per_dst=[522, 0, 0, 414, 1338, 110, 534, 0] combine_rows_per_src=[8072, 8192, 8192, 8012, 8188, 8088, 7604, 6270] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=299407360 dispatch_meta_bytes=186344 combine_payload_bytes=299407360 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=299593704 ingress_bytes=519608320 link_bytes=519608320 token_bytes=599001064 weight_movement_bytes=220200960 total_nvlink_bytes=819202024 weight_to_token_ratio=0.367614 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=14208 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.036843294e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=314230784 dispatch_meta_bytes=188536 combine_payload_bytes=314230784 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=314419320 ingress_bytes=534431744 link_bytes=534431744 token_bytes=628650104 weight_movement_bytes=220200960 total_nvlink_bytes=848851064 weight_to_token_ratio=0.350276 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.294647017e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=391415808 dispatch_meta_bytes=225024 combine_payload_bytes=391415808 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=391640832 ingress_bytes=941918208 link_bytes=941918208 token_bytes=783056640 weight_movement_bytes=550502400 total_nvlink_bytes=1333559040 weight_to_token_ratio=0.703017 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.638150683e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=392146944 dispatch_meta_bytes=228200 combine_payload_bytes=392146944 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=392375144 ingress_bytes=612347904 link_bytes=612347904 token_bytes=784522088 weight_movement_bytes=220200960 total_nvlink_bytes=1004723048 weight_to_token_ratio=0.280682 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.064973719e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=395716608 dispatch_meta_bytes=229360 combine_payload_bytes=395716608 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=395945968 ingress_bytes=615917568 link_bytes=615917568 token_bytes=791662576 weight_movement_bytes=220200960 total_nvlink_bytes=1011863536 weight_to_token_ratio=0.278150 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.071181952e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=390125568 dispatch_meta_bytes=229376 combine_payload_bytes=390125568 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=390354944 ingress_bytes=610326528 link_bytes=610326528 token_bytes=780480512 weight_movement_bytes=220200960 total_nvlink_bytes=1000681472 weight_to_token_ratio=0.282135 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061458214e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63610 payload_rows_remote=55544 dup_rows=1926 dup_rows_remote=1454 dedup_ratio=0.029388 router_tokens_per_expert=[0, 485, 0, 4096, 0, 3736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1146, 0, 0, 2642, 0, 0, 4096, 0, 0, 0, 250, 0, 0, 4019, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7648, 8192, 8192, 8192, 8192, 6936, 8066, 8192] dup_rows_per_dst=[794, 0, 20, 0, 0, 640, 472, 0] combine_rows_per_src=[7648, 8192, 8192, 8192, 8192, 6936, 8066, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64026 payload_rows_remote=56502 dup_rows=1510 dup_rows_remote=850 dedup_ratio=0.023041 router_tokens_per_expert=[0, 316, 0, 4096, 0, 3732, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 773, 0, 35, 3284, 0, 0, 4096, 0, 0, 0, 208, 0, 0, 3908, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7660, 8182, 8192, 8192, 8192, 7524, 7892, 8192] dup_rows_per_dst=[436, 0, 70, 0, 0, 660, 340, 4] combine_rows_per_src=[7660, 8182, 8192, 8192, 8192, 7524, 7892, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=398139392 dispatch_meta_bytes=227992 combine_payload_bytes=398139392 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=398367384 ingress_bytes=838541312 link_bytes=838541312 token_bytes=796506776 weight_movement_bytes=440401920 total_nvlink_bytes=1236908696 weight_to_token_ratio=0.552917 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.458361258e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61856 payload_rows_remote=53664 dup_rows=3680 dup_rows_remote=3680 dedup_ratio=0.056152 router_tokens_per_expert=[0, 545, 0, 4096, 0, 3828, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 993, 0, 0, 2794, 0, 0, 4096, 0, 0, 0, 228, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6104, 8192, 8018, 8192, 8192, 7092, 7874, 8192] dup_rows_per_dst=[906, 2, 2, 0, 0, 2218, 552, 0] combine_rows_per_src=[6104, 8192, 8018, 8192, 8192, 7092, 7874, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63144 payload_rows_remote=55212 dup_rows=2392 dup_rows_remote=1898 dedup_ratio=0.036499 router_tokens_per_expert=[0, 13, 0, 0, 271, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 187, 0, 4052, 3, 0, 0, 0, 0, 4092, 485, 0, 4094, 0, 0, 0, 0, 14, 0, 4096, 0, 0, 3972, 0, 0, 0, 181, 0, 0, 0, 3833, 60, 0, 0, 0, 3484, 0, 0, 0, 3929, 0] payload_rows_per_dst=[7932, 8188, 8192, 8110, 8168, 7912, 7674, 6968] dup_rows_per_dst=[494, 36, 36, 374, 950, 28, 474, 0] combine_rows_per_src=[7932, 8188, 8192, 8110, 8168, 7912, 7674, 6968] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64072 payload_rows_remote=55880 dup_rows=1464 dup_rows_remote=1464 dedup_ratio=0.022339 router_tokens_per_expert=[0, 289, 0, 4095, 0, 3833, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 776, 0, 1, 3186, 0, 0, 4096, 0, 0, 0, 222, 0, 0, 3979, 0, 0, 1, 1, 0, 0, 0, 0] payload_rows_per_dst=[7784, 8192, 8192, 8192, 8192, 7340, 7988, 8192] dup_rows_per_dst=[460, 2, 0, 0, 0, 586, 414, 2] combine_rows_per_src=[7784, 8192, 8192, 8192, 8192, 7340, 7988, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=405006336 dispatch_meta_bytes=229408 combine_payload_bytes=405006336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=405235744 ingress_bytes=515106816 link_bytes=515106816 token_bytes=810242080 weight_movement_bytes=110100480 total_nvlink_bytes=920342560 weight_to_token_ratio=0.135886 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.958554735e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63736 payload_rows_remote=55544 dup_rows=1800 dup_rows_remote=1800 dedup_ratio=0.027466 router_tokens_per_expert=[0, 356, 0, 4095, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 814, 0, 6, 3145, 0, 0, 4096, 0, 0, 0, 298, 0, 0, 3858, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[7736, 8192, 8192, 8192, 8192, 7218, 7822, 8192] dup_rows_per_dst=[564, 0, 34, 0, 0, 712, 490, 0] combine_rows_per_src=[7736, 8192, 8192, 8192, 8192, 7218, 7822, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63170 payload_rows_remote=54978 dup_rows=2366 dup_rows_remote=2352 dedup_ratio=0.036102 router_tokens_per_expert=[0, 424, 0, 4093, 0, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 2, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 1883, 0, 3, 2504, 0, 0, 4096, 0, 0, 0, 209, 0, 0, 3465, 0, 0, 2, 1, 0, 0, 0, 0] payload_rows_per_dst=[7732, 8192, 8192, 8192, 8190, 7432, 7050, 8190] dup_rows_per_dst=[704, 0, 14, 0, 0, 1348, 298, 2] combine_rows_per_src=[7732, 8192, 8192, 8192, 8190, 7432, 7050, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64332 payload_rows_remote=56140 dup_rows=1204 dup_rows_remote=1204 dedup_ratio=0.018372 router_tokens_per_expert=[0, 268, 0, 4096, 0, 3846, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 564, 0, 0, 3335, 0, 0, 4096, 0, 0, 0, 202, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7810, 8192, 8192, 8192, 8192, 7406, 8156, 8192] dup_rows_per_dst=[418, 0, 8, 0, 0, 392, 386, 0] combine_rows_per_src=[7810, 8192, 8192, 8192, 8192, 7406, 8156, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=384663552 dispatch_meta_bytes=229376 combine_payload_bytes=384663552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384892928 ingress_bytes=494764032 link_bytes=494764032 token_bytes=769556480 weight_movement_bytes=110100480 total_nvlink_bytes=879656960 weight_to_token_ratio=0.143070 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.604760263e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=400547840 dispatch_meta_bytes=229376 combine_payload_bytes=400547840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400777216 ingress_bytes=510648320 link_bytes=510648320 token_bytes=801325056 weight_movement_bytes=110100480 total_nvlink_bytes=911425536 weight_to_token_ratio=0.137398 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.881014157e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=395759616 dispatch_meta_bytes=228440 combine_payload_bytes=395759616 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=395988056 ingress_bytes=505860096 link_bytes=505860096 token_bytes=791747672 weight_movement_bytes=110100480 total_nvlink_bytes=901848152 weight_to_token_ratio=0.139060 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.797739066e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=398139392 dispatch_meta_bytes=229376 combine_payload_bytes=398139392 prefetch_bytes=0 reduce_bytes=0 egress_bytes=398368768 ingress_bytes=398139392 link_bytes=398368768 token_bytes=796508160 weight_movement_bytes=0 total_nvlink_bytes=796508160 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.928288080e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=402411520 dispatch_meta_bytes=229376 combine_payload_bytes=402411520 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=402640896 ingress_bytes=512512000 link_bytes=512512000 token_bytes=805052416 weight_movement_bytes=110100480 total_nvlink_bytes=915152896 weight_to_token_ratio=0.136762 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.913426617e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=394082304 dispatch_meta_bytes=229320 combine_payload_bytes=394082304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394311624 ingress_bytes=504182784 link_bytes=504182784 token_bytes=788393928 weight_movement_bytes=110100480 total_nvlink_bytes=898494408 weight_to_token_ratio=0.139652 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.768567852e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65072 payload_rows_remote=56880 dup_rows=464 dup_rows_remote=360 dedup_ratio=0.007080 router_tokens_per_expert=[0, 148, 0, 0, 0, 4055, 0, 0, 3956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 4049, 0, 0, 0, 0, 4096, 0, 4096, 4089, 0, 0, 0, 0, 4096, 59, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7912, 8192, 8102, 8192, 8098, 8192, 8192] dup_rows_per_dst=[214, 0, 80, 0, 64, 0, 104, 2] combine_rows_per_src=[8192, 7912, 8192, 8102, 8192, 8098, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64464 payload_rows_remote=56382 dup_rows=1072 dup_rows_remote=1072 dedup_ratio=0.016357 router_tokens_per_expert=[0, 103, 0, 0, 0, 4051, 0, 0, 3677, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 350, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 4041, 0, 0, 0, 0, 4096, 0, 4096, 4060, 0, 0, 0, 0, 4096, 119, 0, 18, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7376, 8192, 8046, 8192, 8082, 8192, 8192] dup_rows_per_dst=[116, 0, 700, 0, 54, 0, 166, 36] combine_rows_per_src=[8192, 7376, 8192, 8046, 8192, 8082, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=407715840 dispatch_meta_bytes=228960 combine_payload_bytes=407715840 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=407944800 ingress_bytes=848117760 link_bytes=848117760 token_bytes=815660640 weight_movement_bytes=440401920 total_nvlink_bytes=1256062560 weight_to_token_ratio=0.539933 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.475016276e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=404146176 dispatch_meta_bytes=229816 combine_payload_bytes=404146176 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=404375992 ingress_bytes=734447616 link_bytes=734447616 token_bytes=808522168 weight_movement_bytes=330301440 total_nvlink_bytes=1138823608 weight_to_token_ratio=0.408525 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30592 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.277325200e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60806 payload_rows_remote=52614 dup_rows=4730 dup_rows_remote=4730 dedup_ratio=0.072174 router_tokens_per_expert=[0, 123, 0, 0, 0, 4052, 0, 0, 3873, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 4029, 0, 0, 0, 0, 4096, 0, 4096, 4085, 0, 0, 0, 0, 4096, 75, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6994, 7746, 5806, 8182, 7638, 8058, 8190, 8192] dup_rows_per_dst=[154, 0, 210, 472, 40, 2436, 1418, 0] combine_rows_per_src=[6994, 7746, 5806, 8182, 7638, 8058, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64956 payload_rows_remote=56764 dup_rows=580 dup_rows_remote=484 dedup_ratio=0.008850 router_tokens_per_expert=[0, 92, 0, 0, 0, 4057, 0, 0, 3829, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 1, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 4096, 4094, 0, 0, 0, 0, 4096, 4, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7660, 8192, 8148, 8192, 8190, 8190, 8192] dup_rows_per_dst=[106, 0, 370, 0, 96, 0, 6, 2] combine_rows_per_src=[8192, 7660, 8192, 8148, 8192, 8190, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64836 payload_rows_remote=57246 dup_rows=700 dup_rows_remote=700 dedup_ratio=0.010681 router_tokens_per_expert=[0, 73, 0, 0, 0, 4060, 0, 0, 3795, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 237, 0, 0, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 4096, 0, 4096, 4088, 0, 0, 0, 0, 4096, 37, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7590, 8192, 8166, 8192, 8120, 8192, 8192] dup_rows_per_dst=[74, 0, 474, 0, 92, 0, 58, 2] combine_rows_per_src=[8192, 7590, 8192, 8166, 8192, 8120, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63294 payload_rows_remote=55574 dup_rows=2242 dup_rows_remote=1182 dedup_ratio=0.034210 router_tokens_per_expert=[0, 439, 0, 4096, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1391, 0, 3, 2622, 0, 0, 4096, 0, 0, 0, 200, 0, 0, 3846, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[7720, 8190, 8192, 8192, 8192, 7172, 7450, 8186] dup_rows_per_dst=[1060, 8, 26, 0, 2, 860, 286, 0] combine_rows_per_src=[7720, 8190, 8192, 8192, 8192, 7172, 7450, 8186] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65014 payload_rows_remote=56942 dup_rows=522 dup_rows_remote=520 dedup_ratio=0.007965 router_tokens_per_expert=[0, 92, 0, 0, 0, 4053, 0, 0, 3923, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 1, 0, 0, 0, 0, 4036, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 4066, 0, 0, 0, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0, 4096, 30, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7850, 8192, 8072, 8192, 8132, 8192, 8192] dup_rows_per_dst=[98, 0, 314, 2, 62, 0, 42, 4] combine_rows_per_src=[8192, 7850, 8192, 8072, 8192, 8132, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65102 payload_rows_remote=56910 dup_rows=434 dup_rows_remote=376 dedup_ratio=0.006622 router_tokens_per_expert=[0, 127, 0, 0, 0, 4054, 0, 0, 3926, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 4046, 0, 0, 0, 0, 4096, 0, 4096, 4087, 0, 0, 0, 0, 4096, 76, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7858, 8192, 8192, 8192, 8092, 8192, 8192] dup_rows_per_dst=[170, 0, 58, 0, 66, 0, 134, 6] combine_rows_per_src=[8192, 7858, 8192, 8192, 8192, 8092, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=377137152 dispatch_meta_bytes=229376 combine_payload_bytes=377137152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=377366528 ingress_bytes=487237632 link_bytes=487237632 token_bytes=754503680 weight_movement_bytes=110100480 total_nvlink_bytes=864604160 weight_to_token_ratio=0.145924 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.473863789e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=406884352 dispatch_meta_bytes=228992 combine_payload_bytes=406884352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407113344 ingress_bytes=516984832 link_bytes=516984832 token_bytes=813997696 weight_movement_bytes=110100480 total_nvlink_bytes=924098176 weight_to_token_ratio=0.135259 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.991216522e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=410339328 dispatch_meta_bytes=231784 combine_payload_bytes=410339328 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=410571112 ingress_bytes=520439808 link_bytes=520439808 token_bytes=820910440 weight_movement_bytes=110100480 total_nvlink_bytes=931010920 weight_to_token_ratio=0.134120 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.051304238e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60920 payload_rows_remote=52782 dup_rows=4616 dup_rows_remote=4616 dedup_ratio=0.070435 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 4096, 529, 4096, 0, 0, 0, 51, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2087, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 1442, 0, 0, 0, 4069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7206, 8192, 4616, 8192, 8192, 8138, 8192] dup_rows_per_dst=[4, 958, 102, 648, 20, 2884, 0, 0] combine_rows_per_src=[8192, 7206, 8192, 4616, 8192, 8192, 8138, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=398354432 dispatch_meta_bytes=227024 combine_payload_bytes=398354432 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=398581456 ingress_bytes=728655872 link_bytes=728655872 token_bytes=796935888 weight_movement_bytes=330301440 total_nvlink_bytes=1127237328 weight_to_token_ratio=0.414464 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=29568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.267252405e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=408160256 dispatch_meta_bytes=229848 combine_payload_bytes=408160256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408390104 ingress_bytes=518260736 link_bytes=518260736 token_bytes=816550360 weight_movement_bytes=110100480 total_nvlink_bytes=926650840 weight_to_token_ratio=0.134836 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.013406591e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62412 payload_rows_remote=54220 dup_rows=3124 dup_rows_remote=1946 dedup_ratio=0.047668 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 4096, 906, 4081, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2543, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 5086, 8192, 8192, 8174, 8192] dup_rows_per_dst=[0, 1840, 88, 0, 0, 1178, 18, 0] combine_rows_per_src=[8192, 8192, 8192, 5086, 8192, 8192, 8174, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=407930880 dispatch_meta_bytes=229144 combine_payload_bytes=407930880 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408160024 ingress_bytes=518031360 link_bytes=518031360 token_bytes=816090904 weight_movement_bytes=110100480 total_nvlink_bytes=926191384 weight_to_token_ratio=0.134912 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.009417366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=378341376 dispatch_meta_bytes=229592 combine_payload_bytes=378341376 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=378570968 ingress_bytes=488441856 link_bytes=488441856 token_bytes=756912344 weight_movement_bytes=110100480 total_nvlink_bytes=867012824 weight_to_token_ratio=0.145460 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.494807225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=388648960 dispatch_meta_bytes=224664 combine_payload_bytes=388648960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388873624 ingress_bytes=498749440 link_bytes=498749440 token_bytes=777522584 weight_movement_bytes=110100480 total_nvlink_bytes=887623064 weight_to_token_ratio=0.141604 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=21504 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.674073063e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58318 payload_rows_remote=50126 dup_rows=7218 dup_rows_remote=7218 dedup_ratio=0.110138 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 563, 4096, 0, 0, 0, 75, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2221, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1248, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8180, 1126, 8080, 8192, 8192, 8192, 8164, 8192] dup_rows_per_dst=[0, 0, 150, 4454, 6, 2496, 112, 0] combine_rows_per_src=[8180, 1126, 8080, 8192, 8192, 8192, 8164, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60612 payload_rows_remote=52420 dup_rows=4924 dup_rows_remote=4922 dedup_ratio=0.075134 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 681, 4095, 0, 0, 0, 40, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1635, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1740, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 3270, 8192, 8192, 8190, 8192] dup_rows_per_dst=[0, 1362, 80, 0, 2, 3480, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 3270, 8192, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60694 payload_rows_remote=52502 dup_rows=4842 dup_rows_remote=4218 dedup_ratio=0.073883 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4096, 311, 4096, 0, 0, 0, 41, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1676, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 8192, 3352, 8190, 8192] dup_rows_per_dst=[0, 624, 82, 4136, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 8192, 3352, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61394 payload_rows_remote=53202 dup_rows=4142 dup_rows_remote=3876 dedup_ratio=0.063202 router_tokens_per_expert=[0, 186, 0, 0, 0, 4043, 0, 0, 3862, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 2, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 4051, 0, 0, 0, 0, 4096, 0, 4096, 4079, 0, 0, 0, 0, 4096, 50, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7986, 8192, 8150, 8192, 6328, 6214, 8140] dup_rows_per_dst=[266, 3558, 180, 4, 80, 0, 54, 0] combine_rows_per_src=[8192, 7986, 8192, 8150, 8192, 6328, 6214, 8140] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61638 payload_rows_remote=57328 dup_rows=3898 dup_rows_remote=3898 dedup_ratio=0.059479 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 4096, 725, 4094, 0, 0, 0, 43, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 2155, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 4310, 8192, 8192, 8176, 8192] dup_rows_per_dst=[0, 1460, 86, 0, 2, 2346, 4, 0] combine_rows_per_src=[8192, 8192, 8192, 4310, 8192, 8192, 8176, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60848 payload_rows_remote=52656 dup_rows=4688 dup_rows_remote=4580 dedup_ratio=0.071533 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 4096, 305, 4094, 0, 0, 0, 54, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 2132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1614, 0, 0, 0, 4086, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 7334, 8192, 4382, 8172, 8192] dup_rows_per_dst=[4, 610, 108, 3284, 0, 682, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 7334, 8192, 4382, 8172, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=359303168 dispatch_meta_bytes=229376 combine_payload_bytes=359303168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=359532544 ingress_bytes=469403648 link_bytes=469403648 token_bytes=718835712 weight_movement_bytes=110100480 total_nvlink_bytes=828936192 weight_to_token_ratio=0.153165 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.163701475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=375746560 dispatch_meta_bytes=229368 combine_payload_bytes=375746560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=375975928 ingress_bytes=485847040 link_bytes=485847040 token_bytes=751722488 weight_movement_bytes=110100480 total_nvlink_bytes=861822968 weight_to_token_ratio=0.146464 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.449679107e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=376334336 dispatch_meta_bytes=226880 combine_payload_bytes=376334336 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=376561216 ingress_bytes=706635776 link_bytes=706635776 token_bytes=752895552 weight_movement_bytes=330301440 total_nvlink_bytes=1083196992 weight_to_token_ratio=0.438708 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.228955836e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64760 payload_rows_remote=56568 dup_rows=776 dup_rows_remote=776 dedup_ratio=0.011841 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3913, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3941, 0, 1, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2470, 0, 0, 0, 0, 4085, 1933, 0] payload_rows_per_dst=[7826, 8190, 8192, 8192, 8170, 7882, 8192, 8116] dup_rows_per_dst=[0, 0, 0, 84, 0, 2, 0, 690] combine_rows_per_src=[7826, 8190, 8192, 8192, 8170, 7882, 8192, 8116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=381351936 dispatch_meta_bytes=228312 combine_payload_bytes=381351936 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=381580248 ingress_bytes=601552896 link_bytes=601552896 token_bytes=762932184 weight_movement_bytes=220200960 total_nvlink_bytes=983133144 weight_to_token_ratio=0.288625 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.046199425e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=410927104 dispatch_meta_bytes=244904 combine_payload_bytes=410927104 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=411172008 ingress_bytes=851329024 link_bytes=851329024 token_bytes=822099112 weight_movement_bytes=440401920 total_nvlink_bytes=1262501032 weight_to_token_ratio=0.535704 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.480601192e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65016 payload_rows_remote=56872 dup_rows=520 dup_rows_remote=512 dedup_ratio=0.007935 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3949, 0, 0, 0, 0, 0, 0, 4091, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4072, 0, 4, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2990, 0, 0, 0, 0, 4032, 1258, 0] payload_rows_per_dst=[7898, 8182, 8192, 8192, 8106, 8144, 8192, 8110] dup_rows_per_dst=[0, 40, 0, 86, 0, 8, 0, 386] combine_rows_per_src=[7898, 8182, 8192, 8192, 8106, 8144, 8192, 8110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=377438208 dispatch_meta_bytes=228944 combine_payload_bytes=377438208 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=377667152 ingress_bytes=597639168 link_bytes=597639168 token_bytes=755105360 weight_movement_bytes=220200960 total_nvlink_bytes=975306320 weight_to_token_ratio=0.291616 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.039392808e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405479424 dispatch_meta_bytes=229376 combine_payload_bytes=405479424 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=405708800 ingress_bytes=515579904 link_bytes=515579904 token_bytes=811188224 weight_movement_bytes=110100480 total_nvlink_bytes=921288704 weight_to_token_ratio=0.135727 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.966782514e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=407658496 dispatch_meta_bytes=229536 combine_payload_bytes=407658496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407888032 ingress_bytes=517758976 link_bytes=517758976 token_bytes=815546528 weight_movement_bytes=110100480 total_nvlink_bytes=925647008 weight_to_token_ratio=0.135002 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.004680160e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65182 payload_rows_remote=57044 dup_rows=354 dup_rows_remote=354 dedup_ratio=0.005402 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 4078, 1, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3084, 0, 0, 0, 0, 4068, 1126, 0] payload_rows_per_dst=[8090, 8156, 8192, 8190, 8138, 8074, 8192, 8150] dup_rows_per_dst=[0, 2, 2, 80, 0, 0, 0, 270] combine_rows_per_src=[8090, 8156, 8192, 8190, 8138, 8074, 8192, 8150] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64676 payload_rows_remote=56500 dup_rows=860 dup_rows_remote=86 dedup_ratio=0.013123 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 4009, 0, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3065, 0, 0, 0, 0, 3803, 1405, 0] payload_rows_per_dst=[8018, 8186, 8182, 8192, 7610, 8120, 8192, 8176] dup_rows_per_dst=[0, 0, 0, 86, 0, 0, 0, 774] combine_rows_per_src=[8018, 8186, 8182, 8192, 7610, 8120, 8192, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64786 payload_rows_remote=56614 dup_rows=750 dup_rows_remote=750 dedup_ratio=0.011444 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 3880, 0, 0, 0, 0, 0, 0, 4086, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2843, 0, 0, 0, 0, 4024, 1570, 0] payload_rows_per_dst=[7760, 8172, 8192, 8192, 8052, 8070, 8192, 8156] dup_rows_per_dst=[0, 0, 0, 80, 0, 0, 0, 670] combine_rows_per_src=[7760, 8172, 8192, 8192, 8052, 8070, 8192, 8156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60390 payload_rows_remote=52198 dup_rows=5146 dup_rows_remote=5142 dedup_ratio=0.078522 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 4096, 232, 4094, 0, 0, 0, 57, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1911, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1888, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7500, 8192, 8156, 3776, 8190, 8192] dup_rows_per_dst=[4, 1198, 108, 3822, 6, 0, 8, 0] combine_rows_per_src=[8192, 8192, 7500, 8192, 8156, 3776, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=65258 payload_rows_remote=57066 dup_rows=278 dup_rows_remote=188 dedup_ratio=0.004242 router_tokens_per_expert=[0, 0, 4096, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 0, 4089, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4045, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1693, 0, 0, 0, 0, 4081, 2449, 0] payload_rows_per_dst=[8146, 8178, 8192, 8192, 8166, 8090, 8192, 8102] dup_rows_per_dst=[0, 6, 0, 90, 0, 0, 0, 182] combine_rows_per_src=[8146, 8178, 8192, 8192, 8166, 8090, 8192, 8102] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 2, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64762 payload_rows_remote=56570 dup_rows=774 dup_rows_remote=770 dedup_ratio=0.011810 router_tokens_per_expert=[0, 0, 4094, 0, 0, 0, 3924, 0, 0, 0, 0, 0, 0, 4084, 3, 0, 2, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2893, 0, 0, 0, 0, 4020, 1529, 0] payload_rows_per_dst=[7848, 8168, 8192, 8188, 8044, 7956, 8192, 8174] dup_rows_per_dst=[0, 6, 4, 94, 0, 0, 0, 670] combine_rows_per_src=[7848, 8168, 8192, 8188, 8044, 7956, 8192, 8174] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=408891392 dispatch_meta_bytes=229592 combine_payload_bytes=408891392 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=409120984 ingress_bytes=629092352 link_bytes=629092352 token_bytes=818012376 weight_movement_bytes=220200960 total_nvlink_bytes=1038213336 weight_to_token_ratio=0.269190 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.094095068e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=404992000 dispatch_meta_bytes=226344 combine_payload_bytes=404992000 prefetch_bytes=110100480 reduce_bytes=550502400 egress_bytes=405218344 ingress_bytes=1065594880 link_bytes=1065594880 token_bytes=810210344 weight_movement_bytes=660602880 total_nvlink_bytes=1470813224 weight_to_token_ratio=0.815347 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.853244757e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61802 payload_rows_remote=53624 dup_rows=3734 dup_rows_remote=3734 dedup_ratio=0.056976 router_tokens_per_expert=[1, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4059, 0, 0, 13, 4096, 0, 0, 0, 2280, 0, 0, 0, 0, 22, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4089, 0, 0, 1832, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 8118, 4562, 8192, 8192, 8178, 8192] dup_rows_per_dst=[0, 0, 0, 24, 44, 2, 0, 3664] combine_rows_per_src=[8176, 8192, 8118, 4562, 8192, 8192, 8178, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405809152 dispatch_meta_bytes=229456 combine_payload_bytes=405809152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406038608 ingress_bytes=515909632 link_bytes=515909632 token_bytes=811847760 weight_movement_bytes=110100480 total_nvlink_bytes=921948240 weight_to_token_ratio=0.135617 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.972517026e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=374155264 dispatch_meta_bytes=229360 combine_payload_bytes=374155264 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374384624 ingress_bytes=484255744 link_bytes=484255744 token_bytes=748539888 weight_movement_bytes=110100480 total_nvlink_bytes=858640368 weight_to_token_ratio=0.147087 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.422003852e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61950 payload_rows_remote=53760 dup_rows=3586 dup_rows_remote=3584 dedup_ratio=0.054718 router_tokens_per_expert=[45, 0, 0, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 8, 0, 4067, 0, 0, 0, 0, 0, 0, 6, 4056, 0, 0, 66, 4090, 0, 0, 0, 2379, 0, 0, 0, 0, 23, 0, 0, 0, 4093, 1, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4055, 0, 0, 1730, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8150, 8124, 4804, 8192, 8190, 8110, 8190] dup_rows_per_dst=[8, 0, 0, 86, 40, 2, 0, 3450] combine_rows_per_src=[8190, 8150, 8124, 4804, 8192, 8190, 8110, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=409049088 dispatch_meta_bytes=229016 combine_payload_bytes=409049088 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=409278104 ingress_bytes=519149568 link_bytes=519149568 token_bytes=818327192 weight_movement_bytes=110100480 total_nvlink_bytes=928427672 weight_to_token_ratio=0.134543 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.028864842e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405493760 dispatch_meta_bytes=229360 combine_payload_bytes=405493760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=405723120 ingress_bytes=625694720 link_bytes=625694720 token_bytes=811216880 weight_movement_bytes=220200960 total_nvlink_bytes=1031417840 weight_to_token_ratio=0.271445 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.088186027e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=384376832 dispatch_meta_bytes=229432 combine_payload_bytes=384376832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384606264 ingress_bytes=494477312 link_bytes=494477312 token_bytes=768983096 weight_movement_bytes=110100480 total_nvlink_bytes=879083576 weight_to_token_ratio=0.143177 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.599773731e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385351680 dispatch_meta_bytes=229376 combine_payload_bytes=385351680 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=385581056 ingress_bytes=495452160 link_bytes=495452160 token_bytes=770932736 weight_movement_bytes=110100480 total_nvlink_bytes=881033216 weight_to_token_ratio=0.142815 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.616727941e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61152 payload_rows_remote=52960 dup_rows=4384 dup_rows_remote=4332 dedup_ratio=0.066895 router_tokens_per_expert=[2, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 1, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 4084, 0, 0, 4, 4095, 0, 0, 0, 1951, 0, 0, 0, 0, 27, 0, 0, 0, 4095, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4092, 0, 0, 2130, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8168, 4430, 8192, 8192, 8186, 7600] dup_rows_per_dst=[2, 0, 0, 602, 52, 0, 0, 3728] combine_rows_per_src=[8192, 8192, 8168, 4430, 8192, 8192, 8186, 7600] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7303797200 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3653034448 ingress_bytes=5632571392 link_bytes=5632571392 total_nvlink_bytes=9285605840 weight_to_token_ratio=0.271339 prefetch_share_of_ingress=0.195471 dedup_ratio=0.102658 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.795968049e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59782 payload_rows_remote=51590 dup_rows=5754 dup_rows_remote=138 dedup_ratio=0.087799 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 3984, 0, 0, 6, 4096, 0, 0, 0, 1394, 0, 0, 0, 0, 1, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 2808, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7968, 2800, 8056, 8192, 8190, 8192] dup_rows_per_dst=[0, 0, 0, 0, 2, 0, 136, 5616] combine_rows_per_src=[8192, 8192, 7968, 2800, 8056, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61534 payload_rows_remote=53342 dup_rows=4002 dup_rows_remote=4002 dedup_ratio=0.061066 router_tokens_per_expert=[4, 0, 0, 0, 0, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 6, 4096, 0, 0, 0, 1974, 0, 0, 0, 0, 19, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 2134, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8182, 8192, 8128, 8192, 8192, 8192, 8188, 4268] dup_rows_per_dst=[4, 0, 0, 3960, 38, 0, 0, 0] combine_rows_per_src=[8182, 8192, 8128, 8192, 8192, 8192, 8188, 4268] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7493403376 prefetch_bytes=990904320 reduce_bytes=550502400 weight_movement_bytes=1541406720 egress_bytes=3747850992 ingress_bytes=5286959104 link_bytes=5286959104 total_nvlink_bytes=9034810096 weight_to_token_ratio=0.205702 prefetch_share_of_ingress=0.187424 dedup_ratio=0.085623 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=5 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.194891437e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60924 payload_rows_remote=52882 dup_rows=4612 dup_rows_remote=2704 dedup_ratio=0.070374 router_tokens_per_expert=[0, 0, 4095, 0, 0, 0, 4001, 0, 0, 0, 0, 0, 0, 4096, 3, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3973, 0, 0, 0, 0, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 3058, 0, 0, 0, 0, 4056, 1237, 0] payload_rows_per_dst=[8042, 8192, 8192, 7496, 8126, 7990, 8190, 4696] dup_rows_per_dst=[1908, 58, 4, 102, 1124, 1200, 2, 214] combine_rows_per_src=[8042, 8192, 8192, 7496, 8126, 7990, 8190, 4696] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60816 payload_rows_remote=52624 dup_rows=4720 dup_rows_remote=90 dedup_ratio=0.072021 router_tokens_per_expert=[13, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 4067, 0, 0, 14, 4096, 0, 0, 0, 2301, 0, 0, 0, 0, 45, 0, 0, 0, 4094, 1, 0, 0, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4088, 0, 0, 1777, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 8134, 8192, 8192, 8190, 8176, 3554] dup_rows_per_dst=[4, 0, 0, 4630, 86, 0, 0, 0] combine_rows_per_src=[8186, 8192, 8134, 8192, 8192, 8190, 8176, 3554] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62064 payload_rows_remote=53930 dup_rows=3472 dup_rows_remote=3472 dedup_ratio=0.052979 router_tokens_per_expert=[6, 0, 0, 0, 0, 4091, 0, 0, 0, 0, 0, 0, 3, 0, 4093, 0, 0, 0, 0, 0, 0, 1, 4066, 0, 0, 13, 4094, 0, 0, 0, 1703, 0, 0, 0, 0, 20, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 4092, 0, 0, 2394, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8134, 8192, 8190, 8192, 8188, 4788] dup_rows_per_dst=[6, 0, 0, 3428, 38, 0, 0, 0] combine_rows_per_src=[8188, 8192, 8134, 8192, 8190, 8192, 8188, 4788] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=379617280 dispatch_meta_bytes=229168 combine_payload_bytes=379617280 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=379846448 ingress_bytes=930119680 link_bytes=930119680 token_bytes=759463728 weight_movement_bytes=550502400 total_nvlink_bytes=1309966128 weight_to_token_ratio=0.724857 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.617631102e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369797120 dispatch_meta_bytes=206912 combine_payload_bytes=369797120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370004032 ingress_bytes=479897600 link_bytes=479897600 token_bytes=739801152 weight_movement_bytes=110100480 total_nvlink_bytes=849901632 weight_to_token_ratio=0.148824 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.346208560e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=382355456 dispatch_meta_bytes=229376 combine_payload_bytes=382355456 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=382584832 ingress_bytes=492455936 link_bytes=492455936 token_bytes=764940288 weight_movement_bytes=110100480 total_nvlink_bytes=875040768 weight_to_token_ratio=0.143933 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.564618678e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=379058176 dispatch_meta_bytes=222344 combine_payload_bytes=379058176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=379280520 ingress_bytes=599259136 link_bytes=599259136 token_bytes=758338696 weight_movement_bytes=220200960 total_nvlink_bytes=978539656 weight_to_token_ratio=0.290373 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.042210199e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377208832 dispatch_meta_bytes=210856 combine_payload_bytes=377208832 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=377419688 ingress_bytes=707510272 link_bytes=707510272 token_bytes=754628520 weight_movement_bytes=330301440 total_nvlink_bytes=1084929960 weight_to_token_ratio=0.437701 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.230476728e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=386570240 dispatch_meta_bytes=229608 combine_payload_bytes=386570240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386799848 ingress_bytes=496670720 link_bytes=496670720 token_bytes=773370088 weight_movement_bytes=110100480 total_nvlink_bytes=883470568 weight_to_token_ratio=0.142365 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.637920704e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7532848936 prefetch_bytes=1211105280 reduce_bytes=1211105280 weight_movement_bytes=2422210560 egress_bytes=3767570216 ingress_bytes=6187489280 link_bytes=6187489280 total_nvlink_bytes=9955059496 weight_to_token_ratio=0.321553 prefetch_share_of_ingress=0.195735 dedup_ratio=0.082278 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=11 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.076106153e-04 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6963682976 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3482930848 ingress_bytes=5572661248 link_bytes=5572661248 total_nvlink_bytes=9055592096 weight_to_token_ratio=0.300403 prefetch_share_of_ingress=0.177815 dedup_ratio=0.139999 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=6223 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.691774456e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7402807240 prefetch_bytes=1101004800 reduce_bytes=770703360 weight_movement_bytes=1871708160 egress_bytes=3702527944 ingress_bytes=5571987456 link_bytes=5571987456 total_nvlink_bytes=9274515400 weight_to_token_ratio=0.252838 prefetch_share_of_ingress=0.197596 dedup_ratio=0.092700 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=7 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.690602620e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7559296792 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3780785944 ingress_bytes=6090620928 link_bytes=6090850304 total_nvlink_bytes=9871406872 weight_to_token_ratio=0.305863 prefetch_share_of_ingress=0.144616 dedup_ratio=0.081314 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050368 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.059299045e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7506207344 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3754261104 ingress_bytes=5513553920 link_bytes=5513553920 total_nvlink_bytes=9267815024 weight_to_token_ratio=0.234687 prefetch_share_of_ingress=0.199691 dedup_ratio=0.087573 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6604 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.588977091e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61258 payload_rows_remote=53070 dup_rows=4278 dup_rows_remote=4260 dedup_ratio=0.065277 router_tokens_per_expert=[7, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 0, 2, 0, 4094, 0, 0, 0, 0, 0, 0, 1, 4068, 0, 0, 12, 4094, 0, 0, 0, 1934, 0, 0, 0, 0, 37, 0, 0, 0, 4094, 0, 0, 0, 4096, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 4091, 0, 0, 2147, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8182, 8074, 7956, 8188, 8184, 4294] dup_rows_per_dst=[18, 42, 580, 3560, 74, 4, 0, 0] combine_rows_per_src=[8188, 8192, 8182, 8074, 7956, 8188, 8184, 4294] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380405760 dispatch_meta_bytes=229320 combine_payload_bytes=380405760 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380635080 ingress_bytes=490506240 link_bytes=490506240 token_bytes=761040840 weight_movement_bytes=110100480 total_nvlink_bytes=871141320 weight_to_token_ratio=0.144671 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.530710258e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7030226608 prefetch_bytes=880803840 reduce_bytes=1651507200 weight_movement_bytes=2532311040 egress_bytes=3516214960 ingress_bytes=6046322688 link_bytes=6046322688 total_nvlink_bytes=9562537648 weight_to_token_ratio=0.360203 prefetch_share_of_ingress=0.145676 dedup_ratio=0.131921 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=15 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=5842 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.051554960e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:57:47] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 371.4 | throughput per GPU (TFLOP/s/GPU): 332.5 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.669664E+00 | loss scale: 1.0 | grad norm: 4.293 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44630 payload_rows_remote=39590 dup_rows=20906 dup_rows_remote=18422 dedup_ratio=0.319000 router_tokens_per_expert=[1202, 1006, 1080, 907, 631, 1482, 725, 29, 185, 1352, 865, 460, 5, 897, 94, 463, 830, 629, 7, 29, 781, 411, 948, 489, 259, 827, 506, 617, 259, 165, 818, 2, 605, 233, 600, 7, 922, 1, 991, 403, 248, 742, 415, 923, 0, 633, 143, 385, 210, 163, 315, 472, 243, 1222, 53, 518, 612, 863, 413, 94, 407, 767, 200, 5] payload_rows_per_dst=[5886, 4074, 5732, 6338, 5040, 5456, 6246, 5858] dup_rows_per_dst=[2870, 844, 2516, 3532, 2484, 2542, 2850, 3268] combine_rows_per_src=[5886, 4074, 5732, 6338, 5040, 5456, 6246, 5858] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43812 payload_rows_remote=37692 dup_rows=21724 dup_rows_remote=17984 dedup_ratio=0.331482 router_tokens_per_expert=[1074, 1138, 464, 592, 109, 1127, 433, 127, 394, 1213, 866, 693, 1, 1244, 2, 1036, 642, 943, 3, 35, 578, 444, 577, 378, 365, 536, 637, 698, 235, 409, 1121, 1, 1064, 452, 498, 0, 506, 6, 897, 160, 534, 460, 257, 762, 0, 604, 142, 628, 567, 197, 103, 1010, 155, 728, 34, 923, 425, 790, 371, 527, 296, 1310, 247, 0] payload_rows_per_dst=[3016, 5638, 5378, 6448, 4958, 6050, 6120, 6204] dup_rows_per_dst=[434, 2622, 1822, 3810, 2208, 3212, 3740, 3876] combine_rows_per_src=[3016, 5638, 5378, 6448, 4958, 6050, 6120, 6204] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44484 payload_rows_remote=39088 dup_rows=21052 dup_rows_remote=19294 dedup_ratio=0.321228 router_tokens_per_expert=[1211, 962, 473, 959, 95, 1168, 405, 127, 245, 1829, 1298, 322, 5, 935, 29, 1229, 627, 733, 7, 44, 695, 505, 336, 405, 248, 820, 454, 1148, 205, 80, 450, 1, 1142, 731, 622, 17, 729, 0, 1230, 182, 471, 321, 433, 646, 21, 670, 162, 729, 324, 159, 325, 578, 155, 765, 77, 1020, 378, 751, 517, 417, 412, 423, 309, 2] payload_rows_per_dst=[6174, 6942, 5396, 5158, 5792, 4834, 4972, 5216] dup_rows_per_dst=[3526, 4842, 1758, 1654, 3064, 2072, 1834, 2302] combine_rows_per_src=[6174, 6942, 5396, 5158, 5792, 4834, 4972, 5216] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43620 payload_rows_remote=38020 dup_rows=21916 dup_rows_remote=19028 dedup_ratio=0.334412 router_tokens_per_expert=[1276, 1461, 784, 737, 434, 1490, 369, 79, 456, 1362, 303, 388, 32, 874, 114, 715, 674, 515, 22, 51, 850, 305, 396, 1087, 435, 616, 533, 852, 652, 147, 567, 6, 730, 598, 1078, 6, 565, 3, 1161, 211, 314, 733, 614, 797, 15, 613, 106, 506, 107, 560, 331, 311, 309, 511, 96, 591, 554, 543, 621, 404, 276, 637, 280, 5] payload_rows_per_dst=[7054, 5600, 6332, 5394, 4710, 5258, 4342, 4930] dup_rows_per_dst=[6206, 2888, 3790, 2222, 1672, 2138, 1290, 1710] combine_rows_per_src=[7054, 5600, 6332, 5394, 4710, 5258, 4342, 4930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44672 payload_rows_remote=39420 dup_rows=20864 dup_rows_remote=18918 dedup_ratio=0.318359 router_tokens_per_expert=[1126, 1368, 691, 815, 89, 914, 391, 65, 497, 1189, 999, 476, 0, 1092, 29, 1182, 569, 1024, 24, 38, 1082, 320, 459, 706, 485, 478, 334, 616, 342, 361, 409, 9, 634, 658, 771, 38, 893, 0, 1051, 227, 578, 677, 293, 977, 16, 487, 24, 326, 302, 202, 233, 442, 329, 921, 126, 764, 404, 1022, 585, 335, 285, 719, 268, 2] payload_rows_per_dst=[5356, 6182, 5924, 5252, 5464, 4900, 5678, 5916] dup_rows_per_dst=[2180, 3286, 2520, 1946, 3080, 1856, 2420, 3576] combine_rows_per_src=[5356, 6182, 5924, 5252, 5464, 4900, 5678, 5916] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44632 payload_rows_remote=38644 dup_rows=20904 dup_rows_remote=17390 dedup_ratio=0.318970 router_tokens_per_expert=[1538, 437, 472, 982, 325, 488, 560, 177, 384, 1071, 974, 195, 8, 1345, 9, 966, 495, 331, 28, 63, 678, 412, 502, 622, 227, 1032, 336, 1057, 266, 248, 274, 5, 603, 319, 1042, 2, 865, 28, 1385, 662, 273, 674, 313, 1084, 6, 815, 102, 507, 176, 228, 408, 502, 626, 953, 184, 1291, 525, 812, 528, 219, 258, 499, 371, 1] payload_rows_per_dst=[4264, 4582, 5018, 5468, 6120, 6530, 6662, 5988] dup_rows_per_dst=[768, 1034, 1244, 2398, 4022, 3708, 4216, 3514] combine_rows_per_src=[4264, 4582, 5018, 5468, 6120, 6530, 6662, 5988] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44478 payload_rows_remote=38402 dup_rows=21058 dup_rows_remote=17468 dedup_ratio=0.321320 router_tokens_per_expert=[547, 1178, 980, 603, 74, 1134, 472, 36, 809, 1386, 1051, 307, 9, 1294, 70, 1144, 993, 853, 18, 39, 436, 217, 567, 586, 257, 471, 573, 811, 375, 233, 348, 9, 842, 744, 834, 15, 882, 0, 943, 380, 527, 311, 584, 801, 4, 787, 126, 399, 208, 156, 89, 492, 102, 1121, 144, 650, 1121, 714, 760, 199, 329, 363, 259, 2] payload_rows_per_dst=[4472, 5468, 5452, 5582, 5748, 6076, 5974, 5706] dup_rows_per_dst=[1370, 2156, 1966, 2840, 3532, 3590, 2722, 2882] combine_rows_per_src=[4472, 5468, 5452, 5582, 5748, 6076, 5974, 5706] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=283781120 dispatch_meta_bytes=232048 combine_payload_bytes=283781120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=284013168 ingress_bytes=503982080 link_bytes=503982080 token_bytes=567794288 weight_movement_bytes=220200960 total_nvlink_bytes=787995248 weight_to_token_ratio=0.387818 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.765077279e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=270176256 dispatch_meta_bytes=222704 combine_payload_bytes=270176256 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=270398960 ingress_bytes=380276736 link_bytes=380276736 token_bytes=540575216 weight_movement_bytes=110100480 total_nvlink_bytes=650675696 weight_to_token_ratio=0.203673 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.613637887e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280182784 dispatch_meta_bytes=233528 combine_payload_bytes=280182784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280416312 ingress_bytes=390283264 link_bytes=390283264 token_bytes=560599096 weight_movement_bytes=110100480 total_nvlink_bytes=670699576 weight_to_token_ratio=0.196398 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.787667867e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272527360 dispatch_meta_bytes=228192 combine_payload_bytes=272527360 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=272755552 ingress_bytes=602828800 link_bytes=602828800 token_bytes=545282912 weight_movement_bytes=330301440 total_nvlink_bytes=875584352 weight_to_token_ratio=0.605743 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048418432e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=282562560 dispatch_meta_bytes=233352 combine_payload_bytes=282562560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282795912 ingress_bytes=392663040 link_bytes=392663040 token_bytes=565358472 weight_movement_bytes=110100480 total_nvlink_bytes=675458952 weight_to_token_ratio=0.194745 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.829056085e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277000192 dispatch_meta_bytes=224136 combine_payload_bytes=277000192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277224328 ingress_bytes=387100672 link_bytes=387100672 token_bytes=554224520 weight_movement_bytes=110100480 total_nvlink_bytes=664325000 weight_to_token_ratio=0.198657 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.732317357e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275265536 dispatch_meta_bytes=223480 combine_payload_bytes=275265536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=275489016 ingress_bytes=385366016 link_bytes=385366016 token_bytes=550754552 weight_movement_bytes=110100480 total_nvlink_bytes=660855032 weight_to_token_ratio=0.199908 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.702148836e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43566 payload_rows_remote=36804 dup_rows=21970 dup_rows_remote=16772 dedup_ratio=0.335236 router_tokens_per_expert=[1282, 1018, 706, 1080, 146, 1246, 428, 74, 169, 682, 1571, 288, 3, 1156, 74, 746, 1140, 345, 12, 33, 863, 473, 738, 497, 332, 1426, 292, 765, 491, 111, 324, 6, 858, 435, 1234, 7, 1172, 0, 1243, 115, 390, 857, 708, 530, 0, 758, 49, 365, 131, 202, 424, 205, 190, 1082, 127, 874, 434, 601, 349, 225, 358, 130, 196, 2] payload_rows_per_dst=[6762, 6102, 6370, 5254, 5290, 5160, 4856, 3772] dup_rows_per_dst=[5198, 3276, 4318, 2240, 2352, 2154, 1614, 818] combine_rows_per_src=[6762, 6102, 6370, 5254, 5290, 5160, 4856, 3772] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51462 payload_rows_remote=44480 dup_rows=14074 dup_rows_remote=14074 dedup_ratio=0.214752 router_tokens_per_expert=[0, 0, 0, 401, 3715, 0, 443, 738, 2557, 50, 27, 0, 0, 65, 0, 1026, 0, 59, 0, 1, 2719, 3491, 0, 0, 0, 0, 2693, 0, 37, 525, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2972, 0, 0, 0, 154, 0, 0, 1517, 1798, 0, 1572, 0, 0, 0, 3317, 0, 0, 12, 148, 13, 38, 2679] payload_rows_per_dst=[5812, 6052, 7708, 6792, 6982, 6036, 6444, 5636] dup_rows_per_dst=[1770, 1398, 4484, 2732, 0, 216, 3330, 144] combine_rows_per_src=[5812, 6052, 7708, 6792, 6982, 6036, 6444, 5636] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44720 payload_rows_remote=37946 dup_rows=20816 dup_rows_remote=17112 dedup_ratio=0.317627 router_tokens_per_expert=[0, 0, 0, 211, 3934, 0, 830, 1009, 1665, 46, 37, 0, 0, 115, 0, 525, 2, 0, 1, 0, 2629, 3585, 0, 0, 0, 0, 2628, 0, 51, 390, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 2900, 0, 0, 1, 13, 0, 1, 1573, 1755, 0, 1910, 0, 0, 0, 3233, 0, 0, 1, 48, 3, 21, 3648] payload_rows_per_dst=[3758, 4324, 6, 5676, 7952, 8160, 6774, 8070] dup_rows_per_dst=[342, 452, 0, 462, 4478, 5540, 3704, 5838] combine_rows_per_src=[3758, 4324, 6, 5676, 7952, 8160, 6774, 8070] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51856 payload_rows_remote=44312 dup_rows=13680 dup_rows_remote=9460 dedup_ratio=0.208740 router_tokens_per_expert=[0, 0, 0, 365, 3890, 0, 558, 1248, 1816, 209, 45, 0, 1, 261, 0, 944, 3, 22, 0, 0, 2659, 3303, 0, 0, 0, 0, 2652, 1, 62, 647, 1, 3, 0, 0, 1, 0, 0, 1, 0, 3, 0, 0, 2760, 1, 0, 0, 107, 0, 0, 1216, 1920, 0, 1747, 1, 0, 0, 3198, 1, 0, 11, 95, 1, 33, 2982] payload_rows_per_dst=[8044, 6380, 7544, 5888, 6616, 5642, 5632, 6110] dup_rows_per_dst=[4078, 2696, 4220, 844, 0, 94, 1612, 136] combine_rows_per_src=[8044, 6380, 7544, 5888, 6616, 5642, 5632, 6110] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51658 payload_rows_remote=44502 dup_rows=13878 dup_rows_remote=8798 dedup_ratio=0.211761 router_tokens_per_expert=[1, 0, 0, 417, 3689, 0, 588, 1061, 2164, 384, 181, 0, 0, 129, 0, 1539, 5, 33, 0, 9, 2906, 3155, 0, 0, 6, 0, 2181, 8, 55, 1312, 0, 8, 0, 0, 0, 0, 5, 2, 0, 3, 0, 0, 3057, 1, 0, 2, 144, 1, 0, 979, 1721, 0, 1894, 0, 0, 0, 1817, 10, 0, 66, 440, 63, 31, 2701] payload_rows_per_dst=[7830, 7156, 7078, 6084, 6318, 6224, 4988, 5980] dup_rows_per_dst=[3682, 5080, 2462, 1056, 12, 186, 758, 642] combine_rows_per_src=[7830, 7156, 7078, 6084, 6318, 6224, 4988, 5980] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52470 payload_rows_remote=45894 dup_rows=13066 dup_rows_remote=11826 dedup_ratio=0.199371 router_tokens_per_expert=[0, 0, 0, 376, 3636, 0, 655, 1030, 2182, 256, 141, 0, 0, 275, 0, 1088, 1, 15, 0, 1, 2538, 3184, 0, 0, 0, 0, 2766, 0, 66, 1075, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3300, 1, 0, 0, 101, 0, 0, 1030, 2101, 0, 1864, 0, 0, 0, 2360, 0, 0, 17, 70, 9, 71, 2557] payload_rows_per_dst=[7778, 6124, 6930, 6576, 6370, 6644, 6704, 5344] dup_rows_per_dst=[3616, 1760, 2900, 1240, 0, 160, 3286, 104] combine_rows_per_src=[7778, 6124, 6930, 6576, 6370, 6644, 6704, 5344] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=263811072 dispatch_meta_bytes=214304 combine_payload_bytes=263811072 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=264025376 ingress_bytes=704212992 link_bytes=704212992 token_bytes=527836448 weight_movement_bytes=440401920 total_nvlink_bytes=968238368 weight_to_token_ratio=0.834353 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=6784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.224742216e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44956 payload_rows_remote=37014 dup_rows=20580 dup_rows_remote=13840 dedup_ratio=0.314026 router_tokens_per_expert=[0, 0, 1, 400, 3224, 1, 310, 967, 2404, 88, 110, 0, 0, 78, 0, 1044, 0, 9, 0, 1, 2993, 3064, 1, 0, 0, 0, 3127, 1, 225, 731, 0, 0, 0, 0, 0, 0, 1, 5, 0, 1, 0, 0, 3436, 0, 0, 0, 67, 0, 0, 1227, 1861, 0, 1441, 1, 0, 2, 2916, 0, 1, 15, 249, 14, 72, 2680] payload_rows_per_dst=[3116, 6022, 22, 6808, 7828, 7870, 5348, 7942] dup_rows_per_dst=[242, 1426, 0, 1360, 4300, 4818, 1694, 6740] combine_rows_per_src=[3116, 6022, 22, 6808, 7828, 7870, 5348, 7942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318832640 dispatch_meta_bytes=234216 combine_payload_bytes=318832640 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=319066856 ingress_bytes=539033600 link_bytes=539033600 token_bytes=637899496 weight_movement_bytes=220200960 total_nvlink_bytes=858100456 weight_to_token_ratio=0.345197 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=26496 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.374680862e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=271996928 dispatch_meta_bytes=220232 combine_payload_bytes=271996928 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=272217160 ingress_bytes=602298368 link_bytes=602298368 token_bytes=544214088 weight_movement_bytes=330301440 total_nvlink_bytes=874515528 weight_to_token_ratio=0.606933 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047495923e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=317628416 dispatch_meta_bytes=215088 combine_payload_bytes=317628416 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=317843504 ingress_bytes=647929856 link_bytes=647929856 token_bytes=635471920 weight_movement_bytes=330301440 total_nvlink_bytes=965773360 weight_to_token_ratio=0.519773 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.033088 gemm_active_groups=9 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.126856586e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48020 payload_rows_remote=41210 dup_rows=17516 dup_rows_remote=14730 dedup_ratio=0.267273 router_tokens_per_expert=[0, 0, 0, 355, 3857, 0, 559, 1041, 1745, 248, 8, 0, 0, 114, 0, 1379, 2, 14, 0, 0, 2666, 3343, 0, 1, 0, 0, 2570, 2, 93, 951, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 4, 2958, 0, 0, 0, 129, 0, 0, 924, 1821, 2, 2059, 0, 1, 2, 3015, 1, 3, 9, 90, 5, 13, 2779] payload_rows_per_dst=[3470, 5520, 5664, 7014, 6972, 6810, 6520, 6050] dup_rows_per_dst=[440, 1468, 2938, 4518, 1008, 2786, 3098, 1260] combine_rows_per_src=[3470, 5520, 5664, 7014, 6972, 6810, 6520, 6050] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=318990336 dispatch_meta_bytes=213200 combine_payload_bytes=318990336 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=319203536 ingress_bytes=429090816 link_bytes=429090816 token_bytes=638193872 weight_movement_bytes=110100480 total_nvlink_bytes=748294352 weight_to_token_ratio=0.172519 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.462595023e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=328968192 dispatch_meta_bytes=230880 combine_payload_bytes=328968192 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329199072 ingress_bytes=439068672 link_bytes=439068672 token_bytes=658167264 weight_movement_bytes=110100480 total_nvlink_bytes=768267744 weight_to_token_ratio=0.167283 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.636126350e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=265316352 dispatch_meta_bytes=203416 combine_payload_bytes=265316352 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=265519768 ingress_bytes=485517312 link_bytes=485517312 token_bytes=530836120 weight_movement_bytes=220200960 total_nvlink_bytes=751037080 weight_to_token_ratio=0.414819 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=22656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.443944594e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=295393280 dispatch_meta_bytes=223760 combine_payload_bytes=295393280 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295617040 ingress_bytes=405493760 link_bytes=405493760 token_bytes=591010320 weight_movement_bytes=110100480 total_nvlink_bytes=701110800 weight_to_token_ratio=0.186292 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=23552 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.052203409e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50546 payload_rows_remote=42776 dup_rows=14990 dup_rows_remote=11202 dedup_ratio=0.228729 router_tokens_per_expert=[0, 0, 0, 627, 3635, 0, 243, 1274, 1648, 28, 4, 0, 0, 120, 0, 1004, 0, 1, 0, 1, 3207, 3350, 0, 0, 0, 0, 3245, 0, 61, 937, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2101, 0, 0, 0, 74, 0, 0, 1083, 2035, 0, 1739, 0, 0, 0, 3581, 0, 0, 0, 198, 0, 29, 2543] payload_rows_per_dst=[7770, 6638, 7890, 6996, 6700, 4312, 4842, 5398] dup_rows_per_dst=[3788, 3040, 5690, 1490, 0, 38, 802, 142] combine_rows_per_src=[7770, 6638, 7890, 6996, 6700, 4312, 4842, 5398] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58672 payload_rows_remote=50514 dup_rows=6864 dup_rows_remote=5236 dedup_ratio=0.104736 router_tokens_per_expert=[0, 0, 0, 901, 3, 0, 4095, 0, 0, 835, 0, 0, 0, 0, 2095, 0, 2827, 0, 0, 20, 0, 4075, 719, 0, 0, 0, 2777, 4096, 0, 1138, 0, 0, 0, 0, 0, 0, 0, 450, 368, 0, 4076, 0, 229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 4016, 0, 0, 0, 0, 1] payload_rows_per_dst=[8190, 5078, 6228, 6618, 8158, 8174, 8192, 8034] dup_rows_per_dst=[1808, 782, 904, 1212, 1628, 436, 0, 94] combine_rows_per_src=[8190, 5078, 6228, 6618, 8158, 8174, 8192, 8034] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57916 payload_rows_remote=51308 dup_rows=7620 dup_rows_remote=6020 dedup_ratio=0.116272 router_tokens_per_expert=[0, 0, 0, 1123, 6, 0, 4048, 0, 0, 833, 0, 0, 0, 0, 1240, 0, 2909, 0, 0, 20, 0, 4092, 1175, 0, 0, 0, 3434, 4094, 1, 1020, 0, 0, 0, 0, 0, 2, 1, 344, 271, 5, 4068, 0, 133, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 3875, 0, 0, 0, 0, 0] payload_rows_per_dst=[8108, 3592, 6608, 7294, 8186, 8144, 8188, 7796] dup_rows_per_dst=[2246, 554, 1600, 1616, 1244, 260, 0, 100] combine_rows_per_src=[8108, 3592, 6608, 7294, 8186, 8144, 8188, 7796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56488 payload_rows_remote=48296 dup_rows=9048 dup_rows_remote=9048 dedup_ratio=0.138062 router_tokens_per_expert=[0, 0, 0, 1312, 0, 0, 4087, 0, 0, 1485, 0, 0, 0, 0, 1017, 0, 3167, 0, 0, 18, 0, 4087, 812, 0, 0, 0, 2927, 4095, 0, 599, 3, 0, 0, 0, 0, 0, 0, 325, 716, 1, 4086, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 3971, 0, 0, 1, 0, 0] payload_rows_per_dst=[2624, 8180, 6746, 6448, 8182, 8172, 8192, 7944] dup_rows_per_dst=[0, 4998, 1248, 610, 2076, 98, 0, 18] combine_rows_per_src=[2624, 8180, 6746, 6448, 8182, 8172, 8192, 7944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57306 payload_rows_remote=51216 dup_rows=8230 dup_rows_remote=7484 dedup_ratio=0.125580 router_tokens_per_expert=[0, 0, 0, 1279, 0, 0, 3968, 0, 0, 1055, 0, 0, 0, 0, 1305, 0, 2971, 0, 0, 60, 0, 4095, 991, 0, 0, 0, 2664, 4096, 0, 754, 0, 0, 0, 0, 0, 0, 0, 469, 582, 2, 4087, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 4011, 0, 0, 0, 0, 0] payload_rows_per_dst=[7994, 3972, 6636, 6090, 8190, 8180, 8192, 8052] dup_rows_per_dst=[2500, 748, 1408, 746, 2106, 390, 0, 332] combine_rows_per_src=[7994, 3972, 6636, 6090, 8190, 8180, 8192, 8052] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54782 payload_rows_remote=49848 dup_rows=10754 dup_rows_remote=9704 dedup_ratio=0.164093 router_tokens_per_expert=[0, 0, 0, 1234, 20, 0, 4004, 0, 0, 1190, 0, 0, 0, 0, 1802, 0, 2299, 0, 0, 36, 0, 4068, 740, 1, 0, 0, 3292, 4095, 0, 1170, 1, 0, 1, 0, 0, 0, 0, 434, 290, 19, 4004, 3, 251, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 3698, 0, 0, 0, 0, 0] payload_rows_per_dst=[8160, 4934, 7310, 7230, 3402, 8054, 8190, 7502] dup_rows_per_dst=[3924, 1050, 2920, 1696, 576, 462, 0, 126] combine_rows_per_src=[8160, 4934, 7310, 7230, 3402, 8054, 8190, 7502] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54230 payload_rows_remote=46118 dup_rows=11306 dup_rows_remote=10166 dedup_ratio=0.172516 router_tokens_per_expert=[0, 0, 0, 859, 0, 0, 4093, 0, 0, 965, 0, 0, 0, 0, 1679, 0, 2685, 0, 0, 29, 0, 4069, 855, 0, 0, 0, 2618, 4096, 0, 1414, 0, 0, 0, 0, 0, 0, 0, 651, 383, 2, 3993, 0, 243, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 85, 1, 4044, 2, 0, 0, 0, 1] payload_rows_per_dst=[1718, 8188, 5332, 6534, 8164, 7990, 8192, 8112] dup_rows_per_dst=[0, 5286, 820, 1530, 2046, 482, 2, 1140] combine_rows_per_src=[1718, 8188, 5332, 6534, 8164, 7990, 8192, 8112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=306618368 dispatch_meta_bytes=215912 combine_payload_bytes=306618368 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=306834280 ingress_bytes=636919808 link_bytes=636919808 token_bytes=613452648 weight_movement_bytes=330301440 total_nvlink_bytes=943754088 weight_to_token_ratio=0.538430 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.107708301e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=362084352 dispatch_meta_bytes=223000 combine_payload_bytes=362084352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=362307352 ingress_bytes=472184832 link_bytes=472184832 token_bytes=724391704 weight_movement_bytes=110100480 total_nvlink_bytes=834492184 weight_to_token_ratio=0.151990 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.212070839e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=367775744 dispatch_meta_bytes=229312 combine_payload_bytes=367775744 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=368005056 ingress_bytes=808177664 link_bytes=808177664 token_bytes=735780800 weight_movement_bytes=440401920 total_nvlink_bytes=1176182720 weight_to_token_ratio=0.598550 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.405553880e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=346185728 dispatch_meta_bytes=229376 combine_payload_bytes=346185728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346415104 ingress_bytes=456286208 link_bytes=456286208 token_bytes=692600832 weight_movement_bytes=110100480 total_nvlink_bytes=802701312 weight_to_token_ratio=0.158967 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.935567619e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=367116288 dispatch_meta_bytes=234800 combine_payload_bytes=367116288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=367351088 ingress_bytes=587317248 link_bytes=587317248 token_bytes=734467376 weight_movement_bytes=220200960 total_nvlink_bytes=954668336 weight_to_token_ratio=0.299810 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=23936 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.021441291e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=357310464 dispatch_meta_bytes=238208 combine_payload_bytes=357310464 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357548672 ingress_bytes=467410944 link_bytes=467410944 token_bytes=714859136 weight_movement_bytes=110100480 total_nvlink_bytes=824959616 weight_to_token_ratio=0.154017 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=12160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.129045075e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55620 payload_rows_remote=47480 dup_rows=9916 dup_rows_remote=9668 dedup_ratio=0.151306 router_tokens_per_expert=[0, 1, 0, 964, 1, 0, 4050, 0, 0, 773, 0, 0, 0, 0, 1693, 0, 2521, 0, 0, 26, 0, 4080, 1452, 2, 0, 0, 3125, 4094, 0, 529, 0, 0, 0, 0, 0, 0, 0, 490, 611, 0, 4058, 2, 132, 0, 0, 1, 0, 1, 0, 0, 2, 1, 1, 0, 0, 0, 140, 1, 4010, 0, 0, 0, 1, 6] payload_rows_per_dst=[2084, 7992, 6388, 6614, 8168, 8140, 8192, 8042] dup_rows_per_dst=[154, 4734, 1614, 694, 2194, 248, 4, 274] combine_rows_per_src=[2084, 7992, 6388, 6614, 8168, 8140, 8192, 8042] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=330573824 dispatch_meta_bytes=225136 combine_payload_bytes=330573824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=330798960 ingress_bytes=440674304 link_bytes=440674304 token_bytes=661372784 weight_movement_bytes=110100480 total_nvlink_bytes=771473264 weight_to_token_ratio=0.166473 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.664050931e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=340336640 dispatch_meta_bytes=228592 combine_payload_bytes=340336640 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=340565232 ingress_bytes=560537600 link_bytes=560537600 token_bytes=680901872 weight_movement_bytes=220200960 total_nvlink_bytes=901102832 weight_to_token_ratio=0.323396 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.748670790e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57110 payload_rows_remote=49466 dup_rows=8426 dup_rows_remote=5138 dedup_ratio=0.128571 router_tokens_per_expert=[0, 0, 1621, 0, 0, 3964, 0, 0, 0, 0, 0, 670, 0, 0, 0, 0, 0, 0, 1839, 0, 0, 0, 4070, 0, 0, 3679, 0, 3965, 0, 0, 0, 0, 0, 0, 0, 0, 2804, 749, 0, 1913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 3308, 4096, 0] payload_rows_per_dst=[3242, 7950, 8168, 7358, 7644, 8192, 7930, 6626] dup_rows_per_dst=[0, 1318, 3650, 0, 3288, 0, 70, 100] combine_rows_per_src=[3242, 7950, 8168, 7358, 7644, 8192, 7930, 6626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51392 payload_rows_remote=43208 dup_rows=14144 dup_rows_remote=6302 dedup_ratio=0.215820 router_tokens_per_expert=[0, 0, 0, 1584, 0, 0, 4079, 0, 0, 1470, 0, 0, 0, 0, 1254, 0, 2350, 0, 0, 38, 0, 4093, 716, 0, 0, 0, 3049, 4096, 0, 1289, 0, 0, 0, 0, 0, 0, 0, 289, 319, 0, 4096, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 3969, 0, 0, 0, 0, 0] payload_rows_per_dst=[8184, 4482, 8190, 6980, 1202, 6220, 8192, 7942] dup_rows_per_dst=[7842, 966, 1584, 3624, 14, 86, 0, 28] combine_rows_per_src=[8184, 4482, 8190, 6980, 1202, 6220, 8192, 7942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55338 payload_rows_remote=47336 dup_rows=10198 dup_rows_remote=10140 dedup_ratio=0.155609 router_tokens_per_expert=[0, 0, 1444, 0, 0, 3986, 0, 0, 0, 0, 0, 654, 1, 0, 0, 0, 0, 0, 1786, 0, 0, 0, 4082, 0, 0, 3391, 0, 4000, 0, 0, 1, 0, 0, 0, 0, 0, 2597, 1217, 0, 1764, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 3652, 4095, 0] payload_rows_per_dst=[2888, 8002, 8172, 6784, 5560, 8192, 8002, 7738] dup_rows_per_dst=[0, 1280, 3564, 0, 1214, 6, 58, 4076] combine_rows_per_src=[2888, 8002, 8172, 6784, 5560, 8192, 8002, 7738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57686 payload_rows_remote=54540 dup_rows=7850 dup_rows_remote=7850 dedup_ratio=0.119781 router_tokens_per_expert=[0, 0, 1931, 0, 0, 4011, 0, 0, 2, 0, 0, 402, 5, 0, 2, 0, 0, 0, 1573, 0, 0, 0, 4087, 0, 0, 3512, 0, 3994, 0, 0, 0, 0, 0, 0, 0, 0, 2962, 627, 0, 1698, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 32, 2, 0, 0, 0, 0, 79, 0, 0, 0, 0, 3751, 4091, 0] payload_rows_per_dst=[8182, 8032, 3146, 7024, 7590, 8190, 7988, 7534] dup_rows_per_dst=[3854, 812, 0, 0, 2984, 6, 68, 126] combine_rows_per_src=[8182, 8032, 3146, 7024, 7590, 8190, 7988, 7534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58186 payload_rows_remote=51084 dup_rows=7350 dup_rows_remote=7348 dedup_ratio=0.112152 router_tokens_per_expert=[0, 0, 2744, 0, 0, 3783, 0, 0, 2, 0, 0, 890, 0, 0, 0, 0, 0, 0, 1076, 0, 0, 0, 4044, 0, 0, 3551, 0, 3995, 0, 0, 1, 0, 0, 0, 0, 0, 3277, 1130, 0, 969, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 79, 0, 0, 0, 0, 3093, 4095, 0] payload_rows_per_dst=[5494, 7690, 7840, 7102, 7652, 8192, 7992, 6224] dup_rows_per_dst=[390, 1660, 2004, 2, 3100, 4, 70, 120] combine_rows_per_src=[5494, 7690, 7840, 7102, 7652, 8192, 7992, 6224] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53480 payload_rows_remote=45690 dup_rows=12056 dup_rows_remote=11176 dedup_ratio=0.183960 router_tokens_per_expert=[0, 0, 2511, 0, 0, 3885, 0, 0, 8, 0, 0, 435, 7, 0, 0, 0, 0, 0, 972, 0, 0, 0, 4048, 0, 0, 3508, 0, 3948, 0, 0, 14, 0, 0, 0, 0, 0, 3238, 1206, 0, 1800, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 197, 0, 0, 0, 0, 2840, 4083, 0] payload_rows_per_dst=[8164, 7790, 1944, 7978, 5562, 8168, 7918, 5956] dup_rows_per_dst=[4954, 880, 0, 5030, 962, 10, 102, 118] combine_rows_per_src=[8164, 7790, 1944, 7978, 5562, 8168, 7918, 5956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=354572288 dispatch_meta_bytes=218416 combine_payload_bytes=354572288 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=354790704 ingress_bytes=684873728 link_bytes=684873728 token_bytes=709362992 weight_movement_bytes=330301440 total_nvlink_bytes=1039664432 weight_to_token_ratio=0.465631 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=12800 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.191108055e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54554 payload_rows_remote=46514 dup_rows=10982 dup_rows_remote=6352 dedup_ratio=0.167572 router_tokens_per_expert=[0, 0, 1779, 0, 0, 3903, 0, 0, 0, 0, 0, 713, 2, 0, 0, 0, 0, 0, 1638, 0, 0, 0, 4030, 0, 0, 3707, 0, 3984, 0, 0, 0, 0, 0, 0, 0, 0, 3059, 1173, 0, 1375, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 3202, 4094, 0] payload_rows_per_dst=[3558, 7870, 8096, 6116, 4666, 8192, 8016, 8040] dup_rows_per_dst=[0, 1366, 3240, 0, 430, 0, 1316, 4630] combine_rows_per_src=[3558, 7870, 8096, 6116, 4666, 8192, 8016, 8040] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=309714944 dispatch_meta_bytes=198040 combine_payload_bytes=309714944 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=309912984 ingress_bytes=529915904 link_bytes=529915904 token_bytes=619627928 weight_movement_bytes=220200960 total_nvlink_bytes=839828888 weight_to_token_ratio=0.355376 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=20352 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.216109132e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=339304448 dispatch_meta_bytes=229904 combine_payload_bytes=339304448 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=339534352 ingress_bytes=559505408 link_bytes=559505408 token_bytes=678838800 weight_movement_bytes=220200960 total_nvlink_bytes=899039760 weight_to_token_ratio=0.324379 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.730719273e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58712 payload_rows_remote=50528 dup_rows=6824 dup_rows_remote=6824 dedup_ratio=0.104126 router_tokens_per_expert=[0, 0, 2421, 0, 0, 3933, 0, 0, 1, 0, 0, 683, 2, 0, 2, 0, 0, 0, 1332, 0, 0, 0, 4054, 0, 0, 3534, 0, 3925, 0, 1, 4, 0, 0, 0, 0, 0, 2683, 884, 0, 1573, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 1, 0, 0, 0, 47, 0, 0, 0, 0, 3568, 4091, 0] payload_rows_per_dst=[4842, 7904, 8156, 7076, 7552, 8184, 7850, 7148] dup_rows_per_dst=[0, 1338, 2616, 2, 2728, 0, 58, 82] combine_rows_per_src=[4842, 7904, 8156, 7076, 7552, 8184, 7850, 7148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=390942720 dispatch_meta_bytes=249560 combine_payload_bytes=390942720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=391192280 ingress_bytes=611143680 link_bytes=611143680 token_bytes=782135000 weight_movement_bytes=220200960 total_nvlink_bytes=1002335960 weight_to_token_ratio=0.281538 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.062879375e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=327505920 dispatch_meta_bytes=227464 combine_payload_bytes=327505920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327733384 ingress_bytes=437606400 link_bytes=437606400 token_bytes=655239304 weight_movement_bytes=110100480 total_nvlink_bytes=765339784 weight_to_token_ratio=0.168031 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.610695035e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=366170112 dispatch_meta_bytes=233728 combine_payload_bytes=366170112 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=366403840 ingress_bytes=696471552 link_bytes=696471552 token_bytes=732573952 weight_movement_bytes=330301440 total_nvlink_bytes=1062875392 weight_to_token_ratio=0.450878 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.211278579e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=333412352 dispatch_meta_bytes=211464 combine_payload_bytes=333412352 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=333623816 ingress_bytes=663713792 link_bytes=663713792 token_bytes=667036168 weight_movement_bytes=330301440 total_nvlink_bytes=997337608 weight_to_token_ratio=0.495178 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.154307446e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=362184704 dispatch_meta_bytes=229408 combine_payload_bytes=362184704 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=362414112 ingress_bytes=582385664 link_bytes=582385664 token_bytes=724598816 weight_movement_bytes=220200960 total_nvlink_bytes=944799776 weight_to_token_ratio=0.303894 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012864456e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56088 payload_rows_remote=49702 dup_rows=9448 dup_rows_remote=9448 dedup_ratio=0.144165 router_tokens_per_expert=[0, 2, 0, 0, 359, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3034, 0, 2890, 189, 0, 0, 0, 0, 4094, 3193, 0, 4096, 0, 0, 0, 0, 4, 1010, 3167, 0, 0, 3785, 0, 0, 0, 35, 0, 0, 0, 1346, 3829, 0, 0, 0, 168, 0, 0, 0, 1567, 0] payload_rows_per_dst=[8192, 8192, 7570, 7886, 6386, 7192, 2740, 7930] dup_rows_per_dst=[718, 0, 0, 4340, 0, 1170, 22, 3198] combine_rows_per_src=[8192, 8192, 7570, 7886, 6386, 7192, 2740, 7930] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55740 payload_rows_remote=51470 dup_rows=9796 dup_rows_remote=9702 dedup_ratio=0.149475 router_tokens_per_expert=[0, 3, 0, 0, 279, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2837, 0, 1804, 60, 0, 0, 0, 0, 4093, 2940, 0, 4095, 0, 0, 0, 0, 1, 1296, 3458, 0, 0, 4023, 0, 0, 0, 93, 0, 0, 0, 2089, 3868, 0, 0, 0, 108, 0, 0, 0, 1720, 0] payload_rows_per_dst=[8192, 8190, 8048, 3700, 7690, 7730, 4270, 7920] dup_rows_per_dst=[558, 0, 0, 28, 3864, 1780, 94, 3472] combine_rows_per_src=[8192, 8190, 8048, 3700, 7690, 7730, 4270, 7920] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49316 payload_rows_remote=41124 dup_rows=16220 dup_rows_remote=4444 dedup_ratio=0.247498 router_tokens_per_expert=[0, 0, 2296, 0, 0, 3924, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 1842, 0, 0, 0, 4035, 0, 0, 3454, 0, 4007, 0, 0, 0, 0, 0, 0, 0, 0, 2706, 632, 0, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 3476, 4096, 0] payload_rows_per_dst=[8192, 1552, 4586, 7486, 4350, 8192, 8014, 6944] dup_rows_per_dst=[11776, 92, 354, 3578, 254, 8, 84, 74] combine_rows_per_src=[8192, 1552, 4586, 7486, 4350, 8192, 8014, 6944] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56240 payload_rows_remote=48098 dup_rows=9296 dup_rows_remote=9296 dedup_ratio=0.141846 router_tokens_per_expert=[0, 51, 0, 0, 286, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 2471, 0, 2133, 459, 0, 0, 0, 0, 4015, 3006, 0, 4071, 0, 0, 0, 0, 9, 718, 3483, 0, 0, 3891, 0, 0, 0, 402, 0, 0, 0, 2511, 3553, 0, 0, 0, 615, 0, 0, 0, 1089, 0] payload_rows_per_dst=[8132, 8142, 7784, 7322, 6012, 7506, 8082, 3260] dup_rows_per_dst=[572, 0, 8, 2804, 0, 914, 4850, 148] combine_rows_per_src=[8132, 8142, 7784, 7322, 6012, 7506, 8082, 3260] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56812 payload_rows_remote=49882 dup_rows=8724 dup_rows_remote=6820 dedup_ratio=0.133118 router_tokens_per_expert=[0, 2, 0, 0, 315, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2522, 0, 1734, 161, 0, 0, 0, 0, 4088, 2634, 0, 4096, 0, 0, 0, 0, 0, 1127, 3692, 0, 0, 3985, 0, 0, 0, 148, 0, 0, 0, 2428, 3503, 0, 0, 0, 234, 0, 0, 0, 2099, 0] payload_rows_per_dst=[8180, 8192, 7970, 6930, 5268, 7812, 7204, 5256] dup_rows_per_dst=[630, 0, 0, 1904, 0, 1826, 3308, 1056] combine_rows_per_src=[8180, 8192, 7970, 6930, 5268, 7812, 7204, 5256] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=356263936 dispatch_meta_bytes=236600 combine_payload_bytes=356263936 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=356500536 ingress_bytes=796665856 link_bytes=796665856 token_bytes=712764472 weight_movement_bytes=440401920 total_nvlink_bytes=1153166392 weight_to_token_ratio=0.617879 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23040 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.385532953e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57220 payload_rows_remote=49210 dup_rows=8316 dup_rows_remote=8312 dedup_ratio=0.126892 router_tokens_per_expert=[0, 8, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2773, 0, 1856, 82, 0, 0, 0, 0, 4087, 2819, 0, 4092, 0, 0, 0, 0, 0, 1368, 3534, 0, 0, 4005, 0, 2, 0, 73, 0, 0, 0, 1712, 3842, 0, 0, 0, 174, 0, 0, 0, 1955, 0] payload_rows_per_dst=[8186, 8184, 8010, 7320, 5638, 7718, 8048, 4116] dup_rows_per_dst=[772, 0, 4, 2102, 0, 2090, 3206, 142] combine_rows_per_src=[8186, 8184, 8010, 7320, 5638, 7718, 8048, 4116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49568 payload_rows_remote=41614 dup_rows=15968 dup_rows_remote=10832 dedup_ratio=0.243652 router_tokens_per_expert=[0, 2, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3089, 0, 1454, 146, 0, 0, 0, 0, 4094, 2400, 0, 4094, 0, 0, 0, 0, 1, 850, 3544, 0, 0, 3908, 0, 0, 0, 113, 0, 0, 0, 2166, 3740, 0, 0, 0, 230, 0, 0, 0, 2575, 0] payload_rows_per_dst=[3232, 8188, 7816, 3164, 8002, 5246, 5966, 7954] dup_rows_per_dst=[196, 0, 2, 36, 8462, 592, 1544, 5136] combine_rows_per_src=[3232, 8188, 7816, 3164, 8002, 5246, 5966, 7954] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=368936960 dispatch_meta_bytes=244688 combine_payload_bytes=368936960 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=369181648 ingress_bytes=589137920 link_bytes=589137920 token_bytes=738118608 weight_movement_bytes=220200960 total_nvlink_bytes=958319568 weight_to_token_ratio=0.298327 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.024607739e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56758 payload_rows_remote=49180 dup_rows=8778 dup_rows_remote=7398 dedup_ratio=0.133942 router_tokens_per_expert=[0, 6, 0, 0, 334, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2866, 0, 1643, 167, 0, 0, 0, 0, 4090, 3029, 0, 4091, 0, 0, 0, 0, 5, 956, 3517, 0, 0, 3947, 0, 1, 0, 140, 0, 0, 0, 2145, 3865, 0, 0, 0, 129, 0, 0, 0, 1837, 0] payload_rows_per_dst=[8192, 8182, 7894, 5778, 6710, 7578, 4452, 7972] dup_rows_per_dst=[668, 0, 0, 1260, 1662, 1380, 118, 3690] combine_rows_per_src=[8192, 8182, 7894, 5778, 6710, 7578, 4452, 7972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=294776832 dispatch_meta_bytes=182272 combine_payload_bytes=294776832 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=294959104 ingress_bytes=514977792 link_bytes=514977792 token_bytes=589735936 weight_movement_bytes=220200960 total_nvlink_bytes=809936896 weight_to_token_ratio=0.373389 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.956310796e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=344766464 dispatch_meta_bytes=229576 combine_payload_bytes=344766464 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=344996040 ingress_bytes=564967424 link_bytes=564967424 token_bytes=689762504 weight_movement_bytes=220200960 total_nvlink_bytes=909963464 weight_to_token_ratio=0.319242 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.825712715e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=357554176 dispatch_meta_bytes=226808 combine_payload_bytes=357554176 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=357780984 ingress_bytes=577755136 link_bytes=577755136 token_bytes=715335160 weight_movement_bytes=220200960 total_nvlink_bytes=935536120 weight_to_token_ratio=0.307829 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.004811206e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=352737280 dispatch_meta_bytes=230088 combine_payload_bytes=352737280 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=352967368 ingress_bytes=572938240 link_bytes=572938240 token_bytes=705704648 weight_movement_bytes=220200960 total_nvlink_bytes=925905608 weight_to_token_ratio=0.312030 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.964338315e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=298289152 dispatch_meta_bytes=209784 combine_payload_bytes=298289152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298498936 ingress_bytes=408389632 link_bytes=408389632 token_bytes=596788088 weight_movement_bytes=110100480 total_nvlink_bytes=706888568 weight_to_token_ratio=0.184488 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.102567386e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=352522240 dispatch_meta_bytes=226312 combine_payload_bytes=352522240 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=352748552 ingress_bytes=792924160 link_bytes=792924160 token_bytes=705270792 weight_movement_bytes=440401920 total_nvlink_bytes=1145672712 weight_to_token_ratio=0.624444 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.379025528e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62344 payload_rows_remote=56234 dup_rows=3192 dup_rows_remote=3192 dedup_ratio=0.048706 router_tokens_per_expert=[0, 4061, 0, 3370, 0, 477, 0, 403, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 2, 79, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3055, 0, 0, 0, 0, 0, 3915, 0, 2, 54, 0, 0, 4096, 0, 0, 0, 127, 0, 0, 850, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[7518, 8192, 8180, 8192, 6110, 7838, 8192, 8122] dup_rows_per_dst=[982, 0, 150, 0, 0, 104, 1954, 2] combine_rows_per_src=[7518, 8192, 8180, 8192, 6110, 7838, 8192, 8122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62222 payload_rows_remote=54030 dup_rows=3314 dup_rows_remote=658 dedup_ratio=0.050568 router_tokens_per_expert=[0, 4054, 0, 3080, 0, 314, 0, 321, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 13, 0, 0, 4084, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3342, 0, 0, 0, 0, 0, 3879, 0, 4, 58, 0, 0, 4096, 0, 0, 0, 47, 0, 0, 1281, 0, 0, 1, 2, 0, 0, 0, 0] payload_rows_per_dst=[6904, 8192, 8174, 8192, 6684, 7776, 8192, 8108] dup_rows_per_dst=[526, 0, 20, 0, 0, 106, 2656, 6] combine_rows_per_src=[6904, 8192, 8174, 8192, 6684, 7776, 8192, 8108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=403085312 dispatch_meta_bytes=237704 combine_payload_bytes=403085312 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=403323016 ingress_bytes=513185792 link_bytes=513185792 token_bytes=806408328 weight_movement_bytes=110100480 total_nvlink_bytes=916508808 weight_to_token_ratio=0.136532 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23680 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.925144968e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58514 payload_rows_remote=50356 dup_rows=7022 dup_rows_remote=7022 dedup_ratio=0.107147 router_tokens_per_expert=[0, 3992, 0, 2384, 0, 628, 0, 634, 0, 0, 0, 0, 0, 0, 0, 4079, 0, 0, 27, 115, 0, 0, 4049, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2900, 1, 0, 0, 0, 0, 3686, 0, 73, 41, 0, 0, 4096, 0, 0, 0, 110, 0, 0, 1806, 0, 0, 0, 34, 17, 0, 0, 0] payload_rows_per_dst=[6488, 8158, 8150, 8192, 8192, 7528, 3770, 8036] dup_rows_per_dst=[804, 0, 232, 0, 5802, 72, 62, 50] combine_rows_per_src=[6488, 8158, 8150, 8192, 8192, 7528, 3770, 8036] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60128 payload_rows_remote=51936 dup_rows=5408 dup_rows_remote=5408 dedup_ratio=0.082520 router_tokens_per_expert=[0, 4078, 0, 2336, 0, 636, 0, 612, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 26, 0, 0, 4072, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2871, 0, 0, 0, 0, 0, 3957, 0, 3, 74, 0, 0, 4096, 0, 0, 0, 106, 0, 0, 1701, 0, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[6370, 8192, 8148, 8192, 5742, 8008, 7318, 8158] dup_rows_per_dst=[798, 0, 48, 0, 0, 1642, 2906, 14] combine_rows_per_src=[6370, 8192, 8148, 8192, 5742, 8008, 7318, 8158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54544 payload_rows_remote=46352 dup_rows=10992 dup_rows_remote=10068 dedup_ratio=0.167725 router_tokens_per_expert=[0, 0, 0, 0, 462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3097, 0, 2038, 106, 0, 0, 0, 0, 4096, 2958, 0, 4096, 0, 0, 0, 0, 0, 817, 3173, 0, 0, 3966, 0, 0, 0, 136, 0, 0, 0, 1789, 3891, 0, 0, 0, 104, 0, 0, 0, 2039, 0] payload_rows_per_dst=[8192, 8192, 7984, 7674, 5842, 4444, 8034, 4182] dup_rows_per_dst=[924, 74, 2446, 3288, 0, 558, 3598, 104] combine_rows_per_src=[8192, 8192, 7984, 7674, 5842, 4444, 8034, 4182] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58010 payload_rows_remote=49862 dup_rows=7526 dup_rows_remote=7476 dedup_ratio=0.114838 router_tokens_per_expert=[0, 4063, 0, 2643, 0, 450, 0, 549, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 8, 21, 0, 0, 4069, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3273, 0, 0, 0, 0, 0, 3868, 0, 10, 73, 0, 0, 4096, 0, 0, 0, 125, 0, 0, 1298, 0, 0, 0, 29, 0, 0, 0, 0] payload_rows_per_dst=[6694, 8192, 8148, 8192, 7996, 7820, 2820, 8148] dup_rows_per_dst=[590, 0, 50, 0, 5830, 994, 26, 36] combine_rows_per_src=[6694, 8192, 8148, 8192, 7996, 7820, 2820, 8148] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61014 payload_rows_remote=52834 dup_rows=4522 dup_rows_remote=4170 dedup_ratio=0.069000 router_tokens_per_expert=[0, 4088, 0, 3288, 0, 455, 0, 516, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 0, 1, 10, 0, 0, 4080, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 0, 0, 0, 0, 3910, 0, 3, 31, 0, 0, 4096, 0, 0, 0, 133, 0, 0, 1338, 0, 0, 1, 2, 0, 0, 0, 0] payload_rows_per_dst=[7178, 8188, 7990, 8192, 5252, 7842, 8192, 8180] dup_rows_per_dst=[990, 0, 22, 0, 0, 46, 3112, 352] combine_rows_per_src=[7178, 8188, 7990, 8192, 5252, 7842, 8192, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61332 payload_rows_remote=53588 dup_rows=4204 dup_rows_remote=4046 dedup_ratio=0.064148 router_tokens_per_expert=[0, 4077, 0, 2984, 0, 397, 0, 126, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 2, 2, 9, 0, 0, 4044, 0, 0, 0, 0, 4096, 0, 2, 0, 0, 0, 0, 0, 0, 3062, 0, 0, 0, 0, 0, 3839, 0, 12, 100, 0, 0, 4094, 0, 0, 0, 110, 0, 0, 1714, 0, 0, 2, 4, 0, 0, 0, 0] payload_rows_per_dst=[6650, 8184, 8094, 8192, 6124, 7744, 8188, 8156] dup_rows_per_dst=[364, 0, 20, 4, 0, 158, 3648, 10] combine_rows_per_src=[6650, 8184, 8094, 8192, 6124, 7744, 8188, 8156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=387287040 dispatch_meta_bytes=218752 combine_payload_bytes=387287040 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=387505792 ingress_bytes=827688960 link_bytes=827688960 token_bytes=774792832 weight_movement_bytes=440401920 total_nvlink_bytes=1215194752 weight_to_token_ratio=0.568412 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=19712 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.439487233e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360951808 dispatch_meta_bytes=229512 combine_payload_bytes=360951808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=361181320 ingress_bytes=471052288 link_bytes=471052288 token_bytes=722133128 weight_movement_bytes=110100480 total_nvlink_bytes=832233608 weight_to_token_ratio=0.152466 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.192374036e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372277248 dispatch_meta_bytes=229376 combine_payload_bytes=372277248 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=372506624 ingress_bytes=482377728 link_bytes=482377728 token_bytes=744783872 weight_movement_bytes=110100480 total_nvlink_bytes=854884352 weight_to_token_ratio=0.147829 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.389342065e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=332251136 dispatch_meta_bytes=225680 combine_payload_bytes=332251136 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=332476816 ingress_bytes=442351616 link_bytes=442351616 token_bytes=664727952 weight_movement_bytes=110100480 total_nvlink_bytes=774828432 weight_to_token_ratio=0.165632 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.693222146e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=357410816 dispatch_meta_bytes=229352 combine_payload_bytes=357410816 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=357640168 ingress_bytes=687712256 link_bytes=687712256 token_bytes=715050984 weight_movement_bytes=330301440 total_nvlink_bytes=1045352424 weight_to_token_ratio=0.461927 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.196044722e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=378714112 dispatch_meta_bytes=228016 combine_payload_bytes=378714112 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=378942128 ingress_bytes=598915072 link_bytes=598915072 token_bytes=757656240 weight_movement_bytes=220200960 total_nvlink_bytes=977857200 weight_to_token_ratio=0.290634 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041611815e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=384118784 dispatch_meta_bytes=230536 combine_payload_bytes=384118784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=384349320 ingress_bytes=494219264 link_bytes=494219264 token_bytes=768468104 weight_movement_bytes=110100480 total_nvlink_bytes=878568584 weight_to_token_ratio=0.143273 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.595285852e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64036 payload_rows_remote=55934 dup_rows=1500 dup_rows_remote=1412 dedup_ratio=0.022888 router_tokens_per_expert=[0, 4049, 0, 0, 0, 4051, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3578, 0, 4048, 0, 0, 0, 0, 305, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 159, 0, 0, 0, 0, 4096, 0, 4014, 3887, 409, 0, 0, 0, 4096, 24, 0, 5, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8098, 8192, 7156, 8180, 8102, 8192, 8088, 8028] dup_rows_per_dst=[0, 6, 0, 526, 88, 318, 552, 10] combine_rows_per_src=[8098, 8192, 7156, 8180, 8102, 8192, 8088, 8028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63500 payload_rows_remote=55402 dup_rows=2036 dup_rows_remote=1160 dedup_ratio=0.031067 router_tokens_per_expert=[0, 4027, 0, 0, 0, 4041, 0, 0, 16, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3303, 0, 4070, 0, 0, 0, 0, 232, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 4096, 0, 4044, 3889, 578, 0, 0, 0, 4096, 20, 0, 3, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 8192, 6606, 8188, 8082, 8192, 8098, 8088] dup_rows_per_dst=[0, 34, 0, 416, 56, 648, 876, 6] combine_rows_per_src=[8054, 8192, 6606, 8188, 8082, 8192, 8098, 8088] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=400934912 dispatch_meta_bytes=229384 combine_payload_bytes=400934912 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=401164296 ingress_bytes=621135872 link_bytes=621135872 token_bytes=802099208 weight_movement_bytes=220200960 total_nvlink_bytes=1022300168 weight_to_token_ratio=0.274531 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.080257441e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61648 payload_rows_remote=53456 dup_rows=3888 dup_rows_remote=3862 dedup_ratio=0.059326 router_tokens_per_expert=[0, 3383, 0, 0, 0, 4042, 0, 0, 1, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3279, 0, 3746, 0, 0, 0, 0, 1346, 0, 0, 0, 0, 40, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 4096, 0, 3993, 3543, 650, 0, 0, 0, 4096, 391, 0, 45, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6766, 8192, 7198, 7288, 8084, 8192, 7942, 7986] dup_rows_per_dst=[0, 26, 1590, 666, 80, 210, 1226, 90] combine_rows_per_src=[6766, 8192, 7198, 7288, 8084, 8192, 7942, 7986] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63042 payload_rows_remote=54940 dup_rows=2494 dup_rows_remote=1422 dedup_ratio=0.038055 router_tokens_per_expert=[0, 3884, 0, 0, 0, 4023, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3355, 0, 3844, 0, 0, 0, 0, 743, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 4096, 0, 3996, 3740, 660, 0, 0, 0, 4096, 184, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7768, 8192, 6710, 8102, 8046, 8192, 8040, 7992] dup_rows_per_dst=[0, 6, 0, 1072, 50, 220, 1128, 18] combine_rows_per_src=[7768, 8192, 6710, 8102, 8046, 8192, 8040, 7992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63758 payload_rows_remote=56930 dup_rows=1778 dup_rows_remote=1778 dedup_ratio=0.027130 router_tokens_per_expert=[0, 3995, 0, 0, 0, 4053, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3414, 0, 4026, 0, 0, 0, 0, 193, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 4096, 0, 4054, 3867, 648, 0, 0, 0, 4096, 43, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7990, 8192, 6828, 8162, 8106, 8192, 8180, 8108] dup_rows_per_dst=[0, 10, 0, 276, 52, 486, 936, 18] combine_rows_per_src=[7990, 8192, 6828, 8162, 8106, 8192, 8180, 8108] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58462 payload_rows_remote=50728 dup_rows=7074 dup_rows_remote=2128 dedup_ratio=0.107941 router_tokens_per_expert=[0, 4079, 0, 2866, 0, 318, 0, 502, 0, 0, 0, 0, 0, 1, 0, 4096, 0, 0, 0, 1, 0, 0, 4074, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2460, 0, 0, 0, 0, 0, 3921, 0, 0, 8, 0, 0, 4096, 0, 0, 0, 106, 0, 0, 2144, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7734, 8192, 8108, 8188, 5896, 7842, 4344, 8158] dup_rows_per_dst=[4946, 46, 2, 0, 1908, 16, 156, 0] combine_rows_per_src=[7734, 8192, 8108, 8188, 5896, 7842, 4344, 8158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63608 payload_rows_remote=55420 dup_rows=1928 dup_rows_remote=1428 dedup_ratio=0.029419 router_tokens_per_expert=[0, 4023, 0, 0, 0, 4056, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3334, 0, 4076, 0, 0, 0, 0, 181, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 4094, 0, 4063, 3794, 714, 0, 0, 0, 4096, 46, 0, 9, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8046, 8192, 6668, 8172, 8112, 8188, 8104, 8126] dup_rows_per_dst=[0, 10, 0, 342, 54, 500, 1004, 18] combine_rows_per_src=[8046, 8192, 6668, 8172, 8112, 8188, 8104, 8126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62496 payload_rows_remote=54396 dup_rows=3040 dup_rows_remote=2174 dedup_ratio=0.046387 router_tokens_per_expert=[0, 3849, 0, 0, 0, 4027, 0, 0, 14, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3573, 0, 4039, 0, 0, 0, 0, 404, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 125, 0, 0, 0, 0, 4096, 0, 4041, 3755, 605, 0, 0, 0, 4096, 111, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7378, 8066, 7146, 8178, 8056, 8192, 7380, 8100] dup_rows_per_dst=[0, 32, 0, 708, 376, 250, 808, 866] combine_rows_per_src=[7378, 8066, 7146, 8178, 8056, 8192, 7380, 8100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397121536 dispatch_meta_bytes=226248 combine_payload_bytes=397121536 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=397347784 ingress_bytes=947623936 link_bytes=947623936 token_bytes=794469320 weight_movement_bytes=550502400 total_nvlink_bytes=1344971720 weight_to_token_ratio=0.692918 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.648073882e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=408074240 dispatch_meta_bytes=234832 combine_payload_bytes=408074240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=408309072 ingress_bytes=518174720 link_bytes=518174720 token_bytes=816383312 weight_movement_bytes=110100480 total_nvlink_bytes=926483792 weight_to_token_ratio=0.134864 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.011910632e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=393809920 dispatch_meta_bytes=225448 combine_payload_bytes=393809920 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394035368 ingress_bytes=614010880 link_bytes=614010880 token_bytes=787845288 weight_movement_bytes=220200960 total_nvlink_bytes=1008046248 weight_to_token_ratio=0.279498 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067865908e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=397250560 dispatch_meta_bytes=227392 combine_payload_bytes=397250560 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=397477952 ingress_bytes=617451520 link_bytes=617451520 token_bytes=794728512 weight_movement_bytes=220200960 total_nvlink_bytes=1014929472 weight_to_token_ratio=0.277077 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.073849747e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=383172608 dispatch_meta_bytes=229272 combine_payload_bytes=383172608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383401880 ingress_bytes=493273088 link_bytes=493273088 token_bytes=766574488 weight_movement_bytes=110100480 total_nvlink_bytes=876674968 weight_to_token_ratio=0.143627 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.578830295e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363618304 dispatch_meta_bytes=211424 combine_payload_bytes=363618304 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=363829728 ingress_bytes=693919744 link_bytes=693919744 token_bytes=727448032 weight_movement_bytes=330301440 total_nvlink_bytes=1057749472 weight_to_token_ratio=0.454055 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=22784 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.206840565e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59906 payload_rows_remote=51938 dup_rows=5630 dup_rows_remote=5616 dedup_ratio=0.085907 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 4096, 33, 4089, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1397, 0, 7, 0, 0, 0, 0, 3984, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 61, 0, 0, 0, 0, 0, 390, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8192, 2794, 7968, 8192, 8192, 8192] dup_rows_per_dst=[0, 72, 88, 0, 14, 4554, 902, 0] combine_rows_per_src=[8192, 8184, 8192, 2794, 7968, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=389910528 dispatch_meta_bytes=226280 combine_payload_bytes=389910528 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=390136808 ingress_bytes=610111488 link_bytes=610111488 token_bytes=780047336 weight_movement_bytes=220200960 total_nvlink_bytes=1000248296 weight_to_token_ratio=0.282292 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.061084224e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57680 payload_rows_remote=49488 dup_rows=7856 dup_rows_remote=6444 dedup_ratio=0.119873 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 4096, 184, 4088, 0, 0, 0, 44, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 1551, 0, 10, 0, 0, 0, 0, 3991, 0, 0, 0, 0, 0, 0, 1710, 0, 0, 0, 23, 0, 0, 0, 0, 0, 683, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 1314, 8192, 7420, 7988, 8190, 8192, 8192] dup_rows_per_dst=[2, 92, 88, 2828, 14, 3420, 1412, 0] combine_rows_per_src=[8192, 1314, 8192, 7420, 7988, 8190, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=372291584 dispatch_meta_bytes=230216 combine_payload_bytes=372291584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372521800 ingress_bytes=482392064 link_bytes=482392064 token_bytes=744813384 weight_movement_bytes=110100480 total_nvlink_bytes=854913864 weight_to_token_ratio=0.147823 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.389591392e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59890 payload_rows_remote=51698 dup_rows=5646 dup_rows_remote=5568 dedup_ratio=0.086151 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4095, 101, 4087, 0, 0, 0, 39, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4096, 1835, 0, 4, 0, 0, 0, 0, 3954, 0, 0, 0, 0, 0, 0, 1415, 2, 0, 0, 63, 0, 0, 0, 0, 0, 771, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8184, 7912, 2834, 8192, 8192] dup_rows_per_dst=[0, 228, 78, 3670, 4, 0, 1666, 0] combine_rows_per_src=[8192, 8192, 8192, 8184, 7912, 2834, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57494 payload_rows_remote=49302 dup_rows=8042 dup_rows_remote=3450 dedup_ratio=0.122711 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 4093, 154, 3998, 0, 0, 0, 66, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4084, 4096, 1224, 0, 68, 0, 0, 0, 0, 3753, 0, 0, 0, 0, 0, 0, 1767, 0, 0, 0, 385, 0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8168, 7552, 3534, 5472, 8192] dup_rows_per_dst=[0, 4592, 132, 2448, 90, 0, 780, 0] combine_rows_per_src=[8192, 8192, 8192, 8168, 7552, 3534, 5472, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60724 payload_rows_remote=53056 dup_rows=4812 dup_rows_remote=3028 dedup_ratio=0.073425 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 4096, 167, 4083, 0, 0, 0, 83, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1038, 0, 54, 0, 0, 0, 0, 3699, 0, 0, 0, 0, 0, 0, 2190, 0, 0, 0, 223, 0, 0, 0, 0, 0, 736, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8190, 8192, 7668, 7400, 4698, 8192, 8192] dup_rows_per_dst=[0, 340, 166, 1784, 106, 498, 1918, 0] combine_rows_per_src=[8192, 8190, 8192, 7668, 7400, 4698, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=354729984 dispatch_meta_bytes=223728 combine_payload_bytes=354729984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354953712 ingress_bytes=464830464 link_bytes=464830464 token_bytes=709683696 weight_movement_bytes=110100480 total_nvlink_bytes=819784176 weight_to_token_ratio=0.155140 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.084166284e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59272 payload_rows_remote=51086 dup_rows=6264 dup_rows_remote=2492 dedup_ratio=0.095581 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 4095, 186, 4087, 0, 0, 0, 32, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 4096, 1142, 0, 10, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 0, 0, 1886, 1, 0, 0, 155, 0, 0, 0, 0, 0, 867, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 2284, 7842, 8186, 8192, 8192] dup_rows_per_dst=[0, 376, 64, 0, 10, 3772, 2042, 0] combine_rows_per_src=[8192, 8192, 8192, 2284, 7842, 8186, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55166 payload_rows_remote=46974 dup_rows=10370 dup_rows_remote=10370 dedup_ratio=0.158234 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 11, 0, 4096, 32, 4094, 0, 0, 0, 60, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1083, 0, 3, 0, 0, 0, 0, 3915, 0, 0, 0, 0, 0, 0, 2624, 0, 0, 0, 90, 0, 0, 0, 0, 0, 374, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 86, 8192, 6206, 7920, 8192, 8192, 8192] dup_rows_per_dst=[4, 0, 120, 1220, 2850, 5248, 928, 0] combine_rows_per_src=[8186, 86, 8192, 6206, 7920, 8192, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53660 payload_rows_remote=45472 dup_rows=11876 dup_rows_remote=8628 dedup_ratio=0.181213 router_tokens_per_expert=[0, 4077, 0, 0, 0, 4053, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3814, 0, 4003, 0, 0, 0, 0, 241, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 4096, 0, 4041, 3815, 442, 0, 0, 0, 4096, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8188, 8192, 8192, 4970, 8106, 4920, 3010, 8082] dup_rows_per_dst=[3248, 2, 7442, 290, 86, 644, 164, 0] combine_rows_per_src=[8188, 8192, 8192, 4970, 8106, 4920, 3010, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=353396736 dispatch_meta_bytes=211008 combine_payload_bytes=353396736 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=353607744 ingress_bytes=793798656 link_bytes=793798656 token_bytes=707004480 weight_movement_bytes=440401920 total_nvlink_bytes=1147406400 weight_to_token_ratio=0.622912 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.380546420e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=370571264 dispatch_meta_bytes=229064 combine_payload_bytes=370571264 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=370800328 ingress_bytes=480671744 link_bytes=480671744 token_bytes=741371592 weight_movement_bytes=110100480 total_nvlink_bytes=851472072 weight_to_token_ratio=0.148509 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.359672197e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62820 payload_rows_remote=54650 dup_rows=2716 dup_rows_remote=1702 dedup_ratio=0.041443 router_tokens_per_expert=[0, 0, 4051, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 4096, 5, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 750, 0, 0, 0, 0, 0, 508, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 3575, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3399, 0, 0, 0, 0, 189, 4084, 0] payload_rows_per_dst=[8104, 8192, 8192, 7912, 8170, 7150, 8192, 6908] dup_rows_per_dst=[124, 10, 0, 1300, 1014, 0, 0, 268] combine_rows_per_src=[8104, 8192, 8192, 7912, 8170, 7150, 8192, 6908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=380305408 dispatch_meta_bytes=224336 combine_payload_bytes=380305408 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=380529744 ingress_bytes=820707328 link_bytes=820707328 token_bytes=760835152 weight_movement_bytes=440401920 total_nvlink_bytes=1201237072 weight_to_token_ratio=0.578840 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.427345027e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=366184448 dispatch_meta_bytes=214312 combine_payload_bytes=366184448 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366398760 ingress_bytes=476284928 link_bytes=476284928 token_bytes=732583208 weight_movement_bytes=110100480 total_nvlink_bytes=842683688 weight_to_token_ratio=0.150291 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16896 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.283378252e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=336709632 dispatch_meta_bytes=229376 combine_payload_bytes=336709632 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=336939008 ingress_bytes=446810112 link_bytes=446810112 token_bytes=673648640 weight_movement_bytes=110100480 total_nvlink_bytes=783749120 weight_to_token_ratio=0.163439 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.770762724e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=325943296 dispatch_meta_bytes=216400 combine_payload_bytes=325943296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=326159696 ingress_bytes=656244736 link_bytes=656244736 token_bytes=652102992 weight_movement_bytes=330301440 total_nvlink_bytes=982404432 weight_to_token_ratio=0.506517 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.141317530e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58656 payload_rows_remote=50464 dup_rows=6880 dup_rows_remote=6880 dedup_ratio=0.104980 router_tokens_per_expert=[0, 0, 4068, 0, 0, 0, 79, 0, 0, 0, 0, 0, 0, 4096, 4, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1051, 0, 0, 1, 0, 0, 791, 3, 0, 0, 0, 0, 3783, 0, 0, 0, 3538, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2878, 0, 0, 0, 0, 191, 4093, 0] payload_rows_per_dst=[8142, 8192, 8192, 6124, 6116, 7228, 8192, 6470] dup_rows_per_dst=[152, 8, 0, 1482, 1162, 1910, 0, 2166] combine_rows_per_src=[8142, 8192, 8192, 6124, 6116, 7228, 8192, 6470] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=391731200 dispatch_meta_bytes=225408 combine_payload_bytes=391731200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=391956608 ingress_bytes=501831680 link_bytes=501831680 token_bytes=783687808 weight_movement_bytes=110100480 total_nvlink_bytes=893788288 weight_to_token_ratio=0.140490 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27392 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.727678286e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=361725952 dispatch_meta_bytes=229376 combine_payload_bytes=361725952 prefetch_bytes=0 reduce_bytes=0 egress_bytes=361955328 ingress_bytes=361725952 link_bytes=361955328 token_bytes=723681280 weight_movement_bytes=0 total_nvlink_bytes=723681280 weight_to_token_ratio=0.000000 gemm_rows_defined=32768 gemm_rows_charged=34816 gemm_tail_pad_rows=2048 gemm_inner_pad_upper_bound=254 gemm_tail_pad_fraction=0.058824 gemm_active_groups=2 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.294998468e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61872 payload_rows_remote=53682 dup_rows=3664 dup_rows_remote=3528 dedup_ratio=0.055908 router_tokens_per_expert=[0, 0, 4029, 0, 0, 0, 269, 0, 0, 0, 0, 0, 0, 4095, 68, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 892, 0, 0, 0, 0, 0, 661, 44, 0, 0, 0, 0, 3883, 0, 24, 0, 3137, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3395, 0, 0, 0, 0, 67, 4012, 0] payload_rows_per_dst=[8108, 8190, 8192, 7938, 8114, 6322, 8192, 6816] dup_rows_per_dst=[488, 136, 0, 1612, 1320, 0, 0, 108] combine_rows_per_src=[8108, 8190, 8192, 7938, 8114, 6322, 8192, 6816] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62030 payload_rows_remote=53838 dup_rows=3506 dup_rows_remote=3504 dedup_ratio=0.053497 router_tokens_per_expert=[0, 0, 4072, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 4094, 9, 0, 3, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1090, 0, 0, 0, 0, 0, 541, 4, 0, 0, 0, 0, 3783, 0, 7, 0, 3475, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 2978, 0, 0, 0, 0, 331, 4085, 0] payload_rows_per_dst=[8160, 8188, 8192, 7866, 8180, 6964, 8192, 6288] dup_rows_per_dst=[196, 18, 2, 1880, 1080, 0, 0, 330] combine_rows_per_src=[8160, 8188, 8192, 7866, 8180, 6964, 8192, 6288] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62864 payload_rows_remote=55176 dup_rows=2672 dup_rows_remote=2672 dedup_ratio=0.040771 router_tokens_per_expert=[0, 0, 4059, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 4095, 12, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 609, 0, 0, 1, 0, 0, 647, 6, 0, 0, 0, 0, 3878, 0, 3, 0, 3841, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3189, 0, 0, 0, 0, 67, 4066, 0] payload_rows_per_dst=[8132, 8192, 8192, 7910, 8152, 7688, 8192, 6406] dup_rows_per_dst=[190, 22, 2, 1064, 1288, 0, 0, 106] combine_rows_per_src=[8132, 8192, 8192, 7910, 8152, 7688, 8192, 6406] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62092 payload_rows_remote=54290 dup_rows=3444 dup_rows_remote=2248 dedup_ratio=0.052551 router_tokens_per_expert=[0, 0, 4037, 0, 0, 0, 152, 0, 0, 0, 0, 0, 0, 4095, 13, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 688, 0, 0, 0, 0, 0, 842, 3, 0, 0, 0, 0, 3811, 0, 1, 0, 3451, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3256, 0, 0, 0, 0, 145, 4082, 0] payload_rows_per_dst=[8074, 8190, 8192, 7802, 8172, 6904, 8192, 6566] dup_rows_per_dst=[304, 26, 0, 1196, 1682, 0, 0, 236] combine_rows_per_src=[8074, 8190, 8192, 7802, 8172, 6904, 8192, 6566] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51390 payload_rows_remote=43202 dup_rows=14146 dup_rows_remote=6812 dedup_ratio=0.215851 router_tokens_per_expert=[0, 0, 4057, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 4095, 4, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1140, 0, 0, 0, 0, 0, 846, 1, 0, 0, 0, 0, 3899, 0, 1, 0, 2661, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3574, 0, 0, 0, 0, 92, 4092, 0] payload_rows_per_dst=[6494, 8190, 8186, 2280, 1694, 8166, 8192, 8188] dup_rows_per_dst=[212, 8, 2, 0, 0, 6590, 0, 7334] combine_rows_per_src=[6494, 8190, 8186, 2280, 1694, 8166, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53462 payload_rows_remote=45270 dup_rows=12074 dup_rows_remote=12074 dedup_ratio=0.184235 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 145, 4096, 0, 0, 0, 38, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 4096, 1335, 0, 24, 0, 0, 0, 0, 3909, 0, 0, 0, 0, 0, 0, 2109, 0, 0, 0, 55, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7386, 8192, 7826, 4218, 1264, 8192] dup_rows_per_dst=[0, 9294, 70, 2670, 40, 0, 0, 0] combine_rows_per_src=[8192, 8192, 7386, 8192, 7826, 4218, 1264, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63884 payload_rows_remote=55692 dup_rows=1652 dup_rows_remote=1652 dedup_ratio=0.025208 router_tokens_per_expert=[6, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 0, 0, 0, 0, 0, 3779, 0, 0, 3592, 4096, 0, 0, 0, 2, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 0, 15, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8190, 8192, 7558, 7184, 8192, 8192, 8184, 8192] dup_rows_per_dst=[2, 1616, 0, 4, 0, 0, 0, 30] combine_rows_per_src=[8190, 8192, 7558, 7184, 8192, 8192, 8184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=384792576 dispatch_meta_bytes=228840 combine_payload_bytes=384792576 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=385021416 ingress_bytes=604993536 link_bytes=604993536 token_bytes=769813992 weight_movement_bytes=220200960 total_nvlink_bytes=990014952 weight_to_token_ratio=0.286044 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.052183263e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=385910784 dispatch_meta_bytes=229368 combine_payload_bytes=385910784 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=386140152 ingress_bytes=496011264 link_bytes=496011264 token_bytes=772050936 weight_movement_bytes=110100480 total_nvlink_bytes=882151416 weight_to_token_ratio=0.142608 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.626451679e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=395501568 dispatch_meta_bytes=231392 combine_payload_bytes=395501568 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=395732960 ingress_bytes=615702528 link_bytes=615702528 token_bytes=791234528 weight_movement_bytes=220200960 total_nvlink_bytes=1011435488 weight_to_token_ratio=0.278300 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070807962e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=389150720 dispatch_meta_bytes=226152 combine_payload_bytes=389150720 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=389376872 ingress_bytes=499251200 link_bytes=499251200 token_bytes=778527592 weight_movement_bytes=110100480 total_nvlink_bytes=888628072 weight_to_token_ratio=0.141421 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=25856 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.682799495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=309671936 dispatch_meta_bytes=200056 combine_payload_bytes=309671936 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=309871992 ingress_bytes=639973376 link_bytes=639973376 token_bytes=619543928 weight_movement_bytes=330301440 total_nvlink_bytes=949845368 weight_to_token_ratio=0.533136 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.113018958e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64086 payload_rows_remote=55898 dup_rows=1450 dup_rows_remote=1450 dedup_ratio=0.022125 router_tokens_per_expert=[4, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 2, 0, 361, 0, 0, 0, 0, 0, 0, 1, 3893, 0, 0, 3935, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4093, 0, 0, 4, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 7442, 7788, 7904, 8190, 8190, 8188, 8192] dup_rows_per_dst=[2, 658, 0, 782, 0, 0, 0, 8] combine_rows_per_src=[8192, 7442, 7788, 7904, 8190, 8190, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=324495360 dispatch_meta_bytes=229376 combine_payload_bytes=324495360 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324724736 ingress_bytes=434595840 link_bytes=434595840 token_bytes=649220096 weight_movement_bytes=110100480 total_nvlink_bytes=759320576 weight_to_token_ratio=0.169589 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.558336445e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=399200256 dispatch_meta_bytes=229376 combine_payload_bytes=399200256 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=399429632 ingress_bytes=949702656 link_bytes=949702656 token_bytes=798629888 weight_movement_bytes=550502400 total_nvlink_bytes=1349132288 weight_to_token_ratio=0.689309 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.651689118e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=400676864 dispatch_meta_bytes=229392 combine_payload_bytes=400676864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=400906256 ingress_bytes=510777344 link_bytes=510777344 token_bytes=801583120 weight_movement_bytes=110100480 total_nvlink_bytes=911683600 weight_to_token_ratio=0.137354 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.883258097e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7287851568 prefetch_bytes=1101004800 reduce_bytes=1321205760 weight_movement_bytes=2422210560 egress_bytes=3645073968 ingress_bytes=6064988160 link_bytes=6064988160 total_nvlink_bytes=9710062128 weight_to_token_ratio=0.332363 prefetch_share_of_ingress=0.181535 dedup_ratio=0.113538 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=12 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048897 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.054801193e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63828 payload_rows_remote=55636 dup_rows=1708 dup_rows_remote=240 dedup_ratio=0.026062 router_tokens_per_expert=[72, 0, 0, 0, 0, 4033, 0, 0, 0, 0, 0, 0, 44, 0, 705, 0, 0, 0, 0, 0, 0, 13, 3584, 0, 0, 3822, 4081, 0, 0, 0, 6, 0, 0, 0, 0, 4096, 0, 0, 0, 4065, 15, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 4022, 0, 0, 93, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 8192, 7194, 7644, 8130, 8188, 8102, 8192] dup_rows_per_dst=[24, 1468, 0, 12, 0, 16, 2, 186] combine_rows_per_src=[8186, 8192, 7194, 7644, 8130, 8188, 8102, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64616 payload_rows_remote=56428 dup_rows=920 dup_rows_remote=920 dedup_ratio=0.014038 router_tokens_per_expert=[15, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 0, 9, 0, 443, 0, 0, 0, 0, 0, 0, 6, 3811, 0, 0, 3926, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 4087, 0, 0, 3, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8192, 8192, 7634, 7852, 8182, 8188, 8184, 8192] dup_rows_per_dst=[8, 894, 0, 0, 0, 0, 4, 14] combine_rows_per_src=[8192, 8192, 7634, 7852, 8182, 8188, 8184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64278 payload_rows_remote=56684 dup_rows=1258 dup_rows_remote=1258 dedup_ratio=0.019196 router_tokens_per_expert=[16, 0, 0, 0, 0, 4060, 0, 0, 0, 0, 0, 0, 9, 0, 578, 0, 0, 0, 0, 0, 0, 4, 3793, 0, 0, 3812, 4092, 0, 0, 0, 7, 0, 0, 0, 0, 4096, 0, 0, 0, 4087, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 4079, 0, 0, 36, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[8152, 8192, 7594, 7624, 8174, 8186, 8164, 8192] dup_rows_per_dst=[0, 1166, 0, 14, 0, 0, 4, 74] combine_rows_per_src=[8152, 8192, 7594, 7624, 8174, 8186, 8164, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64490 payload_rows_remote=56722 dup_rows=1046 dup_rows_remote=1044 dedup_ratio=0.015961 router_tokens_per_expert=[6, 0, 0, 0, 0, 4083, 0, 0, 0, 0, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 1, 3793, 0, 0, 3884, 4096, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 12, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8176, 8192, 7588, 7768, 8192, 8192, 8190, 8192] dup_rows_per_dst=[2, 1018, 0, 2, 0, 0, 0, 24] combine_rows_per_src=[8176, 8192, 7588, 7768, 8192, 8192, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57620 payload_rows_remote=49428 dup_rows=7916 dup_rows_remote=7870 dedup_ratio=0.120789 router_tokens_per_expert=[6, 0, 0, 0, 0, 4087, 0, 0, 0, 0, 0, 0, 4, 0, 601, 0, 0, 0, 0, 0, 0, 2, 3716, 0, 0, 3861, 4094, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4094, 0, 0, 0, 4094, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 4086, 0, 0, 23, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 1210, 7436, 8192, 8040, 8188, 8176, 8192] dup_rows_per_dst=[0, 0, 0, 7718, 0, 4, 148, 46] combine_rows_per_src=[8186, 1210, 7436, 8192, 8040, 8188, 8176, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6998557792 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3500416096 ingress_bytes=5700151296 link_bytes=5700380672 total_nvlink_bytes=9200567392 weight_to_token_ratio=0.314638 prefetch_share_of_ingress=0.173838 dedup_ratio=0.142087 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.913899540e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62238 payload_rows_remote=54090 dup_rows=3298 dup_rows_remote=2790 dedup_ratio=0.050323 router_tokens_per_expert=[0, 0, 4074, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 743, 0, 0, 0, 0, 0, 602, 0, 0, 0, 0, 0, 3842, 0, 0, 0, 3307, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 3511, 0, 0, 0, 0, 182, 4094, 0] payload_rows_per_dst=[8148, 7974, 8192, 7886, 8188, 6614, 8192, 7044] dup_rows_per_dst=[508, 0, 0, 1284, 1204, 0, 0, 302] combine_rows_per_src=[8148, 7974, 8192, 7886, 8188, 6614, 8192, 7044] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=398798848 dispatch_meta_bytes=223504 combine_payload_bytes=398798848 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=399022352 ingress_bytes=508899328 link_bytes=508899328 token_bytes=797821200 weight_movement_bytes=110100480 total_nvlink_bytes=907921680 weight_to_token_ratio=0.138001 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.850596309e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406310912 dispatch_meta_bytes=231768 combine_payload_bytes=406310912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406542680 ingress_bytes=516411392 link_bytes=516411392 token_bytes=812853592 weight_movement_bytes=110100480 total_nvlink_bytes=922954072 weight_to_token_ratio=0.135449 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30336 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.981243458e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=404475904 dispatch_meta_bytes=229392 combine_payload_bytes=404475904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=404705296 ingress_bytes=514576384 link_bytes=514576384 token_bytes=809181200 weight_movement_bytes=110100480 total_nvlink_bytes=919281680 weight_to_token_ratio=0.136064 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.949329650e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406583296 dispatch_meta_bytes=231064 combine_payload_bytes=406583296 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=406814360 ingress_bytes=736884736 link_bytes=736884736 token_bytes=813397656 weight_movement_bytes=330301440 total_nvlink_bytes=1143699096 weight_to_token_ratio=0.406076 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28288 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.281563753e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=354299904 dispatch_meta_bytes=229192 combine_payload_bytes=354299904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=354529096 ingress_bytes=464400384 link_bytes=464400384 token_bytes=708829000 weight_movement_bytes=110100480 total_nvlink_bytes=818929480 weight_to_token_ratio=0.155327 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.076686485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=387717120 dispatch_meta_bytes=227520 combine_payload_bytes=387717120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=387944640 ingress_bytes=607918080 link_bytes=607918080 token_bytes=775661760 weight_movement_bytes=220200960 total_nvlink_bytes=995862720 weight_to_token_ratio=0.283888 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.057269526e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7006685016 prefetch_bytes=1211105280 reduce_bytes=660602880 weight_movement_bytes=1871708160 egress_bytes=3504471896 ingress_bytes=5373921280 link_bytes=5373921280 total_nvlink_bytes=8878393176 weight_to_token_ratio=0.267132 prefetch_share_of_ingress=0.225367 dedup_ratio=0.140723 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=6 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.050000 gemm_inner_pad_upper_bound=6731 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.346132964e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7277401880 prefetch_bytes=990904320 reduce_bytes=1101004800 weight_movement_bytes=2091909120 egress_bytes=3639856920 ingress_bytes=5729454080 link_bytes=5729454080 total_nvlink_bytes=9369311000 weight_to_token_ratio=0.287453 prefetch_share_of_ingress=0.172949 dedup_ratio=0.119373 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=10 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.964462978e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7148731904 prefetch_bytes=990904320 reduce_bytes=880803840 weight_movement_bytes=1871708160 egress_bytes=3575498240 ingress_bytes=5444941824 link_bytes=5444941824 total_nvlink_bytes=9020440064 weight_to_token_ratio=0.261824 prefetch_share_of_ingress=0.181986 dedup_ratio=0.122192 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=8 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047059 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.469649371e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6549982824 prefetch_bytes=1321205760 reduce_bytes=550502400 weight_movement_bytes=1871708160 egress_bytes=3276084840 ingress_bytes=5145606144 link_bytes=5145606144 total_nvlink_bytes=8421690984 weight_to_token_ratio=0.285758 prefetch_share_of_ingress=0.256764 dedup_ratio=0.182700 prefetch_slots=24 unused_prefetch_slots=136 slot_utilization=0.150000 reduce_slots=5 remote_demand=12 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=7366 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.949055391e-05 prefetch_bytes_per_gemm_flop=2.297794118e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7291291784 prefetch_bytes=990904320 reduce_bytes=1211105280 weight_movement_bytes=2202009600 egress_bytes=3646793864 ingress_bytes=5846507520 link_bytes=5846507520 total_nvlink_bytes=9493301384 weight_to_token_ratio=0.302005 prefetch_share_of_ingress=0.169487 dedup_ratio=0.115109 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=11 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.016803816e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59710 payload_rows_remote=51518 dup_rows=5826 dup_rows_remote=5794 dedup_ratio=0.088898 router_tokens_per_expert=[0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 0, 0, 0, 0, 3932, 0, 0, 3805, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8106, 2836, 8192, 8192, 8190, 7810] dup_rows_per_dst=[32, 912, 4638, 0, 244, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8106, 2836, 8192, 8192, 8190, 7810] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369281024 dispatch_meta_bytes=229248 combine_payload_bytes=369281024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369510272 ingress_bytes=479381504 link_bytes=479381504 token_bytes=738791296 weight_movement_bytes=110100480 total_nvlink_bytes=848891776 weight_to_token_ratio=0.149028 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.337232802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6558605088 prefetch_bytes=880803840 reduce_bytes=1541406720 weight_movement_bytes=2422210560 egress_bytes=3280377632 ingress_bytes=5700438016 link_bytes=5700438016 total_nvlink_bytes=8980815648 weight_to_token_ratio=0.369318 prefetch_share_of_ingress=0.154515 dedup_ratio=0.180762 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=14 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.051103 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.913999271e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none - [2026-08-26 16:57:48] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 366.5 | throughput per GPU (TFLOP/s/GPU): 337.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.600756E+00 | loss scale: 1.0 | grad norm: 4.394 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43906 payload_rows_remote=38206 dup_rows=21630 dup_rows_remote=19228 dedup_ratio=0.330048 router_tokens_per_expert=[822, 810, 780, 608, 487, 774, 576, 155, 716, 666, 396, 416, 15, 1031, 249, 726, 586, 873, 189, 99, 803, 372, 262, 867, 386, 1425, 601, 717, 1161, 396, 914, 10, 214, 633, 405, 12, 936, 7, 1237, 393, 207, 645, 721, 1154, 2, 473, 178, 770, 247, 251, 120, 759, 192, 540, 206, 678, 606, 512, 252, 338, 296, 426, 453, 17] payload_rows_per_dst=[6108, 5492, 5700, 6638, 5270, 5730, 4518, 4450] dup_rows_per_dst=[3916, 2938, 2402, 4582, 2404, 2570, 1468, 1350] combine_rows_per_src=[6108, 5492, 5700, 6638, 5270, 5730, 4518, 4450] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44332 payload_rows_remote=38618 dup_rows=21204 dup_rows_remote=18112 dedup_ratio=0.323547 router_tokens_per_expert=[1508, 545, 615, 242, 479, 539, 555, 182, 286, 1004, 414, 707, 7, 467, 68, 823, 540, 1144, 41, 46, 644, 211, 850, 535, 683, 559, 341, 841, 486, 288, 756, 24, 963, 602, 358, 17, 731, 12, 1190, 309, 476, 780, 258, 995, 4, 645, 249, 729, 220, 449, 481, 704, 277, 901, 138, 704, 566, 903, 421, 367, 426, 1071, 378, 14] payload_rows_per_dst=[5858, 5178, 5762, 5586, 5714, 5482, 5232, 5520] dup_rows_per_dst=[2898, 2374, 2260, 2370, 3092, 2790, 2648, 2772] combine_rows_per_src=[5858, 5178, 5762, 5586, 5714, 5482, 5232, 5520] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43586 payload_rows_remote=37340 dup_rows=21950 dup_rows_remote=17908 dedup_ratio=0.334930 router_tokens_per_expert=[1075, 1352, 876, 288, 320, 944, 510, 126, 668, 990, 281, 521, 16, 904, 160, 674, 337, 826, 3, 72, 612, 491, 738, 453, 457, 293, 569, 546, 416, 423, 811, 12, 998, 1067, 526, 36, 377, 12, 696, 845, 619, 522, 284, 751, 0, 558, 137, 628, 612, 182, 482, 992, 213, 763, 117, 708, 574, 672, 594, 677, 316, 824, 210, 12] payload_rows_per_dst=[3394, 5444, 6142, 5406, 5482, 5372, 6246, 6100] dup_rows_per_dst=[846, 2760, 2810, 1872, 2240, 3018, 4042, 4362] combine_rows_per_src=[3394, 5444, 6142, 5406, 5482, 5372, 6246, 6100] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44086 payload_rows_remote=38912 dup_rows=21450 dup_rows_remote=19820 dedup_ratio=0.327301 router_tokens_per_expert=[1290, 898, 488, 662, 443, 791, 435, 233, 393, 822, 505, 524, 15, 786, 93, 963, 473, 1227, 8, 94, 1017, 410, 739, 421, 600, 424, 349, 639, 364, 353, 663, 10, 794, 525, 570, 12, 1086, 3, 932, 550, 315, 584, 480, 1195, 7, 481, 48, 397, 248, 194, 702, 626, 442, 794, 250, 538, 360, 952, 583, 322, 350, 866, 392, 38] payload_rows_per_dst=[6368, 5410, 6082, 5174, 5644, 5090, 5202, 5116] dup_rows_per_dst=[4112, 2792, 2696, 1630, 3300, 1924, 2386, 2610] combine_rows_per_src=[6368, 5410, 6082, 5174, 5644, 5090, 5202, 5116] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44918 payload_rows_remote=39072 dup_rows=20618 dup_rows_remote=17374 dedup_ratio=0.314606 router_tokens_per_expert=[1230, 1038, 382, 875, 297, 611, 589, 23, 259, 831, 631, 307, 15, 970, 32, 958, 529, 810, 3, 31, 847, 236, 893, 435, 292, 588, 356, 772, 779, 253, 810, 5, 579, 568, 426, 3, 1005, 0, 1035, 572, 653, 674, 319, 1523, 0, 773, 136, 424, 449, 390, 526, 445, 291, 758, 90, 401, 460, 1341, 634, 262, 455, 579, 304, 6] payload_rows_per_dst=[4980, 5484, 6180, 5390, 5362, 5846, 5962, 5714] dup_rows_per_dst=[1508, 2522, 2162, 2320, 2928, 3244, 3198, 2736] combine_rows_per_src=[4980, 5484, 6180, 5390, 5362, 5846, 5962, 5714] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44290 payload_rows_remote=38124 dup_rows=21246 dup_rows_remote=17356 dedup_ratio=0.324188 router_tokens_per_expert=[765, 1545, 1158, 521, 495, 1109, 823, 54, 777, 926, 606, 313, 29, 1004, 188, 781, 469, 524, 9, 73, 760, 245, 617, 313, 374, 685, 430, 751, 663, 332, 537, 17, 866, 920, 524, 3, 719, 4, 1130, 446, 248, 723, 292, 1231, 0, 375, 139, 438, 213, 383, 580, 510, 196, 660, 72, 720, 918, 532, 442, 268, 478, 455, 384, 6] payload_rows_per_dst=[4670, 5404, 5946, 6118, 4872, 5670, 5444, 6166] dup_rows_per_dst=[1432, 1992, 2292, 3312, 2092, 3482, 2754, 3890] combine_rows_per_src=[4670, 5404, 5946, 6118, 4872, 5670, 5444, 6166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273860608 dispatch_meta_bytes=229736 combine_payload_bytes=273860608 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274090344 ingress_bytes=383961088 link_bytes=383961088 token_bytes=547950952 weight_movement_bytes=110100480 total_nvlink_bytes=658051432 weight_to_token_ratio=0.200931 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6656 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.677714828e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276813824 dispatch_meta_bytes=226920 combine_payload_bytes=276813824 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=277040744 ingress_bytes=497014784 link_bytes=497014784 token_bytes=553854568 weight_movement_bytes=220200960 total_nvlink_bytes=774055528 weight_to_token_ratio=0.397579 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.643904543e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=267653120 dispatch_meta_bytes=220992 combine_payload_bytes=267653120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=267874112 ingress_bytes=377753600 link_bytes=377753600 token_bytes=535527232 weight_movement_bytes=110100480 total_nvlink_bytes=645627712 weight_to_token_ratio=0.205593 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.569756402e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278921216 dispatch_meta_bytes=234928 combine_payload_bytes=278921216 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279156144 ingress_bytes=389021696 link_bytes=389021696 token_bytes=558077360 weight_movement_bytes=110100480 total_nvlink_bytes=668177840 weight_to_token_ratio=0.197285 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.036765 gemm_active_groups=10 gemm_peak_group_rows=6144 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.765727124e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=280068096 dispatch_meta_bytes=225784 combine_payload_bytes=280068096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=280293880 ingress_bytes=390168576 link_bytes=390168576 token_bytes=560361976 weight_movement_bytes=110100480 total_nvlink_bytes=670462456 weight_to_token_ratio=0.196481 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=8320 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.785673254e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=273272832 dispatch_meta_bytes=221920 combine_payload_bytes=273272832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=273494752 ingress_bytes=383373312 link_bytes=383373312 token_bytes=546767584 weight_movement_bytes=110100480 total_nvlink_bytes=656868064 weight_to_token_ratio=0.201366 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.667492436e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43862 payload_rows_remote=37960 dup_rows=21674 dup_rows_remote=17830 dedup_ratio=0.330719 router_tokens_per_expert=[1142, 330, 589, 349, 489, 1100, 777, 71, 225, 939, 710, 534, 13, 1179, 117, 623, 946, 488, 14, 16, 316, 448, 695, 964, 273, 444, 491, 1137, 438, 563, 605, 20, 864, 342, 442, 12, 811, 0, 1047, 385, 557, 573, 284, 653, 2, 718, 472, 462, 450, 376, 358, 408, 368, 1606, 91, 662, 522, 884, 552, 175, 228, 1057, 360, 2] payload_rows_per_dst=[5902, 5500, 5370, 5762, 5188, 5078, 5820, 5242] dup_rows_per_dst=[3844, 3128, 2404, 2180, 2618, 2364, 2818, 2318] combine_rows_per_src=[5902, 5500, 5370, 5762, 5188, 5078, 5820, 5242] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49316 payload_rows_remote=41580 dup_rows=16220 dup_rows_remote=11544 dedup_ratio=0.247498 router_tokens_per_expert=[1, 0, 0, 48, 1044, 3, 2522, 125, 1496, 1031, 63, 0, 0, 2099, 0, 3483, 2, 1398, 0, 0, 940, 3422, 0, 0, 0, 1, 1228, 77, 1013, 180, 0, 31, 1, 8, 0, 0, 0, 16, 0, 0, 0, 1, 1007, 0, 3, 0, 1410, 1, 0, 2174, 801, 0, 2537, 0, 0, 0, 1450, 0, 0, 36, 678, 545, 979, 914] payload_rows_per_dst=[6644, 6392, 7736, 5254, 6984, 4280, 6808, 5218] dup_rows_per_dst=[842, 2986, 4676, 2706, 32, 564, 3328, 1086] combine_rows_per_src=[6644, 6392, 7736, 5254, 6984, 4280, 6808, 5218] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49284 payload_rows_remote=43018 dup_rows=16252 dup_rows_remote=16238 dedup_ratio=0.247986 router_tokens_per_expert=[0, 0, 0, 62, 1625, 1, 2392, 242, 2280, 732, 52, 0, 0, 1903, 0, 3129, 0, 696, 0, 2, 739, 3380, 0, 0, 1, 0, 1369, 39, 1488, 59, 1, 2, 0, 0, 0, 1, 0, 10, 0, 0, 0, 3, 595, 2, 0, 1, 2093, 0, 0, 2200, 664, 0, 2099, 0, 0, 0, 1893, 0, 0, 57, 543, 368, 787, 1258] payload_rows_per_dst=[6966, 6212, 4722, 6152, 6266, 6618, 6686, 5662] dup_rows_per_dst=[1678, 2456, 1316, 3552, 14, 2366, 3240, 1630] combine_rows_per_src=[6966, 6212, 4722, 6152, 6266, 6618, 6686, 5662] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48938 payload_rows_remote=43010 dup_rows=16598 dup_rows_remote=13460 dedup_ratio=0.253265 router_tokens_per_expert=[0, 0, 0, 108, 1566, 0, 1960, 199, 1765, 923, 64, 0, 0, 1996, 0, 3169, 0, 1334, 0, 0, 834, 3150, 0, 0, 0, 0, 800, 230, 1692, 91, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 682, 1, 0, 0, 1668, 0, 0, 2123, 719, 0, 2295, 0, 0, 0, 1719, 0, 1, 43, 827, 698, 646, 1462] payload_rows_per_dst=[6336, 6422, 7336, 5928, 6340, 4296, 6682, 5598] dup_rows_per_dst=[1330, 3074, 3300, 3138, 0, 408, 3592, 1756] combine_rows_per_src=[6336, 6422, 7336, 5928, 6340, 4296, 6682, 5598] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46510 payload_rows_remote=39790 dup_rows=19026 dup_rows_remote=15682 dedup_ratio=0.290314 router_tokens_per_expert=[0, 0, 0, 13, 1899, 0, 2879, 151, 1541, 1731, 70, 0, 0, 2648, 0, 2907, 0, 917, 0, 0, 385, 3321, 0, 0, 0, 0, 423, 57, 1357, 108, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 0, 1832, 0, 0, 2451, 713, 0, 1867, 0, 0, 1, 1814, 0, 0, 39, 847, 224, 1125, 1014] payload_rows_per_dst=[7590, 3568, 2492, 3476, 7468, 7706, 6720, 7490] dup_rows_per_dst=[2294, 34, 112, 418, 3642, 3464, 3344, 5718] combine_rows_per_src=[7590, 3568, 2492, 3476, 7468, 7706, 6720, 7490] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47368 payload_rows_remote=39362 dup_rows=18168 dup_rows_remote=13590 dedup_ratio=0.277222 router_tokens_per_expert=[0, 0, 0, 99, 1066, 0, 2151, 172, 1000, 1059, 9, 0, 0, 3247, 0, 3314, 0, 1175, 0, 0, 524, 3612, 0, 0, 0, 0, 710, 60, 1225, 23, 0, 27, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 379, 0, 0, 0, 2301, 0, 0, 2349, 755, 0, 2233, 0, 0, 0, 2231, 3, 1, 56, 803, 478, 711, 988] payload_rows_per_dst=[5930, 3856, 3238, 6096, 7440, 8006, 6894, 5908] dup_rows_per_dst=[1046, 614, 160, 2434, 3362, 4578, 3780, 2194] combine_rows_per_src=[5930, 3856, 3238, 6096, 7440, 8006, 6894, 5908] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47800 payload_rows_remote=40518 dup_rows=17736 dup_rows_remote=13520 dedup_ratio=0.270630 router_tokens_per_expert=[0, 0, 0, 67, 1597, 0, 2546, 181, 1142, 1885, 17, 0, 1, 2547, 0, 3539, 4, 1121, 0, 0, 496, 3397, 0, 0, 0, 0, 995, 151, 1627, 112, 1, 5, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 304, 0, 0, 4, 1949, 0, 0, 2204, 695, 0, 1569, 0, 0, 2, 1938, 0, 0, 29, 538, 155, 787, 1160] payload_rows_per_dst=[7172, 3792, 3096, 4544, 7858, 7784, 6272, 7282] dup_rows_per_dst=[1610, 14, 146, 1238, 4320, 3524, 2668, 4216] combine_rows_per_src=[7172, 3792, 3096, 4544, 7858, 7784, 6272, 7282] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=298045440 dispatch_meta_bytes=212496 combine_payload_bytes=298045440 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=298257936 ingress_bytes=518246400 link_bytes=518246400 token_bytes=596303376 weight_movement_bytes=220200960 total_nvlink_bytes=816504336 weight_to_token_ratio=0.369277 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=14720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.013157265e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=308353024 dispatch_meta_bytes=237024 combine_payload_bytes=308353024 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=308590048 ingress_bytes=528553984 link_bytes=528553984 token_bytes=616943072 weight_movement_bytes=220200960 total_nvlink_bytes=837144032 weight_to_token_ratio=0.356923 gemm_rows_defined=33664 gemm_rows_charged=34816 gemm_tail_pad_rows=1152 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.033088 gemm_active_groups=10 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.192423104e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=272097280 dispatch_meta_bytes=223160 combine_payload_bytes=272097280 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=272320440 ingress_bytes=822599680 link_bytes=822599680 token_bytes=544417720 weight_movement_bytes=550502400 total_nvlink_bytes=1094920120 weight_to_token_ratio=1.011176 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=5888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.430636138e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=308295680 dispatch_meta_bytes=225880 combine_payload_bytes=308295680 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308521560 ingress_bytes=418396160 link_bytes=418396160 token_bytes=616817240 weight_movement_bytes=110100480 total_nvlink_bytes=726917720 weight_to_token_ratio=0.178498 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=11520 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.276597366e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=285214720 dispatch_meta_bytes=221888 combine_payload_bytes=285214720 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=285436608 ingress_bytes=615516160 link_bytes=615516160 token_bytes=570651328 weight_movement_bytes=330301440 total_nvlink_bytes=900952768 weight_to_token_ratio=0.578815 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=14080 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.070483837e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=282146816 dispatch_meta_bytes=211808 combine_payload_bytes=282146816 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=282358624 ingress_bytes=392247296 link_bytes=392247296 token_bytes=564505440 weight_movement_bytes=110100480 total_nvlink_bytes=674605920 weight_to_token_ratio=0.195039 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=15616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.821825613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=290433024 dispatch_meta_bytes=216152 combine_payload_bytes=290433024 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=290649176 ingress_bytes=510633984 link_bytes=510633984 token_bytes=581082200 weight_movement_bytes=220200960 total_nvlink_bytes=801283160 weight_to_token_ratio=0.378950 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=9728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.880764830e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44844 payload_rows_remote=38792 dup_rows=20692 dup_rows_remote=17626 dedup_ratio=0.315735 router_tokens_per_expert=[1301, 799, 585, 291, 471, 587, 591, 225, 284, 1222, 899, 336, 9, 849, 212, 748, 717, 747, 44, 131, 764, 263, 453, 545, 531, 628, 381, 683, 628, 252, 459, 36, 784, 702, 1121, 15, 819, 13, 938, 273, 497, 924, 374, 769, 0, 760, 107, 507, 354, 208, 409, 596, 188, 1061, 168, 915, 432, 1253, 561, 253, 470, 264, 350, 12] payload_rows_per_dst=[6220, 6052, 5490, 5174, 5822, 5316, 5518, 5252] dup_rows_per_dst=[3480, 3066, 1838, 2022, 3508, 2560, 2280, 1938] combine_rows_per_src=[6220, 6052, 5490, 5174, 5822, 5316, 5518, 5252] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55724 payload_rows_remote=49046 dup_rows=9812 dup_rows_remote=7954 dedup_ratio=0.149719 router_tokens_per_expert=[0, 0, 0, 1, 1, 0, 658, 0, 0, 147, 0, 0, 0, 1, 204, 0, 2563, 1, 0, 2330, 1, 1936, 4006, 0, 0, 0, 2524, 4017, 0, 2876, 1, 0, 2, 1, 0, 0, 0, 3376, 44, 4, 2266, 0, 2647, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2857, 0, 139, 159, 0, 0, 0, 3] payload_rows_per_dst=[5572, 8064, 6678, 7518, 6774, 7000, 8012, 6106] dup_rows_per_dst=[874, 674, 1858, 3284, 80, 2826, 6, 210] combine_rows_per_src=[5572, 8064, 6678, 7518, 6774, 7000, 8012, 6106] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55030 payload_rows_remote=48716 dup_rows=10506 dup_rows_remote=10450 dedup_ratio=0.160309 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 1560, 0, 0, 123, 0, 0, 0, 0, 482, 0, 2702, 0, 0, 2702, 0, 1718, 3801, 0, 0, 0, 2492, 4033, 0, 2105, 0, 0, 0, 1, 0, 0, 0, 3155, 28, 1, 2741, 0, 2158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2826, 1, 45, 91, 0, 0, 0, 1] payload_rows_per_dst=[6360, 8144, 6888, 7040, 6314, 6874, 7602, 5808] dup_rows_per_dst=[2168, 1132, 1952, 2154, 56, 2924, 0, 120] combine_rows_per_src=[6360, 8144, 6888, 7040, 6314, 6874, 7602, 5808] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 2, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351561728 dispatch_meta_bytes=228000 combine_payload_bytes=351561728 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=351789728 ingress_bytes=902064128 link_bytes=902064128 token_bytes=703351456 weight_movement_bytes=550502400 total_nvlink_bytes=1253853856 weight_to_token_ratio=0.782685 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=16512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.568837883e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=349196288 dispatch_meta_bytes=236664 combine_payload_bytes=349196288 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349432952 ingress_bytes=459296768 link_bytes=459296768 token_bytes=698629240 weight_movement_bytes=110100480 total_nvlink_bytes=808729720 weight_to_token_ratio=0.157595 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.987926209e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56042 payload_rows_remote=49100 dup_rows=9494 dup_rows_remote=7452 dedup_ratio=0.144867 router_tokens_per_expert=[0, 0, 0, 4, 0, 1, 967, 0, 0, 136, 0, 0, 0, 0, 451, 0, 3063, 0, 0, 2613, 0, 1835, 3957, 0, 0, 0, 2146, 3985, 0, 2346, 0, 0, 0, 0, 0, 0, 0, 3509, 32, 1, 2169, 0, 2336, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2838, 0, 238, 121, 0, 0, 0, 17] payload_rows_per_dst=[6508, 8090, 6888, 6942, 7030, 6548, 7914, 6122] dup_rows_per_dst=[1562, 1054, 2008, 2042, 54, 2462, 6, 306] combine_rows_per_src=[6508, 8090, 6888, 6942, 7030, 6548, 7914, 6122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54048 payload_rows_remote=48320 dup_rows=11488 dup_rows_remote=10710 dedup_ratio=0.175293 router_tokens_per_expert=[0, 0, 0, 15, 0, 0, 1139, 0, 0, 131, 0, 0, 0, 0, 190, 0, 3326, 0, 0, 2981, 0, 1552, 4053, 0, 0, 0, 2498, 4028, 0, 1376, 0, 0, 0, 0, 0, 0, 0, 3210, 46, 0, 2913, 0, 2509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2618, 0, 125, 58, 0, 0, 0, 0] payload_rows_per_dst=[6980, 8144, 7020, 3628, 7500, 5728, 8106, 6942] dup_rows_per_dst=[1980, 554, 2046, 412, 2720, 778, 0, 2998] combine_rows_per_src=[6980, 8144, 7020, 3628, 7500, 5728, 8106, 6942] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53620 payload_rows_remote=45716 dup_rows=11916 dup_rows_remote=11902 dedup_ratio=0.181824 router_tokens_per_expert=[0, 0, 0, 6, 0, 0, 1220, 0, 0, 126, 0, 0, 0, 0, 253, 0, 3011, 0, 0, 2758, 0, 2836, 3952, 0, 0, 0, 1974, 4055, 0, 1689, 0, 0, 0, 0, 0, 0, 0, 3225, 60, 0, 2795, 0, 1788, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 2932, 0, 45, 36, 0, 0, 0, 0] payload_rows_per_dst=[6544, 8132, 5900, 6042, 7660, 3682, 7904, 7756] dup_rows_per_dst=[1930, 736, 1060, 1352, 2858, 106, 14, 3860] combine_rows_per_src=[6544, 8132, 5900, 6042, 7660, 3682, 7904, 7756] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278061056 dispatch_meta_bytes=225672 combine_payload_bytes=278061056 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=278286728 ingress_bytes=498262016 link_bytes=498262016 token_bytes=556347784 weight_movement_bytes=220200960 total_nvlink_bytes=776548744 weight_to_token_ratio=0.395797 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665595958e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49846 payload_rows_remote=42142 dup_rows=15690 dup_rows_remote=11396 dedup_ratio=0.239410 router_tokens_per_expert=[0, 0, 0, 8, 2, 0, 719, 0, 0, 110, 0, 0, 0, 0, 84, 0, 2862, 0, 0, 2855, 0, 2658, 4028, 0, 0, 0, 2334, 3973, 0, 1850, 1, 0, 0, 0, 0, 0, 0, 3220, 41, 4, 2705, 0, 2019, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3154, 0, 38, 102, 0, 0, 0, 0] payload_rows_per_dst=[6066, 1976, 5340, 7206, 7730, 5736, 8088, 7704] dup_rows_per_dst=[1074, 68, 18, 2786, 3468, 1460, 2522, 4294] combine_rows_per_src=[6066, 1976, 5340, 7206, 7730, 5736, 8088, 7704] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45328 payload_rows_remote=38738 dup_rows=20208 dup_rows_remote=18450 dedup_ratio=0.308350 router_tokens_per_expert=[0, 0, 0, 0, 1491, 0, 2016, 185, 1833, 1079, 73, 0, 0, 2563, 0, 2751, 2, 1014, 0, 0, 530, 3491, 0, 0, 0, 0, 1406, 78, 1536, 65, 0, 18, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 1038, 0, 0, 0, 2171, 0, 0, 2021, 577, 0, 1646, 0, 0, 0, 2212, 0, 0, 69, 618, 493, 653, 1130] payload_rows_per_dst=[6590, 7292, 8008, 6616, 5510, 5332, 1154, 4826] dup_rows_per_dst=[1758, 6132, 6108, 4014, 10, 1086, 0, 1100] combine_rows_per_src=[6590, 7292, 8008, 6616, 5510, 5332, 1154, 4826] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=351948800 dispatch_meta_bytes=226208 combine_payload_bytes=351948800 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=352175008 ingress_bytes=682250240 link_bytes=682250240 token_bytes=704123808 weight_movement_bytes=330301440 total_nvlink_bytes=1034425248 weight_to_token_ratio=0.469096 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=15360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.186545378e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=346357760 dispatch_meta_bytes=236120 combine_payload_bytes=346357760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=346593880 ingress_bytes=566558720 link_bytes=566558720 token_bytes=692951640 weight_movement_bytes=220200960 total_nvlink_bytes=913152600 weight_to_token_ratio=0.317772 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.853387970e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=327692288 dispatch_meta_bytes=230472 combine_payload_bytes=327692288 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=327922760 ingress_bytes=547893248 link_bytes=547893248 token_bytes=655615048 weight_movement_bytes=220200960 total_nvlink_bytes=875816008 weight_to_token_ratio=0.335869 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.528764712e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=302073856 dispatch_meta_bytes=214152 combine_payload_bytes=302073856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302288008 ingress_bytes=412174336 link_bytes=412174336 token_bytes=604361864 weight_movement_bytes=110100480 total_nvlink_bytes=714462344 weight_to_token_ratio=0.182176 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=23424 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.168389613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52938 payload_rows_remote=45236 dup_rows=12598 dup_rows_remote=10310 dedup_ratio=0.192230 router_tokens_per_expert=[22, 0, 3565, 0, 0, 63, 0, 0, 3, 0, 0, 2065, 6, 0, 0, 0, 0, 0, 169, 0, 0, 0, 1479, 0, 0, 533, 0, 3127, 0, 0, 3, 0, 0, 0, 0, 0, 4013, 2922, 0, 172, 17, 0, 0, 0, 0, 0, 176, 0, 0, 0, 3183, 1, 0, 0, 0, 0, 3815, 0, 0, 0, 0, 3347, 4087, 0] payload_rows_per_dst=[7192, 8148, 7702, 7236, 8102, 8192, 6366, 0] dup_rows_per_dst=[108, 3630, 2288, 3616, 2586, 368, 2, 0] combine_rows_per_src=[7192, 8148, 7702, 7236, 8102, 8192, 6366, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=277673984 dispatch_meta_bytes=228752 combine_payload_bytes=277673984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=277902736 ingress_bytes=387774464 link_bytes=387774464 token_bytes=555576720 weight_movement_bytes=110100480 total_nvlink_bytes=665677200 weight_to_token_ratio=0.198173 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=18560 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.744035708e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52286 payload_rows_remote=45946 dup_rows=13250 dup_rows_remote=13012 dedup_ratio=0.202179 router_tokens_per_expert=[65, 0, 3429, 0, 0, 488, 0, 0, 6, 0, 0, 1807, 0, 0, 0, 0, 0, 0, 186, 0, 0, 0, 1952, 0, 0, 705, 0, 3185, 0, 0, 0, 0, 0, 0, 0, 0, 4022, 3095, 0, 194, 4, 0, 0, 1, 0, 0, 178, 0, 0, 3, 2408, 1, 0, 0, 0, 0, 3498, 0, 0, 0, 0, 3446, 4095, 0] payload_rows_per_dst=[7102, 3680, 4170, 6760, 6340, 8192, 8168, 7874] dup_rows_per_dst=[862, 12, 106, 1020, 238, 364, 4700, 5948] combine_rows_per_src=[7102, 3680, 4170, 6760, 6340, 8192, 8168, 7874] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=324251648 dispatch_meta_bytes=222184 combine_payload_bytes=324251648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324473832 ingress_bytes=434352128 link_bytes=434352128 token_bytes=648725480 weight_movement_bytes=110100480 total_nvlink_bytes=758825960 weight_to_token_ratio=0.169718 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=16768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.554097892e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48342 payload_rows_remote=41056 dup_rows=17194 dup_rows_remote=11788 dedup_ratio=0.262360 router_tokens_per_expert=[0, 0, 0, 39, 1701, 0, 2016, 119, 1916, 1145, 136, 0, 0, 2782, 0, 3135, 1, 671, 0, 0, 693, 2954, 0, 0, 0, 0, 1469, 178, 1493, 122, 0, 10, 0, 0, 1, 0, 5, 2, 0, 2, 0, 1, 571, 0, 1, 0, 2189, 0, 0, 2226, 767, 0, 1780, 0, 0, 4, 1713, 4, 0, 31, 230, 305, 647, 1709] payload_rows_per_dst=[6570, 7286, 7772, 6286, 6282, 5180, 3936, 5030] dup_rows_per_dst=[1180, 5406, 5318, 3684, 8, 344, 432, 822] combine_rows_per_src=[6570, 7286, 7772, 6286, 6282, 5180, 3936, 5030] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=329340928 dispatch_meta_bytes=235832 combine_payload_bytes=329340928 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=329576760 ingress_bytes=769742848 link_bytes=769742848 token_bytes=658917688 weight_movement_bytes=440401920 total_nvlink_bytes=1099319608 weight_to_token_ratio=0.668372 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=15488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.338709414e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53558 payload_rows_remote=45366 dup_rows=11978 dup_rows_remote=11570 dedup_ratio=0.182770 router_tokens_per_expert=[68, 0, 2856, 0, 0, 271, 0, 0, 0, 0, 0, 2563, 0, 0, 0, 0, 0, 0, 569, 0, 0, 0, 2586, 0, 0, 540, 0, 3095, 0, 0, 0, 0, 0, 0, 0, 0, 3916, 2550, 0, 518, 2, 0, 0, 0, 0, 0, 202, 0, 0, 1, 1579, 0, 0, 0, 0, 0, 3916, 0, 0, 0, 0, 3440, 4096, 0] payload_rows_per_dst=[6034, 5126, 5850, 6588, 5660, 8192, 8076, 8032] dup_rows_per_dst=[356, 0, 460, 682, 476, 408, 2916, 6680] combine_rows_per_src=[6034, 5126, 5850, 6588, 5660, 8192, 8076, 8032] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50446 payload_rows_remote=43554 dup_rows=15090 dup_rows_remote=14202 dedup_ratio=0.230255 router_tokens_per_expert=[90, 0, 3060, 0, 0, 131, 0, 0, 6, 0, 0, 2469, 5, 0, 0, 0, 0, 0, 274, 0, 0, 0, 1879, 0, 0, 658, 0, 3231, 0, 0, 1, 0, 0, 0, 0, 0, 4032, 3378, 0, 116, 10, 0, 0, 0, 0, 0, 152, 0, 0, 1, 2410, 0, 0, 0, 0, 0, 3718, 0, 0, 0, 0, 3055, 4092, 0] payload_rows_per_dst=[6396, 8060, 7376, 6892, 7720, 8192, 5544, 266] dup_rows_per_dst=[166, 4336, 2774, 888, 5536, 316, 1074, 0] combine_rows_per_src=[6396, 8060, 7376, 6892, 7720, 8192, 5544, 266] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53268 payload_rows_remote=45174 dup_rows=12268 dup_rows_remote=8914 dedup_ratio=0.187195 router_tokens_per_expert=[84, 0, 2984, 0, 0, 261, 0, 0, 0, 0, 0, 2676, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 2691, 0, 0, 762, 0, 2682, 0, 0, 0, 0, 0, 0, 0, 0, 3896, 2578, 0, 342, 4, 0, 0, 0, 0, 0, 187, 0, 0, 8, 1820, 0, 0, 0, 0, 0, 3676, 0, 0, 0, 0, 3539, 4096, 0] payload_rows_per_dst=[6240, 5352, 5830, 6066, 5502, 8192, 8094, 7992] dup_rows_per_dst=[418, 0, 516, 822, 338, 382, 3354, 6438] combine_rows_per_src=[6240, 5352, 5830, 6066, 5502, 8192, 8094, 7992] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47410 payload_rows_remote=39288 dup_rows=18126 dup_rows_remote=3942 dedup_ratio=0.276581 router_tokens_per_expert=[53, 0, 3505, 0, 0, 256, 0, 0, 4, 0, 0, 1886, 4, 0, 0, 0, 0, 0, 464, 0, 0, 0, 2326, 0, 0, 533, 0, 2925, 0, 1, 2, 0, 0, 0, 0, 0, 3987, 2686, 0, 225, 1, 0, 0, 0, 0, 0, 199, 0, 0, 1, 2294, 0, 0, 0, 0, 0, 3772, 0, 0, 0, 0, 3553, 4091, 0] payload_rows_per_dst=[7128, 3780, 5240, 6292, 5590, 5300, 5958, 8122] dup_rows_per_dst=[500, 8, 340, 630, 232, 248, 1984, 14184] combine_rows_per_src=[7128, 3780, 5240, 6292, 5590, 5300, 5958, 8122] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55514 payload_rows_remote=48940 dup_rows=10022 dup_rows_remote=8218 dedup_ratio=0.152924 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 1180, 0, 0, 109, 0, 0, 0, 0, 559, 0, 3008, 0, 0, 2581, 0, 2022, 3717, 0, 0, 0, 2660, 3997, 0, 1597, 0, 0, 0, 0, 0, 0, 0, 3218, 173, 0, 2345, 0, 2226, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3188, 0, 18, 164, 0, 0, 0, 3] payload_rows_per_dst=[6574, 8076, 6964, 6802, 6554, 6636, 7434, 6474] dup_rows_per_dst=[1804, 1254, 2242, 1712, 228, 2506, 4, 272] combine_rows_per_src=[6574, 8076, 6964, 6802, 6554, 6636, 7434, 6474] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=294289408 dispatch_meta_bytes=211376 combine_payload_bytes=294289408 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=294500784 ingress_bytes=734691328 link_bytes=734691328 token_bytes=588790192 weight_movement_bytes=440401920 total_nvlink_bytes=1029192112 weight_to_token_ratio=0.747978 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=12544 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.277749055e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=325183488 dispatch_meta_bytes=227744 combine_payload_bytes=325183488 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=325411232 ingress_bytes=435283968 link_bytes=435283968 token_bytes=650594720 weight_movement_bytes=110100480 total_nvlink_bytes=760695200 weight_to_token_ratio=0.169231 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.570304123e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57344 payload_rows_remote=49170 dup_rows=8192 dup_rows_remote=7906 dedup_ratio=0.125000 router_tokens_per_expert=[0, 10, 0, 0, 1366, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144, 0, 0, 0, 0, 1689, 0, 39, 3301, 0, 12, 0, 0, 4086, 3793, 0, 4087, 0, 0, 0, 0, 0, 4056, 553, 0, 0, 1666, 0, 0, 0, 2737, 0, 0, 0, 644, 3953, 0, 0, 0, 148, 0, 0, 82, 402, 0] payload_rows_per_dst=[8134, 8174, 8174, 7416, 7586, 3972, 5924, 7964] dup_rows_per_dst=[2730, 0, 286, 2666, 0, 466, 838, 1206] combine_rows_per_src=[8134, 8174, 8174, 7416, 7586, 3972, 5924, 7964] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=281616384 dispatch_meta_bytes=172920 combine_payload_bytes=281616384 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=281789304 ingress_bytes=722018304 link_bytes=722018304 token_bytes=563405688 weight_movement_bytes=440401920 total_nvlink_bytes=1003807608 weight_to_token_ratio=0.781678 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=14848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.255708582e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=312195072 dispatch_meta_bytes=231024 combine_payload_bytes=312195072 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=312426096 ingress_bytes=422295552 link_bytes=422295552 token_bytes=624621168 weight_movement_bytes=110100480 total_nvlink_bytes=734721648 weight_to_token_ratio=0.176268 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24448 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.344414206e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=323807232 dispatch_meta_bytes=216352 combine_payload_bytes=323807232 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=324023584 ingress_bytes=433907712 link_bytes=433907712 token_bytes=647830816 weight_movement_bytes=110100480 total_nvlink_bytes=757931296 weight_to_token_ratio=0.169953 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=18432 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.546368767e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57190 payload_rows_remote=49256 dup_rows=8346 dup_rows_remote=8346 dedup_ratio=0.127350 router_tokens_per_expert=[0, 4, 0, 0, 1864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 1668, 0, 108, 2644, 0, 7, 0, 0, 4092, 3967, 0, 4088, 0, 0, 0, 0, 0, 4067, 475, 0, 0, 1841, 0, 0, 0, 2636, 0, 0, 0, 388, 3970, 0, 0, 0, 109, 0, 0, 253, 463, 0] payload_rows_per_dst=[7930, 8176, 8186, 6804, 7934, 4512, 5654, 7994] dup_rows_per_dst=[3482, 0, 246, 2050, 0, 578, 394, 1596] combine_rows_per_src=[7930, 8176, 8186, 6804, 7934, 4512, 5654, 7994] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=350801920 dispatch_meta_bytes=228632 combine_payload_bytes=350801920 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=351030552 ingress_bytes=460902400 link_bytes=460902400 token_bytes=701832472 weight_movement_bytes=110100480 total_nvlink_bytes=811932952 weight_to_token_ratio=0.156876 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=24064 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.015850790e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=352450560 dispatch_meta_bytes=228304 combine_payload_bytes=352450560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=352678864 ingress_bytes=462551040 link_bytes=462551040 token_bytes=705129424 weight_movement_bytes=110100480 total_nvlink_bytes=815229904 weight_to_token_ratio=0.156142 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31744 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.044523351e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=353067008 dispatch_meta_bytes=230408 combine_payload_bytes=353067008 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=353297416 ingress_bytes=793468928 link_bytes=793468928 token_bytes=706364424 weight_movement_bytes=440401920 total_nvlink_bytes=1146766344 weight_to_token_ratio=0.623477 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.379972969e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55148 payload_rows_remote=46956 dup_rows=10388 dup_rows_remote=5232 dedup_ratio=0.158508 router_tokens_per_expert=[0, 0, 0, 0, 2343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 1889, 0, 12, 2756, 0, 2, 0, 0, 4095, 3846, 0, 4095, 0, 0, 0, 0, 0, 4092, 183, 0, 0, 1858, 0, 0, 0, 2252, 0, 0, 0, 174, 4030, 0, 0, 0, 72, 0, 0, 494, 440, 0] payload_rows_per_dst=[4686, 8190, 8192, 5220, 7852, 8192, 4734, 8082] dup_rows_per_dst=[0, 0, 268, 908, 1948, 5156, 118, 1990] combine_rows_per_src=[4686, 8190, 8192, 5220, 7852, 8192, 4734, 8082] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53216 payload_rows_remote=45080 dup_rows=12320 dup_rows_remote=10158 dedup_ratio=0.187988 router_tokens_per_expert=[0, 10, 0, 0, 1215, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 2225, 0, 7, 2927, 0, 5, 0, 0, 4085, 3903, 0, 4091, 0, 0, 0, 0, 0, 4084, 173, 0, 0, 1910, 0, 0, 0, 2567, 0, 0, 0, 299, 4045, 0, 0, 0, 91, 0, 0, 264, 749, 0] payload_rows_per_dst=[2450, 8182, 6350, 4468, 7870, 8192, 7568, 8136] dup_rows_per_dst=[0, 0, 178, 6, 1814, 4786, 3374, 2162] combine_rows_per_src=[2450, 8182, 6350, 4468, 7870, 8192, 7568, 8136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56534 payload_rows_remote=48622 dup_rows=9002 dup_rows_remote=5194 dedup_ratio=0.137360 router_tokens_per_expert=[0, 6, 0, 0, 1288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 2177, 0, 28, 3646, 0, 9, 0, 0, 4089, 3813, 0, 4090, 0, 0, 0, 0, 2, 4039, 379, 0, 0, 1287, 0, 0, 0, 2276, 0, 0, 0, 608, 4013, 0, 0, 0, 105, 0, 0, 127, 662, 0] payload_rows_per_dst=[8112, 8180, 8180, 7912, 7626, 3164, 5284, 8076] dup_rows_per_dst=[2554, 0, 246, 3808, 0, 172, 484, 1738] combine_rows_per_src=[8112, 8180, 8180, 7912, 7626, 3164, 5284, 8076] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55804 payload_rows_remote=47720 dup_rows=9732 dup_rows_remote=8378 dedup_ratio=0.148499 router_tokens_per_expert=[0, 0, 0, 61, 5, 0, 1249, 0, 0, 138, 0, 0, 0, 0, 561, 0, 3455, 0, 0, 2608, 0, 1729, 3956, 0, 0, 0, 1832, 4020, 0, 1475, 1, 0, 0, 0, 0, 0, 0, 3221, 29, 5, 2616, 1, 2518, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 2930, 0, 294, 55, 0, 1, 0, 2] payload_rows_per_dst=[7440, 8084, 6786, 5754, 6478, 7090, 7922, 6250] dup_rows_per_dst=[2100, 1354, 1888, 862, 32, 3180, 2, 314] combine_rows_per_src=[7440, 8084, 6786, 5754, 6478, 7090, 7922, 6250] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52298 payload_rows_remote=47206 dup_rows=13238 dup_rows_remote=12836 dedup_ratio=0.201996 router_tokens_per_expert=[0, 0, 0, 0, 1388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 2004, 0, 3, 3268, 0, 0, 0, 0, 4095, 3865, 0, 4096, 0, 0, 0, 0, 0, 4073, 306, 0, 0, 1378, 0, 0, 0, 2292, 0, 0, 0, 455, 4019, 0, 0, 0, 97, 0, 0, 484, 818, 0] payload_rows_per_dst=[2776, 8192, 8190, 4014, 7730, 8192, 5092, 8112] dup_rows_per_dst=[0, 0, 254, 0, 0, 9858, 402, 2724] combine_rows_per_src=[2776, 8192, 8190, 4014, 7730, 8192, 5092, 8112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=336580608 dispatch_meta_bytes=208752 combine_payload_bytes=336580608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=336789360 ingress_bytes=556781568 link_bytes=556781568 token_bytes=673369968 weight_movement_bytes=220200960 total_nvlink_bytes=893570928 weight_to_token_ratio=0.327013 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=13568 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.683347216e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49952 payload_rows_remote=42114 dup_rows=15584 dup_rows_remote=10636 dedup_ratio=0.237793 router_tokens_per_expert=[69, 0, 3112, 0, 0, 16, 0, 0, 4, 0, 0, 2207, 2, 0, 0, 0, 0, 0, 452, 0, 0, 0, 2279, 0, 0, 572, 0, 2924, 0, 0, 0, 0, 0, 0, 0, 0, 3847, 3196, 0, 580, 17, 0, 0, 0, 0, 0, 213, 0, 0, 3, 1974, 0, 0, 0, 0, 0, 3803, 0, 0, 0, 0, 3407, 4091, 0] payload_rows_per_dst=[7838, 8010, 7856, 6318, 7786, 8192, 3952, 0] dup_rows_per_dst=[4948, 4022, 4420, 674, 1068, 450, 2, 0] combine_rows_per_src=[7838, 8010, 7856, 6318, 7786, 8192, 3952, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60798 payload_rows_remote=54878 dup_rows=4738 dup_rows_remote=4736 dedup_ratio=0.072296 router_tokens_per_expert=[0, 3800, 0, 0, 0, 143, 1683, 4094, 0, 0, 0, 0, 0, 0, 0, 4042, 0, 0, 1, 2960, 0, 0, 4095, 0, 0, 0, 0, 4096, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 542, 0, 43, 0, 0, 0, 4096, 0, 0, 14, 110, 0, 0, 34, 0, 0, 0, 2989, 24, 0, 0, 0] payload_rows_per_dst=[8012, 8084, 5920, 8192, 8190, 8192, 8192, 6016] dup_rows_per_dst=[3240, 0, 2, 2, 0, 1168, 316, 10] combine_rows_per_src=[8012, 8084, 5920, 8192, 8190, 8192, 8192, 6016] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=323133440 dispatch_meta_bytes=220952 combine_payload_bytes=323133440 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=323354392 ingress_bytes=433233920 link_bytes=433233920 token_bytes=646487832 weight_movement_bytes=110100480 total_nvlink_bytes=756588312 weight_to_token_ratio=0.170306 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.534650416e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61362 payload_rows_remote=53172 dup_rows=4174 dup_rows_remote=4174 dedup_ratio=0.063690 router_tokens_per_expert=[0, 3452, 0, 0, 0, 38, 1645, 4095, 0, 0, 0, 0, 0, 30, 0, 3971, 0, 0, 3, 3347, 0, 0, 4094, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 670, 0, 33, 0, 0, 0, 4095, 0, 0, 4, 121, 0, 0, 24, 0, 0, 0, 3023, 27, 0, 0, 0] payload_rows_per_dst=[7864, 7942, 6694, 8192, 8190, 8192, 8190, 6098] dup_rows_per_dst=[2406, 60, 6, 0, 0, 1402, 298, 2] combine_rows_per_src=[7864, 7942, 6694, 8192, 8190, 8192, 8190, 6098] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=348522496 dispatch_meta_bytes=215264 combine_payload_bytes=348522496 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=348737760 ingress_bytes=678823936 link_bytes=678823936 token_bytes=697260256 weight_movement_bytes=330301440 total_nvlink_bytes=1027561696 weight_to_token_ratio=0.473713 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=17152 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.180586472e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=342056960 dispatch_meta_bytes=224392 combine_payload_bytes=342056960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342281352 ingress_bytes=452157440 link_bytes=452157440 token_bytes=684338312 weight_movement_bytes=110100480 total_nvlink_bytes=794438792 weight_to_token_ratio=0.160886 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29056 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.863761553e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=338372608 dispatch_meta_bytes=240168 combine_payload_bytes=338372608 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=338612776 ingress_bytes=558573568 link_bytes=558573568 token_bytes=676985384 weight_movement_bytes=220200960 total_nvlink_bytes=897186344 weight_to_token_ratio=0.325267 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.714513043e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=301873152 dispatch_meta_bytes=211000 combine_payload_bytes=301873152 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=302084152 ingress_bytes=411973632 link_bytes=411973632 token_bytes=603957304 weight_movement_bytes=110100480 total_nvlink_bytes=714057784 weight_to_token_ratio=0.182298 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.164899041e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=393365504 dispatch_meta_bytes=238456 combine_payload_bytes=393365504 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=393603960 ingress_bytes=613566464 link_bytes=613566464 token_bytes=786969464 weight_movement_bytes=220200960 total_nvlink_bytes=1007170424 weight_to_token_ratio=0.279809 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067092995e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=381136896 dispatch_meta_bytes=229384 combine_payload_bytes=381136896 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=381366280 ingress_bytes=601337856 link_bytes=601337856 token_bytes=762503176 weight_movement_bytes=220200960 total_nvlink_bytes=982704136 weight_to_token_ratio=0.288787 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.045825435e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60448 payload_rows_remote=52256 dup_rows=5088 dup_rows_remote=4248 dedup_ratio=0.077637 router_tokens_per_expert=[0, 3763, 0, 0, 0, 22, 1514, 4096, 0, 0, 0, 0, 0, 13, 0, 4030, 0, 0, 0, 3691, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 413, 0, 7, 0, 0, 0, 4096, 0, 0, 4, 125, 0, 0, 41, 0, 0, 0, 2688, 73, 0, 0, 0] payload_rows_per_dst=[4242, 8060, 7382, 8192, 8192, 8192, 8192, 7996] dup_rows_per_dst=[246, 26, 0, 0, 6, 840, 340, 3630] combine_rows_per_src=[4242, 8060, 7382, 8192, 8192, 8192, 8192, 7996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59706 payload_rows_remote=51534 dup_rows=5830 dup_rows_remote=1504 dedup_ratio=0.088959 router_tokens_per_expert=[0, 3823, 0, 0, 0, 29, 1952, 4096, 0, 0, 0, 0, 0, 20, 0, 4049, 0, 0, 3, 3354, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 561, 0, 27, 0, 0, 0, 4095, 0, 0, 1, 110, 0, 0, 29, 0, 0, 0, 2324, 102, 0, 0, 0] payload_rows_per_dst=[3960, 8100, 6710, 8190, 8192, 8192, 8190, 8172] dup_rows_per_dst=[2, 38, 4, 0, 2, 1176, 282, 4326] combine_rows_per_src=[3960, 8100, 6710, 8190, 8192, 8192, 8190, 8172] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61104 payload_rows_remote=52912 dup_rows=4432 dup_rows_remote=4432 dedup_ratio=0.067627 router_tokens_per_expert=[0, 3725, 0, 0, 0, 72, 1596, 4096, 0, 0, 0, 0, 0, 2, 0, 4061, 0, 0, 0, 3461, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 592, 0, 20, 0, 0, 0, 4096, 0, 0, 5, 121, 0, 0, 38, 0, 0, 0, 2618, 73, 0, 0, 0] payload_rows_per_dst=[7922, 8122, 6922, 8192, 8192, 8192, 8192, 5370] dup_rows_per_dst=[2864, 4, 0, 0, 0, 1224, 328, 12] combine_rows_per_src=[7922, 8122, 6922, 8192, 8192, 8192, 8192, 5370] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58926 payload_rows_remote=50734 dup_rows=6610 dup_rows_remote=6230 dedup_ratio=0.100861 router_tokens_per_expert=[0, 3926, 0, 0, 0, 39, 1180, 4096, 0, 0, 0, 0, 0, 19, 0, 4079, 0, 0, 0, 3697, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 589, 0, 5, 0, 0, 0, 4096, 0, 0, 32, 122, 0, 0, 36, 0, 0, 0, 2580, 80, 0, 0, 0] payload_rows_per_dst=[2438, 8160, 7394, 8192, 8192, 8192, 8192, 8166] dup_rows_per_dst=[0, 36, 0, 0, 0, 1188, 380, 5006] combine_rows_per_src=[2438, 8160, 7394, 8192, 8192, 8192, 8192, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52126 payload_rows_remote=44088 dup_rows=13410 dup_rows_remote=10076 dedup_ratio=0.204620 router_tokens_per_expert=[49, 0, 3476, 0, 0, 241, 0, 0, 0, 0, 0, 1940, 5, 0, 0, 0, 0, 0, 378, 0, 0, 0, 1543, 0, 0, 559, 0, 3222, 0, 2, 2, 0, 0, 0, 0, 0, 3999, 3099, 0, 182, 30, 0, 0, 0, 0, 0, 169, 0, 0, 0, 2673, 0, 0, 0, 0, 0, 3741, 0, 0, 0, 0, 3383, 4075, 0] payload_rows_per_dst=[7508, 8038, 7520, 7510, 8026, 8178, 5346, 0] dup_rows_per_dst=[2740, 3334, 3088, 3542, 336, 370, 0, 0] combine_rows_per_src=[7508, 8038, 7520, 7510, 8026, 8178, 5346, 0] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=374571008 dispatch_meta_bytes=226016 combine_payload_bytes=374571008 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374797024 ingress_bytes=484671488 link_bytes=484671488 token_bytes=749368032 weight_movement_bytes=110100480 total_nvlink_bytes=859468512 weight_to_token_ratio=0.146924 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=28160 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.429234324e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54096 payload_rows_remote=45988 dup_rows=11440 dup_rows_remote=8248 dedup_ratio=0.174561 router_tokens_per_expert=[0, 7, 0, 0, 1596, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 113, 0, 0, 0, 0, 1914, 0, 25, 3027, 0, 0, 0, 0, 4095, 3872, 0, 4087, 0, 0, 0, 0, 0, 4047, 261, 0, 0, 2006, 0, 0, 0, 1896, 0, 0, 0, 637, 4023, 0, 0, 0, 91, 0, 0, 444, 627, 0] payload_rows_per_dst=[8108, 8192, 8190, 7260, 7648, 4262, 8190, 2246] dup_rows_per_dst=[3192, 78, 226, 2672, 0, 272, 4922, 78] combine_rows_per_src=[8108, 8192, 8190, 7260, 7648, 4262, 8190, 2246] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60494 payload_rows_remote=54154 dup_rows=5042 dup_rows_remote=3364 dedup_ratio=0.076935 router_tokens_per_expert=[0, 3994, 0, 0, 0, 3380, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2565, 0, 4096, 0, 0, 0, 0, 1459, 0, 0, 0, 0, 425, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4095, 0, 263, 218, 4086, 0, 0, 0, 4096, 3982, 0, 107, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6760, 8192, 6340, 6806, 8000, 8190, 8014, 8192] dup_rows_per_dst=[2, 2, 1678, 1416, 838, 0, 912, 194] combine_rows_per_src=[6760, 8192, 6340, 6806, 8000, 8190, 8014, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61122 payload_rows_remote=53028 dup_rows=4414 dup_rows_remote=3174 dedup_ratio=0.067352 router_tokens_per_expert=[0, 4041, 0, 0, 0, 3173, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2924, 0, 4077, 0, 0, 0, 0, 1026, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 241, 307, 4095, 0, 0, 0, 4096, 4023, 0, 42, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6346, 8192, 5848, 8192, 8094, 8192, 8066, 8192] dup_rows_per_dst=[0, 2, 0, 2014, 1240, 0, 1076, 82] combine_rows_per_src=[6346, 8192, 5848, 8192, 8094, 8192, 8066, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=369395712 dispatch_meta_bytes=212152 combine_payload_bytes=369395712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369607864 ingress_bytes=479496192 link_bytes=479496192 token_bytes=739003576 weight_movement_bytes=110100480 total_nvlink_bytes=849104056 weight_to_token_ratio=0.148985 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.339227415e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=379273216 dispatch_meta_bytes=229376 combine_payload_bytes=379273216 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=379502592 ingress_bytes=489373696 link_bytes=489373696 token_bytes=758775808 weight_movement_bytes=110100480 total_nvlink_bytes=868876288 weight_to_token_ratio=0.145103 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.511013455e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=363661312 dispatch_meta_bytes=227856 combine_payload_bytes=363661312 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=363889168 ingress_bytes=804063232 link_bytes=804063232 token_bytes=727550480 weight_movement_bytes=440401920 total_nvlink_bytes=1167952400 weight_to_token_ratio=0.605321 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.398398206e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=316022784 dispatch_meta_bytes=216656 combine_payload_bytes=316022784 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=316239440 ingress_bytes=426123264 link_bytes=426123264 token_bytes=632262224 weight_movement_bytes=110100480 total_nvlink_bytes=742362704 weight_to_token_ratio=0.174137 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=17664 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.410984413e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=329641984 dispatch_meta_bytes=216944 combine_payload_bytes=329641984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=329858928 ingress_bytes=439742464 link_bytes=439742464 token_bytes=659500912 weight_movement_bytes=110100480 total_nvlink_bytes=769601392 weight_to_token_ratio=0.166945 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=20096 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.647844701e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=388175872 dispatch_meta_bytes=230072 combine_payload_bytes=388175872 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=388405944 ingress_bytes=498276352 link_bytes=498276352 token_bytes=776581816 weight_movement_bytes=110100480 total_nvlink_bytes=886682296 weight_to_token_ratio=0.141776 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.665845285e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=380104704 dispatch_meta_bytes=224808 combine_payload_bytes=380104704 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380329512 ingress_bytes=490205184 link_bytes=490205184 token_bytes=760434216 weight_movement_bytes=110100480 total_nvlink_bytes=870534696 weight_to_token_ratio=0.144786 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=25728 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.525474399e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61594 payload_rows_remote=53402 dup_rows=3942 dup_rows_remote=3942 dedup_ratio=0.060150 router_tokens_per_expert=[0, 4070, 0, 0, 0, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3046, 0, 4094, 0, 0, 0, 0, 584, 0, 0, 0, 0, 961, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 157, 269, 4096, 0, 0, 0, 4096, 3992, 0, 23, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6568, 8192, 6092, 8192, 8146, 8192, 8020, 8192] dup_rows_per_dst=[0, 0, 0, 1164, 1916, 0, 816, 46] combine_rows_per_src=[6568, 8192, 6092, 8192, 8146, 8192, 8020, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58968 payload_rows_remote=50776 dup_rows=6568 dup_rows_remote=6502 dedup_ratio=0.100220 router_tokens_per_expert=[0, 4071, 0, 0, 0, 2382, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2592, 0, 4090, 0, 0, 0, 0, 1383, 0, 0, 0, 0, 1406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 152, 318, 4094, 0, 0, 0, 4096, 4052, 0, 35, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4764, 8178, 5184, 8190, 8148, 8192, 8120, 8192] dup_rows_per_dst=[0, 2, 0, 2756, 2806, 0, 938, 66] combine_rows_per_src=[4764, 8178, 5184, 8190, 8148, 8192, 8120, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382785536 dispatch_meta_bytes=229376 combine_payload_bytes=382785536 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=383014912 ingress_bytes=492886016 link_bytes=492886016 token_bytes=765800448 weight_movement_bytes=110100480 total_nvlink_bytes=875900928 weight_to_token_ratio=0.143772 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.572098476e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61598 payload_rows_remote=53406 dup_rows=3938 dup_rows_remote=2518 dedup_ratio=0.060089 router_tokens_per_expert=[0, 4044, 0, 0, 0, 3265, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3083, 0, 4089, 0, 0, 0, 0, 717, 0, 0, 0, 0, 792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 129, 313, 4095, 0, 0, 0, 4096, 4000, 0, 49, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6530, 8192, 6166, 8192, 8110, 8192, 8024, 8192] dup_rows_per_dst=[0, 0, 0, 1420, 1562, 0, 860, 96] combine_rows_per_src=[6530, 8192, 6166, 8192, 8110, 8192, 8024, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61670 payload_rows_remote=53556 dup_rows=3866 dup_rows_remote=2520 dedup_ratio=0.058990 router_tokens_per_expert=[0, 4048, 0, 0, 0, 3433, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2911, 0, 4096, 0, 0, 0, 0, 498, 0, 0, 0, 0, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 118, 566, 4096, 0, 0, 0, 4096, 4046, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6866, 8192, 5822, 8192, 8100, 8192, 8114, 8192] dup_rows_per_dst=[0, 0, 0, 996, 1520, 0, 1346, 4] combine_rows_per_src=[6866, 8192, 5822, 8192, 8100, 8192, 8114, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64746 payload_rows_remote=56810 dup_rows=790 dup_rows_remote=706 dedup_ratio=0.012054 router_tokens_per_expert=[0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 33, 0, 4085, 45, 14, 0, 0, 0, 52, 3958, 0, 0, 0, 0, 4096, 0, 0, 0, 4083, 4096, 27, 0, 4082, 0, 0, 0, 0, 3851, 0, 0, 0, 0, 0, 0, 15, 1, 0, 220, 8, 0, 0, 0, 0, 0, 4090, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7936, 8170, 7702, 8180, 8182, 8192] dup_rows_per_dst=[22, 162, 84, 50, 0, 456, 14, 2] combine_rows_per_src=[8192, 8192, 7936, 8170, 7702, 8180, 8182, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64868 payload_rows_remote=57134 dup_rows=668 dup_rows_remote=668 dedup_ratio=0.010193 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 40, 0, 4095, 12, 13, 0, 0, 0, 38, 4002, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 28, 0, 4093, 0, 0, 0, 0, 3867, 0, 0, 0, 0, 0, 0, 24, 0, 0, 174, 1, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8004, 8182, 7734, 8186, 8186, 8192] dup_rows_per_dst=[10, 128, 76, 56, 0, 396, 2, 0] combine_rows_per_src=[8192, 8192, 8004, 8182, 7734, 8186, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53672 payload_rows_remote=45500 dup_rows=11864 dup_rows_remote=11864 dedup_ratio=0.181030 router_tokens_per_expert=[0, 7, 0, 0, 1686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 135, 0, 0, 0, 0, 1861, 0, 20, 3276, 0, 9, 0, 0, 4045, 3764, 0, 4086, 0, 0, 0, 0, 4, 3998, 260, 0, 0, 1617, 0, 0, 0, 2075, 0, 0, 0, 809, 3947, 0, 0, 0, 154, 0, 0, 335, 675, 0] payload_rows_per_dst=[8022, 8172, 8102, 7530, 7528, 3622, 7700, 2996] dup_rows_per_dst=[3360, 0, 268, 2802, 0, 140, 4858, 436] combine_rows_per_src=[8022, 8172, 8102, 7530, 7528, 3622, 7700, 2996] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53562 payload_rows_remote=45500 dup_rows=11974 dup_rows_remote=8434 dedup_ratio=0.182709 router_tokens_per_expert=[0, 3550, 0, 0, 0, 35, 1632, 4094, 0, 0, 0, 0, 0, 18, 0, 4053, 0, 0, 0, 3406, 0, 0, 4096, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 367, 0, 35, 0, 0, 0, 4094, 0, 0, 7, 106, 0, 0, 29, 0, 0, 0, 3129, 21, 0, 0, 0] payload_rows_per_dst=[8062, 8108, 8192, 8192, 8188, 804, 5724, 6292] dup_rows_per_dst=[3540, 620, 7602, 0, 0, 0, 204, 8] combine_rows_per_src=[8062, 8108, 8192, 8192, 8188, 804, 5724, 6292] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=363962368 dispatch_meta_bytes=229112 combine_payload_bytes=363962368 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364191480 ingress_bytes=474062848 link_bytes=474062848 token_bytes=728153848 weight_movement_bytes=110100480 total_nvlink_bytes=838254328 weight_to_token_ratio=0.151205 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.244732626e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=382814208 dispatch_meta_bytes=223696 combine_payload_bytes=382814208 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=383037904 ingress_bytes=603015168 link_bytes=603015168 token_bytes=765852112 weight_movement_bytes=220200960 total_nvlink_bytes=986053072 weight_to_token_ratio=0.287524 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=23168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.048742556e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=383889408 dispatch_meta_bytes=224304 combine_payload_bytes=383889408 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=384113712 ingress_bytes=934391808 link_bytes=934391808 token_bytes=768003120 weight_movement_bytes=550502400 total_nvlink_bytes=1318505520 weight_to_token_ratio=0.716797 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.625061035e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=407214080 dispatch_meta_bytes=230064 combine_payload_bytes=407214080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=407444144 ingress_bytes=517314560 link_bytes=517314560 token_bytes=814658224 weight_movement_bytes=110100480 total_nvlink_bytes=924758704 weight_to_token_ratio=0.135149 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.996951035e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=409536512 dispatch_meta_bytes=231208 combine_payload_bytes=409536512 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=409767720 ingress_bytes=739837952 link_bytes=739837952 token_bytes=819304232 weight_movement_bytes=330301440 total_nvlink_bytes=1149605672 weight_to_token_ratio=0.403149 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.286699881e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=326144000 dispatch_meta_bytes=229456 combine_payload_bytes=326144000 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=326373456 ingress_bytes=546344960 link_bytes=546344960 token_bytes=652517456 weight_movement_bytes=220200960 total_nvlink_bytes=872718416 weight_to_token_ratio=0.337464 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.501837437e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=326144000 dispatch_meta_bytes=215736 combine_payload_bytes=326144000 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=326359736 ingress_bytes=766545920 link_bytes=766545920 token_bytes=652503736 weight_movement_bytes=440401920 total_nvlink_bytes=1092905656 weight_to_token_ratio=0.674942 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=19072 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.333149430e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64842 payload_rows_remote=56670 dup_rows=694 dup_rows_remote=282 dedup_ratio=0.010590 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 22, 0, 4096, 10, 33, 0, 0, 0, 44, 3918, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 29, 0, 4086, 0, 0, 0, 0, 3940, 0, 0, 0, 0, 0, 0, 6, 0, 0, 200, 1, 0, 0, 0, 0, 0, 4094, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7836, 8190, 7880, 8172, 8188, 8192] dup_rows_per_dst=[2, 130, 88, 58, 0, 412, 2, 2] combine_rows_per_src=[8192, 8192, 7836, 8190, 7880, 8172, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406210560 dispatch_meta_bytes=227808 combine_payload_bytes=406210560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406438368 ingress_bytes=516311040 link_bytes=516311040 token_bytes=812648928 weight_movement_bytes=110100480 total_nvlink_bytes=922749408 weight_to_token_ratio=0.135483 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=30976 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.979498171e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63226 payload_rows_remote=55034 dup_rows=2310 dup_rows_remote=2310 dedup_ratio=0.035248 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 28, 0, 4095, 10, 19, 0, 0, 0, 32, 3985, 0, 0, 0, 0, 4096, 0, 0, 0, 4091, 4096, 17, 0, 4094, 0, 0, 0, 0, 3910, 0, 0, 0, 0, 0, 0, 12, 0, 0, 180, 3, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8106, 6542, 8020, 8184, 7802, 8188, 8192, 8192] dup_rows_per_dst=[8, 98, 1678, 118, 0, 384, 24, 0] combine_rows_per_src=[8106, 6542, 8020, 8184, 7802, 8188, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64894 payload_rows_remote=56716 dup_rows=642 dup_rows_remote=578 dedup_ratio=0.009796 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 16, 0, 4080, 13, 10, 0, 0, 0, 46, 4003, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 4096, 35, 0, 4088, 0, 0, 0, 0, 3901, 0, 0, 0, 0, 0, 0, 49, 0, 0, 152, 8, 0, 0, 0, 0, 0, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8182, 8006, 8178, 7802, 8176, 8166, 8192] dup_rows_per_dst=[14, 56, 92, 64, 0, 402, 14, 0] combine_rows_per_src=[8192, 8182, 8006, 8178, 7802, 8176, 8166, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64764 payload_rows_remote=56630 dup_rows=772 dup_rows_remote=770 dedup_ratio=0.011780 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 4093, 14, 54, 0, 0, 0, 52, 3936, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 4096, 32, 0, 4069, 0, 0, 0, 0, 3903, 0, 0, 0, 0, 0, 0, 68, 0, 0, 173, 1, 0, 0, 0, 0, 0, 4067, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7872, 8190, 7806, 8186, 8134, 8192] dup_rows_per_dst=[0, 168, 104, 64, 0, 434, 2, 0] combine_rows_per_src=[8192, 8192, 7872, 8190, 7806, 8186, 8134, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58698 payload_rows_remote=50506 dup_rows=6838 dup_rows_remote=6838 dedup_ratio=0.104340 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1271, 21, 0, 1, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 59, 4092, 0, 0, 4, 0, 0, 4086, 2, 0, 14, 0, 0, 2102, 0, 4, 0, 4085, 0, 0, 0, 4096, 0, 0, 0, 7, 0, 2169, 0, 0, 0, 0, 0, 4080, 0] payload_rows_per_dst=[8188, 8186, 8192, 5234, 8176, 8192, 8192, 4338] dup_rows_per_dst=[2, 2558, 0, 38, 36, 4190, 14, 0] combine_rows_per_src=[8188, 8186, 8192, 5234, 8176, 8192, 8192, 4338] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59142 payload_rows_remote=50952 dup_rows=6394 dup_rows_remote=6332 dedup_ratio=0.097565 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1742, 28, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3356, 0, 0, 40, 4095, 0, 0, 1, 0, 0, 4091, 4, 0, 30, 0, 0, 1365, 0, 0, 0, 4082, 0, 0, 0, 4096, 0, 0, 0, 26, 0, 1615, 0, 0, 0, 0, 17, 4079, 0] payload_rows_per_dst=[8190, 8192, 8192, 6730, 8190, 8192, 8192, 3264] dup_rows_per_dst=[10, 3506, 0, 62, 62, 2702, 52, 0] combine_rows_per_src=[8190, 8192, 8192, 6730, 8190, 8192, 8192, 3264] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=394483712 dispatch_meta_bytes=229376 combine_payload_bytes=394483712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394713088 ingress_bytes=504584192 link_bytes=504584192 token_bytes=789196800 weight_movement_bytes=110100480 total_nvlink_bytes=899297280 weight_to_token_ratio=0.139510 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.775548997e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60126 payload_rows_remote=51992 dup_rows=5410 dup_rows_remote=5368 dedup_ratio=0.082550 router_tokens_per_expert=[0, 3774, 0, 0, 0, 36, 1559, 4096, 0, 0, 0, 0, 0, 21, 0, 4067, 0, 0, 4, 3760, 0, 0, 4061, 0, 0, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 739, 0, 81, 0, 0, 0, 4096, 0, 0, 2, 129, 0, 0, 31, 0, 0, 4, 2197, 15, 0, 0, 0] payload_rows_per_dst=[7882, 8134, 7626, 8192, 8192, 7484, 8192, 4424] dup_rows_per_dst=[2856, 42, 1082, 0, 0, 1098, 324, 8] combine_rows_per_src=[7882, 8134, 7626, 8192, 8192, 7484, 8192, 4424] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50532 payload_rows_remote=42340 dup_rows=15004 dup_rows_remote=6816 dedup_ratio=0.228943 router_tokens_per_expert=[0, 4033, 0, 0, 0, 3338, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3132, 0, 4087, 0, 0, 0, 0, 571, 0, 0, 0, 0, 1201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 98, 177, 4089, 0, 0, 0, 4096, 3770, 0, 80, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8186, 5656, 2402, 8192, 2228, 7484] dup_rows_per_dst=[8188, 0, 6252, 424, 0, 6, 120, 14] combine_rows_per_src=[8192, 8192, 8186, 5656, 2402, 8192, 2228, 7484] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406540288 dispatch_meta_bytes=229176 combine_payload_bytes=406540288 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=406769464 ingress_bytes=846942208 link_bytes=846942208 token_bytes=813309752 weight_movement_bytes=440401920 total_nvlink_bytes=1253711672 weight_to_token_ratio=0.541493 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.472971798e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=405923840 dispatch_meta_bytes=229600 combine_payload_bytes=405923840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406153440 ingress_bytes=516024320 link_bytes=516024320 token_bytes=812077280 weight_movement_bytes=110100480 total_nvlink_bytes=922177760 weight_to_token_ratio=0.135579 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.974511639e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=362027008 dispatch_meta_bytes=229376 combine_payload_bytes=362027008 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=362256384 ingress_bytes=472127488 link_bytes=472127488 token_bytes=724283392 weight_movement_bytes=110100480 total_nvlink_bytes=834383872 weight_to_token_ratio=0.152013 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.211073533e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=365223936 dispatch_meta_bytes=229136 combine_payload_bytes=365223936 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=365453072 ingress_bytes=585424896 link_bytes=585424896 token_bytes=730677008 weight_movement_bytes=220200960 total_nvlink_bytes=950877968 weight_to_token_ratio=0.301366 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.018150180e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58834 payload_rows_remote=50642 dup_rows=6702 dup_rows_remote=3636 dedup_ratio=0.102264 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1051, 29, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3675, 0, 0, 60, 4096, 0, 0, 0, 0, 0, 4095, 1, 0, 73, 0, 0, 1538, 0, 0, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 10, 0, 1747, 0, 0, 0, 0, 13, 4095, 0] payload_rows_per_dst=[8192, 3520, 8192, 7372, 8192, 8192, 8192, 6982] dup_rows_per_dst=[4, 642, 0, 98, 146, 3066, 20, 2726] combine_rows_per_src=[8192, 3520, 8192, 7372, 8192, 8192, 8192, 6982] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=372678656 dispatch_meta_bytes=229440 combine_payload_bytes=372678656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=372908096 ingress_bytes=482779136 link_bytes=482779136 token_bytes=745586752 weight_movement_bytes=110100480 total_nvlink_bytes=855687232 weight_to_token_ratio=0.147670 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.396323210e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=303493120 dispatch_meta_bytes=196624 combine_payload_bytes=303493120 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=303689744 ingress_bytes=523694080 link_bytes=523694080 token_bytes=607182864 weight_movement_bytes=220200960 total_nvlink_bytes=827383824 weight_to_token_ratio=0.362660 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25472 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.107901380e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=363001856 dispatch_meta_bytes=217112 combine_payload_bytes=363001856 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=363218968 ingress_bytes=693303296 link_bytes=693303296 token_bytes=726220824 weight_movement_bytes=330301440 total_nvlink_bytes=1056522264 weight_to_token_ratio=0.454822 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=20224 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.205768461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52488 payload_rows_remote=44304 dup_rows=13048 dup_rows_remote=574 dedup_ratio=0.199097 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1134, 31, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3393, 0, 0, 25, 4093, 0, 0, 4, 0, 0, 4088, 3, 0, 36, 0, 0, 1415, 0, 1, 0, 4091, 0, 0, 0, 4096, 0, 0, 0, 24, 0, 2157, 0, 0, 0, 0, 2, 4079, 0] payload_rows_per_dst=[8172, 2300, 8190, 6796, 7004, 3650, 8192, 8184] dup_rows_per_dst=[0, 30, 0, 54, 62, 378, 50, 12474] combine_rows_per_src=[8172, 2300, 8190, 6796, 7004, 3650, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60316 payload_rows_remote=53276 dup_rows=5220 dup_rows_remote=5186 dedup_ratio=0.079651 router_tokens_per_expert=[0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1147, 20, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 3511, 0, 0, 26, 4096, 0, 0, 0, 0, 0, 4089, 6, 0, 89, 0, 0, 1324, 0, 0, 0, 4078, 0, 0, 0, 4096, 0, 0, 0, 29, 0, 2067, 0, 0, 0, 0, 2, 4087, 0] payload_rows_per_dst=[8192, 8184, 8192, 7040, 8188, 8190, 8192, 4138] dup_rows_per_dst=[10, 2324, 0, 34, 180, 2614, 58, 0] combine_rows_per_src=[8192, 8184, 8192, 7040, 8188, 8190, 8192, 4138] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62426 payload_rows_remote=56684 dup_rows=3110 dup_rows_remote=3110 dedup_ratio=0.047455 router_tokens_per_expert=[16, 0, 0, 0, 0, 4073, 0, 0, 0, 0, 0, 30, 24, 0, 0, 0, 0, 0, 0, 0, 0, 10, 2861, 0, 0, 4096, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 0, 0, 0, 3554, 0, 0, 473, 0, 0, 1, 0, 981, 0, 0, 4004, 0, 0, 75, 0, 0, 0, 0, 299, 0] payload_rows_per_dst=[8178, 8192, 5742, 8178, 8172, 7600, 8172, 8192] dup_rows_per_dst=[0, 108, 0, 0, 0, 454, 1800, 748] combine_rows_per_src=[8178, 8192, 5742, 8178, 8172, 7600, 8172, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55798 payload_rows_remote=47606 dup_rows=9738 dup_rows_remote=9688 dedup_ratio=0.148590 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1277, 8, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2872, 0, 0, 36, 4096, 0, 0, 0, 0, 0, 4094, 0, 0, 95, 0, 0, 1374, 0, 0, 0, 4092, 0, 0, 0, 4096, 0, 0, 0, 25, 0, 2508, 0, 0, 0, 0, 1, 4096, 0] payload_rows_per_dst=[8192, 2570, 8192, 5778, 8188, 6494, 8192, 8192] dup_rows_per_dst=[4, 0, 0, 38, 190, 1970, 50, 7486] combine_rows_per_src=[8192, 2570, 8192, 5778, 8188, 6494, 8192, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62966 payload_rows_remote=54782 dup_rows=2570 dup_rows_remote=2570 dedup_ratio=0.039215 router_tokens_per_expert=[15, 0, 0, 0, 0, 4075, 0, 0, 0, 0, 0, 40, 21, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3059, 0, 0, 4096, 4092, 0, 0, 0, 1, 0, 0, 0, 0, 4096, 0, 0, 0, 4092, 3, 0, 0, 3532, 0, 0, 639, 0, 0, 0, 0, 630, 0, 0, 4075, 0, 0, 28, 0, 0, 0, 0, 268, 0] payload_rows_per_dst=[8180, 8192, 6130, 8186, 8184, 7718, 8184, 8192] dup_rows_per_dst=[0, 122, 0, 0, 0, 630, 1226, 592] combine_rows_per_src=[8180, 8192, 6130, 8186, 8184, 7718, 8184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=317571072 dispatch_meta_bytes=179512 combine_payload_bytes=317571072 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=317750584 ingress_bytes=537772032 link_bytes=537772032 token_bytes=635321656 weight_movement_bytes=220200960 total_nvlink_bytes=855522616 weight_to_token_ratio=0.346598 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=16128 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.352740119e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53934 payload_rows_remote=45742 dup_rows=11602 dup_rows_remote=11594 dedup_ratio=0.177032 router_tokens_per_expert=[0, 3984, 0, 0, 0, 3121, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2903, 0, 4081, 0, 0, 0, 0, 963, 0, 0, 0, 0, 882, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 167, 270, 4090, 0, 0, 0, 4096, 4004, 0, 109, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7338, 8192, 8168, 1926, 3888, 8188, 8042, 8192] dup_rows_per_dst=[3982, 8, 5800, 0, 766, 0, 840, 206] combine_rows_per_src=[7338, 8192, 8168, 1926, 3888, 8188, 8042, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=381882368 dispatch_meta_bytes=233848 combine_payload_bytes=381882368 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=382116216 ingress_bytes=602083328 link_bytes=602083328 token_bytes=763998584 weight_movement_bytes=220200960 total_nvlink_bytes=984199544 weight_to_token_ratio=0.288222 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=25216 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.047121933e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61260 payload_rows_remote=53068 dup_rows=4276 dup_rows_remote=4264 dedup_ratio=0.065247 router_tokens_per_expert=[0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 45, 0, 4089, 1, 139, 0, 0, 0, 34, 3785, 0, 0, 0, 0, 4096, 0, 0, 0, 4085, 4096, 18, 0, 4077, 0, 0, 0, 0, 3898, 0, 0, 0, 0, 0, 0, 12, 0, 0, 276, 16, 0, 0, 0, 0, 0, 4095, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 7570, 7798, 8000, 5130, 8190, 8188] dup_rows_per_dst=[12, 816, 68, 34, 2976, 338, 32, 0] combine_rows_per_src=[8192, 8192, 7570, 7798, 8000, 5130, 8190, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=406310912 dispatch_meta_bytes=239176 combine_payload_bytes=406310912 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=406550088 ingress_bytes=516411392 link_bytes=516411392 token_bytes=812861000 weight_movement_bytes=110100480 total_nvlink_bytes=922961480 weight_to_token_ratio=0.135448 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=24960 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.981243458e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=341239808 dispatch_meta_bytes=229176 combine_payload_bytes=341239808 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=341468984 ingress_bytes=561440768 link_bytes=561440768 token_bytes=682708792 weight_movement_bytes=220200960 total_nvlink_bytes=902909752 weight_to_token_ratio=0.322540 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.764378367e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=392677376 dispatch_meta_bytes=229408 combine_payload_bytes=392677376 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=392906784 ingress_bytes=722978816 link_bytes=722978816 token_bytes=785584160 weight_movement_bytes=330301440 total_nvlink_bytes=1115885600 weight_to_token_ratio=0.420453 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.257379071e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63258 payload_rows_remote=55744 dup_rows=2278 dup_rows_remote=1752 dedup_ratio=0.034760 router_tokens_per_expert=[4, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 39, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3306, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4095, 0, 0, 0, 3274, 0, 0, 746, 0, 0, 0, 0, 553, 0, 0, 4084, 0, 0, 15, 0, 0, 0, 0, 277, 0] payload_rows_per_dst=[8176, 8190, 6614, 8192, 8190, 7514, 8190, 8192] dup_rows_per_dst=[2, 82, 0, 0, 0, 526, 1084, 584] combine_rows_per_src=[8176, 8190, 6614, 8192, 8190, 7514, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=327878656 dispatch_meta_bytes=229344 combine_payload_bytes=327878656 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=328108000 ingress_bytes=437979136 link_bytes=437979136 token_bytes=655986656 weight_movement_bytes=110100480 total_nvlink_bytes=766087136 weight_to_token_ratio=0.167840 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.617177527e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7116814584 prefetch_bytes=880803840 reduce_bytes=880803840 weight_movement_bytes=1761607680 egress_bytes=3559551224 ingress_bytes=5318871040 link_bytes=5318871040 total_nvlink_bytes=8878422264 weight_to_token_ratio=0.247528 prefetch_share_of_ingress=0.165600 dedup_ratio=0.135757 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=8 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6477 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.250391543e-05 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=380391424 dispatch_meta_bytes=229328 combine_payload_bytes=380391424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=380620752 ingress_bytes=490491904 link_bytes=490491904 token_bytes=761012176 weight_movement_bytes=110100480 total_nvlink_bytes=871112656 weight_to_token_ratio=0.144676 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.055147 gemm_active_groups=3 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.530460931e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7093211784 prefetch_bytes=1101004800 reduce_bytes=1541406720 weight_movement_bytes=2642411520 egress_bytes=3547761288 ingress_bytes=6187862016 link_bytes=6187862016 total_nvlink_bytes=9735623304 weight_to_token_ratio=0.372527 prefetch_share_of_ingress=0.177930 dedup_ratio=0.133939 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=14 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.076170977e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56928 payload_rows_remote=48736 dup_rows=8608 dup_rows_remote=8028 dedup_ratio=0.131348 router_tokens_per_expert=[17, 0, 0, 0, 0, 4085, 0, 0, 0, 0, 0, 40, 13, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2987, 0, 0, 4096, 4092, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4089, 7, 0, 0, 3157, 0, 0, 884, 0, 0, 0, 0, 862, 0, 0, 4049, 0, 0, 37, 0, 0, 0, 0, 253, 0] payload_rows_per_dst=[8192, 7288, 5982, 8180, 3522, 7388, 8184, 8192] dup_rows_per_dst=[12, 92, 0, 0, 0, 708, 7216, 580] combine_rows_per_src=[8192, 7288, 5982, 8180, 3522, 7388, 8184, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=399572992 dispatch_meta_bytes=229984 combine_payload_bytes=399572992 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=399802976 ingress_bytes=619773952 link_bytes=619773952 token_bytes=799375968 weight_movement_bytes=220200960 total_nvlink_bytes=1019576928 weight_to_token_ratio=0.275466 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=26752 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.077888838e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63190 payload_rows_remote=55010 dup_rows=2346 dup_rows_remote=2346 dedup_ratio=0.035797 router_tokens_per_expert=[16, 0, 0, 0, 0, 4074, 0, 0, 0, 0, 0, 26, 24, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3171, 0, 0, 4095, 4090, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4090, 0, 0, 0, 3381, 0, 0, 843, 0, 0, 0, 0, 621, 0, 0, 4066, 0, 0, 42, 0, 0, 0, 0, 127, 0] payload_rows_per_dst=[8176, 8190, 6354, 8180, 8180, 7730, 8188, 8192] dup_rows_per_dst=[4, 100, 0, 0, 0, 718, 1186, 338] combine_rows_per_src=[8176, 8190, 6354, 8180, 8180, 7730, 8188, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=63176 payload_rows_remote=54986 dup_rows=2360 dup_rows_remote=1572 dedup_ratio=0.036011 router_tokens_per_expert=[0, 0, 0, 0, 0, 4088, 0, 0, 0, 0, 0, 22, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3082, 0, 0, 4095, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 0, 3541, 0, 0, 897, 0, 0, 0, 0, 396, 0, 0, 4093, 0, 0, 25, 0, 0, 0, 0, 233, 0] payload_rows_per_dst=[8176, 8190, 6164, 8192, 8192, 7880, 8190, 8192] dup_rows_per_dst=[0, 60, 0, 0, 0, 996, 788, 516] combine_rows_per_src=[8176, 8190, 6164, 8192, 8192, 7880, 8190, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=349339648 dispatch_meta_bytes=227056 combine_payload_bytes=349339648 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349566704 ingress_bytes=459440128 link_bytes=459440128 token_bytes=698906352 weight_movement_bytes=110100480 total_nvlink_bytes=809006832 weight_to_token_ratio=0.157533 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=30464 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.990419475e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6995197944 prefetch_bytes=990904320 reduce_bytes=660602880 weight_movement_bytes=1651507200 egress_bytes=3498719224 ingress_bytes=5147985920 link_bytes=5147985920 total_nvlink_bytes=8646705144 weight_to_token_ratio=0.236092 prefetch_share_of_ingress=0.192484 dedup_ratio=0.139380 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=6 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049265 gemm_inner_pad_upper_bound=7112 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.953194213e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=64960 payload_rows_remote=56776 dup_rows=576 dup_rows_remote=466 dedup_ratio=0.008789 router_tokens_per_expert=[0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 86, 0, 4038, 18, 5, 0, 0, 0, 44, 4007, 0, 0, 0, 0, 4096, 0, 0, 0, 4088, 4096, 12, 0, 4007, 0, 0, 0, 0, 3917, 0, 0, 0, 0, 0, 0, 137, 0, 0, 104, 11, 0, 0, 0, 0, 0, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8184, 8014, 8176, 7834, 8182, 8186, 8192] dup_rows_per_dst=[18, 110, 88, 24, 0, 314, 22, 0] combine_rows_per_src=[8192, 8184, 8014, 8176, 7834, 8182, 8186, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394311680 dispatch_meta_bytes=229424 combine_payload_bytes=394311680 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=394541104 ingress_bytes=724613120 link_bytes=724613120 token_bytes=788852784 weight_movement_bytes=330301440 total_nvlink_bytes=1119154224 weight_to_token_ratio=0.418711 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.260221394e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=394139648 dispatch_meta_bytes=226232 combine_payload_bytes=394139648 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=394365880 ingress_bytes=614340608 link_bytes=614340608 token_bytes=788505528 weight_movement_bytes=220200960 total_nvlink_bytes=1008706488 weight_to_token_ratio=0.279264 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=24704 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.068439359e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56338 payload_rows_remote=48148 dup_rows=9198 dup_rows_remote=9194 dedup_ratio=0.140350 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1301, 40, 0, 0, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2874, 0, 0, 136, 4095, 0, 0, 0, 0, 0, 4086, 3, 0, 92, 0, 0, 1462, 0, 2, 0, 4073, 0, 0, 0, 4096, 0, 0, 0, 33, 0, 2328, 0, 0, 0, 0, 0, 4049, 0] payload_rows_per_dst=[8190, 8190, 8192, 8146, 8192, 2924, 7848, 4656] dup_rows_per_dst=[4, 2590, 0, 6020, 522, 4, 58, 0] combine_rows_per_src=[8190, 8190, 8192, 8146, 8192, 2924, 7848, 4656] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6532687400 prefetch_bytes=1101004800 reduce_bytes=550502400 weight_movement_bytes=1651507200 egress_bytes=3267405352 ingress_bytes=4916789248 link_bytes=4916789248 total_nvlink_bytes=8184194600 weight_to_token_ratio=0.252807 prefetch_share_of_ingress=0.223928 dedup_ratio=0.185367 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=5 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048162 gemm_inner_pad_upper_bound=7493 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.551105175e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=406970368 dispatch_meta_bytes=228968 combine_payload_bytes=406970368 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=407199336 ingress_bytes=627171328 link_bytes=627171328 token_bytes=814169704 weight_movement_bytes=220200960 total_nvlink_bytes=1034370664 weight_to_token_ratio=0.270461 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.090754091e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7091688872 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3546983848 ingress_bytes=5856815104 link_bytes=5856815104 total_nvlink_bytes=9403798952 weight_to_token_ratio=0.326031 prefetch_share_of_ingress=0.150390 dedup_ratio=0.134601 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.047794 gemm_inner_pad_upper_bound=7620 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.018596475e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6865455008 prefetch_bytes=990904320 reduce_bytes=1541406720 weight_movement_bytes=2532311040 egress_bytes=3433861024 ingress_bytes=5963905024 link_bytes=5963905024 total_nvlink_bytes=9397766048 weight_to_token_ratio=0.368848 prefetch_share_of_ingress=0.166150 dedup_ratio=0.155249 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=14 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.048529 gemm_inner_pad_upper_bound=6985 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.037221173e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=345124864 dispatch_meta_bytes=229368 combine_payload_bytes=345124864 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=345354232 ingress_bytes=455225344 link_bytes=455225344 token_bytes=690479096 weight_movement_bytes=110100480 total_nvlink_bytes=800579576 weight_to_token_ratio=0.159455 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.917117449e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55954 payload_rows_remote=47762 dup_rows=9582 dup_rows_remote=5030 dedup_ratio=0.146210 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2239, 67, 0, 1, 4096, 0, 0, 0, 0, 0, 0, 0, 0, 2955, 0, 0, 53, 4092, 0, 0, 2, 0, 0, 4081, 7, 0, 14, 0, 0, 1404, 0, 3, 0, 4022, 0, 0, 0, 4096, 0, 0, 0, 21, 0, 1529, 0, 0, 0, 0, 19, 4066, 0] payload_rows_per_dst=[8184, 8192, 8192, 7802, 8166, 4134, 8192, 3092] dup_rows_per_dst=[2, 4552, 2, 4294, 42, 644, 42, 4] combine_rows_per_src=[8184, 8192, 8192, 7802, 8166, 4134, 8192, 3092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53064 payload_rows_remote=44872 dup_rows=12472 dup_rows_remote=12368 dedup_ratio=0.190308 router_tokens_per_expert=[26, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 0, 26, 44, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3553, 0, 0, 4096, 4093, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4080, 4, 0, 0, 3492, 0, 0, 445, 0, 0, 0, 0, 493, 0, 0, 4078, 0, 0, 10, 0, 0, 0, 0, 181, 0] payload_rows_per_dst=[8192, 8192, 8180, 8186, 8192, 7438, 986, 3698] dup_rows_per_dst=[104, 140, 7098, 60, 4732, 226, 0, 112] combine_rows_per_src=[8192, 8192, 8180, 8186, 8192, 7438, 986, 3698] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=342358016 dispatch_meta_bytes=211168 combine_payload_bytes=342358016 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=342569184 ingress_bytes=452458496 link_bytes=452458496 token_bytes=684927200 weight_movement_bytes=110100480 total_nvlink_bytes=795027680 weight_to_token_ratio=0.160748 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.868997412e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=321642496 dispatch_meta_bytes=228960 combine_payload_bytes=321642496 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=321871456 ingress_bytes=431742976 link_bytes=431742976 token_bytes=643513952 weight_movement_bytes=110100480 total_nvlink_bytes=753614432 weight_to_token_ratio=0.171093 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.508720448e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6419976952 prefetch_bytes=1101004800 reduce_bytes=880803840 weight_movement_bytes=1981808640 egress_bytes=3211092728 ingress_bytes=5190692864 link_bytes=5190692864 total_nvlink_bytes=8401785592 weight_to_token_ratio=0.308694 prefetch_share_of_ingress=0.212111 dedup_ratio=0.201190 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=8 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=6858 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.027468613e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58066 payload_rows_remote=49874 dup_rows=7470 dup_rows_remote=7264 dedup_ratio=0.113983 router_tokens_per_expert=[90, 0, 0, 0, 0, 3982, 0, 0, 0, 0, 0, 39, 64, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2868, 0, 0, 4096, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4086, 4, 0, 0, 2912, 0, 0, 1089, 0, 0, 0, 0, 795, 0, 0, 4029, 0, 0, 72, 0, 0, 0, 0, 440, 0] payload_rows_per_dst=[8144, 8192, 7972, 8182, 8172, 7168, 2044, 8192] dup_rows_per_dst=[0, 206, 5300, 0, 0, 842, 98, 1024] combine_rows_per_src=[8144, 8192, 7972, 8182, 8172, 7168, 2044, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=357496832 dispatch_meta_bytes=228552 combine_payload_bytes=357496832 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=357725384 ingress_bytes=467597312 link_bytes=467597312 token_bytes=715222216 weight_movement_bytes=110100480 total_nvlink_bytes=825322696 weight_to_token_ratio=0.153939 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.132286321e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6730148496 prefetch_bytes=1101004800 reduce_bytes=660602880 weight_movement_bytes=1761607680 egress_bytes=3366191760 ingress_bytes=5125564416 link_bytes=5125564416 total_nvlink_bytes=8491756176 weight_to_token_ratio=0.261749 prefetch_share_of_ingress=0.214807 dedup_ratio=0.164081 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=6 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.052206 gemm_inner_pad_upper_bound=6350 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.914199530e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:57:48] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 679.1 | throughput per GPU (TFLOP/s/GPU): 181.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.541231E+00 | loss scale: 1.0 | grad norm: 4.296 | number of skipped iterations: 0 | number of nan iterations: 0 | -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44062 payload_rows_remote=38988 dup_rows=21474 dup_rows_remote=20084 dedup_ratio=0.327667 router_tokens_per_expert=[1062, 959, 1235, 328, 432, 826, 623, 58, 644, 550, 359, 513, 17, 772, 377, 830, 515, 562, 39, 75, 580, 555, 697, 209, 550, 695, 671, 442, 699, 579, 737, 22, 619, 702, 767, 21, 1205, 34, 275, 335, 467, 803, 701, 814, 7, 936, 106, 561, 395, 314, 703, 406, 410, 1045, 241, 475, 663, 554, 418, 173, 298, 666, 421, 21] payload_rows_per_dst=[6380, 5442, 5074, 5992, 5302, 5816, 5412, 4644] dup_rows_per_dst=[4666, 2682, 1390, 2798, 2614, 2974, 2566, 1784] combine_rows_per_src=[6380, 5442, 5074, 5992, 5302, 5816, 5412, 4644] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43958 payload_rows_remote=38494 dup_rows=21578 dup_rows_remote=18494 dedup_ratio=0.329254 router_tokens_per_expert=[629, 1220, 402, 508, 281, 1021, 1109, 178, 390, 525, 439, 270, 13, 890, 196, 518, 443, 563, 29, 72, 863, 176, 541, 315, 638, 512, 534, 832, 807, 277, 618, 19, 666, 621, 753, 26, 783, 0, 1140, 285, 539, 622, 452, 612, 3, 539, 487, 473, 423, 491, 735, 785, 371, 776, 658, 443, 967, 1198, 647, 188, 340, 460, 449, 8] payload_rows_per_dst=[5128, 5134, 5738, 5808, 5464, 5220, 5876, 5590] dup_rows_per_dst=[2020, 2456, 2706, 2666, 3084, 2234, 3488, 2924] combine_rows_per_src=[5128, 5134, 5738, 5808, 5464, 5220, 5876, 5590] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44366 payload_rows_remote=38880 dup_rows=21170 dup_rows_remote=18646 dedup_ratio=0.323029 router_tokens_per_expert=[1132, 727, 465, 755, 622, 709, 376, 73, 253, 425, 695, 448, 58, 699, 230, 689, 998, 581, 70, 66, 618, 549, 590, 372, 456, 451, 477, 497, 873, 383, 748, 28, 423, 711, 402, 20, 852, 8, 1127, 455, 511, 578, 469, 691, 9, 722, 160, 865, 650, 493, 686, 603, 390, 959, 259, 578, 785, 700, 596, 333, 351, 830, 429, 10] payload_rows_per_dst=[4886, 5630, 6014, 5606, 5296, 5486, 5884, 5564] dup_rows_per_dst=[1960, 2782, 3128, 2220, 2700, 2524, 3352, 2504] combine_rows_per_src=[4886, 5630, 6014, 5606, 5296, 5486, 5884, 5564] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43528 payload_rows_remote=38572 dup_rows=22008 dup_rows_remote=19866 dedup_ratio=0.335815 router_tokens_per_expert=[364, 911, 668, 942, 115, 1167, 791, 74, 860, 698, 842, 591, 20, 1013, 266, 1061, 768, 444, 19, 47, 759, 219, 692, 323, 309, 385, 559, 555, 682, 397, 310, 6, 915, 1381, 581, 45, 1128, 0, 860, 416, 1030, 481, 357, 428, 23, 477, 357, 727, 548, 336, 149, 797, 145, 552, 168, 854, 541, 568, 450, 177, 244, 574, 588, 14] payload_rows_per_dst=[4464, 7138, 5724, 4842, 6106, 5724, 4956, 4574] dup_rows_per_dst=[1444, 5898, 2640, 1564, 3836, 2746, 2142, 1738] combine_rows_per_src=[4464, 7138, 5724, 4842, 6106, 5724, 4956, 4574] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44056 payload_rows_remote=38568 dup_rows=21480 dup_rows_remote=19482 dedup_ratio=0.327759 router_tokens_per_expert=[1011, 1171, 465, 192, 349, 692, 518, 69, 423, 401, 204, 240, 44, 842, 91, 802, 648, 776, 30, 90, 554, 675, 551, 170, 337, 393, 508, 568, 861, 369, 681, 26, 801, 963, 449, 12, 817, 73, 800, 697, 472, 597, 469, 749, 6, 972, 132, 618, 576, 571, 363, 1141, 366, 646, 189, 713, 1128, 812, 436, 420, 521, 1215, 277, 16] payload_rows_per_dst=[4754, 4432, 6110, 5488, 5872, 5350, 5738, 6312] dup_rows_per_dst=[2032, 1662, 3026, 1998, 3352, 2680, 3392, 3338] combine_rows_per_src=[4754, 4432, 6110, 5488, 5872, 5350, 5738, 6312] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=43626 payload_rows_remote=37144 dup_rows=21910 dup_rows_remote=16824 dedup_ratio=0.334320 router_tokens_per_expert=[800, 816, 543, 372, 1001, 1056, 666, 83, 332, 493, 308, 290, 18, 633, 165, 560, 541, 388, 19, 86, 748, 550, 725, 385, 408, 953, 500, 897, 714, 271, 739, 46, 1069, 621, 676, 7, 740, 9, 708, 407, 401, 872, 445, 704, 0, 790, 145, 637, 621, 209, 643, 680, 206, 680, 225, 566, 981, 1183, 592, 213, 418, 852, 351, 11] payload_rows_per_dst=[6482, 4202, 5194, 6286, 4786, 5504, 5216, 5956] dup_rows_per_dst=[5086, 1396, 1690, 3384, 2180, 2484, 2444, 3246] combine_rows_per_src=[6482, 4202, 5194, 6286, 4786, 5504, 5216, 5956] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44568 payload_rows_remote=38768 dup_rows=20968 dup_rows_remote=18534 dedup_ratio=0.319946 router_tokens_per_expert=[773, 1023, 268, 520, 970, 658, 872, 222, 375, 469, 681, 224, 3, 1018, 471, 523, 888, 557, 105, 89, 762, 834, 688, 382, 450, 952, 398, 622, 892, 336, 566, 19, 669, 661, 1101, 9, 773, 26, 971, 192, 384, 758, 333, 747, 1, 781, 168, 591, 326, 225, 719, 336, 359, 824, 255, 518, 989, 728, 479, 209, 469, 304, 212, 41] payload_rows_per_dst=[5196, 5836, 6496, 5744, 5002, 5256, 5238, 5800] dup_rows_per_dst=[2054, 3008, 4160, 2726, 2024, 2444, 2118, 2434] combine_rows_per_src=[5196, 5836, 6496, 5744, 5002, 5256, 5238, 5800] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=279465984 dispatch_meta_bytes=236288 combine_payload_bytes=279465984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=279702272 ingress_bytes=389566464 link_bytes=389566464 token_bytes=559168256 weight_movement_bytes=110100480 total_nvlink_bytes=669268736 weight_to_token_ratio=0.196900 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=5632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.775201536e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=275924992 dispatch_meta_bytes=227952 combine_payload_bytes=275924992 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=276152944 ingress_bytes=826427392 link_bytes=826427392 token_bytes=552077936 weight_movement_bytes=550502400 total_nvlink_bytes=1102580336 weight_to_token_ratio=0.997146 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.044118 gemm_active_groups=9 gemm_peak_group_rows=6016 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.437293159e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=278691840 dispatch_meta_bytes=230104 combine_payload_bytes=278691840 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278921944 ingress_bytes=388792320 link_bytes=388792320 token_bytes=557613784 weight_movement_bytes=110100480 total_nvlink_bytes=667714264 weight_to_token_ratio=0.197449 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.761737898e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276484096 dispatch_meta_bytes=233752 combine_payload_bytes=276484096 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276717848 ingress_bytes=386584576 link_bytes=386584576 token_bytes=553201944 weight_movement_bytes=110100480 total_nvlink_bytes=663302424 weight_to_token_ratio=0.199024 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6272 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.723341599e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=276455424 dispatch_meta_bytes=232200 combine_payload_bytes=276455424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=276687624 ingress_bytes=386555904 link_bytes=386555904 token_bytes=553143048 weight_movement_bytes=110100480 total_nvlink_bytes=663243528 weight_to_token_ratio=0.199045 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.722842946e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=266248192 dispatch_meta_bytes=215872 combine_payload_bytes=266248192 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=266464064 ingress_bytes=596549632 link_bytes=596549632 token_bytes=532712256 weight_movement_bytes=330301440 total_nvlink_bytes=863013696 weight_to_token_ratio=0.620037 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=6400 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.037497926e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=277889024 dispatch_meta_bytes=229208 combine_payload_bytes=277889024 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=278118232 ingress_bytes=387989504 link_bytes=387989504 token_bytes=556007256 weight_movement_bytes=110100480 total_nvlink_bytes=666107736 weight_to_token_ratio=0.198020 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=7168 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.747775608e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50760 payload_rows_remote=44070 dup_rows=14776 dup_rows_remote=12830 dedup_ratio=0.225464 router_tokens_per_expert=[0, 0, 0, 4, 184, 0, 1876, 0, 1206, 1724, 93, 0, 6, 2949, 0, 3093, 19, 2607, 0, 0, 143, 1537, 0, 0, 0, 0, 51, 3407, 1070, 21, 0, 557, 0, 0, 0, 0, 1, 260, 0, 0, 0, 9, 129, 0, 0, 0, 2374, 0, 0, 989, 334, 0, 1762, 0, 0, 136, 298, 12, 0, 115, 798, 2561, 2408, 35] payload_rows_per_dst=[6950, 7764, 6690, 7264, 6072, 4978, 5072, 5970] dup_rows_per_dst=[2300, 4480, 1946, 2948, 348, 46, 1370, 1338] combine_rows_per_src=[6950, 7764, 6690, 7264, 6072, 4978, 5072, 5970] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50422 payload_rows_remote=44204 dup_rows=15114 dup_rows_remote=15052 dedup_ratio=0.230621 router_tokens_per_expert=[0, 0, 0, 2, 154, 0, 1445, 10, 636, 1590, 34, 0, 4, 3057, 0, 2910, 1, 2818, 0, 0, 90, 2139, 0, 0, 0, 0, 116, 2476, 1773, 12, 0, 462, 0, 0, 0, 0, 5, 78, 0, 0, 0, 99, 3, 0, 0, 0, 3000, 0, 0, 1144, 57, 0, 1579, 0, 0, 42, 389, 81, 0, 110, 1108, 2945, 2366, 33] payload_rows_per_dst=[6998, 6066, 5086, 6024, 6218, 7308, 6610, 6112] dup_rows_per_dst=[2114, 1772, 704, 2316, 62, 2744, 3340, 2062] combine_rows_per_src=[6998, 6066, 5086, 6024, 6218, 7308, 6610, 6112] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48652 payload_rows_remote=41170 dup_rows=16884 dup_rows_remote=12886 dedup_ratio=0.257629 router_tokens_per_expert=[0, 0, 0, 30, 104, 0, 1510, 37, 704, 1247, 196, 0, 20, 2840, 0, 2805, 28, 2847, 0, 0, 52, 2032, 0, 0, 0, 0, 150, 2886, 1892, 8, 0, 745, 0, 0, 0, 0, 0, 298, 0, 0, 0, 68, 9, 0, 0, 0, 2777, 0, 0, 1262, 99, 0, 1142, 0, 0, 32, 398, 14, 0, 191, 783, 2719, 2827, 16] payload_rows_per_dst=[6932, 3848, 4206, 5196, 7334, 7482, 7120, 6534] dup_rows_per_dst=[1868, 486, 18, 1010, 3936, 3998, 3644, 1924] combine_rows_per_src=[6932, 3848, 4206, 5196, 7334, 7482, 7120, 6534] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51564 payload_rows_remote=44880 dup_rows=13972 dup_rows_remote=11774 dedup_ratio=0.213196 router_tokens_per_expert=[0, 0, 0, 1, 86, 0, 1837, 17, 924, 1343, 89, 0, 16, 3353, 0, 3050, 7, 2552, 0, 2, 29, 2658, 0, 0, 3, 0, 70, 3040, 1085, 5, 0, 238, 0, 0, 0, 0, 42, 98, 0, 10, 0, 28, 13, 5, 0, 0, 2631, 0, 0, 1142, 176, 0, 1424, 0, 0, 11, 335, 108, 0, 248, 704, 2595, 2744, 49] payload_rows_per_dst=[7012, 7458, 7550, 6684, 6850, 5310, 4446, 6254] dup_rows_per_dst=[2060, 3386, 2946, 2198, 156, 44, 1060, 2122] combine_rows_per_src=[7012, 7458, 7550, 6684, 6850, 5310, 4446, 6254] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48308 payload_rows_remote=41534 dup_rows=17228 dup_rows_remote=13806 dedup_ratio=0.262878 router_tokens_per_expert=[0, 0, 0, 2, 168, 0, 1660, 37, 519, 1605, 81, 0, 26, 2680, 0, 3106, 10, 2446, 0, 0, 41, 2181, 0, 0, 0, 0, 161, 3050, 1424, 8, 0, 496, 0, 0, 0, 0, 0, 70, 0, 1, 0, 55, 14, 0, 0, 0, 2561, 0, 0, 1244, 100, 0, 1282, 0, 0, 26, 641, 3, 4, 246, 812, 2748, 3197, 63] payload_rows_per_dst=[6732, 4136, 4432, 3740, 7608, 7522, 6774, 7364] dup_rows_per_dst=[2100, 326, 32, 438, 4106, 3838, 3422, 2966] combine_rows_per_src=[6732, 4136, 4432, 3740, 7608, 7522, 6774, 7364] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48136 payload_rows_remote=41298 dup_rows=17400 dup_rows_remote=15602 dedup_ratio=0.265503 router_tokens_per_expert=[0, 0, 0, 4, 174, 0, 1340, 14, 949, 1556, 25, 0, 4, 3190, 0, 3153, 2, 2607, 0, 0, 92, 2308, 0, 0, 0, 0, 41, 2950, 1955, 8, 0, 612, 0, 0, 0, 0, 0, 100, 0, 0, 0, 84, 7, 0, 0, 0, 2729, 0, 0, 1350, 126, 0, 1147, 0, 0, 47, 376, 111, 0, 124, 662, 2786, 2083, 52] payload_rows_per_dst=[6838, 7622, 7808, 7156, 6492, 5618, 4364, 2238] dup_rows_per_dst=[1798, 3752, 6376, 3976, 88, 22, 976, 412] combine_rows_per_src=[6838, 7622, 7808, 7156, 6492, 5618, 4364, 2238] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=45434 payload_rows_remote=37458 dup_rows=20102 dup_rows_remote=11166 dedup_ratio=0.306732 router_tokens_per_expert=[0, 0, 0, 3, 138, 0, 1425, 143, 803, 1153, 75, 0, 7, 3145, 0, 2863, 53, 2888, 0, 0, 162, 1558, 0, 0, 0, 0, 136, 2887, 2177, 42, 0, 572, 0, 0, 0, 0, 6, 104, 0, 0, 0, 70, 43, 8, 2, 0, 2643, 0, 0, 1181, 68, 1, 1281, 0, 0, 131, 327, 123, 0, 144, 873, 3177, 2313, 43] payload_rows_per_dst=[3368, 3712, 3422, 5192, 7050, 7520, 7194, 7976] dup_rows_per_dst=[50, 364, 124, 662, 2274, 3786, 3906, 8936] combine_rows_per_src=[3368, 3712, 3422, 5192, 7050, 7520, 7194, 7976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=315893760 dispatch_meta_bytes=227600 combine_payload_bytes=315893760 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=316121360 ingress_bytes=536094720 link_bytes=536094720 token_bytes=632015120 weight_movement_bytes=220200960 total_nvlink_bytes=852216080 weight_to_token_ratio=0.348411 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.323568905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=316854272 dispatch_meta_bytes=237024 combine_payload_bytes=316854272 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=317091296 ingress_bytes=537055232 link_bytes=537055232 token_bytes=633945568 weight_movement_bytes=220200960 total_nvlink_bytes=854146528 weight_to_token_ratio=0.347350 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.040441 gemm_active_groups=9 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.340273789e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=295106560 dispatch_meta_bytes=216224 combine_payload_bytes=295106560 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=295322784 ingress_bytes=405207040 link_bytes=405207040 token_bytes=590429344 weight_movement_bytes=110100480 total_nvlink_bytes=700529824 weight_to_token_ratio=0.186475 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.047216877e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=321699840 dispatch_meta_bytes=226616 combine_payload_bytes=321699840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=321926456 ingress_bytes=541900800 link_bytes=541900800 token_bytes=643626296 weight_movement_bytes=220200960 total_nvlink_bytes=863827256 weight_to_token_ratio=0.342125 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=12928 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.424546186e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=297715712 dispatch_meta_bytes=221360 combine_payload_bytes=297715712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=297937072 ingress_bytes=407816192 link_bytes=407816192 token_bytes=595652784 weight_movement_bytes=110100480 total_nvlink_bytes=705753264 weight_to_token_ratio=0.184840 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=11008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.092594321e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=296024064 dispatch_meta_bytes=227600 combine_payload_bytes=296024064 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=296251664 ingress_bytes=406124544 link_bytes=406124544 token_bytes=592275728 weight_movement_bytes=110100480 total_nvlink_bytes=702376208 weight_to_token_ratio=0.185894 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=18816 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.063173780e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=268498944 dispatch_meta_bytes=194496 combine_payload_bytes=268498944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=268693440 ingress_bytes=598800384 link_bytes=598800384 token_bytes=537192384 weight_movement_bytes=330301440 total_nvlink_bytes=867493824 weight_to_token_ratio=0.614866 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=15872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.041412354e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=44746 payload_rows_remote=39296 dup_rows=20790 dup_rows_remote=18402 dedup_ratio=0.317230 router_tokens_per_expert=[609, 989, 329, 202, 544, 1128, 628, 63, 888, 497, 619, 361, 17, 572, 178, 787, 778, 369, 86, 100, 579, 387, 630, 294, 419, 637, 400, 753, 846, 390, 745, 17, 795, 572, 819, 26, 647, 1, 697, 402, 712, 684, 561, 783, 1, 1035, 157, 614, 544, 462, 435, 511, 310, 663, 205, 727, 829, 1347, 310, 566, 398, 499, 588, 27] payload_rows_per_dst=[5930, 5450, 5050, 5912, 5284, 5894, 5166, 6060] dup_rows_per_dst=[3054, 2388, 1396, 2502, 2634, 3200, 2548, 3068] combine_rows_per_src=[5930, 5450, 5050, 5912, 5284, 5894, 5166, 6060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57380 payload_rows_remote=50244 dup_rows=8156 dup_rows_remote=7650 dedup_ratio=0.124451 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 185, 0, 0, 18, 0, 0, 0, 0, 502, 0, 2824, 0, 0, 3757, 0, 40, 3936, 0, 0, 0, 2214, 568, 0, 1353, 0, 0, 0, 0, 0, 0, 1757, 3339, 62, 0, 265, 0, 3556, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 0, 3690, 0, 0, 4087, 0, 0, 0, 1] payload_rows_per_dst=[7872, 8174, 5698, 6334, 7222, 7136, 7564, 7380] dup_rows_per_dst=[370, 1040, 30, 1936, 3094, 506, 1178, 2] combine_rows_per_src=[7872, 8174, 5698, 6334, 7222, 7136, 7564, 7380] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57244 payload_rows_remote=51620 dup_rows=8292 dup_rows_remote=8224 dedup_ratio=0.126526 router_tokens_per_expert=[0, 0, 0, 7, 0, 0, 239, 0, 0, 0, 0, 0, 0, 0, 740, 0, 2777, 0, 0, 3787, 0, 69, 3948, 0, 0, 0, 869, 855, 0, 2128, 14, 0, 0, 0, 0, 0, 1446, 3532, 79, 0, 290, 0, 3505, 0, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 3772, 0, 0, 4074, 0, 0, 0, 11] payload_rows_per_dst=[7900, 8150, 5624, 6044, 7320, 7054, 7608, 7544] dup_rows_per_dst=[488, 1478, 68, 1688, 2794, 536, 1218, 22] combine_rows_per_src=[7900, 8150, 5624, 6044, 7320, 7054, 7608, 7544] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56544 payload_rows_remote=49120 dup_rows=8992 dup_rows_remote=5594 dedup_ratio=0.137207 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 184, 0, 0, 1, 0, 0, 0, 0, 976, 0, 2670, 0, 0, 3851, 0, 34, 4014, 0, 0, 0, 1329, 457, 0, 1234, 0, 0, 0, 0, 0, 0, 1692, 3521, 198, 0, 275, 0, 3666, 0, 0, 0, 0, 0, 0, 0, 852, 0, 0, 1, 0, 0, 3720, 0, 0, 4086, 0, 1, 0, 0] payload_rows_per_dst=[8030, 8186, 5380, 4992, 7424, 7352, 7738, 7442] dup_rows_per_dst=[378, 1940, 28, 1048, 3398, 530, 1670, 0] combine_rows_per_src=[8030, 8186, 5380, 4992, 7424, 7352, 7738, 7442] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57380 payload_rows_remote=51530 dup_rows=8156 dup_rows_remote=6666 dedup_ratio=0.124451 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 154, 0, 0, 0, 0, 0, 0, 0, 622, 0, 3450, 0, 0, 3684, 0, 60, 3943, 0, 0, 0, 1136, 684, 0, 1847, 3, 0, 0, 0, 0, 0, 1557, 3456, 52, 5, 275, 0, 3208, 0, 0, 0, 0, 0, 0, 0, 993, 0, 0, 0, 0, 0, 3553, 0, 0, 4075, 0, 0, 0, 8] payload_rows_per_dst=[7892, 8154, 6946, 5850, 7420, 6474, 7530, 7114] dup_rows_per_dst=[308, 1240, 74, 1490, 2720, 492, 1824, 8] combine_rows_per_src=[7892, 8154, 6946, 5850, 7420, 6474, 7530, 7114] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54892 payload_rows_remote=47150 dup_rows=10644 dup_rows_remote=9550 dedup_ratio=0.162415 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 132, 0, 0, 0, 0, 0, 0, 0, 383, 0, 2601, 0, 0, 3853, 0, 124, 4063, 0, 0, 0, 2021, 679, 0, 1506, 0, 0, 0, 0, 0, 0, 2540, 2921, 62, 0, 302, 2, 3307, 0, 0, 0, 0, 0, 0, 0, 565, 0, 0, 0, 0, 0, 3620, 0, 0, 4086, 0, 0, 0, 0] payload_rows_per_dst=[8126, 5866, 5362, 6534, 6728, 6920, 7742, 7614] dup_rows_per_dst=[266, 462, 88, 1878, 2818, 1802, 1094, 2236] combine_rows_per_src=[8126, 5866, 5362, 6534, 6728, 6920, 7742, 7614] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=46312 payload_rows_remote=38264 dup_rows=19224 dup_rows_remote=18868 dedup_ratio=0.293335 router_tokens_per_expert=[0, 0, 0, 2, 2, 0, 175, 0, 0, 1, 0, 0, 0, 0, 645, 0, 3427, 0, 0, 3813, 0, 57, 4023, 0, 0, 0, 775, 695, 0, 1656, 0, 0, 0, 0, 0, 0, 1534, 3201, 109, 0, 420, 0, 3198, 0, 0, 0, 0, 0, 0, 0, 1263, 0, 0, 0, 0, 0, 3688, 0, 0, 4081, 0, 0, 0, 3] payload_rows_per_dst=[8048, 8164, 8188, 8020, 3212, 6528, 2526, 1626] dup_rows_per_dst=[356, 1290, 12162, 4634, 74, 708, 0, 0] combine_rows_per_src=[8048, 8164, 8188, 8020, 3212, 6528, 2526, 1626] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=360148992 dispatch_meta_bytes=231576 combine_payload_bytes=360148992 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360380568 ingress_bytes=470249472 link_bytes=470249472 token_bytes=720529560 weight_movement_bytes=110100480 total_nvlink_bytes=830630040 weight_to_token_ratio=0.152805 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=27136 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.178411746e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=281673728 dispatch_meta_bytes=230792 combine_payload_bytes=281673728 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=281904520 ingress_bytes=391774208 link_bytes=391774208 token_bytes=563578248 weight_movement_bytes=110100480 total_nvlink_bytes=673678728 weight_to_token_ratio=0.195360 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.040441 gemm_active_groups=10 gemm_peak_group_rows=6528 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.813597835e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47686 payload_rows_remote=39498 dup_rows=17850 dup_rows_remote=8876 dedup_ratio=0.272369 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 163, 0, 0, 4, 0, 0, 0, 0, 993, 0, 3457, 0, 0, 3802, 0, 18, 4028, 0, 0, 0, 1315, 430, 0, 1611, 0, 0, 3, 0, 0, 0, 1236, 3266, 118, 5, 284, 0, 3478, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 3, 0, 0, 3735, 1, 0, 4077, 0, 1, 0, 4] payload_rows_per_dst=[6854, 1994, 7134, 5430, 2658, 7784, 7644, 8188] dup_rows_per_dst=[274, 0, 1076, 1282, 66, 4746, 1432, 8974] combine_rows_per_src=[6854, 1994, 7134, 5430, 2658, 7784, 7644, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=370012160 dispatch_meta_bytes=239376 combine_payload_bytes=370012160 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=370251536 ingress_bytes=700313600 link_bytes=700313600 token_bytes=740263696 weight_movement_bytes=330301440 total_nvlink_bytes=1070565136 weight_to_token_ratio=0.446194 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.044118 gemm_active_groups=8 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.217960532e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=352092160 dispatch_meta_bytes=218856 combine_payload_bytes=352092160 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=352311016 ingress_bytes=682393600 link_bytes=682393600 token_bytes=704403176 weight_movement_bytes=330301440 total_nvlink_bytes=1034704616 weight_to_token_ratio=0.468910 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.186794705e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=369367040 dispatch_meta_bytes=232784 combine_payload_bytes=369367040 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369599824 ingress_bytes=479467520 link_bytes=479467520 token_bytes=738966864 weight_movement_bytes=110100480 total_nvlink_bytes=849067344 weight_to_token_ratio=0.148992 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=13440 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.338728761e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=337971200 dispatch_meta_bytes=226800 combine_payload_bytes=337971200 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338198000 ingress_bytes=448071680 link_bytes=448071680 token_bytes=676169200 weight_movement_bytes=110100480 total_nvlink_bytes=786269680 weight_to_token_ratio=0.162830 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=26112 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.792703466e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=274276352 dispatch_meta_bytes=228528 combine_payload_bytes=274276352 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=274504880 ingress_bytes=384376832 link_bytes=384376832 token_bytes=548781232 weight_movement_bytes=110100480 total_nvlink_bytes=658881712 weight_to_token_ratio=0.200627 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=31360 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=6.684945300e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=283121664 dispatch_meta_bytes=193496 combine_payload_bytes=283121664 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=283315160 ingress_bytes=613423104 link_bytes=613423104 token_bytes=566436824 weight_movement_bytes=330301440 total_nvlink_bytes=896738264 weight_to_token_ratio=0.583121 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=26624 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.066843669e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55604 payload_rows_remote=47542 dup_rows=9932 dup_rows_remote=6724 dedup_ratio=0.151550 router_tokens_per_expert=[1878, 0, 2073, 0, 267, 0, 0, 0, 0, 0, 0, 2173, 0, 0, 0, 0, 0, 0, 66, 0, 0, 0, 72, 0, 0, 54, 0, 180, 0, 0, 0, 0, 0, 0, 0, 0, 1908, 1027, 0, 55, 1662, 0, 0, 0, 0, 0, 3973, 0, 0, 3505, 4036, 0, 0, 0, 0, 0, 3976, 0, 0, 0, 0, 1776, 4087, 0] payload_rows_per_dst=[6808, 4346, 8178, 8072, 5156, 8062, 7010, 7972] dup_rows_per_dst=[1628, 0, 272, 468, 824, 3208, 0, 3532] combine_rows_per_src=[6808, 4346, 8178, 8072, 5156, 8062, 7010, 7972] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50784 payload_rows_remote=42680 dup_rows=14752 dup_rows_remote=14562 dedup_ratio=0.225098 router_tokens_per_expert=[1312, 0, 1371, 0, 274, 0, 0, 0, 1, 0, 0, 2970, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 78, 0, 0, 76, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 2940, 1138, 0, 28, 1855, 0, 1, 0, 0, 0, 3779, 0, 0, 3328, 4059, 0, 0, 0, 0, 0, 3975, 0, 0, 0, 0, 1422, 4051, 0] payload_rows_per_dst=[5094, 7898, 8104, 8120, 6828, 4324, 7572, 2844] dup_rows_per_dst=[820, 5602, 190, 334, 2166, 424, 5216, 0] combine_rows_per_src=[5094, 7898, 8104, 8120, 6828, 4324, 7572, 2844] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56520 payload_rows_remote=50818 dup_rows=9016 dup_rows_remote=8468 dedup_ratio=0.137573 router_tokens_per_expert=[2432, 0, 1721, 0, 139, 0, 0, 0, 14, 0, 0, 2335, 11, 0, 0, 0, 0, 0, 51, 0, 0, 0, 49, 0, 0, 65, 0, 172, 0, 1, 0, 0, 0, 0, 0, 0, 2637, 462, 0, 26, 1644, 0, 0, 0, 0, 0, 3982, 0, 0, 3456, 4061, 0, 0, 0, 0, 0, 3907, 0, 0, 0, 0, 1519, 4084, 0] payload_rows_per_dst=[6934, 4686, 8168, 8122, 5702, 8036, 6912, 7960] dup_rows_per_dst=[1650, 34, 200, 476, 548, 3216, 0, 2892] combine_rows_per_src=[6934, 4686, 8168, 8122, 5702, 8036, 6912, 7960] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52438 payload_rows_remote=46208 dup_rows=13098 dup_rows_remote=10800 dedup_ratio=0.199860 router_tokens_per_expert=[2373, 0, 959, 0, 129, 0, 0, 0, 7, 0, 0, 2732, 2, 0, 0, 0, 0, 0, 44, 0, 0, 0, 82, 0, 0, 54, 0, 129, 0, 1, 0, 0, 0, 0, 0, 0, 2444, 550, 0, 26, 1569, 0, 0, 0, 0, 0, 3929, 0, 0, 3668, 3960, 0, 0, 0, 0, 0, 3995, 0, 0, 0, 0, 2030, 4085, 0] payload_rows_per_dst=[6230, 7938, 8176, 7932, 7628, 3138, 7336, 4060] dup_rows_per_dst=[2298, 5402, 246, 356, 4796, 0, 0, 0] combine_rows_per_src=[6230, 7938, 8176, 7932, 7628, 3138, 7336, 4060] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56412 payload_rows_remote=48310 dup_rows=9124 dup_rows_remote=8640 dedup_ratio=0.139221 router_tokens_per_expert=[2437, 0, 1018, 0, 117, 2, 0, 0, 6, 0, 0, 2557, 11, 0, 3, 0, 0, 0, 3, 0, 0, 0, 46, 0, 0, 47, 0, 199, 0, 2, 1, 0, 0, 0, 0, 0, 2697, 799, 0, 60, 1945, 0, 0, 0, 0, 0, 3997, 0, 0, 3492, 4044, 0, 0, 0, 0, 0, 3948, 0, 0, 0, 0, 1254, 4083, 0] payload_rows_per_dst=[6108, 6130, 8170, 8102, 6230, 7148, 7324, 7200] dup_rows_per_dst=[1040, 1538, 94, 484, 882, 2222, 526, 2338] combine_rows_per_src=[6108, 6130, 8170, 8102, 6230, 7148, 7324, 7200] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55436 payload_rows_remote=48466 dup_rows=10100 dup_rows_remote=10100 dedup_ratio=0.154114 router_tokens_per_expert=[2668, 0, 1672, 0, 155, 0, 0, 0, 7, 0, 0, 1867, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 74, 0, 0, 29, 0, 208, 0, 0, 0, 0, 0, 0, 0, 0, 2426, 229, 0, 88, 1868, 0, 0, 0, 0, 0, 3995, 0, 0, 3485, 4008, 0, 0, 0, 0, 0, 3929, 0, 0, 0, 0, 1958, 4088, 0] payload_rows_per_dst=[7188, 3748, 8176, 8016, 5250, 8126, 6970, 7962] dup_rows_per_dst=[1802, 0, 176, 474, 236, 3600, 0, 3812] combine_rows_per_src=[7188, 3748, 8176, 8016, 5250, 8126, 6970, 7962] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48520 payload_rows_remote=41194 dup_rows=17016 dup_rows_remote=13978 dedup_ratio=0.259644 router_tokens_per_expert=[0, 0, 0, 5, 133, 0, 1407, 34, 1075, 1196, 112, 0, 3, 3347, 0, 2796, 26, 2443, 0, 6, 322, 1779, 1, 0, 1, 0, 162, 2929, 1495, 69, 0, 710, 0, 7, 0, 2, 5, 108, 2, 0, 0, 102, 6, 3, 0, 0, 2549, 0, 1, 1533, 73, 0, 1380, 0, 0, 132, 534, 102, 0, 147, 789, 2629, 2521, 92] payload_rows_per_dst=[6576, 7326, 7744, 6984, 6872, 5254, 5038, 2726] dup_rows_per_dst=[1840, 3038, 6452, 3748, 70, 66, 1200, 602] combine_rows_per_src=[6576, 7326, 7744, 6984, 6872, 5254, 5038, 2726] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52262 payload_rows_remote=44126 dup_rows=13274 dup_rows_remote=9532 dedup_ratio=0.202545 router_tokens_per_expert=[2491, 0, 1920, 0, 247, 0, 0, 0, 14, 0, 0, 2529, 9, 0, 0, 0, 0, 0, 125, 0, 0, 0, 56, 0, 0, 38, 0, 259, 0, 8, 1, 0, 0, 0, 0, 0, 1982, 604, 0, 109, 1719, 0, 0, 0, 0, 0, 3959, 0, 0, 3433, 4048, 0, 0, 0, 0, 0, 4043, 0, 0, 0, 0, 1098, 4076, 0] payload_rows_per_dst=[7222, 5090, 6632, 5192, 4814, 8082, 7094, 8136] dup_rows_per_dst=[2094, 14, 286, 518, 576, 3274, 2770, 3742] combine_rows_per_src=[7222, 5090, 6632, 5192, 4814, 8082, 7094, 8136] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=340781056 dispatch_meta_bytes=217064 combine_payload_bytes=340781056 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=340998120 ingress_bytes=560982016 link_bytes=560982016 token_bytes=681779176 weight_movement_bytes=220200960 total_nvlink_bytes=901980136 weight_to_token_ratio=0.322980 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.756399915e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=305930240 dispatch_meta_bytes=228968 combine_payload_bytes=305930240 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=306159208 ingress_bytes=416030720 link_bytes=416030720 token_bytes=612089448 weight_movement_bytes=110100480 total_nvlink_bytes=722189928 weight_to_token_ratio=0.179876 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=31872 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.235458474e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=364263424 dispatch_meta_bytes=237144 combine_payload_bytes=364263424 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=364500568 ingress_bytes=474363904 link_bytes=474363904 token_bytes=728763992 weight_movement_bytes=110100480 total_nvlink_bytes=838864472 weight_to_token_ratio=0.151078 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=19328 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.249968485e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=346286080 dispatch_meta_bytes=227800 combine_payload_bytes=346286080 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=346513880 ingress_bytes=456386560 link_bytes=456386560 token_bytes=692799960 weight_movement_bytes=110100480 total_nvlink_bytes=802900440 weight_to_token_ratio=0.158921 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=30848 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.937312905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=331218944 dispatch_meta_bytes=228032 combine_payload_bytes=331218944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=331446976 ingress_bytes=441319424 link_bytes=441319424 token_bytes=662665920 weight_movement_bytes=110100480 total_nvlink_bytes=772766400 weight_to_token_ratio=0.166148 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=18176 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.675270629e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=347404288 dispatch_meta_bytes=234264 combine_payload_bytes=347404288 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=347638552 ingress_bytes=567605248 link_bytes=567605248 token_bytes=695042840 weight_movement_bytes=220200960 total_nvlink_bytes=915243800 weight_to_token_ratio=0.316816 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27776 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.871588813e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=295278592 dispatch_meta_bytes=220688 combine_payload_bytes=295278592 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=295499280 ingress_bytes=735680512 link_bytes=735680512 token_bytes=590777872 weight_movement_bytes=440401920 total_nvlink_bytes=1031179792 weight_to_token_ratio=0.745461 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=13824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.279469409e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=316295168 dispatch_meta_bytes=214632 combine_payload_bytes=316295168 prefetch_bytes=0 reduce_bytes=550502400 egress_bytes=316509800 ingress_bytes=866797568 link_bytes=866797568 token_bytes=632804968 weight_movement_bytes=550502400 total_nvlink_bytes=1183307368 weight_to_token_ratio=0.869940 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=19456 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.507503534e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54956 payload_rows_remote=47878 dup_rows=10580 dup_rows_remote=8512 dedup_ratio=0.161438 router_tokens_per_expert=[0, 0, 199, 0, 2011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3349, 0, 4, 1220, 0, 0, 0, 0, 50, 0, 0, 208, 0, 683, 0, 0, 3841, 2686, 0, 1385, 0, 0, 0, 0, 0, 4053, 44, 0, 0, 60, 2522, 0, 0, 2129, 0, 0, 0, 38, 3767, 0, 0, 0, 58, 0, 406, 3979, 76, 0] payload_rows_per_dst=[8018, 7682, 7078, 8158, 6568, 5178, 4298, 7976] dup_rows_per_dst=[3936, 0, 2068, 1830, 1574, 74, 36, 1062] combine_rows_per_src=[8018, 7682, 7078, 8158, 6568, 5178, 4298, 7976] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52194 payload_rows_remote=47700 dup_rows=13342 dup_rows_remote=13020 dedup_ratio=0.203583 router_tokens_per_expert=[0, 0, 186, 0, 1688, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3251, 0, 0, 2146, 0, 0, 0, 0, 5, 0, 0, 308, 0, 669, 0, 0, 3815, 3002, 0, 630, 0, 0, 0, 0, 0, 4076, 22, 0, 0, 2, 2129, 0, 0, 2398, 0, 0, 0, 14, 3684, 0, 0, 0, 45, 0, 653, 3978, 67, 0] payload_rows_per_dst=[3654, 7630, 4292, 8166, 7904, 4494, 8042, 8012] dup_rows_per_dst=[94, 0, 0, 1950, 5352, 322, 4150, 1474] combine_rows_per_src=[3654, 7630, 4292, 8166, 7904, 4494, 8042, 8012] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=52950 payload_rows_remote=45918 dup_rows=12586 dup_rows_remote=9158 dedup_ratio=0.192047 router_tokens_per_expert=[0, 3, 114, 0, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3489, 0, 0, 2487, 0, 0, 0, 0, 5, 0, 0, 230, 0, 680, 0, 0, 3876, 2688, 0, 1191, 0, 0, 0, 0, 0, 4064, 5, 0, 0, 25, 1941, 0, 0, 1884, 0, 0, 0, 15, 3831, 0, 0, 0, 52, 0, 315, 4000, 90, 0] payload_rows_per_dst=[4946, 7752, 6264, 8186, 7032, 3934, 6808, 8028] dup_rows_per_dst=[1052, 0, 2986, 1772, 3428, 8, 2454, 886] combine_rows_per_src=[4946, 7752, 6264, 8186, 7032, 3934, 6808, 8028] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=53844 payload_rows_remote=45658 dup_rows=11692 dup_rows_remote=10434 dedup_ratio=0.178406 router_tokens_per_expert=[0, 4, 120, 0, 2405, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3258, 0, 0, 2338, 0, 0, 0, 0, 1, 0, 0, 155, 0, 493, 0, 0, 3996, 2667, 0, 1007, 0, 0, 0, 1, 3, 4073, 27, 0, 0, 0, 2160, 0, 0, 1902, 0, 0, 0, 14, 3846, 0, 0, 0, 68, 0, 188, 3974, 68, 0] payload_rows_per_dst=[8074, 7992, 7302, 8186, 6122, 4352, 3806, 8010] dup_rows_per_dst=[4676, 0, 3890, 1258, 1228, 28, 26, 586] combine_rows_per_src=[8074, 7992, 7302, 8186, 6122, 4352, 3806, 8010] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49784 payload_rows_remote=41634 dup_rows=15752 dup_rows_remote=11326 dedup_ratio=0.240356 router_tokens_per_expert=[0, 1, 192, 0, 2156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3093, 0, 0, 1986, 0, 0, 0, 0, 39, 0, 0, 81, 0, 593, 0, 0, 3986, 2843, 0, 1106, 0, 0, 0, 0, 0, 4082, 9, 0, 0, 0, 2709, 0, 0, 1499, 0, 0, 0, 16, 3939, 0, 0, 0, 56, 0, 326, 4035, 21, 0] payload_rows_per_dst=[8150, 8096, 7364, 8184, 4074, 5436, 5270, 3210] dup_rows_per_dst=[4426, 3226, 4810, 1406, 474, 0, 1150, 260] combine_rows_per_src=[8150, 8096, 7364, 8184, 4074, 5436, 5270, 3210] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=49332 payload_rows_remote=41274 dup_rows=16204 dup_rows_remote=11426 dedup_ratio=0.247253 router_tokens_per_expert=[0, 0, 68, 0, 1653, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3420, 0, 0, 2021, 0, 0, 0, 0, 0, 0, 0, 52, 0, 864, 0, 0, 3899, 2434, 0, 731, 0, 0, 0, 0, 0, 4067, 10, 0, 0, 0, 2791, 0, 0, 2549, 0, 0, 0, 2, 3867, 0, 0, 0, 47, 0, 332, 3958, 3, 0] payload_rows_per_dst=[3388, 7798, 4042, 4418, 5596, 8080, 8058, 7952] dup_rows_per_dst=[54, 0, 0, 968, 734, 8942, 4778, 728] combine_rows_per_src=[3388, 7798, 4042, 4418, 5596, 8080, 8058, 7952] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=47940 payload_rows_remote=39966 dup_rows=17596 dup_rows_remote=16840 dedup_ratio=0.268494 router_tokens_per_expert=[0, 8, 143, 0, 1569, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3636, 0, 0, 2077, 0, 0, 0, 0, 37, 0, 0, 413, 0, 536, 0, 0, 3891, 3024, 0, 948, 0, 0, 0, 0, 0, 4036, 41, 0, 0, 2, 2143, 0, 0, 2182, 0, 0, 0, 21, 3696, 0, 0, 0, 73, 0, 279, 3944, 69, 0] payload_rows_per_dst=[3388, 7782, 4154, 1832, 6648, 8188, 7974, 7974] dup_rows_per_dst=[52, 0, 0, 140, 1296, 11528, 3824, 756] combine_rows_per_src=[3388, 7782, 4154, 1832, 6648, 8188, 7974, 7974] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=343189504 dispatch_meta_bytes=225560 combine_payload_bytes=343189504 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=343415064 ingress_bytes=673490944 link_bytes=673490944 token_bytes=686604568 weight_movement_bytes=330301440 total_nvlink_bytes=1016906008 weight_to_token_ratio=0.481065 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=16640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.171311522e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55436 payload_rows_remote=47340 dup_rows=10100 dup_rows_remote=8798 dedup_ratio=0.154114 router_tokens_per_expert=[2, 0, 0, 6, 7, 1, 184, 0, 0, 17, 0, 2, 0, 0, 637, 3, 3291, 0, 0, 3795, 4, 37, 3987, 3, 1, 0, 1102, 440, 5, 2001, 0, 0, 0, 0, 3, 0, 1678, 3189, 60, 3, 282, 0, 3195, 2, 0, 4, 1, 0, 2, 0, 1088, 1, 0, 0, 2, 0, 3685, 0, 1, 4040, 0, 0, 0, 7] payload_rows_per_dst=[8002, 8096, 6792, 5796, 6050, 6482, 6846, 7372] dup_rows_per_dst=[372, 1302, 886, 2580, 2538, 486, 1922, 14] combine_rows_per_src=[8002, 8096, 6792, 5796, 6050, 6482, 6846, 7372] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=341913600 dispatch_meta_bytes=242880 combine_payload_bytes=341913600 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=342156480 ingress_bytes=562114560 link_bytes=562114560 token_bytes=684070080 weight_movement_bytes=220200960 total_nvlink_bytes=904271040 weight_to_token_ratio=0.321898 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=18944 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.776096718e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=329140224 dispatch_meta_bytes=220304 combine_payload_bytes=329140224 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=329360528 ingress_bytes=659441664 link_bytes=659441664 token_bytes=658500752 weight_movement_bytes=330301440 total_nvlink_bytes=988802192 weight_to_token_ratio=0.501596 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=20608 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.146877513e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=327276544 dispatch_meta_bytes=224368 combine_payload_bytes=327276544 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=327500912 ingress_bytes=437377024 link_bytes=437377024 token_bytes=654777456 weight_movement_bytes=110100480 total_nvlink_bytes=764877936 weight_to_token_ratio=0.168149 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=26240 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.606705809e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=298432512 dispatch_meta_bytes=211840 combine_payload_bytes=298432512 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=298644352 ingress_bytes=408532992 link_bytes=408532992 token_bytes=597076864 weight_movement_bytes=110100480 total_nvlink_bytes=707177344 weight_to_token_ratio=0.184399 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=16384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.105060652e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=295852032 dispatch_meta_bytes=210800 combine_payload_bytes=295852032 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=296062832 ingress_bytes=516052992 link_bytes=516052992 token_bytes=591914864 weight_movement_bytes=220200960 total_nvlink_bytes=812115824 weight_to_token_ratio=0.372015 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=17024 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.975010292e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=286476288 dispatch_meta_bytes=227224 combine_payload_bytes=286476288 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=286703512 ingress_bytes=506677248 link_bytes=506677248 token_bytes=573179800 weight_movement_bytes=220200960 total_nvlink_bytes=793380760 weight_to_token_ratio=0.384174 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.811950684e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=339333120 dispatch_meta_bytes=224552 combine_payload_bytes=339333120 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=339557672 ingress_bytes=449433600 link_bytes=449433600 token_bytes=678890792 weight_movement_bytes=110100480 total_nvlink_bytes=788991272 weight_to_token_ratio=0.162177 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=27264 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.816389495e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56570 payload_rows_remote=49464 dup_rows=8966 dup_rows_remote=8964 dedup_ratio=0.136810 router_tokens_per_expert=[0, 43, 0, 0, 0, 0, 4035, 4096, 0, 0, 0, 0, 0, 1535, 0, 1188, 0, 0, 1, 3553, 0, 0, 4096, 0, 0, 0, 0, 3121, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 897, 0, 0, 0, 16, 0, 0, 3596, 136, 0, 0, 0, 0, 0, 0, 4094, 2303, 0, 0, 0] payload_rows_per_dst=[8070, 5012, 7106, 8192, 8192, 8192, 7200, 4606] dup_rows_per_dst=[86, 434, 2, 6238, 112, 1798, 296, 0] combine_rows_per_src=[8070, 5012, 7106, 8192, 8192, 8192, 7200, 4606] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57138 payload_rows_remote=50284 dup_rows=8398 dup_rows_remote=7550 dedup_ratio=0.128143 router_tokens_per_expert=[0, 90, 0, 0, 0, 4, 4076, 4096, 0, 0, 0, 0, 0, 2578, 0, 1104, 0, 0, 4, 3491, 0, 0, 4094, 0, 0, 0, 0, 3243, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 12, 0, 0, 3454, 120, 0, 0, 0, 0, 0, 0, 4096, 1806, 0, 0, 0] payload_rows_per_dst=[8160, 6496, 6984, 6854, 8192, 8192, 7668, 4592] dup_rows_per_dst=[180, 868, 6, 848, 90, 906, 4784, 716] combine_rows_per_src=[8160, 6496, 6984, 6854, 8192, 8192, 7668, 4592] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59298 payload_rows_remote=51106 dup_rows=6238 dup_rows_remote=4724 dedup_ratio=0.095184 router_tokens_per_expert=[0, 37, 0, 0, 0, 1, 4073, 4096, 0, 0, 0, 0, 0, 2519, 0, 707, 0, 0, 0, 3564, 0, 0, 4096, 1, 0, 0, 0, 3024, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 757, 0, 0, 0, 6, 0, 0, 3722, 117, 0, 0, 1, 0, 0, 0, 4096, 1908, 0, 0, 0] payload_rows_per_dst=[8146, 5948, 7130, 6048, 8192, 8192, 7450, 8192] dup_rows_per_dst=[76, 504, 0, 0, 86, 1514, 242, 3816] combine_rows_per_src=[8146, 5948, 7130, 6048, 8192, 8192, 7450, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=48728 payload_rows_remote=40548 dup_rows=16808 dup_rows_remote=14702 dedup_ratio=0.256470 router_tokens_per_expert=[0, 76, 0, 0, 0, 3, 4085, 4096, 0, 0, 0, 0, 0, 3134, 0, 1097, 0, 0, 1, 3553, 0, 0, 4094, 0, 0, 0, 0, 3292, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 236, 0, 0, 0, 3, 0, 0, 3592, 107, 0, 0, 0, 0, 0, 0, 4096, 1252, 0, 0, 0] payload_rows_per_dst=[8180, 8192, 8188, 6584, 7422, 472, 7186, 2504] dup_rows_per_dst=[2106, 8462, 5942, 0, 80, 0, 218, 0] combine_rows_per_src=[8180, 8192, 8188, 6584, 7422, 472, 7186, 2504] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60716 payload_rows_remote=52524 dup_rows=4820 dup_rows_remote=4726 dedup_ratio=0.073547 router_tokens_per_expert=[0, 85, 0, 0, 0, 1, 4081, 4096, 0, 0, 0, 0, 0, 2772, 0, 741, 0, 0, 3, 3657, 0, 0, 4095, 0, 0, 0, 0, 3452, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 504, 0, 0, 0, 25, 0, 0, 3607, 121, 0, 0, 0, 0, 0, 0, 4096, 1385, 0, 0, 0] payload_rows_per_dst=[8166, 6512, 7316, 6904, 8192, 8192, 7242, 8192] dup_rows_per_dst=[168, 514, 4, 0, 94, 1006, 264, 2770] combine_rows_per_src=[8166, 6512, 7316, 6904, 8192, 8192, 7242, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59228 payload_rows_remote=51478 dup_rows=6308 dup_rows_remote=6134 dedup_ratio=0.096252 router_tokens_per_expert=[0, 68, 0, 0, 0, 0, 4074, 4096, 0, 0, 0, 0, 0, 2356, 0, 1011, 0, 0, 0, 3382, 0, 0, 4096, 0, 0, 0, 0, 3021, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 619, 0, 0, 0, 0, 0, 0, 3874, 88, 0, 0, 0, 0, 0, 0, 4096, 1943, 0, 0, 0] payload_rows_per_dst=[8148, 5948, 6764, 6042, 8192, 8192, 7750, 8192] dup_rows_per_dst=[136, 786, 0, 0, 88, 1238, 174, 3886] combine_rows_per_src=[8148, 5948, 6764, 6042, 8192, 8192, 7750, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=354557952 dispatch_meta_bytes=233712 combine_payload_bytes=354557952 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=354791664 ingress_bytes=574758912 link_bytes=574758912 token_bytes=709349616 weight_movement_bytes=220200960 total_nvlink_bytes=929550576 weight_to_token_ratio=0.310427 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.996002796e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56114 payload_rows_remote=47922 dup_rows=9422 dup_rows_remote=2340 dedup_ratio=0.143768 router_tokens_per_expert=[0, 85, 0, 0, 0, 2, 4012, 4096, 0, 0, 0, 0, 0, 2461, 0, 837, 0, 0, 2, 3473, 0, 0, 4096, 0, 0, 0, 0, 3021, 0, 0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 627, 0, 0, 0, 4, 0, 0, 3545, 107, 0, 0, 0, 0, 0, 0, 4096, 2240, 0, 0, 0] payload_rows_per_dst=[5472, 5982, 6946, 6042, 8192, 8192, 7096, 8192] dup_rows_per_dst=[124, 614, 4, 0, 128, 1254, 216, 7082] combine_rows_per_src=[5472, 5982, 6946, 6042, 8192, 8192, 7096, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=360435712 dispatch_meta_bytes=231336 combine_payload_bytes=360435712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=360667048 ingress_bytes=470536192 link_bytes=470536192 token_bytes=721102760 weight_movement_bytes=110100480 total_nvlink_bytes=831203240 weight_to_token_ratio=0.152683 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=25088 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.183398278e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=366327808 dispatch_meta_bytes=223320 combine_payload_bytes=366327808 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=366551128 ingress_bytes=476428288 link_bytes=476428288 token_bytes=732878936 weight_movement_bytes=110100480 total_nvlink_bytes=842979416 weight_to_token_ratio=0.150230 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=27904 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.285871518e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=290648064 dispatch_meta_bytes=221000 combine_payload_bytes=290648064 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=290869064 ingress_bytes=620949504 link_bytes=620949504 token_bytes=581517128 weight_movement_bytes=330301440 total_nvlink_bytes=911818568 weight_to_token_ratio=0.568000 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=31232 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.079933316e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=376492032 dispatch_meta_bytes=229000 combine_payload_bytes=376492032 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=376721032 ingress_bytes=486592512 link_bytes=486592512 token_bytes=753213064 weight_movement_bytes=110100480 total_nvlink_bytes=863313544 weight_to_token_ratio=0.146174 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32384 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.462644091e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=368994304 dispatch_meta_bytes=230448 combine_payload_bytes=368994304 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=369224752 ingress_bytes=479094784 link_bytes=479094784 token_bytes=738219056 weight_movement_bytes=110100480 total_nvlink_bytes=848319536 weight_to_token_ratio=0.149143 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.036765 gemm_active_groups=9 gemm_peak_group_rows=29184 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.332246269e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51924 payload_rows_remote=43876 dup_rows=13612 dup_rows_remote=8550 dedup_ratio=0.207703 router_tokens_per_expert=[2556, 0, 1609, 0, 138, 0, 3, 4, 3, 0, 0, 2561, 4, 0, 0, 1, 0, 0, 96, 0, 0, 0, 53, 0, 0, 40, 4, 341, 0, 1, 0, 2, 0, 0, 0, 1, 2262, 670, 0, 28, 1332, 2, 4, 0, 0, 0, 3986, 0, 0, 3397, 4047, 1, 0, 5, 0, 0, 3989, 0, 0, 4, 3, 1536, 4085, 0] payload_rows_per_dst=[6860, 8048, 8176, 8108, 8182, 2672, 6798, 3080] dup_rows_per_dst=[1760, 5062, 292, 762, 5718, 4, 8, 6] combine_rows_per_src=[6860, 8048, 8176, 8108, 8182, 2672, 6798, 3080] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=343504896 dispatch_meta_bytes=201048 combine_payload_bytes=343504896 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=343705944 ingress_bytes=783906816 link_bytes=783906816 token_bytes=687210840 weight_movement_bytes=440401920 total_nvlink_bytes=1127612760 weight_to_token_ratio=0.640854 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=17280 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.363342884e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=54810 payload_rows_remote=46618 dup_rows=10726 dup_rows_remote=10700 dedup_ratio=0.163666 router_tokens_per_expert=[0, 1300, 0, 0, 0, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2777, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4096, 0, 0, 0, 0, 714, 0, 0, 0, 0, 3159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4096, 0, 0, 5, 4096, 4094, 0, 4089, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5694, 8192, 8192, 8158, 8192, 12, 8178] dup_rows_per_dst=[3064, 44, 26, 1428, 6164, 0, 0, 0] combine_rows_per_src=[8192, 5694, 8192, 8192, 8158, 8192, 12, 8178] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=314503168 dispatch_meta_bytes=209704 combine_payload_bytes=314503168 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=314712872 ingress_bytes=424603648 link_bytes=424603648 token_bytes=629216040 weight_movement_bytes=110100480 total_nvlink_bytes=739316520 weight_to_token_ratio=0.174980 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=19840 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.384555792e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59258 payload_rows_remote=51066 dup_rows=6278 dup_rows_remote=5926 dedup_ratio=0.095795 router_tokens_per_expert=[0, 1333, 0, 0, 0, 168, 0, 2, 0, 0, 0, 4, 0, 0, 0, 3282, 0, 0, 0, 0, 0, 0, 0, 22, 0, 4096, 0, 0, 0, 0, 176, 0, 0, 0, 0, 3243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 5, 0, 4091, 0, 0, 0, 4096, 4095, 0, 4063, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6572, 8192, 8192, 7304, 8184, 4496, 8126] dup_rows_per_dst=[2996, 0, 44, 352, 2886, 0, 0, 0] combine_rows_per_src=[8192, 6572, 8192, 8192, 7304, 8184, 4496, 8126] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=334157824 dispatch_meta_bytes=229272 combine_payload_bytes=334157824 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=334387096 ingress_bytes=444258304 link_bytes=444258304 token_bytes=668544920 weight_movement_bytes=110100480 total_nvlink_bytes=778645400 weight_to_token_ratio=0.164687 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32640 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.726382586e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57424 payload_rows_remote=49232 dup_rows=8112 dup_rows_remote=8112 dedup_ratio=0.123779 router_tokens_per_expert=[0, 752, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3042, 0, 0, 0, 0, 0, 0, 0, 57, 0, 4095, 0, 0, 0, 0, 583, 0, 0, 0, 0, 3575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 4093, 0, 4082, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3258, 6084, 8192, 8192, 7150, 8192, 8192, 8164] dup_rows_per_dst=[396, 0, 114, 1164, 0, 0, 6438, 0] combine_rows_per_src=[3258, 6084, 8192, 8192, 7150, 8192, 8192, 8164] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55414 payload_rows_remote=47222 dup_rows=10122 dup_rows_remote=6940 dedup_ratio=0.154449 router_tokens_per_expert=[0, 1469, 0, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3318, 0, 0, 0, 0, 0, 0, 0, 2, 0, 4096, 0, 0, 0, 0, 177, 0, 0, 0, 0, 3254, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 2, 0, 4096, 0, 0, 1, 4096, 4069, 0, 3970, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 8192, 8192, 6508, 8192, 6, 7940] dup_rows_per_dst=[3182, 6582, 4, 354, 0, 0, 0, 0] combine_rows_per_src=[8192, 8192, 8192, 8192, 6508, 8192, 6, 7940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56414 payload_rows_remote=48222 dup_rows=9122 dup_rows_remote=934 dedup_ratio=0.139191 router_tokens_per_expert=[0, 1147, 0, 0, 0, 197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2837, 0, 0, 0, 0, 0, 0, 0, 5, 0, 4091, 0, 0, 0, 0, 467, 0, 0, 0, 0, 3564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 4096, 0, 0, 1, 4096, 4093, 0, 4078, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2688, 5674, 8192, 8192, 7128, 8192, 8192, 8156] dup_rows_per_dst=[0, 0, 10, 924, 0, 0, 8188, 0] combine_rows_per_src=[2688, 5674, 8192, 8192, 7128, 8192, 8192, 8156] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61790 payload_rows_remote=55034 dup_rows=3746 dup_rows_remote=3746 dedup_ratio=0.057159 router_tokens_per_expert=[0, 1436, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2965, 0, 0, 0, 0, 0, 0, 0, 12, 0, 4096, 0, 0, 0, 0, 244, 0, 0, 0, 0, 3378, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4094, 0, 2, 0, 4096, 0, 0, 0, 4096, 4089, 0, 4081, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 5930, 8192, 8192, 6756, 8188, 8178, 8162] dup_rows_per_dst=[3230, 0, 24, 488, 0, 0, 4, 0] combine_rows_per_src=[8192, 5930, 8192, 8192, 6756, 8188, 8178, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=51116 payload_rows_remote=42958 dup_rows=14420 dup_rows_remote=13898 dedup_ratio=0.220032 router_tokens_per_expert=[0, 1195, 0, 0, 0, 211, 0, 0, 0, 0, 0, 5, 0, 0, 0, 3380, 0, 0, 0, 0, 0, 0, 0, 13, 0, 4095, 0, 0, 0, 0, 203, 0, 0, 0, 0, 3229, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 1, 0, 4094, 0, 0, 1, 4096, 4073, 0, 4076, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2806, 6770, 7912, 2628, 6458, 8192, 8192, 8158] dup_rows_per_dst=[6, 0, 26, 228, 0, 8, 13630, 522] combine_rows_per_src=[2806, 6770, 7912, 2628, 6458, 8192, 8192, 8158] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=366041088 dispatch_meta_bytes=227968 combine_payload_bytes=366041088 prefetch_bytes=0 reduce_bytes=330301440 egress_bytes=366269056 ingress_bytes=696342528 link_bytes=696342528 token_bytes=732310144 weight_movement_bytes=330301440 total_nvlink_bytes=1062611584 weight_to_token_ratio=0.451040 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=29952 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.211054185e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=352894976 dispatch_meta_bytes=229376 combine_payload_bytes=352894976 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=353124352 ingress_bytes=462995456 link_bytes=462995456 token_bytes=706019328 weight_movement_bytes=110100480 total_nvlink_bytes=816119808 weight_to_token_ratio=0.155945 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.052252477e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=338487296 dispatch_meta_bytes=216648 combine_payload_bytes=338487296 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=338703944 ingress_bytes=448587776 link_bytes=448587776 token_bytes=677191240 weight_movement_bytes=110100480 total_nvlink_bytes=787291720 weight_to_token_ratio=0.162584 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.801679225e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=345655296 dispatch_meta_bytes=196624 combine_payload_bytes=345655296 prefetch_bytes=110100480 reduce_bytes=440401920 egress_bytes=345851920 ingress_bytes=896157696 link_bytes=896157696 token_bytes=691507216 weight_movement_bytes=550502400 total_nvlink_bytes=1242009616 weight_to_token_ratio=0.796091 gemm_rows_defined=33536 gemm_rows_charged=34816 gemm_tail_pad_rows=1280 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.036765 gemm_active_groups=8 gemm_peak_group_rows=18688 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.558565626e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=394483712 dispatch_meta_bytes=235120 combine_payload_bytes=394483712 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=394718832 ingress_bytes=504584192 link_bytes=504584192 token_bytes=789202544 weight_movement_bytes=110100480 total_nvlink_bytes=899303024 weight_to_token_ratio=0.139509 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=27008 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.775548997e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=307922944 dispatch_meta_bytes=227424 combine_payload_bytes=307922944 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=308150368 ingress_bytes=418023424 link_bytes=418023424 token_bytes=616073312 weight_movement_bytes=110100480 total_nvlink_bytes=726173792 weight_to_token_ratio=0.178713 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.270114874e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55082 payload_rows_remote=47514 dup_rows=10454 dup_rows_remote=10444 dedup_ratio=0.159515 router_tokens_per_expert=[0, 0, 337, 3, 1782, 0, 0, 0, 1, 3, 1, 0, 0, 0, 0, 0, 3386, 2, 1, 2248, 3, 0, 0, 3, 6, 0, 0, 222, 4, 250, 0, 0, 3784, 2901, 0, 850, 0, 1, 1, 0, 0, 4059, 12, 0, 0, 22, 2452, 0, 0, 2386, 0, 0, 1, 44, 3692, 0, 0, 0, 87, 0, 236, 3916, 72, 0] payload_rows_per_dst=[7878, 7568, 7376, 8146, 6420, 4958, 4796, 7940] dup_rows_per_dst=[3750, 10, 3910, 936, 1086, 14, 66, 682] combine_rows_per_src=[7878, 7568, 7376, 8146, 6420, 4958, 4796, 7940] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59296 payload_rows_remote=51776 dup_rows=6240 dup_rows_remote=6034 dedup_ratio=0.095215 router_tokens_per_expert=[0, 0, 0, 0, 0, 18, 0, 98, 0, 0, 3704, 0, 2604, 575, 0, 0, 1, 0, 139, 19, 0, 0, 0, 0, 4096, 0, 0, 0, 801, 4096, 1085, 0, 4093, 0, 0, 0, 0, 2814, 0, 0, 0, 0, 0, 0, 887, 0, 0, 3896, 1, 0, 0, 0, 0, 0, 3841, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 6030, 7520, 8192, 5628, 7860, 7682, 8192] dup_rows_per_dst=[226, 328, 206, 3772, 0, 1706, 2, 0] combine_rows_per_src=[8192, 6030, 7520, 8192, 5628, 7860, 7682, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=340580352 dispatch_meta_bytes=231832 combine_payload_bytes=340580352 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=340812184 ingress_bytes=560781312 link_bytes=560781312 token_bytes=681392536 weight_movement_bytes=220200960 total_nvlink_bytes=901593496 weight_to_token_ratio=0.323163 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=31104 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=9.752909342e-05 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371130368 dispatch_meta_bytes=231240 combine_payload_bytes=371130368 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=371361608 ingress_bytes=591331328 link_bytes=591331328 token_bytes=742491976 weight_movement_bytes=220200960 total_nvlink_bytes=962692936 weight_to_token_ratio=0.296570 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=29312 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.028422437e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58804 payload_rows_remote=50914 dup_rows=6732 dup_rows_remote=6732 dedup_ratio=0.102722 router_tokens_per_expert=[0, 0, 0, 0, 0, 44, 0, 114, 0, 0, 3869, 0, 2975, 587, 0, 0, 2, 0, 66, 4, 0, 0, 0, 0, 4096, 0, 0, 0, 792, 4096, 1226, 0, 4093, 0, 0, 0, 0, 2514, 0, 0, 0, 0, 0, 0, 349, 0, 0, 3996, 0, 0, 0, 0, 0, 0, 3945, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 6626, 7760, 3928, 8192, 8030, 7890, 8192] dup_rows_per_dst=[316, 498, 122, 108, 5028, 660, 0, 0] combine_rows_per_src=[8186, 6626, 7760, 3928, 8192, 8030, 7890, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55242 payload_rows_remote=47050 dup_rows=10294 dup_rows_remote=10082 dedup_ratio=0.157074 router_tokens_per_expert=[0, 0, 0, 0, 0, 16, 0, 91, 0, 0, 3270, 0, 3236, 365, 0, 0, 6, 0, 124, 3, 0, 0, 0, 0, 4096, 0, 0, 0, 837, 4096, 1145, 0, 4095, 0, 0, 0, 0, 3046, 0, 0, 0, 0, 0, 0, 448, 0, 0, 4034, 0, 0, 0, 0, 0, 0, 3860, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8192, 8192, 6730, 5882, 6092, 4254, 7720, 8180] dup_rows_per_dst=[212, 7202, 76, 2284, 0, 520, 0, 0] combine_rows_per_src=[8192, 8192, 6730, 5882, 6092, 4254, 7720, 8180] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59982 payload_rows_remote=51790 dup_rows=5554 dup_rows_remote=1490 dedup_ratio=0.084747 router_tokens_per_expert=[0, 0, 0, 0, 0, 48, 0, 45, 0, 0, 3719, 0, 3062, 531, 0, 0, 10, 0, 59, 9, 0, 0, 0, 0, 4096, 0, 0, 0, 1210, 4096, 822, 0, 4086, 0, 0, 0, 0, 2718, 0, 0, 0, 0, 0, 0, 332, 0, 0, 4035, 0, 0, 0, 0, 0, 0, 3889, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[8178, 6664, 7470, 8192, 5436, 8072, 7778, 8192] dup_rows_per_dst=[180, 522, 124, 4064, 0, 662, 0, 2] combine_rows_per_src=[8178, 6664, 7470, 8192, 5436, 8072, 7778, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59578 payload_rows_remote=54880 dup_rows=5958 dup_rows_remote=5958 dedup_ratio=0.090912 router_tokens_per_expert=[0, 0, 0, 0, 0, 33, 0, 101, 0, 0, 3716, 0, 3086, 616, 0, 0, 3, 0, 92, 15, 0, 0, 0, 0, 4096, 0, 0, 0, 988, 4096, 1024, 0, 4092, 0, 0, 0, 0, 2349, 0, 0, 0, 0, 0, 0, 488, 0, 0, 4009, 1, 0, 0, 0, 0, 0, 3963, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8186, 6794, 7528, 8192, 4698, 8062, 7926, 8192] dup_rows_per_dst=[266, 610, 124, 4024, 0, 932, 2, 0] combine_rows_per_src=[8186, 6794, 7528, 8192, 4698, 8062, 7926, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59102 payload_rows_remote=50978 dup_rows=6434 dup_rows_remote=5910 dedup_ratio=0.098175 router_tokens_per_expert=[0, 0, 0, 0, 0, 52, 0, 184, 0, 0, 3625, 0, 2747, 962, 0, 0, 1, 0, 91, 20, 0, 0, 0, 0, 4096, 0, 0, 0, 881, 4096, 941, 0, 4080, 0, 0, 0, 0, 2657, 0, 0, 0, 0, 0, 0, 268, 0, 0, 4056, 0, 0, 0, 0, 0, 0, 4011, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8164, 6564, 7338, 5998, 6700, 8124, 8022, 8192] dup_rows_per_dst=[468, 854, 136, 1650, 2802, 524, 0, 0] combine_rows_per_src=[8164, 6564, 7338, 5998, 6700, 8124, 8022, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=50162 payload_rows_remote=41970 dup_rows=15374 dup_rows_remote=15374 dedup_ratio=0.234589 router_tokens_per_expert=[0, 0, 0, 0, 0, 82, 0, 80, 0, 0, 3694, 0, 2922, 937, 0, 0, 14, 0, 125, 39, 0, 0, 0, 0, 4096, 0, 0, 0, 665, 4096, 1078, 0, 4087, 0, 0, 0, 0, 2265, 0, 0, 0, 0, 0, 0, 604, 0, 0, 4004, 2, 0, 0, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5978, 2646, 8030, 3370, 8192, 5710, 8044, 8192] dup_rows_per_dst=[232, 182, 4604, 116, 6818, 868, 2554, 0] combine_rows_per_src=[5978, 2646, 8030, 3370, 8192, 5710, 8044, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=364951552 dispatch_meta_bytes=230584 combine_payload_bytes=364951552 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=365182136 ingress_bytes=475052032 link_bytes=475052032 token_bytes=730133688 weight_movement_bytes=110100480 total_nvlink_bytes=840234168 weight_to_token_ratio=0.150795 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.261936163e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=337254400 dispatch_meta_bytes=228528 combine_payload_bytes=337254400 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=337482928 ingress_bytes=447354880 link_bytes=447354880 token_bytes=674737328 weight_movement_bytes=110100480 total_nvlink_bytes=784837808 weight_to_token_ratio=0.163175 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=31616 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.780237136e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=371230720 dispatch_meta_bytes=213120 combine_payload_bytes=371230720 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=371443840 ingress_bytes=811632640 link_bytes=811632640 token_bytes=742674560 weight_movement_bytes=440401920 total_nvlink_bytes=1183076480 weight_to_token_ratio=0.592994 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=16000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.411562652e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=393379840 dispatch_meta_bytes=243352 combine_payload_bytes=393379840 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=393623192 ingress_bytes=613580800 link_bytes=613580800 token_bytes=787003032 weight_movement_bytes=220200960 total_nvlink_bytes=1007203992 weight_to_token_ratio=0.279797 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=21376 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.067117928e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=365410304 dispatch_meta_bytes=227552 combine_payload_bytes=365410304 prefetch_bytes=0 reduce_bytes=220200960 egress_bytes=365637856 ingress_bytes=585611264 link_bytes=585611264 token_bytes=731048160 weight_movement_bytes=220200960 total_nvlink_bytes=951249120 weight_to_token_ratio=0.301213 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=28672 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.018474305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=300840960 dispatch_meta_bytes=229376 combine_payload_bytes=300840960 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=301070336 ingress_bytes=410941440 link_bytes=410941440 token_bytes=601911296 weight_movement_bytes=110100480 total_nvlink_bytes=712011776 weight_to_token_ratio=0.182918 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.146947524e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55886 payload_rows_remote=48656 dup_rows=9650 dup_rows_remote=7616 dedup_ratio=0.147247 router_tokens_per_expert=[2, 87, 0, 1, 1, 0, 4068, 4091, 0, 0, 0, 0, 0, 2529, 0, 1021, 1, 0, 1, 3688, 3, 1, 4091, 0, 0, 2, 0, 2879, 0, 0, 0, 1, 5, 45, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 884, 0, 1, 0, 8, 3, 2, 3748, 110, 4, 0, 0, 2, 0, 0, 4092, 1394, 0, 2, 0] payload_rows_per_dst=[8148, 7230, 7474, 7078, 8192, 7452, 7516, 2796] dup_rows_per_dst=[170, 2034, 1670, 4706, 92, 744, 234, 0] combine_rows_per_src=[8148, 7230, 7474, 7078, 8192, 7452, 7516, 2796] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60036 payload_rows_remote=51976 dup_rows=5500 dup_rows_remote=5498 dedup_ratio=0.083923 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 238, 4052, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4012, 0, 0, 316, 4030, 0, 0, 3, 0, 0, 4090, 0, 0, 2758, 0, 0, 1, 0, 2, 0, 4025, 0, 0, 0, 4096, 0, 0, 1008, 4096, 0, 15, 0, 0, 0, 0, 0, 25, 0] payload_rows_per_dst=[8192, 8116, 8060, 8082, 7350, 8054, 3998, 8184] dup_rows_per_dst=[0, 464, 2, 574, 3700, 2, 682, 76] combine_rows_per_src=[8192, 8116, 8060, 8082, 7350, 8054, 3998, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=372563968 dispatch_meta_bytes=229896 combine_payload_bytes=372563968 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=372793864 ingress_bytes=592764928 link_bytes=592764928 token_bytes=745357832 weight_movement_bytes=220200960 total_nvlink_bytes=965558792 weight_to_token_ratio=0.295430 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.030915703e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=348766208 dispatch_meta_bytes=225088 combine_payload_bytes=348766208 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=348991296 ingress_bytes=458866688 link_bytes=458866688 token_bytes=697757504 weight_movement_bytes=110100480 total_nvlink_bytes=807857984 weight_to_token_ratio=0.157792 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=19968 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.980446410e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61504 payload_rows_remote=53312 dup_rows=4032 dup_rows_remote=1428 dedup_ratio=0.061523 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 298, 3819, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3892, 0, 0, 477, 4070, 0, 0, 0, 0, 0, 4089, 0, 0, 2534, 0, 0, 0, 0, 6, 0, 4032, 0, 0, 0, 4096, 0, 0, 1302, 4096, 0, 34, 0, 0, 0, 0, 0, 21, 0] payload_rows_per_dst=[8192, 7740, 8140, 7912, 5068, 8076, 8192, 8184] dup_rows_per_dst=[0, 494, 4, 826, 0, 0, 2604, 104] combine_rows_per_src=[8192, 7740, 8140, 7912, 5068, 8076, 8192, 8184] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57856 payload_rows_remote=49664 dup_rows=7680 dup_rows_remote=7672 dedup_ratio=0.117188 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 3976, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 3806, 0, 0, 535, 4028, 0, 0, 4, 0, 0, 4087, 0, 0, 2814, 0, 0, 2, 0, 0, 0, 4054, 0, 0, 0, 4096, 0, 0, 1139, 4096, 0, 7, 0, 0, 0, 0, 0, 30, 0] payload_rows_per_dst=[8192, 7968, 8060, 7316, 8192, 8112, 2278, 7738] dup_rows_per_dst=[8, 266, 652, 616, 6080, 0, 0, 58] combine_rows_per_src=[8192, 7968, 8060, 7316, 8192, 8112, 2278, 7738] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61344 payload_rows_remote=56540 dup_rows=4192 dup_rows_remote=4182 dedup_ratio=0.063965 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 205, 3920, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 3923, 0, 0, 620, 4040, 0, 0, 7, 0, 0, 4081, 5, 0, 2395, 0, 0, 13, 0, 4, 0, 3991, 0, 0, 0, 4096, 0, 0, 1318, 4096, 0, 2, 0, 0, 0, 0, 0, 50, 0] payload_rows_per_dst=[8192, 7884, 8082, 8026, 4804, 8002, 8192, 8162] dup_rows_per_dst=[0, 366, 2, 1060, 10, 14, 2636, 104] combine_rows_per_src=[8192, 7884, 8082, 8026, 4804, 8002, 8192, 8162] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61970 payload_rows_remote=53928 dup_rows=3566 dup_rows_remote=3566 dedup_ratio=0.054413 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 467, 3936, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3820, 0, 0, 602, 4034, 0, 0, 0, 0, 0, 4093, 0, 0, 2724, 0, 0, 0, 0, 0, 0, 4021, 0, 0, 0, 4096, 0, 0, 850, 4096, 0, 8, 0, 0, 0, 0, 0, 21, 0] payload_rows_per_dst=[8192, 7950, 8068, 7890, 5448, 8042, 8192, 8188] dup_rows_per_dst=[0, 856, 0, 954, 0, 0, 1700, 56] combine_rows_per_src=[8192, 7950, 8068, 7890, 5448, 8042, 8192, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56904 payload_rows_remote=48738 dup_rows=8632 dup_rows_remote=8564 dedup_ratio=0.131714 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 3963, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 3558, 0, 0, 1098, 4059, 0, 0, 14, 0, 0, 4082, 13, 0, 2565, 0, 0, 11, 0, 1, 0, 3949, 0, 0, 0, 4096, 0, 0, 897, 4096, 0, 13, 0, 0, 0, 0, 0, 22, 0] payload_rows_per_dst=[8192, 6502, 8126, 4638, 5166, 7922, 8192, 8166] dup_rows_per_dst=[0, 554, 6, 758, 18, 1518, 5710, 68] combine_rows_per_src=[8192, 6502, 8126, 4638, 5166, 7922, 8192, 8166] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=62050 payload_rows_remote=54276 dup_rows=3486 dup_rows_remote=3290 dedup_ratio=0.053192 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 3873, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 0, 0, 132, 4074, 0, 0, 7, 0, 0, 4088, 3, 0, 2828, 0, 0, 11, 0, 5, 0, 4003, 0, 0, 0, 4096, 0, 0, 1519, 4096, 0, 1, 0, 0, 0, 0, 0, 49, 0] payload_rows_per_dst=[8192, 7868, 8154, 7774, 5670, 8024, 8192, 8176] dup_rows_per_dst=[0, 132, 0, 196, 6, 14, 3038, 100] combine_rows_per_src=[8192, 7868, 8154, 7774, 5670, 8024, 8192, 8176] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=12 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.250000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=382140416 dispatch_meta_bytes=218960 combine_payload_bytes=382140416 prefetch_bytes=110100480 reduce_bytes=330301440 egress_bytes=382359376 ingress_bytes=822542336 link_bytes=822542336 token_bytes=764499792 weight_movement_bytes=440401920 total_nvlink_bytes=1204901712 weight_to_token_ratio=0.576065 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=21888 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.430536407e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=355991552 dispatch_meta_bytes=229344 combine_payload_bytes=355991552 prefetch_bytes=220200960 reduce_bytes=0 egress_bytes=356220896 ingress_bytes=576192512 link_bytes=576192512 token_bytes=712212448 weight_movement_bytes=220200960 total_nvlink_bytes=932413408 weight_to_token_ratio=0.309179 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.051471 gemm_active_groups=4 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.002093546e-04 prefetch_bytes_per_gemm_flop=3.829656863e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=405278720 dispatch_meta_bytes=242888 combine_payload_bytes=405278720 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=405521608 ingress_bytes=625479680 link_bytes=625479680 token_bytes=810800328 weight_movement_bytes=220200960 total_nvlink_bytes=1031001288 weight_to_token_ratio=0.271585 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=21632 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.087812037e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=349353984 dispatch_meta_bytes=229208 combine_payload_bytes=349353984 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=349583192 ingress_bytes=459454464 link_bytes=459454464 token_bytes=698937176 weight_movement_bytes=110100480 total_nvlink_bytes=809037656 weight_to_token_ratio=0.157526 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32512 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.990668802e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60380 payload_rows_remote=52188 dup_rows=5156 dup_rows_remote=4776 dedup_ratio=0.078674 router_tokens_per_expert=[11, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 4052, 4038, 0, 0, 0, 0, 0, 0, 0, 0, 165, 25, 0, 0, 2160, 4089, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3703, 65, 0, 0, 0, 0, 0, 155, 0, 0, 0, 0, 4096, 0, 0, 2015, 0, 0, 2, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8104, 8076, 8192, 8192, 7406, 8192, 4030, 8188] dup_rows_per_dst=[26, 0, 380, 4306, 0, 440, 0, 4] combine_rows_per_src=[8104, 8076, 8192, 8192, 7406, 8192, 4030, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=386555904 dispatch_meta_bytes=229976 combine_payload_bytes=386555904 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=386785880 ingress_bytes=496656384 link_bytes=496656384 token_bytes=773341784 weight_movement_bytes=110100480 total_nvlink_bytes=883442264 weight_to_token_ratio=0.142370 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.637671377e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=389050368 dispatch_meta_bytes=230264 combine_payload_bytes=389050368 prefetch_bytes=0 reduce_bytes=440401920 egress_bytes=389280632 ingress_bytes=829452288 link_bytes=829452288 token_bytes=778331000 weight_movement_bytes=440401920 total_nvlink_bytes=1218732920 weight_to_token_ratio=0.565829 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=28416 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.442553950e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=56080 payload_rows_remote=47892 dup_rows=9456 dup_rows_remote=3650 dedup_ratio=0.144287 router_tokens_per_expert=[0, 1335, 0, 0, 0, 153, 0, 0, 0, 0, 3, 0, 1, 0, 0, 2919, 0, 0, 0, 0, 0, 2, 0, 19, 0, 4092, 0, 0, 0, 0, 315, 2, 0, 1, 5, 3502, 0, 0, 4, 0, 0, 0, 0, 3, 2, 0, 4088, 5, 1, 2, 4083, 0, 0, 1, 4091, 4074, 0, 4055, 0, 0, 6, 4, 0, 0] payload_rows_per_dst=[8188, 8188, 8186, 8186, 7018, 8186, 8, 8120] dup_rows_per_dst=[2954, 5806, 38, 632, 6, 10, 0, 10] combine_rows_per_src=[8188, 8188, 8186, 8186, 7018, 8186, 8, 8120] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374083584 dispatch_meta_bytes=227856 combine_payload_bytes=374083584 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=374311440 ingress_bytes=484184064 link_bytes=484184064 token_bytes=748395024 weight_movement_bytes=110100480 total_nvlink_bytes=858495504 weight_to_token_ratio=0.147115 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.420757219e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=343289856 dispatch_meta_bytes=206168 combine_payload_bytes=343289856 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=343496024 ingress_bytes=453390336 link_bytes=453390336 token_bytes=686785880 weight_movement_bytes=110100480 total_nvlink_bytes=796886360 weight_to_token_ratio=0.160313 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.044118 gemm_active_groups=6 gemm_peak_group_rows=24576 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.885203642e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61250 payload_rows_remote=53086 dup_rows=4286 dup_rows_remote=380 dedup_ratio=0.065399 router_tokens_per_expert=[24, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 4071, 4042, 0, 0, 0, 0, 0, 0, 0, 0, 69, 1, 0, 0, 2332, 4082, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3808, 67, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 4096, 0, 0, 1953, 0, 0, 2, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8146, 8084, 8192, 4664, 7616, 8192, 8164, 8192] dup_rows_per_dst=[86, 0, 140, 0, 0, 150, 3906, 4] combine_rows_per_src=[8146, 8084, 8192, 4664, 7616, 8192, 8164, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6844280456 prefetch_bytes=1211105280 reduce_bytes=770703360 weight_movement_bytes=1981808640 egress_bytes=3423295112 ingress_bytes=5402793984 link_bytes=5402793984 total_nvlink_bytes=8826089096 weight_to_token_ratio=0.289557 prefetch_share_of_ingress=0.224163 dedup_ratio=0.162448 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=7 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=9017 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.396347345e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=61940 payload_rows_remote=53808 dup_rows=3596 dup_rows_remote=3346 dedup_ratio=0.054871 router_tokens_per_expert=[16, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 4065, 4050, 0, 0, 0, 0, 0, 0, 0, 0, 63, 33, 0, 0, 1345, 4087, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3846, 27, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 4096, 0, 0, 2635, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8132, 8100, 8192, 8192, 7692, 8192, 5270, 8170] dup_rows_per_dst=[250, 22, 192, 2672, 0, 460, 0, 0] combine_rows_per_src=[8132, 8100, 8192, 8192, 7692, 8192, 5270, 8170] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=55426 payload_rows_remote=47234 dup_rows=10110 dup_rows_remote=10110 dedup_ratio=0.154266 router_tokens_per_expert=[86, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 4050, 4059, 0, 0, 0, 0, 0, 0, 0, 0, 169, 10, 0, 0, 1950, 4056, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3811, 79, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 4096, 0, 0, 2121, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[7008, 8118, 6482, 4190, 7694, 5630, 8112, 8192] dup_rows_per_dst=[234, 692, 330, 164, 1666, 140, 6884, 0] combine_rows_per_src=[7008, 8118, 6482, 4190, 7694, 5630, 8112, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58438 payload_rows_remote=50528 dup_rows=7098 dup_rows_remote=3068 dedup_ratio=0.108307 router_tokens_per_expert=[39, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 4068, 4052, 0, 0, 0, 0, 0, 0, 0, 0, 56, 11, 0, 0, 1638, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3816, 61, 0, 0, 0, 0, 0, 96, 0, 0, 0, 0, 4096, 0, 0, 2482, 0, 0, 1, 0, 0, 0, 0, 4094, 0] payload_rows_per_dst=[8138, 8104, 8192, 4116, 7910, 8192, 5598, 8188] dup_rows_per_dst=[232, 0, 134, 1132, 4030, 314, 1254, 2] combine_rows_per_src=[8138, 8104, 8192, 4116, 7910, 8192, 5598, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=59786 payload_rows_remote=51594 dup_rows=5750 dup_rows_remote=5620 dedup_ratio=0.087738 router_tokens_per_expert=[10, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 4061, 4053, 0, 0, 0, 0, 0, 0, 0, 0, 81, 12, 0, 0, 1831, 4091, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3568, 42, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 4096, 0, 0, 2658, 0, 0, 1, 0, 0, 0, 0, 4095, 0] payload_rows_per_dst=[8122, 8106, 8192, 3662, 7136, 8192, 8186, 8190] dup_rows_per_dst=[120, 0, 186, 0, 0, 130, 5312, 2] combine_rows_per_src=[8122, 8106, 8192, 3662, 7136, 8192, 8186, 8190] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60900 payload_rows_remote=52708 dup_rows=4636 dup_rows_remote=644 dedup_ratio=0.070740 router_tokens_per_expert=[57, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 4068, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 69, 21, 0, 0, 2008, 4084, 0, 0, 0, 0, 0, 0, 0, 0, 4096, 0, 0, 0, 3621, 22, 0, 0, 0, 0, 0, 148, 0, 0, 0, 0, 4096, 0, 0, 2313, 0, 0, 0, 0, 0, 0, 0, 4096, 0] payload_rows_per_dst=[8136, 8128, 8192, 8192, 7242, 8192, 4626, 8192] dup_rows_per_dst=[124, 0, 180, 3992, 0, 340, 0, 0] combine_rows_per_src=[8136, 8128, 8192, 8192, 7242, 8192, 4626, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=16 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=380520448 dispatch_meta_bytes=213864 combine_payload_bytes=380520448 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=380734312 ingress_bytes=600721408 link_bytes=600721408 token_bytes=761254760 weight_movement_bytes=220200960 total_nvlink_bytes=981455720 weight_to_token_ratio=0.289261 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.047794 gemm_active_groups=6 gemm_peak_group_rows=18304 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.044753330e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=385695744 dispatch_meta_bytes=228616 combine_payload_bytes=385695744 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=385924360 ingress_bytes=495796224 link_bytes=495796224 token_bytes=771620104 weight_movement_bytes=110100480 total_nvlink_bytes=881720584 weight_to_token_ratio=0.142687 gemm_rows_defined=33024 gemm_rows_charged=34816 gemm_tail_pad_rows=1792 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.051471 gemm_active_groups=5 gemm_peak_group_rows=32000 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.622711780e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=338573312 dispatch_meta_bytes=229376 combine_payload_bytes=338573312 prefetch_bytes=0 reduce_bytes=110100480 egress_bytes=338802688 ingress_bytes=448673792 link_bytes=448673792 token_bytes=677376000 weight_movement_bytes=110100480 total_nvlink_bytes=787476480 weight_to_token_ratio=0.162540 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=7.803175184e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=362184704 dispatch_meta_bytes=214384 combine_payload_bytes=362184704 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=362399088 ingress_bytes=582385664 link_bytes=582385664 token_bytes=724583792 weight_movement_bytes=220200960 total_nvlink_bytes=944784752 weight_to_token_ratio=0.303900 gemm_rows_defined=32896 gemm_rows_charged=34816 gemm_tail_pad_rows=1920 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=29824 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.012864456e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=369825792 dispatch_meta_bytes=228856 combine_payload_bytes=369825792 prefetch_bytes=110100480 reduce_bytes=0 egress_bytes=370054648 ingress_bytes=479926272 link_bytes=479926272 token_bytes=739880440 weight_movement_bytes=110100480 total_nvlink_bytes=849980920 weight_to_token_ratio=0.148808 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.040441 gemm_active_groups=8 gemm_peak_group_rows=31488 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=8.346707213e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=377810944 dispatch_meta_bytes=213408 combine_payload_bytes=377810944 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=378024352 ingress_bytes=708112384 link_bytes=708112384 token_bytes=755835296 weight_movement_bytes=330301440 total_nvlink_bytes=1086136736 weight_to_token_ratio=0.437002 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=17408 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.231523900e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6797596144 prefetch_bytes=1101004800 reduce_bytes=1101004800 weight_movement_bytes=2202009600 egress_bytes=3399906800 ingress_bytes=5599698944 link_bytes=5599698944 total_nvlink_bytes=8999605744 weight_to_token_ratio=0.323939 prefetch_share_of_ingress=0.196619 dedup_ratio=0.162756 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=10 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.738797456e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=57968 payload_rows_remote=50456 dup_rows=7568 dup_rows_remote=3256 dedup_ratio=0.115479 router_tokens_per_expert=[12, 0, 0, 0, 0, 48, 0, 134, 0, 0, 3698, 4, 2905, 506, 4, 1, 34, 0, 125, 21, 5, 0, 1, 0, 4096, 0, 0, 0, 533, 4091, 746, 4, 4086, 0, 1, 0, 0, 2973, 0, 1, 0, 0, 0, 1, 815, 0, 0, 3960, 0, 0, 0, 0, 0, 0, 3958, 0, 0, 1, 0, 0, 0, 0, 4, 0] payload_rows_per_dst=[8176, 7512, 7484, 4800, 5950, 7938, 7916, 8192] dup_rows_per_dst=[384, 4312, 284, 964, 0, 1614, 0, 10] combine_rows_per_src=[8176, 7512, 7484, 4800, 5950, 7938, 7916, 8192] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6350790248 prefetch_bytes=1211105280 reduce_bytes=440401920 weight_movement_bytes=1651507200 egress_bytes=3176513128 ingress_bytes=4825784320 link_bytes=4825784320 total_nvlink_bytes=8002297448 weight_to_token_ratio=0.260048 prefetch_share_of_ingress=0.250965 dedup_ratio=0.207343 prefetch_slots=22 unused_prefetch_slots=138 slot_utilization=0.137500 reduce_slots=4 remote_demand=11 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.046691 gemm_inner_pad_upper_bound=8255 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.392832638e-05 prefetch_bytes_per_gemm_flop=2.106311275e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7142494184 prefetch_bytes=990904320 reduce_bytes=1431306240 weight_movement_bytes=2422210560 egress_bytes=3572400104 ingress_bytes=5992304640 link_bytes=5992304640 total_nvlink_bytes=9564704744 weight_to_token_ratio=0.339127 prefetch_share_of_ingress=0.165363 dedup_ratio=0.142059 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=13 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045588 gemm_inner_pad_upper_bound=8509 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.042160333e-04 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6147129856 prefetch_bytes=770703360 reduce_bytes=1651507200 weight_movement_bytes=2422210560 egress_bytes=3074652672 ingress_bytes=5494687744 link_bytes=5494687744 total_nvlink_bytes=8569340416 weight_to_token_ratio=0.394039 prefetch_share_of_ingress=0.140263 dedup_ratio=0.225446 prefetch_slots=14 unused_prefetch_slots=146 slot_utilization=0.087500 reduce_slots=15 remote_demand=7 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.045956 gemm_inner_pad_upper_bound=8255 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.556165708e-05 prefetch_bytes_per_gemm_flop=1.340379902e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6917590592 prefetch_bytes=990904320 reduce_bytes=440401920 weight_movement_bytes=1431306240 egress_bytes=3459933760 ingress_bytes=4888963072 link_bytes=4888963072 total_nvlink_bytes=8348896832 weight_to_token_ratio=0.206908 prefetch_share_of_ingress=0.202682 dedup_ratio=0.151953 prefetch_slots=18 unused_prefetch_slots=142 slot_utilization=0.112500 reduce_slots=4 remote_demand=9 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.043750 gemm_inner_pad_upper_bound=8763 gemm_flops_step=57498874675200 bytes_per_gemm_flop=8.502710878e-05 prefetch_bytes_per_gemm_flop=1.723345588e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=7013567384 prefetch_bytes=880803840 reduce_bytes=1431306240 weight_movement_bytes=2312110080 egress_bytes=3507913624 ingress_bytes=5817763840 link_bytes=5817763840 total_nvlink_bytes=9325677464 weight_to_token_ratio=0.329662 prefetch_share_of_ingress=0.151399 dedup_ratio=0.141565 prefetch_slots=16 unused_prefetch_slots=144 slot_utilization=0.100000 reduce_slots=13 remote_demand=8 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=8636 gemm_flops_step=57498874675200 bytes_per_gemm_flop=1.011804818e-04 prefetch_bytes_per_gemm_flop=1.531862745e-05 anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=361668608 dispatch_meta_bytes=214848 combine_payload_bytes=361668608 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=361883456 ingress_bytes=691970048 link_bytes=691970048 token_bytes=723552064 weight_movement_bytes=330301440 total_nvlink_bytes=1053853504 weight_to_token_ratio=0.456500 gemm_rows_defined=33280 gemm_rows_charged=34816 gemm_tail_pad_rows=1536 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.044118 gemm_active_groups=7 gemm_peak_group_rows=21120 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.203449723e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=58054 payload_rows_remote=50242 dup_rows=7482 dup_rows_remote=6918 dedup_ratio=0.114166 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 1, 325, 3858, 0, 11, 3, 0, 0, 2, 0, 0, 0, 0, 3832, 0, 2, 622, 4074, 1, 0, 8, 1, 4, 4010, 0, 0, 2914, 1, 0, 2, 5, 1, 0, 3866, 3, 0, 0, 4085, 0, 0, 963, 4091, 2, 36, 0, 0, 0, 2, 0, 38, 0] payload_rows_per_dst=[8184, 7812, 8156, 7944, 8188, 7748, 1930, 8092] dup_rows_per_dst=[0, 564, 24, 970, 5836, 8, 0, 80] combine_rows_per_src=[8184, 7812, 8156, 7944, 8188, 7748, 1930, 8092] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=360134656 dispatch_meta_bytes=228640 combine_payload_bytes=360134656 prefetch_bytes=110100480 reduce_bytes=110100480 egress_bytes=360363296 ingress_bytes=580335616 link_bytes=580335616 token_bytes=720497952 weight_movement_bytes=220200960 total_nvlink_bytes=940698912 weight_to_token_ratio=0.305623 gemm_rows_defined=33408 gemm_rows_charged=34816 gemm_tail_pad_rows=1408 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.040441 gemm_active_groups=7 gemm_peak_group_rows=30720 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.009299085e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=64 B=8 K=8 R=8 NvS=34800 hidden_size=3584 assignments=32768 payload_rows=60344 payload_rows_remote=52238 dup_rows=5192 dup_rows_remote=5192 dedup_ratio=0.079224 router_tokens_per_expert=[119, 0, 0, 2, 0, 149, 0, 0, 0, 3, 0, 4069, 4050, 0, 0, 0, 0, 0, 1, 0, 5, 70, 13, 0, 5, 1957, 4063, 1, 5, 0, 0, 0, 0, 0, 0, 4091, 0, 0, 0, 3572, 85, 1, 4, 0, 1, 1, 207, 0, 0, 0, 0, 4091, 0, 2, 2091, 4, 0, 11, 1, 0, 0, 5, 4089, 0] payload_rows_per_dst=[8142, 8106, 8192, 8192, 7144, 8190, 4190, 8188] dup_rows_per_dst=[536, 0, 168, 3870, 0, 590, 4, 24] combine_rows_per_src=[8142, 8106, 8192, 8192, 7144, 8190, 4190, 8188] -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=14 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.125000 weight_bytes_per_expert=55050240 grad_bytes_per_expert=110100480 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=374441984 dispatch_meta_bytes=229720 combine_payload_bytes=374441984 prefetch_bytes=110100480 reduce_bytes=220200960 egress_bytes=374671704 ingress_bytes=704743424 link_bytes=704743424 token_bytes=749113688 weight_movement_bytes=330301440 total_nvlink_bytes=1079415128 weight_to_token_ratio=0.440923 gemm_rows_defined=33152 gemm_rows_charged=34816 gemm_tail_pad_rows=1664 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.047794 gemm_active_groups=5 gemm_peak_group_rows=32256 gemm_flops_fwd=1916629155840 gemm_flops_step=5749887467520 bytes_per_gemm_flop=1.225664725e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 -WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=10 plans=10 dispatch_calls=20 combine_calls=20 prefetch_calls=20 reduce_calls=10 token_bytes=6721562576 prefetch_bytes=1101004800 reduce_bytes=990904320 weight_movement_bytes=2091909120 egress_bytes=3361892304 ingress_bytes=5451579392 link_bytes=5451579392 total_nvlink_bytes=8813471696 weight_to_token_ratio=0.311224 prefetch_share_of_ingress=0.201961 dedup_ratio=0.169861 prefetch_slots=20 unused_prefetch_slots=140 slot_utilization=0.125000 reduce_slots=9 remote_demand=10 gemm_rows_charged=348160 gemm_tail_pad_fraction=0.044853 gemm_inner_pad_upper_bound=8128 gemm_flops_step=57498874675200 bytes_per_gemm_flop=9.481193193e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 anomalies=none - [2026-08-26 16:57:49] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 369.5 | throughput per GPU (TFLOP/s/GPU): 334.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 8.467325E+00 | loss scale: 1.0 | grad norm: 4.141 | number of skipped iterations: 0 | number of nan iterations: 0 | -[after training is done] datetime: 2026-08-26 16:57:49 -[rank0]:[W826 16:57:50.702931611 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank7]:[W826 16:57:51.915943334 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank4]:[W826 16:57:51.935282504 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank6]:[W826 16:57:51.952886960 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank2]:[W826 16:57:52.249123351 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank3]:[W826 16:57:52.280919334 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank1]:[W826 16:57:52.291579202 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -[rank5]:[W826 16:57:52.845587230 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) -W0826 16:57:53.783000 290089 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:57:53.783000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290182 closing signal SIGTERM -W0826 16:57:53.784000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290183 closing signal SIGTERM -W0826 16:57:53.785000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290184 closing signal SIGTERM -W0826 16:57:53.787000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290185 closing signal SIGTERM -W0826 16:57:53.789000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290186 closing signal SIGTERM -W0826 16:57:53.790000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290187 closing signal SIGTERM -W0826 16:57:53.791000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290188 closing signal SIGTERM -W0826 16:57:53.792000 290089 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 290189 closing signal SIGTERM -Traceback (most recent call last): - File "/usr/local/bin/torchrun", line 8, in - sys.exit(main()) - ^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper - return f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main - run(args) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run - elastic_launch( - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ - return launch_agent(self._config, self._entrypoint, list(args)) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent - result = agent.run() - ^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper - result = f(*args, **kwargs) - ^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run - result = self._invoke_run(role) - ^^^^^^^^^^^^^^^^^^^^^^ - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run - time.sleep(monitor_interval) - File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler - raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 290089 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv index f6e789ef49bec32ab35274e17dfdd7f643c11802..866d3faba4992503d45c95ca26a6c55c79cd9f13 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/mem_samples.csv @@ -38,110 +38,102 @@ 5, 4 6, 4 7, 4 -0, 4 -1, 4 -2, 4 -3, 4 -4, 4 -5, 4 -6, 4 -7, 4 -0, 821 -1, 819 -2, 819 -3, 819 -4, 819 -5, 821 -6, 819 -7, 819 -0, 1537 -1, 1537 -2, 1537 -3, 1537 -4, 1537 -5, 1537 -6, 1537 -7, 1537 -0, 2551 -1, 2551 -2, 2551 -3, 2551 -4, 2551 -5, 2551 -6, 2551 -7, 2551 -0, 14899 -1, 14901 -2, 14901 -3, 14901 -4, 14901 -5, 14901 -6, 14901 -7, 14901 -0, 70421 -1, 70395 -2, 70425 -3, 70451 -4, 70369 -5, 70445 -6, 70397 -7, 70417 -0, 72743 -1, 72743 -2, 72743 -3, 72743 -4, 72743 -5, 72743 -6, 72743 -7, 72743 +0, 725 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 1079 +1, 1079 +2, 1079 +3, 1079 +4, 1079 +5, 1079 +6, 1079 +7, 1079 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 8385 +1, 8395 +2, 8375 +3, 8391 +4, 8385 +5, 8383 +6, 8393 +7, 8387 +0, 70075 +1, 70075 +2, 70075 +3, 70075 +4, 70075 +5, 70075 +6, 70075 +7, 70075 +0, 71553 +1, 71553 +2, 71553 +3, 71553 +4, 71553 +5, 71553 +6, 71553 +7, 71553 0, 73289 1, 73289 2, 73289 3, 73289 4, 73289 5, 73289 -6, 73289 +6, 73291 7, 73289 -0, 73291 -1, 73291 -2, 73291 -3, 73291 -4, 73291 -5, 73291 +0, 73289 +1, 73289 +2, 73289 +3, 73289 +4, 73289 +5, 73289 6, 73291 -7, 73291 -0, 94717 -1, 94815 -2, 94405 -3, 106753 -4, 94629 -5, 94827 -6, 108453 -7, 95571 -0, 131701 -1, 134425 -2, 131723 -3, 140655 -4, 131747 -5, 134239 -6, 140757 -7, 135495 -0, 160705 -1, 160761 -2, 160735 -3, 160765 -4, 160745 -5, 160763 -6, 160743 -7, 160771 -0, 161649 -1, 161685 -2, 161685 -3, 161685 -4, 161697 -5, 161685 -6, 161685 -7, 161685 +7, 73289 +0, 79131 +1, 79165 +2, 78593 +3, 79165 +4, 79177 +5, 79165 +6, 79165 +7, 79165 +0, 103955 +1, 102715 +2, 104165 +3, 103693 +4, 109621 +5, 104067 +6, 110713 +7, 102913 +0, 141611 +1, 140655 +2, 142039 +3, 140945 +4, 146927 +5, 142059 +6, 153203 +7, 151853 +0, 160807 +1, 160905 +2, 160905 +3, 160905 +4, 160901 +5, 160905 +6, 160873 +7, 160865 0, 161649 1, 161685 2, 161685 @@ -150,22 +142,30 @@ 5, 161685 6, 161685 7, 161685 -0, 170517 +0, 161659 +1, 161695 +2, 161695 +3, 161695 +4, 161707 +5, 161695 +6, 161695 +7, 161695 +0, 170717 1, 170649 -2, 170457 +2, 170649 3, 170649 4, 170637 -5, 170265 +5, 170649 6, 170649 7, 170649 -0, 171217 -1, 171279 -2, 171359 -3, 171329 -4, 171299 -5, 171331 -6, 171335 -7, 171295 +0, 174943 +1, 175027 +2, 175027 +3, 175027 +4, 175015 +5, 175027 +6, 175027 +7, 175027 0, 182495 1, 182575 2, 182575 @@ -183,7 +183,7 @@ 6, 182575 7, 182575 0, 182495 -1, 182575 +1, 182573 2, 182575 3, 182575 4, 182589 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log index 68ecfabd77f7f8b006355ff7907dc549b5cba07e..3b8a23ead5901df143907fee2cdb3bbd256fc5a9 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/train.log @@ -1,6 +1,6 @@ -[16:52:05] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:52:05] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) -[16:52:05] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[18:04:12] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:04:12] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +[18:04:12] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN3_235B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=1536 m=16 @@ -9,10 +9,19 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 16:52:06.827000 280115 torch/distributed/run.py:766] -W0826 16:52:06.827000 280115 torch/distributed/run.py:766] ***************************************** -W0826 16:52:06.827000 280115 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:52:06.827000 280115 torch/distributed/run.py:766] ***************************************** +W0826 18:04:14.190000 452537 torch/distributed/run.py:766] +W0826 18:04:14.190000 452537 torch/distributed/run.py:766] ***************************************** +W0826 18:04:14.190000 452537 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:04:14.190000 452537 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -572,49 +581,40 @@ INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 01 is connected to 7 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 6 is connected to 7 peer ranks. 5Expected number of connected peer ranks is : is connected to 77 peer ranks. -Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank 0 is connected to 71 peer ranks. is connected to Expected number of connected peer ranks is : 77 peer ranks. -Expected number of connected peer ranks is : 7 -[Gloo] Rank [Gloo] Rank [Gloo] Rank 325 is connected to is connected to is connected to 777 peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777 - - +[Gloo] Rank [Gloo] Rank 2 is connected to 07 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 6 is connected to 57 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : +7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 0 is connected to 00 is connected to peer ranks. 0Expected number of connected peer ranks is : peer ranks. 0Expected number of connected peer ranks is : +0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank3]:[W826 16:52:24.953105271 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:52:24.953175033 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:52:24.955798695 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:52:24.955797615 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank5]:[W826 16:52:24.955816274 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:52:24.958990785 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:52:24.959299675 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[rank6]:[W826 18:04:32.310929546 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:04:32.311049529 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:04:32.311050915 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:04:32.312968552 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:04:32.313057109 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:04:32.313057880 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:04:32.313058639 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -622,10 +622,10 @@ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.036 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:52:25.104976490 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.614 seconds -time to initialize megatron (seconds): 7.280 -[after megatron is initialized] datetime: 2026-08-26 16:52:30 +[rank0]:[W826 18:04:32.464090524 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 4.024 seconds +time to initialize megatron (seconds): 7.635 +[after megatron is initialized] datetime: 2026-08-26 18:04:38 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -650,130 +650,130 @@ Params for bucket 1 (206045184 elements, 206045184 padded size): module.output_layer.weight Params for bucket 2 (71840000 elements, 71840000 padded size): module.decoder.final_layernorm.weight - module.decoder.layers.11.pre_mlp_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.q_layernorm.weight module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight module.decoder.layers.11.self_attention.linear_qkv.weight module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.11.self_attention.linear_proj.weight Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.k_layernorm.weight module.decoder.layers.10.self_attention.q_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.10.self_attention.linear_proj.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.pre_mlp_layernorm.weight - module.decoder.layers.10.self_attention.k_layernorm.weight Params for bucket 4 (71835904 elements, 71835904 padded size): module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.k_layernorm.weight module.decoder.layers.9.self_attention.q_layernorm.weight - module.decoder.layers.9.self_attention.linear_qkv.weight - module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.9.self_attention.linear_proj.weight Params for bucket 5 (71835904 elements, 71835904 padded size): module.decoder.layers.8.mlp.router.weight - module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.q_layernorm.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.8.self_attention.linear_proj.weight Params for bucket 6 (71835904 elements, 71835904 padded size): module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.7.pre_mlp_layernorm.weight module.decoder.layers.7.self_attention.q_layernorm.weight module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.7.self_attention.linear_proj.weight Params for bucket 7 (71835904 elements, 71835904 padded size): - module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.self_attention.linear_qkv.weight module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.6.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.6.self_attention.linear_proj.weight Params for bucket 8 (71835904 elements, 71835904 padded size): module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.q_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_proj.weight Params for bucket 9 (71835904 elements, 71835904 padded size): - module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.k_layernorm.weight module.decoder.layers.4.self_attention.q_layernorm.weight module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.self_attention.linear_proj.weight Params for bucket 10 (71835904 elements, 71835904 padded size): module.decoder.layers.3.self_attention.q_layernorm.weight - module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.linear_qkv.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.3.self_attention.linear_proj.weight Params for bucket 11 (71835904 elements, 71835904 padded size): module.decoder.layers.2.mlp.router.weight - module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.2.self_attention.linear_proj.weight Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.weight module.decoder.layers.1.self_attention.q_layernorm.weight module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_proj.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.1.pre_mlp_layernorm.weight - module.decoder.layers.1.self_attention.k_layernorm.weight Params for bucket 13 (71835904 elements, 71835904 padded size): module.decoder.layers.0.mlp.router.weight module.decoder.layers.0.self_attention.k_layernorm.weight - module.decoder.layers.0.self_attention.linear_proj.weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.q_layernorm.weight module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.linear_proj.weight Params for bucket 14 (206045184 elements, 206045184 padded size): module.embedding.word_embeddings.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 Params for bucket 1 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight12 - module.decoder.layers.11.mlp.experts.linear_fc2.weight14 - module.decoder.layers.11.mlp.experts.linear_fc2.weight10 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 - module.decoder.layers.11.mlp.experts.linear_fc2.weight11 module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 Params for bucket 2 (44040192 elements, 44040192 padded size): - module.decoder.layers.11.mlp.experts.linear_fc2.weight8 - module.decoder.layers.11.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc2.weight2 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 module.decoder.layers.11.mlp.experts.linear_fc2.weight5 module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight13 - module.decoder.layers.11.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight14 Params for bucket 4 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight12 - module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight10 Params for bucket 5 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight8 @@ -783,15 +783,15 @@ Params for bucket 5 (50331648 elements, 50331648 padded size): Params for bucket 6 (50331648 elements, 50331648 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight4 module.decoder.layers.11.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 Params for bucket 8 (44040192 elements, 44040192 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 @@ -803,203 +803,203 @@ Params for bucket 8 (44040192 elements, 44040192 padded size): Params for bucket 9 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc2.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight1 Params for bucket 10 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 module.decoder.layers.10.mlp.experts.linear_fc1.weight10 module.decoder.layers.10.mlp.experts.linear_fc1.weight11 - module.decoder.layers.10.mlp.experts.linear_fc1.weight12 Params for bucket 11 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight7 module.decoder.layers.10.mlp.experts.linear_fc1.weight8 - module.decoder.layers.10.mlp.experts.linear_fc1.weight6 Params for bucket 12 (50331648 elements, 50331648 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight4 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 Params for bucket 13 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 module.decoder.layers.10.mlp.experts.linear_fc1.weight0 - module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc1.weight1 - module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 Params for bucket 14 (44040192 elements, 44040192 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight9 - module.decoder.layers.9.mlp.experts.linear_fc2.weight11 - module.decoder.layers.9.mlp.experts.linear_fc2.weight7 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 - module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 Params for bucket 15 (50331648 elements, 50331648 padded size): - module.decoder.layers.9.mlp.experts.linear_fc2.weight5 - module.decoder.layers.9.mlp.experts.linear_fc2.weight3 - module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 - module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 Params for bucket 16 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight12 - module.decoder.layers.9.mlp.experts.linear_fc1.weight13 Params for bucket 17 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight8 - module.decoder.layers.9.mlp.experts.linear_fc1.weight9 Params for bucket 18 (50331648 elements, 50331648 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 - module.decoder.layers.9.mlp.experts.linear_fc1.weight3 Params for bucket 19 (44040192 elements, 44040192 padded size): module.decoder.layers.9.mlp.experts.linear_fc1.weight2 - module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 Params for bucket 20 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 - module.decoder.layers.8.mlp.experts.linear_fc2.weight14 - module.decoder.layers.8.mlp.experts.linear_fc2.weight11 - module.decoder.layers.8.mlp.experts.linear_fc2.weight9 Params for bucket 21 (44040192 elements, 44040192 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight6 - module.decoder.layers.8.mlp.experts.linear_fc2.weight4 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 module.decoder.layers.8.mlp.experts.linear_fc2.weight7 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 Params for bucket 22 (44040192 elements, 44040192 padded size): module.decoder.layers.8.mlp.experts.linear_fc2.weight0 module.decoder.layers.8.mlp.experts.linear_fc1.weight14 - module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 Params for bucket 23 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight12 - module.decoder.layers.8.mlp.experts.linear_fc1.weight10 module.decoder.layers.8.mlp.experts.linear_fc1.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 Params for bucket 24 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight8 - module.decoder.layers.8.mlp.experts.linear_fc1.weight7 module.decoder.layers.8.mlp.experts.linear_fc1.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 Params for bucket 25 (50331648 elements, 50331648 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight4 - module.decoder.layers.8.mlp.experts.linear_fc1.weight2 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 Params for bucket 26 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight13 - module.decoder.layers.7.mlp.experts.linear_fc2.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 - module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 Params for bucket 27 (44040192 elements, 44040192 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 - module.decoder.layers.7.mlp.experts.linear_fc2.weight9 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.7.mlp.experts.linear_fc1.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 Params for bucket 29 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 - module.decoder.layers.7.mlp.experts.linear_fc1.weight10 Params for bucket 30 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 module.decoder.layers.7.mlp.experts.linear_fc1.weight6 module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 Params for bucket 31 (50331648 elements, 50331648 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 Params for bucket 32 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight13 module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 Params for bucket 33 (44040192 elements, 44040192 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 - module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 Params for bucket 34 (50331648 elements, 50331648 padded size): - module.decoder.layers.6.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc2.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight2 - module.decoder.layers.6.mlp.experts.linear_fc1.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight5 module.decoder.layers.6.mlp.experts.linear_fc2.weight3 module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 Params for bucket 35 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight14 module.decoder.layers.6.mlp.experts.linear_fc1.weight12 - module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 Params for bucket 36 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight10 module.decoder.layers.6.mlp.experts.linear_fc1.weight9 - module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 Params for bucket 37 (50331648 elements, 50331648 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight6 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 Params for bucket 38 (44040192 elements, 44040192 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 Params for bucket 39 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 - module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 module.decoder.layers.5.mlp.experts.linear_fc2.weight8 Params for bucket 40 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight7 module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 Params for bucket 41 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight14 module.decoder.layers.5.mlp.experts.linear_fc1.weight15 - module.decoder.layers.5.mlp.experts.linear_fc1.weight13 Params for bucket 42 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight12 module.decoder.layers.5.mlp.experts.linear_fc1.weight10 @@ -1008,13 +1008,13 @@ Params for bucket 42 (50331648 elements, 50331648 padded size): Params for bucket 43 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 - module.decoder.layers.5.mlp.experts.linear_fc1.weight5 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 44 (50331648 elements, 50331648 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 Params for bucket 45 (44040192 elements, 44040192 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight0 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 @@ -1027,21 +1027,21 @@ Params for bucket 46 (44040192 elements, 44040192 padded size): module.decoder.layers.4.mlp.experts.linear_fc2.weight8 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight5 Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 module.decoder.layers.4.mlp.experts.linear_fc1.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 module.decoder.layers.4.mlp.experts.linear_fc1.weight14 Params for bucket 48 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight12 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight9 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 @@ -1049,31 +1049,31 @@ Params for bucket 49 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight8 Params for bucket 50 (50331648 elements, 50331648 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight3 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 Params for bucket 51 (44040192 elements, 44040192 padded size): module.decoder.layers.4.mlp.experts.linear_fc1.weight1 - module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 - module.decoder.layers.3.mlp.experts.linear_fc2.weight11 - module.decoder.layers.3.mlp.experts.linear_fc2.weight9 - module.decoder.layers.3.mlp.experts.linear_fc2.weight7 Params for bucket 53 (50331648 elements, 50331648 padded size): - module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.3.mlp.experts.linear_fc1.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 Params for bucket 54 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc1.weight11 @@ -1086,40 +1086,40 @@ Params for bucket 55 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight7 Params for bucket 56 (50331648 elements, 50331648 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight6 - module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 module.decoder.layers.3.mlp.experts.linear_fc1.weight4 Params for bucket 57 (44040192 elements, 44040192 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 Params for bucket 58 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight13 - module.decoder.layers.2.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc2.weight9 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight10 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 Params for bucket 59 (44040192 elements, 44040192 padded size): - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight5 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 module.decoder.layers.2.mlp.experts.linear_fc2.weight4 module.decoder.layers.2.mlp.experts.linear_fc2.weight2 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 Params for bucket 60 (44040192 elements, 44040192 padded size): module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 module.decoder.layers.2.mlp.experts.linear_fc1.weight13 module.decoder.layers.2.mlp.experts.linear_fc1.weight14 - module.decoder.layers.2.mlp.experts.linear_fc1.weight15 Params for bucket 61 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight12 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 module.decoder.layers.2.mlp.experts.linear_fc1.weight10 module.decoder.layers.2.mlp.experts.linear_fc1.weight11 - module.decoder.layers.2.mlp.experts.linear_fc1.weight9 Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight8 module.decoder.layers.2.mlp.experts.linear_fc1.weight6 @@ -1127,19 +1127,19 @@ Params for bucket 62 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight5 Params for bucket 63 (50331648 elements, 50331648 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight4 - module.decoder.layers.2.mlp.experts.linear_fc1.weight2 module.decoder.layers.2.mlp.experts.linear_fc1.weight3 module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 Params for bucket 64 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight11 - module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight14 Params for bucket 65 (44040192 elements, 44040192 padded size): - module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight8 @@ -1147,70 +1147,70 @@ Params for bucket 65 (44040192 elements, 44040192 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight4 Params for bucket 66 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc2.weight3 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight14 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 Params for bucket 67 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight13 - module.decoder.layers.1.mlp.experts.linear_fc1.weight12 module.decoder.layers.1.mlp.experts.linear_fc1.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 Params for bucket 68 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight9 - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 Params for bucket 69 (50331648 elements, 50331648 padded size): module.decoder.layers.1.mlp.experts.linear_fc1.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight2 module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 Params for bucket 70 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight13 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.1.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 Params for bucket 71 (44040192 elements, 44040192 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight11 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 Params for bucket 72 (50331648 elements, 50331648 padded size): - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 - module.decoder.layers.0.mlp.experts.linear_fc2.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 Params for bucket 73 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight14 - module.decoder.layers.0.mlp.experts.linear_fc1.weight11 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 Params for bucket 74 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight10 - module.decoder.layers.0.mlp.experts.linear_fc1.weight8 module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 Params for bucket 75 (50331648 elements, 50331648 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight3 Params for bucket 76 (37748736 elements, 37748736 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight2 - module.decoder.layers.0.mlp.experts.linear_fc1.weight1 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:52:33 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:04:41 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -1218,7 +1218,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -1231,14 +1231,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:52:33 +[after dataloaders are built] datetime: 2026-08-26 18:04:41 done with setup ... -(min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (2897.35, 2919.50) - train/valid/test-data-iterators-setup ..........: (62.92, 66.14) training ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3092.61, 3106.58) + train/valid/test-data-iterators-setup ..........: (62.93, 82.08) Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 16:52:33 +[before the start of training step] datetime: 2026-08-26 18:04:41 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 sm_count : num_sms=32, num_sms_dedup=148 @@ -1247,8 +1247,8 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 2.52 MiB total_size : 548.52 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 16:52:58] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 24408.4 | throughput per GPU (TFLOP/s/GPU): 7.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116256E+01 | loss scale: 1.0 | grad norm: 44.738 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.85 - + [2026-08-26 18:05:06] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 25021.7 | throughput per GPU (TFLOP/s/GPU): 7.2 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.116256E+01 | loss scale: 1.0 | grad norm: 44.737 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 Number of parameters in embedding layers in billions: 0.41 Total number of parameters in billions: 30.26 Number of parameters in most loaded shard in billions: 30.2590 @@ -1397,8 +1397,8 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank0]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 0 has a total capacity of 178.35 GiB of which 136.81 MiB is free. Including non-PyTorch memory, this process has 178.20 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.72 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 42.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) ['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 42.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank4]: Traceback (most recent call last): [rank4]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank4]: pretrain( @@ -1538,288 +1538,431 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank4]: out = torch.empty( [rank4]: ^^^^^^^^^^^^ [rank4]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 4 has a total capacity of 178.35 GiB of which 42.81 MiB is free. Including non-PyTorch memory, this process has 178.29 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.81 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -[rank3]: Traceback (most recent call last): -[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank3]: pretrain( -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank3]: iteration, num_floating_point_operations_so_far = train( -[rank3]: ^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank3]: ) = train_step( -[rank3]: ^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank3]: losses_reduced = forward_backward_func( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank3]: output_tensor, num_tokens = forward_step( -[rank3]: ^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank3]: output_tensor = model( -[rank3]: ^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank3]: return forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank3]: return self.module(*inputs, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank3]: outputs = self.module(*inputs, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank3]: hidden_states = self.decoder( -[rank3]: ^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank3]: hidden_states, context = layer( -[rank3]: ^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank3]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank3]: ^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank3]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank3]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank3]: ^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank3]: return self._call_impl(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank3]: return inner() -[rank3]: ^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank3]: result = forward_call(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank3]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank3]: return fn(*args, **kwargs) -[rank3]: ^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank3]: out = linear_fn(*args) -[rank3]: ^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank3]: out = torch.empty( -[rank3]: ^^^^^^^^^^^^ -[rank3]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank1]: Traceback (most recent call last): -[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank1]: pretrain( -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank1]: iteration, num_floating_point_operations_so_far = train( -[rank1]: ^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank1]: ) = train_step( -[rank1]: ^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank1]: losses_reduced = forward_backward_func( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank1]: output_tensor, num_tokens = forward_step( -[rank1]: ^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank1]: output_tensor = model( -[rank1]: ^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank1]: return forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank1]: return self.module(*inputs, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank1]: outputs = self.module(*inputs, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank1]: hidden_states = self.decoder( -[rank1]: ^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank1]: hidden_states, context = layer( -[rank1]: ^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank1]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank1]: ^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank1]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank1]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank1]: ^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank1]: return self._call_impl(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank1]: return inner() -[rank1]: ^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank1]: result = forward_call(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank1]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank1]: return fn(*args, **kwargs) -[rank1]: ^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank1]: out = linear_fn(*args) -[rank1]: ^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank1]: out = torch.empty( -[rank1]: ^^^^^^^^^^^^ -[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +[rank6]: Traceback (most recent call last): +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank6]: pretrain( +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank6]: iteration, num_floating_point_operations_so_far = train( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank6]: ) = train_step( +[rank6]: ^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank6]: losses_reduced = forward_backward_func( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank6]: output_tensor, num_tokens = forward_step( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank6]: output_tensor = model( +[rank6]: ^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank6]: return forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank6]: return self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank6]: outputs = self.module(*inputs, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank6]: hidden_states = self.decoder( +[rank6]: ^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank6]: hidden_states, context = layer( +[rank6]: ^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank6]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank6]: ^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank6]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank6]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank6]: ^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank6]: return self._call_impl(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank6]: return inner() +[rank6]: ^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank6]: result = forward_call(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank6]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank6]: return fn(*args, **kwargs) +[rank6]: ^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank6]: out = linear_fn(*args) +[rank6]: ^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank6]: out = torch.empty( +[rank6]: ^^^^^^^^^^^^ +[rank6]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank5]: Traceback (most recent call last): +[rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank5]: pretrain( +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank5]: iteration, num_floating_point_operations_so_far = train( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank5]: ) = train_step( +[rank5]: ^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank5]: losses_reduced = forward_backward_func( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank5]: output_tensor, num_tokens = forward_step( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank5]: output_tensor = model( +[rank5]: ^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank5]: return forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank5]: return self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank5]: outputs = self.module(*inputs, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank5]: hidden_states = self.decoder( +[rank5]: ^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank5]: hidden_states, context = layer( +[rank5]: ^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank5]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank5]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank5]: ^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank5]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank5]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank5]: ^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank5]: return self._call_impl(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank5]: return inner() +[rank5]: ^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank5]: result = forward_call(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank5]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank5]: return fn(*args, **kwargs) +[rank5]: ^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank5]: out = linear_fn(*args) +[rank5]: ^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank5]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank5]: out = torch.empty( +[rank5]: ^^^^^^^^^^^^ +[rank5]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) ['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank3]: Traceback (most recent call last): +[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank3]: pretrain( +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank3]: iteration, num_floating_point_operations_so_far = train( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank3]: ) = train_step( +[rank3]: ^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank3]: losses_reduced = forward_backward_func( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank3]: output_tensor, num_tokens = forward_step( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank3]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank3]: output_tensor = model( +[rank3]: ^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank3]: return forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank3]: return self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank3]: outputs = self.module(*inputs, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank3]: hidden_states = self.decoder( +[rank3]: ^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank3]: hidden_states, context = layer( +[rank3]: ^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank3]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank3]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank3]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank3]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank3]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank3]: ^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank3]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank3]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank3]: ^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank3]: return self._call_impl(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank3]: return inner() +[rank3]: ^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank3]: result = forward_call(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank3]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank3]: return fn(*args, **kwargs) +[rank3]: ^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank3]: out = linear_fn(*args) +[rank3]: ^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank3]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank3]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank3]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank3]: out = torch.empty( +[rank3]: ^^^^^^^^^^^^ +[rank3]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 3 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) +['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] [rank2]: Traceback (most recent call last): [rank2]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank2]: pretrain( @@ -1959,151 +2102,147 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank2]: out = torch.empty( [rank2]: ^^^^^^^^^^^^ [rank2]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 2 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) ['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] -[rank5]: Traceback (most recent call last): -[rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank5]: pretrain( -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank5]: iteration, num_floating_point_operations_so_far = train( -[rank5]: ^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank5]: ) = train_step( -[rank5]: ^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank5]: losses_reduced = forward_backward_func( -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank5]: output_tensor, num_tokens = forward_step( -[rank5]: ^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank5]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank5]: output_tensor = model( -[rank5]: ^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank5]: return forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank5]: return self.module(*inputs, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank5]: outputs = self.module(*inputs, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank5]: hidden_states = self.decoder( -[rank5]: ^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank5]: hidden_states, context = layer( -[rank5]: ^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank5]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank5]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank5]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank5]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank5]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank5]: ^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank5]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank5]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank5]: ^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank5]: return self._call_impl(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank5]: return inner() -[rank5]: ^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank5]: result = forward_call(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank5]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank5]: return fn(*args, **kwargs) -[rank5]: ^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank5]: out = linear_fn(*args) -[rank5]: ^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank5]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank5]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank5]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank5]: out = torch.empty( -[rank5]: ^^^^^^^^^^^^ -[rank5]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 5 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -['Traceback (most recent call last):\n', ' File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step\n output_tensor = model(\n ^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl\n return forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward\n return self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward\n outputs = self.module(*inputs, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward\n hidden_states = self.decoder(\n ^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward\n hidden_states, context = layer(\n ^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__\n return super(MegatronModule, self).__call__(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward\n output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None))\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp\n mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward\n output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute\n output, shared_expert_output, mlp_bias = self.experts_compute(\n ^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute\n expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward\n intermediate_parallel, bias_parallel = self.linear_fc1(\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl\n return self._call_impl(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl\n return inner()\n ^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner\n result = forward_call(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward\n out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch)\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn\n return fn(*args, **kwargs)\n ^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward\n out = linear_fn(*args)\n ^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply\n return super().apply(*args, **kwargs) # type: ignore[misc]\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n', ' File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward\n out = torch.empty(\n ^^^^^^^^^^^^\n', 'torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)\n'] +[rank1]: Traceback (most recent call last): +[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in +[rank1]: pretrain( +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain +[rank1]: iteration, num_floating_point_operations_so_far = train( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train +[rank1]: ) = train_step( +[rank1]: ^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step +[rank1]: losses_reduced = forward_backward_func( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining +[rank1]: output_tensor, num_tokens = forward_step( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step +[rank1]: output_tensor, loss_func = forward_step_func(data_iterator, model) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step +[rank1]: output_tensor = model( +[rank1]: ^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl +[rank1]: return forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward +[rank1]: return self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward +[rank1]: outputs = self.module(*inputs, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward +[rank1]: hidden_states = self.decoder( +[rank1]: ^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward +[rank1]: hidden_states, context = layer( +[rank1]: ^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ +[rank1]: return super(MegatronModule, self).__call__(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward +[rank1]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp +[rank1]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward +[rank1]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute +[rank1]: output, shared_expert_output, mlp_bias = self.experts_compute( +[rank1]: ^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute +[rank1]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward +[rank1]: intermediate_parallel, bias_parallel = self.linear_fc1( +[rank1]: ^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl +[rank1]: return self._call_impl(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl +[rank1]: return inner() +[rank1]: ^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner +[rank1]: result = forward_call(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward +[rank1]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn +[rank1]: return fn(*args, **kwargs) +[rank1]: ^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward +[rank1]: out = linear_fn(*args) +[rank1]: ^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply +[rank1]: return super().apply(*args, **kwargs) # type: ignore[misc] +[rank1]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +[rank1]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward +[rank1]: out = torch.empty( +[rank1]: ^^^^^^^^^^^^ +[rank1]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 1 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) [rank7]: Traceback (most recent call last): [rank7]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in [rank7]: pretrain( @@ -2243,154 +2382,15 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 408.00 MiB. GP [rank7]: out = torch.empty( [rank7]: ^^^^^^^^^^^^ [rank7]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -[rank6]: Traceback (most recent call last): -[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 394, in -[rank6]: pretrain( -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 917, in pretrain -[rank6]: iteration, num_floating_point_operations_so_far = train( -[rank6]: ^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 2326, in train -[rank6]: ) = train_step( -[rank6]: ^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/training/training.py", line 1433, in train_step -[rank6]: losses_reduced = forward_backward_func( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 542, in forward_backward_no_pipelining -[rank6]: output_tensor, num_tokens = forward_step( -[rank6]: ^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/pipeline_parallel/schedules.py", line 361, in forward_step -[rank6]: output_tensor, loss_func = forward_step_func(data_iterator, model) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/pretrain_gpt.py", line 315, in forward_step -[rank6]: output_tensor = model( -[rank6]: ^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1762, in _call_impl -[rank6]: return forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/distributed/data_parallel_base.py", line 22, in forward -[rank6]: return self.module(*inputs, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/module.py", line 237, in forward -[rank6]: outputs = self.module(*inputs, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/models/gpt/gpt_model.py", line 379, in forward -[rank6]: hidden_states = self.decoder( -[rank6]: ^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_block.py", line 581, in forward -[rank6]: hidden_states, context = layer( -[rank6]: ^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 872, in __call__ -[rank6]: return super(MegatronModule, self).__call__(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 440, in forward -[rank6]: output = self._forward_mlp(hidden_states, kwargs.get("inference_context", None)) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/transformer_layer.py", line 617, in _forward_mlp -[rank6]: mlp_output_with_bias = self.mlp(pre_mlp_layernorm_output) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 565, in forward -[rank6]: output, mlp_bias = dispatch_and_compute(hidden_states, probs, residual, metadata) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 538, in dispatch_and_compute -[rank6]: output, shared_expert_output, mlp_bias = self.experts_compute( -[rank6]: ^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/moe_layer.py", line 320, in experts_compute -[rank6]: expert_output, mlp_bias = self.experts(dispatched_input, tokens_per_expert, permuted_probs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/transformer/moe/experts.py", line 875, in forward -[rank6]: intermediate_parallel, bias_parallel = self.linear_fc1( -[rank6]: ^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1751, in _wrapped_call_impl -[rank6]: return self._call_impl(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1857, in _call_impl -[rank6]: return inner() -[rank6]: ^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/nn/modules/module.py", line 1805, in inner -[rank6]: result = forward_call(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/workspace/Megatron-MoE-Echo/megatron/core/extensions/transformer_engine.py", line 1123, in forward -[rank6]: out = super().forward(x, m_splits, is_first_microbatch=_is_first_microbatch) -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/_dynamo/eval_frame.py", line 850, in _fn -[rank6]: return fn(*args, **kwargs) -[rank6]: ^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 900, in forward -[rank6]: out = linear_fn(*args) -[rank6]: ^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/torch/autograd/function.py", line 575, in apply -[rank6]: return super().apply(*args, **kwargs) # type: ignore[misc] -[rank6]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -[rank6]: File "/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/module/grouped_linear.py", line 173, in forward -[rank6]: out = torch.empty( -[rank6]: ^^^^^^^^^^^^ -[rank6]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 408.00 MiB. GPU 6 has a total capacity of 178.35 GiB of which 56.81 MiB is free. Including non-PyTorch memory, this process has 178.28 GiB memory in use. Of the allocated memory 146.08 GiB is allocated by PyTorch, and 3.80 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -W0826 16:53:05.493000 280115 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:53:05.494000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280211 closing signal SIGTERM -W0826 16:53:05.496000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280212 closing signal SIGTERM -W0826 16:53:05.505000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280213 closing signal SIGTERM -W0826 16:53:05.507000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280214 closing signal SIGTERM -W0826 16:53:05.516000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280215 closing signal SIGTERM -W0826 16:53:05.522000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280216 closing signal SIGTERM -W0826 16:53:05.530000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280217 closing signal SIGTERM -W0826 16:53:05.538000 280115 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 280218 closing signal SIGTERM +W0826 18:05:12.860000 452537 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:05:12.861000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452633 closing signal SIGTERM +W0826 18:05:12.862000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452634 closing signal SIGTERM +W0826 18:05:12.879000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452635 closing signal SIGTERM +W0826 18:05:12.888000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452636 closing signal SIGTERM +W0826 18:05:12.888000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452637 closing signal SIGTERM +W0826 18:05:12.907000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452638 closing signal SIGTERM +W0826 18:05:12.911000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452639 closing signal SIGTERM +W0826 18:05:12.914000 452537 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 452640 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -2418,4 +2418,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 280115 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 452537 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv index afb16dfe172d258eb475186409a329020fea78d8..fcaffbb8d41f0efb8e4a25593ed7aa6f2a1adcb6 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/mem_samples.csv @@ -37,71 +37,79 @@ 4, 4 5, 4 6, 4 -7, 4 +7, 1 0, 4 -1, 629 -2, 629 -3, 629 -4, 629 -5, 629 -6, 629 -7, 629 -0, 943 -1, 937 -2, 935 -3, 975 -4, 947 -5, 945 -6, 941 -7, 931 -0, 1831 -1, 1831 -2, 1831 -3, 1831 -4, 1831 -5, 1831 -6, 1831 -7, 1831 -0, 3789 -1, 3793 -2, 3789 -3, 3781 -4, 3785 -5, 3803 -6, 3809 -7, 3787 -0, 15563 -1, 15565 -2, 15565 -3, 15565 -4, 15565 -5, 15565 -6, 15565 -7, 15565 -0, 57141 -1, 57099 -2, 57365 -3, 57003 -4, 57267 -5, 57177 -6, 57159 -7, 57365 -0, 67479 -1, 67481 -2, 67459 -3, 67409 -4, 67481 -5, 67459 -6, 67481 -7, 67495 -0, 68527 -1, 68565 -2, 68543 -3, 68493 -4, 68529 -5, 68543 -6, 68565 -7, 68543 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 821 +1, 821 +2, 819 +3, 821 +4, 819 +5, 821 +6, 821 +7, 821 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 5611 +1, 5613 +2, 5613 +3, 5613 +4, 5613 +5, 5613 +6, 5613 +7, 5613 +0, 19683 +1, 19685 +2, 19685 +3, 19685 +4, 19685 +5, 19685 +6, 19685 +7, 19685 +0, 66997 +1, 67059 +2, 67037 +3, 66987 +4, 66927 +5, 67037 +6, 66799 +7, 66713 +0, 67881 +1, 67919 +2, 67753 +3, 67703 +4, 67739 +5, 67753 +6, 67775 +7, 67753 +0, 68673 +1, 68711 +2, 68689 +3, 68639 +4, 68675 +5, 68689 +6, 68711 +7, 68689 0, 68673 1, 68711 2, 68689 @@ -110,75 +118,83 @@ 5, 68689 6, 68711 7, 68689 -0, 68675 -1, 68713 -2, 68691 -3, 68641 -4, 68677 -5, 68691 -6, 68713 -7, 68691 -0, 87971 -1, 87467 -2, 84739 -3, 87193 -4, 85637 -5, 86851 -6, 85601 -7, 86233 -0, 125825 -1, 125839 -2, 125529 -3, 125837 -4, 125833 -5, 125825 -6, 125839 -7, 125825 -0, 155829 -1, 155843 -2, 155761 -3, 155841 -4, 155835 -5, 155829 -6, 155843 -7, 155829 -0, 156475 -1, 156489 -2, 156475 -3, 156487 -4, 156483 -5, 156475 -6, 156489 -7, 156475 +0, 69737 +1, 69717 +2, 69723 +3, 69707 +4, 69697 +5, 69723 +6, 69695 +7, 69723 +0, 94413 +1, 94553 +2, 94391 +3, 94217 +4, 92389 +5, 94391 +6, 94177 +7, 91431 +0, 133221 +1, 133257 +2, 131337 +3, 131821 +4, 130575 +5, 132595 +6, 133257 +7, 129597 +0, 156087 +1, 156099 +2, 156085 +3, 156099 +4, 156081 +5, 156085 +6, 156101 +7, 156083 0, 156865 -1, 156879 -2, 156865 +1, 156829 +2, 156851 3, 156877 4, 156873 -5, 156865 -6, 156879 -7, 156865 -0, 156879 -1, 156893 -2, 156879 -3, 156891 -4, 156887 -5, 156879 -6, 156893 -7, 156879 -0, 167327 -1, 167239 -2, 167349 -3, 167187 -4, 167269 -5, 167333 -6, 167479 -7, 167391 -0, 169195 -1, 168833 -2, 168939 -3, 168923 -4, 168847 -5, 168939 -6, 169109 -7, 169083 +5, 156781 +6, 156823 +7, 156833 +0, 156877 +1, 156891 +2, 156877 +3, 156889 +4, 156885 +5, 156877 +6, 156891 +7, 156877 +0, 157525 +1, 157539 +2, 157525 +3, 157537 +4, 157533 +5, 157525 +6, 157539 +7, 157525 +0, 167365 +1, 167279 +2, 167361 +3, 167281 +4, 167249 +5, 167341 +6, 167559 +7, 167359 +0, 175927 +1, 175457 +2, 175671 +3, 175511 +4, 175867 +5, 175959 +6, 175841 +7, 175671 +0, 182485 +1, 182583 +2, 23094 +3, 182529 +4, 182597 +5, 182517 +6, 182543 +7, 182517 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log index 9ba25e83c05ddaa9b036d730a04bf1152aca1108..23e930b21ce4ee1043c5638082877c9eb2746c06 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log +++ b/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/train.log @@ -1,6 +1,6 @@ -[16:53:17] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) -[16:53:17] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) -[16:53:17] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads +[18:05:24] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[18:05:24] WARN: moonep_backend_token_routing_analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +[18:05:24] WARN: moonep_backend_token_routing_analysis lacks --moe-debug-drop-logging; probe will have no pre/post rank loads ============================================================== QWEN3_30B / moonep (slots/rank=16 policy=matched) E=128 K=8 H'=768 m=16 @@ -9,16 +9,10 @@ memory_dump=none snapshots=/workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1/memory_snapshots/snapshot.pickle repo=/workspace/Megatron-MoE-Echo @ 35ae6e7f ============================================================== -W0826 16:53:18.872000 282417 torch/distributed/run.py:766] -W0826 16:53:18.872000 282417 torch/distributed/run.py:766] ***************************************** -W0826 16:53:18.872000 282417 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. -W0826 16:53:18.872000 282417 torch/distributed/run.py:766] ***************************************** -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +W0826 18:05:25.892000 454848 torch/distributed/run.py:766] +W0826 18:05:25.892000 454848 torch/distributed/run.py:766] ***************************************** +W0826 18:05:25.892000 454848 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0826 18:05:25.892000 454848 torch/distributed/run.py:766] ***************************************** /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled @@ -26,8 +20,6 @@ W0826 16:53:18.872000 282417 torch/distributed/run.py:766] ********************* /workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it -/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled - warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer Number of virtual stages per pipeline stage: None @@ -543,6 +535,8 @@ using torch.bfloat16 for parameters ... use_dist_ckpt_deprecated ........................ False use_distributed_optimizer ....................... True use_flash_attn .................................. False +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") use_legacy_models ............................... False use_mp_args_from_checkpoint_args ................ False use_one_sent_docs ............................... False @@ -578,43 +572,49 @@ using torch.bfloat16 for parameters ... -------------------- end of arguments --------------------- INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 > building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS > initializing torch distributed ... +[Gloo] Rank [Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 77 + is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 14 is connected to is connected to 77 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 77 + [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 5 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank -7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 -[Gloo] Rank 6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 > initialized tensor model parallel with size 1 > initialized pipeline model parallel with size 1 > setting random seeds to 1234 ... [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0[Gloo] Rank is connected to 00 peer ranks. is connected to Expected number of connected peer ranks is : 00 peer ranks. +Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 - [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 [Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 -[rank5]:[W826 16:53:37.532812924 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank7]:[W826 16:53:37.532833608 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank6]:[W826 16:53:37.532923439 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank3]:[W826 16:53:37.532937394 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank1]:[W826 16:53:37.533025935 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank2]:[W826 16:53:37.534558799 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. -[rank4]:[W826 16:53:37.534974220 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W826 18:05:44.397588065 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W826 18:05:44.399483613 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W826 18:05:44.399507355 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W826 18:05:44.399576605 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W826 18:05:44.399628962 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W826 18:05:44.399642483 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W826 18:05:44.399661386 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. > compiling dataset index builder ... make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' make: Nothing to be done for 'default'. @@ -622,10 +622,10 @@ make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' >>> done with dataset index builder. Compilation time: 0.033 seconds WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. > compiling and loading fused kernels ... -[rank0]:[W826 16:53:37.685130694 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. ->>> done with compiling and loading fused kernels. Compilation time: 3.558 seconds -time to initialize megatron (seconds): 6.772 -[after megatron is initialized] datetime: 2026-08-26 16:53:43 +[rank0]:[W826 18:05:44.545649875 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.586 seconds +time to initialize megatron (seconds): 7.005 +[after megatron is initialized] datetime: 2026-08-26 18:05:50 building GPT model ... /usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. warnings.warn( @@ -651,65 +651,63 @@ Params for bucket 1 (103022592 elements, 103022592 padded size): Params for bucket 2 (49033984 elements, 49033984 padded size): module.decoder.layers.47.self_attention.k_layernorm.weight module.decoder.layers.47.self_attention.linear_qkv.weight + module.decoder.layers.46.self_attention.k_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.47.mlp.router.weight module.decoder.layers.47.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.46.mlp.router.weight + module.decoder.layers.46.self_attention.q_layernorm.weight module.decoder.layers.46.self_attention.linear_proj.weight module.decoder.layers.45.mlp.router.weight module.decoder.layers.45.self_attention.k_layernorm.weight - module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.linear_qkv.weight + module.decoder.final_layernorm.weight module.decoder.layers.47.self_attention.q_layernorm.weight - module.decoder.layers.47.mlp.router.weight - module.decoder.layers.45.pre_mlp_layernorm.weight - module.decoder.layers.45.self_attention.q_layernorm.weight - module.decoder.layers.46.self_attention.k_layernorm.weight module.decoder.layers.47.pre_mlp_layernorm.weight module.decoder.layers.47.self_attention.linear_proj.weight - module.decoder.layers.46.self_attention.linear_qkv.layer_norm_weight - module.decoder.final_layernorm.weight - module.decoder.layers.45.self_attention.linear_qkv.weight - module.decoder.layers.46.mlp.router.weight - module.decoder.layers.46.self_attention.q_layernorm.weight + module.decoder.layers.46.pre_mlp_layernorm.weight + module.decoder.layers.46.self_attention.linear_qkv.weight + module.decoder.layers.45.pre_mlp_layernorm.weight + module.decoder.layers.45.self_attention.q_layernorm.weight Params for bucket 3 (46672384 elements, 46672384 padded size): - module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.44.mlp.router.weight module.decoder.layers.44.pre_mlp_layernorm.weight module.decoder.layers.44.self_attention.k_layernorm.weight - module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.mlp.router.weight module.decoder.layers.44.self_attention.q_layernorm.weight - module.decoder.layers.44.self_attention.linear_qkv.weight module.decoder.layers.44.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.pre_mlp_layernorm.weight + module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.43.self_attention.linear_proj.weight + module.decoder.layers.45.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.45.self_attention.linear_proj.weight + module.decoder.layers.44.self_attention.linear_qkv.weight module.decoder.layers.44.self_attention.linear_proj.weight module.decoder.layers.43.mlp.router.weight - module.decoder.layers.43.pre_mlp_layernorm.weight module.decoder.layers.43.self_attention.k_layernorm.weight module.decoder.layers.43.self_attention.q_layernorm.weight module.decoder.layers.43.self_attention.linear_qkv.weight - module.decoder.layers.43.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.43.self_attention.linear_proj.weight Params for bucket 4 (49031936 elements, 49031936 padded size): - module.decoder.layers.42.self_attention.linear_qkv.weight module.decoder.layers.42.self_attention.k_layernorm.weight + module.decoder.layers.42.self_attention.linear_qkv.weight module.decoder.layers.42.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.41.pre_mlp_layernorm.weight - module.decoder.layers.41.self_attention.linear_qkv.weight - module.decoder.layers.40.self_attention.linear_qkv.weight - module.decoder.layers.41.self_attention.q_layernorm.weight - module.decoder.layers.40.mlp.router.weight module.decoder.layers.40.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_qkv.weight + module.decoder.layers.41.self_attention.k_layernorm.weight + module.decoder.layers.41.self_attention.linear_qkv.weight module.decoder.layers.42.mlp.router.weight module.decoder.layers.42.self_attention.q_layernorm.weight + module.decoder.layers.40.mlp.router.weight + module.decoder.layers.41.mlp.router.weight + module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.42.pre_mlp_layernorm.weight module.decoder.layers.42.self_attention.linear_proj.weight - module.decoder.layers.41.self_attention.linear_proj.weight module.decoder.layers.40.pre_mlp_layernorm.weight module.decoder.layers.40.self_attention.q_layernorm.weight - module.decoder.layers.42.pre_mlp_layernorm.weight - module.decoder.layers.41.mlp.router.weight - module.decoder.layers.41.self_attention.k_layernorm.weight - module.decoder.layers.41.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.41.pre_mlp_layernorm.weight + module.decoder.layers.41.self_attention.q_layernorm.weight + module.decoder.layers.41.self_attention.linear_proj.weight Params for bucket 5 (46672384 elements, 46672384 padded size): module.decoder.layers.39.mlp.router.weight - module.decoder.layers.40.self_attention.linear_proj.weight - module.decoder.layers.39.self_attention.k_layernorm.weight module.decoder.layers.39.self_attention.q_layernorm.weight module.decoder.layers.39.self_attention.linear_qkv.weight module.decoder.layers.39.self_attention.linear_qkv.layer_norm_weight @@ -718,111 +716,111 @@ Params for bucket 5 (46672384 elements, 46672384 padded size): module.decoder.layers.38.pre_mlp_layernorm.weight module.decoder.layers.38.self_attention.k_layernorm.weight module.decoder.layers.38.self_attention.q_layernorm.weight + module.decoder.layers.38.self_attention.linear_qkv.weight module.decoder.layers.38.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.38.self_attention.linear_proj.weight module.decoder.layers.40.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.39.self_attention.k_layernorm.weight + module.decoder.layers.40.self_attention.linear_proj.weight module.decoder.layers.39.pre_mlp_layernorm.weight - module.decoder.layers.38.self_attention.linear_proj.weight - module.decoder.layers.38.self_attention.linear_qkv.weight Params for bucket 6 (49031936 elements, 49031936 padded size): - module.decoder.layers.37.mlp.router.weight - module.decoder.layers.37.self_attention.q_layernorm.weight + module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.36.pre_mlp_layernorm.weight module.decoder.layers.36.self_attention.q_layernorm.weight module.decoder.layers.36.self_attention.linear_proj.weight + module.decoder.layers.35.pre_mlp_layernorm.weight + module.decoder.layers.37.mlp.router.weight + module.decoder.layers.37.self_attention.q_layernorm.weight module.decoder.layers.37.self_attention.linear_proj.weight - module.decoder.layers.35.self_attention.linear_qkv.weight module.decoder.layers.37.pre_mlp_layernorm.weight - module.decoder.layers.35.self_attention.k_layernorm.weight module.decoder.layers.36.self_attention.k_layernorm.weight module.decoder.layers.36.self_attention.linear_qkv.weight module.decoder.layers.36.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.35.self_attention.k_layernorm.weight + module.decoder.layers.35.self_attention.linear_qkv.weight + module.decoder.layers.36.mlp.router.weight module.decoder.layers.35.mlp.router.weight module.decoder.layers.35.self_attention.q_layernorm.weight module.decoder.layers.37.self_attention.linear_qkv.weight module.decoder.layers.37.self_attention.k_layernorm.weight - module.decoder.layers.37.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.36.mlp.router.weight - module.decoder.layers.35.pre_mlp_layernorm.weight Params for bucket 7 (46672384 elements, 46672384 padded size): - module.decoder.layers.34.self_attention.k_layernorm.weight - module.decoder.layers.34.pre_mlp_layernorm.weight - module.decoder.layers.34.mlp.router.weight - module.decoder.layers.34.self_attention.q_layernorm.weight - module.decoder.layers.34.self_attention.linear_qkv.weight - module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.34.self_attention.linear_proj.weight + module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.33.mlp.router.weight module.decoder.layers.33.pre_mlp_layernorm.weight module.decoder.layers.33.self_attention.k_layernorm.weight module.decoder.layers.33.self_attention.q_layernorm.weight module.decoder.layers.33.self_attention.linear_qkv.weight module.decoder.layers.33.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.35.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.35.self_attention.linear_proj.weight + module.decoder.layers.34.mlp.router.weight + module.decoder.layers.34.pre_mlp_layernorm.weight + module.decoder.layers.34.self_attention.k_layernorm.weight + module.decoder.layers.34.self_attention.q_layernorm.weight + module.decoder.layers.34.self_attention.linear_qkv.weight + module.decoder.layers.34.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.34.self_attention.linear_proj.weight module.decoder.layers.33.self_attention.linear_proj.weight Params for bucket 8 (49031936 elements, 49031936 padded size): - module.decoder.layers.31.self_attention.linear_qkv.weight - module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.32.pre_mlp_layernorm.weight - module.decoder.layers.32.self_attention.linear_proj.weight - module.decoder.layers.31.pre_mlp_layernorm.weight + module.decoder.layers.31.mlp.router.weight module.decoder.layers.31.self_attention.k_layernorm.weight - module.decoder.layers.30.self_attention.k_layernorm.weight - module.decoder.layers.30.self_attention.linear_qkv.weight - module.decoder.layers.31.self_attention.q_layernorm.weight - module.decoder.layers.31.self_attention.linear_proj.weight - module.decoder.layers.30.mlp.router.weight module.decoder.layers.30.self_attention.q_layernorm.weight - module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.30.mlp.router.weight + module.decoder.layers.31.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.32.self_attention.linear_qkv.weight - module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.32.self_attention.k_layernorm.weight + module.decoder.layers.31.pre_mlp_layernorm.weight module.decoder.layers.30.pre_mlp_layernorm.weight + module.decoder.layers.31.self_attention.q_layernorm.weight module.decoder.layers.32.mlp.router.weight module.decoder.layers.32.self_attention.q_layernorm.weight - module.decoder.layers.31.mlp.router.weight + module.decoder.layers.32.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_qkv.weight + module.decoder.layers.31.self_attention.linear_proj.weight + module.decoder.layers.32.pre_mlp_layernorm.weight + module.decoder.layers.32.self_attention.linear_proj.weight + module.decoder.layers.30.self_attention.k_layernorm.weight + module.decoder.layers.31.self_attention.linear_qkv.weight Params for bucket 9 (46672384 elements, 46672384 padded size): module.decoder.layers.30.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.30.self_attention.linear_proj.weight + module.decoder.layers.28.self_attention.k_layernorm.weight + module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.28.self_attention.linear_proj.weight module.decoder.layers.29.self_attention.k_layernorm.weight module.decoder.layers.29.mlp.router.weight module.decoder.layers.29.pre_mlp_layernorm.weight - module.decoder.layers.30.self_attention.linear_proj.weight module.decoder.layers.29.self_attention.q_layernorm.weight module.decoder.layers.29.self_attention.linear_qkv.weight module.decoder.layers.29.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.29.self_attention.linear_proj.weight module.decoder.layers.28.mlp.router.weight module.decoder.layers.28.pre_mlp_layernorm.weight - module.decoder.layers.28.self_attention.k_layernorm.weight module.decoder.layers.28.self_attention.q_layernorm.weight module.decoder.layers.28.self_attention.linear_qkv.weight - module.decoder.layers.28.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.28.self_attention.linear_proj.weight Params for bucket 10 (49031936 elements, 49031936 padded size): + module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.25.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.q_layernorm.weight module.decoder.layers.27.pre_mlp_layernorm.weight + module.decoder.layers.27.self_attention.linear_proj.weight + module.decoder.layers.25.pre_mlp_layernorm.weight + module.decoder.layers.26.mlp.router.weight module.decoder.layers.26.self_attention.k_layernorm.weight - module.decoder.layers.25.mlp.router.weight - module.decoder.layers.25.self_attention.linear_qkv.weight - module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.25.self_attention.q_layernorm.weight module.decoder.layers.27.self_attention.linear_qkv.weight - module.decoder.layers.25.pre_mlp_layernorm.weight - module.decoder.layers.26.pre_mlp_layernorm.weight - module.decoder.layers.26.self_attention.q_layernorm.weight - module.decoder.layers.27.self_attention.k_layernorm.weight module.decoder.layers.27.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.25.self_attention.k_layernorm.weight - module.decoder.layers.25.self_attention.q_layernorm.weight - module.decoder.layers.26.self_attention.linear_proj.weight + module.decoder.layers.26.pre_mlp_layernorm.weight + module.decoder.layers.26.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.26.self_attention.q_layernorm.weight module.decoder.layers.27.mlp.router.weight - module.decoder.layers.27.self_attention.q_layernorm.weight - module.decoder.layers.27.self_attention.linear_proj.weight - module.decoder.layers.26.mlp.router.weight - module.decoder.layers.26.self_attention.linear_qkv.weight + module.decoder.layers.27.self_attention.k_layernorm.weight + module.decoder.layers.25.mlp.router.weight Params for bucket 11 (46672384 elements, 46672384 padded size): - module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.24.self_attention.k_layernorm.weight module.decoder.layers.24.mlp.router.weight - module.decoder.layers.25.self_attention.linear_proj.weight module.decoder.layers.24.pre_mlp_layernorm.weight + module.decoder.layers.24.self_attention.k_layernorm.weight module.decoder.layers.24.self_attention.q_layernorm.weight module.decoder.layers.24.self_attention.linear_qkv.weight module.decoder.layers.24.self_attention.linear_qkv.layer_norm_weight @@ -831,146 +829,145 @@ Params for bucket 11 (46672384 elements, 46672384 padded size): module.decoder.layers.23.pre_mlp_layernorm.weight module.decoder.layers.23.self_attention.k_layernorm.weight module.decoder.layers.23.self_attention.q_layernorm.weight - module.decoder.layers.23.self_attention.linear_qkv.weight module.decoder.layers.23.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.23.self_attention.linear_proj.weight + module.decoder.layers.25.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.25.self_attention.linear_proj.weight + module.decoder.layers.23.self_attention.linear_qkv.weight Params for bucket 12 (49031936 elements, 49031936 padded size): - module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.q_layernorm.weight + module.decoder.layers.21.self_attention.linear_qkv.weight + module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.21.self_attention.linear_proj.weight + module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.21.pre_mlp_layernorm.weight + module.decoder.layers.21.self_attention.k_layernorm.weight module.decoder.layers.20.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.linear_qkv.weight + module.decoder.layers.22.self_attention.k_layernorm.weight module.decoder.layers.22.self_attention.linear_proj.weight - module.decoder.layers.21.self_attention.linear_proj.weight module.decoder.layers.20.self_attention.k_layernorm.weight - module.decoder.layers.21.self_attention.linear_qkv.weight - module.decoder.layers.22.self_attention.k_layernorm.weight - module.decoder.layers.20.mlp.router.weight - module.decoder.layers.20.self_attention.q_layernorm.weight - module.decoder.layers.21.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.22.mlp.router.weight module.decoder.layers.22.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.21.mlp.router.weight - module.decoder.layers.21.self_attention.k_layernorm.weight - module.decoder.layers.20.pre_mlp_layernorm.weight + module.decoder.layers.20.mlp.router.weight + module.decoder.layers.20.self_attention.q_layernorm.weight module.decoder.layers.22.pre_mlp_layernorm.weight module.decoder.layers.22.self_attention.q_layernorm.weight - module.decoder.layers.21.pre_mlp_layernorm.weight - module.decoder.layers.21.self_attention.q_layernorm.weight Params for bucket 13 (46672384 elements, 46672384 padded size): module.decoder.layers.20.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.19.self_attention.k_layernorm.weight module.decoder.layers.19.mlp.router.weight module.decoder.layers.19.pre_mlp_layernorm.weight - module.decoder.layers.20.self_attention.linear_proj.weight + module.decoder.layers.19.self_attention.k_layernorm.weight module.decoder.layers.19.self_attention.q_layernorm.weight module.decoder.layers.19.self_attention.linear_qkv.weight + module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.19.self_attention.linear_proj.weight module.decoder.layers.18.mlp.router.weight module.decoder.layers.18.pre_mlp_layernorm.weight module.decoder.layers.18.self_attention.linear_qkv.weight module.decoder.layers.18.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.18.self_attention.q_layernorm.weight - module.decoder.layers.19.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.20.self_attention.linear_proj.weight module.decoder.layers.18.self_attention.k_layernorm.weight module.decoder.layers.18.self_attention.linear_proj.weight Params for bucket 14 (49031936 elements, 49031936 padded size): - module.decoder.layers.16.self_attention.linear_qkv.weight - module.decoder.layers.15.self_attention.linear_qkv.weight module.decoder.layers.17.mlp.router.weight - module.decoder.layers.15.pre_mlp_layernorm.weight - module.decoder.layers.16.mlp.router.weight - module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.k_layernorm.weight + module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.16.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.16.self_attention.linear_proj.weight + module.decoder.layers.16.self_attention.q_layernorm.weight + module.decoder.layers.17.pre_mlp_layernorm.weight module.decoder.layers.17.self_attention.q_layernorm.weight + module.decoder.layers.17.self_attention.linear_proj.weight + module.decoder.layers.16.mlp.router.weight module.decoder.layers.15.self_attention.k_layernorm.weight + module.decoder.layers.15.self_attention.linear_qkv.weight module.decoder.layers.16.pre_mlp_layernorm.weight - module.decoder.layers.16.self_attention.q_layernorm.weight - module.decoder.layers.16.self_attention.linear_proj.weight - module.decoder.layers.17.self_attention.linear_qkv.weight - module.decoder.layers.17.self_attention.linear_proj.weight module.decoder.layers.15.self_attention.q_layernorm.weight - module.decoder.layers.17.pre_mlp_layernorm.weight - module.decoder.layers.17.self_attention.k_layernorm.weight - module.decoder.layers.17.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.15.mlp.router.weight + module.decoder.layers.17.self_attention.linear_qkv.weight + module.decoder.layers.16.self_attention.k_layernorm.weight + module.decoder.layers.15.pre_mlp_layernorm.weight + module.decoder.layers.16.self_attention.linear_qkv.weight Params for bucket 15 (46672384 elements, 46672384 padded size): + module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.15.self_attention.linear_proj.weight module.decoder.layers.14.mlp.router.weight - module.decoder.layers.14.pre_mlp_layernorm.weight - module.decoder.layers.14.self_attention.k_layernorm.weight - module.decoder.layers.14.self_attention.linear_qkv.weight - module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.14.self_attention.linear_proj.weight module.decoder.layers.13.mlp.router.weight module.decoder.layers.13.pre_mlp_layernorm.weight module.decoder.layers.13.self_attention.k_layernorm.weight module.decoder.layers.13.self_attention.q_layernorm.weight - module.decoder.layers.13.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_qkv.weight module.decoder.layers.13.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.15.self_attention.linear_proj.weight + module.decoder.layers.14.self_attention.k_layernorm.weight + module.decoder.layers.14.pre_mlp_layernorm.weight module.decoder.layers.14.self_attention.q_layernorm.weight - module.decoder.layers.13.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.weight + module.decoder.layers.14.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.14.self_attention.linear_proj.weight + module.decoder.layers.13.self_attention.linear_proj.weight Params for bucket 16 (49031936 elements, 49031936 padded size): - module.decoder.layers.12.self_attention.linear_qkv.weight - module.decoder.layers.12.self_attention.linear_proj.weight - module.decoder.layers.11.self_attention.linear_qkv.weight - module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.pre_mlp_layernorm.weight module.decoder.layers.12.self_attention.q_layernorm.weight - module.decoder.layers.12.self_attention.k_layernorm.weight - module.decoder.layers.10.mlp.router.weight - module.decoder.layers.10.self_attention.q_layernorm.weight - module.decoder.layers.11.self_attention.k_layernorm.weight - module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.10.pre_mlp_layernorm.weight module.decoder.layers.10.self_attention.linear_qkv.weight module.decoder.layers.11.mlp.router.weight module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.12.self_attention.linear_proj.weight + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.11.pre_mlp_layernorm.weight module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.layers.12.self_attention.linear_qkv.weight + module.decoder.layers.12.self_attention.k_layernorm.weight module.decoder.layers.10.self_attention.k_layernorm.weight - module.decoder.layers.12.pre_mlp_layernorm.weight - module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.12.mlp.router.weight + module.decoder.layers.12.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight Params for bucket 17 (46672384 elements, 46672384 padded size): module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.k_layernorm.weight module.decoder.layers.9.mlp.router.weight module.decoder.layers.10.self_attention.linear_proj.weight + module.decoder.layers.9.pre_mlp_layernorm.weight module.decoder.layers.9.self_attention.q_layernorm.weight module.decoder.layers.9.self_attention.linear_qkv.weight module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_qkv.weight module.decoder.layers.8.mlp.router.weight module.decoder.layers.8.pre_mlp_layernorm.weight module.decoder.layers.8.self_attention.k_layernorm.weight module.decoder.layers.8.self_attention.q_layernorm.weight - module.decoder.layers.8.self_attention.linear_qkv.weight - module.decoder.layers.8.self_attention.linear_proj.weight module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.9.self_attention.k_layernorm.weight - module.decoder.layers.9.pre_mlp_layernorm.weight - module.decoder.layers.9.self_attention.linear_proj.weight + module.decoder.layers.8.self_attention.linear_proj.weight Params for bucket 18 (49031936 elements, 49031936 padded size): - module.decoder.layers.7.mlp.router.weight - module.decoder.layers.7.self_attention.linear_qkv.weight - module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.5.pre_mlp_layernorm.weight - module.decoder.layers.7.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.6.mlp.router.weight module.decoder.layers.6.self_attention.q_layernorm.weight - module.decoder.layers.6.self_attention.linear_proj.weight - module.decoder.layers.5.self_attention.linear_qkv.weight - module.decoder.layers.7.self_attention.q_layernorm.weight - module.decoder.layers.7.pre_mlp_layernorm.weight - module.decoder.layers.6.pre_mlp_layernorm.weight module.decoder.layers.5.mlp.router.weight module.decoder.layers.5.self_attention.k_layernorm.weight - module.decoder.layers.7.self_attention.linear_proj.weight - module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.5.pre_mlp_layernorm.weight module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_proj.weight + module.decoder.layers.7.self_attention.k_layernorm.weight module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight Params for bucket 19 (46672384 elements, 46672384 padded size): module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.5.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.weight - module.decoder.layers.3.self_attention.linear_proj.weight - module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.mlp.router.weight module.decoder.layers.4.pre_mlp_layernorm.weight module.decoder.layers.4.self_attention.k_layernorm.weight @@ -978,55 +975,58 @@ Params for bucket 19 (46672384 elements, 46672384 padded size): module.decoder.layers.4.self_attention.linear_qkv.weight module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_proj.weight module.decoder.layers.3.mlp.router.weight module.decoder.layers.3.pre_mlp_layernorm.weight module.decoder.layers.3.self_attention.k_layernorm.weight module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight Params for bucket 20 (49031936 elements, 49031936 padded size): + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.0.self_attention.k_layernorm.weight module.decoder.layers.1.self_attention.q_layernorm.weight - module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.2.self_attention.linear_qkv.weight module.decoder.layers.2.self_attention.k_layernorm.weight - module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.1.pre_mlp_layernorm.weight module.decoder.layers.0.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.q_layernorm.weight - module.decoder.layers.2.self_attention.linear_proj.weight + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight module.decoder.layers.1.self_attention.linear_qkv.weight - module.decoder.layers.0.mlp.router.weight - module.decoder.layers.1.self_attention.k_layernorm.weight - module.decoder.layers.0.pre_mlp_layernorm.weight module.decoder.layers.2.mlp.router.weight - module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.linear_qkv.weight - module.decoder.layers.2.pre_mlp_layernorm.weight - module.decoder.layers.1.mlp.router.weight - module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_proj.weight Params for bucket 21 (111413248 elements, 111413248 padded size): module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight - module.decoder.layers.0.self_attention.linear_proj.weight module.embedding.word_embeddings.weight + module.decoder.layers.0.self_attention.linear_proj.weight INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 89 Params for bucket 1 (40894464 elements, 40894464 padded size): - module.decoder.layers.47.mlp.experts.linear_fc2.weight9 - module.decoder.layers.47.mlp.experts.linear_fc1.weight11 - module.decoder.layers.47.mlp.experts.linear_fc2.weight14 - module.decoder.layers.47.mlp.experts.linear_fc2.weight6 - module.decoder.layers.47.mlp.experts.linear_fc2.weight11 - module.decoder.layers.47.mlp.experts.linear_fc2.weight3 - module.decoder.layers.47.mlp.experts.linear_fc1.weight13 module.decoder.layers.47.mlp.experts.linear_fc2.weight8 module.decoder.layers.47.mlp.experts.linear_fc2.weight1 + module.decoder.layers.47.mlp.experts.linear_fc1.weight11 module.decoder.layers.47.mlp.experts.linear_fc2.weight13 module.decoder.layers.47.mlp.experts.linear_fc2.weight5 - module.decoder.layers.47.mlp.experts.linear_fc1.weight15 module.decoder.layers.47.mlp.experts.linear_fc2.weight10 module.decoder.layers.47.mlp.experts.linear_fc2.weight2 - module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc1.weight13 module.decoder.layers.47.mlp.experts.linear_fc2.weight15 module.decoder.layers.47.mlp.experts.linear_fc2.weight7 module.decoder.layers.47.mlp.experts.linear_fc2.weight12 module.decoder.layers.47.mlp.experts.linear_fc2.weight4 + module.decoder.layers.47.mlp.experts.linear_fc1.weight15 + module.decoder.layers.47.mlp.experts.linear_fc2.weight9 + module.decoder.layers.47.mlp.experts.linear_fc1.weight12 + module.decoder.layers.47.mlp.experts.linear_fc2.weight14 + module.decoder.layers.47.mlp.experts.linear_fc2.weight6 + module.decoder.layers.47.mlp.experts.linear_fc2.weight11 + module.decoder.layers.47.mlp.experts.linear_fc2.weight3 module.decoder.layers.47.mlp.experts.linear_fc2.weight0 module.decoder.layers.47.mlp.experts.linear_fc1.weight14 Params for bucket 2 (40894464 elements, 40894464 padded size): @@ -1084,32 +1084,33 @@ Params for bucket 4 (40894464 elements, 40894464 padded size): module.decoder.layers.45.mlp.experts.linear_fc2.weight11 module.decoder.layers.45.mlp.experts.linear_fc2.weight9 Params for bucket 5 (40894464 elements, 40894464 padded size): - module.decoder.layers.45.mlp.experts.linear_fc2.weight6 module.decoder.layers.45.mlp.experts.linear_fc2.weight4 + module.decoder.layers.45.mlp.experts.linear_fc2.weight6 module.decoder.layers.45.mlp.experts.linear_fc2.weight2 module.decoder.layers.45.mlp.experts.linear_fc2.weight1 - module.decoder.layers.45.mlp.experts.linear_fc1.weight14 - module.decoder.layers.45.mlp.experts.linear_fc1.weight12 - module.decoder.layers.45.mlp.experts.linear_fc1.weight10 - module.decoder.layers.45.mlp.experts.linear_fc1.weight8 - module.decoder.layers.45.mlp.experts.linear_fc2.weight7 - module.decoder.layers.45.mlp.experts.linear_fc2.weight5 - module.decoder.layers.45.mlp.experts.linear_fc2.weight3 - module.decoder.layers.45.mlp.experts.linear_fc2.weight0 module.decoder.layers.45.mlp.experts.linear_fc1.weight15 module.decoder.layers.45.mlp.experts.linear_fc1.weight13 module.decoder.layers.45.mlp.experts.linear_fc1.weight11 module.decoder.layers.45.mlp.experts.linear_fc1.weight9 module.decoder.layers.45.mlp.experts.linear_fc1.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight7 + module.decoder.layers.45.mlp.experts.linear_fc2.weight3 + module.decoder.layers.45.mlp.experts.linear_fc2.weight5 + module.decoder.layers.45.mlp.experts.linear_fc2.weight0 + module.decoder.layers.45.mlp.experts.linear_fc1.weight14 + module.decoder.layers.45.mlp.experts.linear_fc1.weight12 + module.decoder.layers.45.mlp.experts.linear_fc1.weight10 + module.decoder.layers.45.mlp.experts.linear_fc1.weight8 Params for bucket 6 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc2.weight9 module.decoder.layers.45.mlp.experts.linear_fc1.weight6 module.decoder.layers.44.mlp.experts.linear_fc2.weight14 module.decoder.layers.44.mlp.experts.linear_fc2.weight6 module.decoder.layers.45.mlp.experts.linear_fc1.weight1 module.decoder.layers.44.mlp.experts.linear_fc2.weight11 - module.decoder.layers.45.mlp.experts.linear_fc1.weight0 module.decoder.layers.44.mlp.experts.linear_fc2.weight8 module.decoder.layers.45.mlp.experts.linear_fc1.weight5 + module.decoder.layers.45.mlp.experts.linear_fc1.weight2 module.decoder.layers.44.mlp.experts.linear_fc2.weight13 module.decoder.layers.44.mlp.experts.linear_fc2.weight5 module.decoder.layers.45.mlp.experts.linear_fc1.weight3 @@ -1117,27 +1118,30 @@ Params for bucket 6 (40894464 elements, 40894464 padded size): module.decoder.layers.44.mlp.experts.linear_fc2.weight15 module.decoder.layers.44.mlp.experts.linear_fc2.weight7 module.decoder.layers.45.mlp.experts.linear_fc1.weight4 + module.decoder.layers.45.mlp.experts.linear_fc1.weight0 module.decoder.layers.44.mlp.experts.linear_fc2.weight12 module.decoder.layers.44.mlp.experts.linear_fc2.weight4 - module.decoder.layers.45.mlp.experts.linear_fc1.weight2 - module.decoder.layers.44.mlp.experts.linear_fc2.weight9 Params for bucket 7 (40894464 elements, 40894464 padded size): - module.decoder.layers.44.mlp.experts.linear_fc1.weight15 - module.decoder.layers.44.mlp.experts.linear_fc2.weight2 - module.decoder.layers.44.mlp.experts.linear_fc2.weight1 - module.decoder.layers.44.mlp.experts.linear_fc1.weight13 - module.decoder.layers.44.mlp.experts.linear_fc1.weight11 - module.decoder.layers.44.mlp.experts.linear_fc1.weight9 - module.decoder.layers.44.mlp.experts.linear_fc1.weight7 - module.decoder.layers.44.mlp.experts.linear_fc1.weight5 module.decoder.layers.44.mlp.experts.linear_fc2.weight3 - module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc2.weight1 + module.decoder.layers.44.mlp.experts.linear_fc2.weight2 module.decoder.layers.44.mlp.experts.linear_fc1.weight14 module.decoder.layers.44.mlp.experts.linear_fc1.weight12 module.decoder.layers.44.mlp.experts.linear_fc1.weight10 module.decoder.layers.44.mlp.experts.linear_fc1.weight8 module.decoder.layers.44.mlp.experts.linear_fc1.weight6 + module.decoder.layers.44.mlp.experts.linear_fc1.weight15 + module.decoder.layers.44.mlp.experts.linear_fc2.weight0 + module.decoder.layers.44.mlp.experts.linear_fc1.weight13 + module.decoder.layers.44.mlp.experts.linear_fc1.weight11 + module.decoder.layers.44.mlp.experts.linear_fc1.weight9 + module.decoder.layers.44.mlp.experts.linear_fc1.weight7 + module.decoder.layers.44.mlp.experts.linear_fc1.weight5 Params for bucket 8 (40894464 elements, 40894464 padded size): + module.decoder.layers.44.mlp.experts.linear_fc1.weight4 + module.decoder.layers.43.mlp.experts.linear_fc2.weight11 + module.decoder.layers.43.mlp.experts.linear_fc2.weight3 + module.decoder.layers.43.mlp.experts.linear_fc2.weight0 module.decoder.layers.44.mlp.experts.linear_fc1.weight2 module.decoder.layers.43.mlp.experts.linear_fc2.weight8 module.decoder.layers.43.mlp.experts.linear_fc2.weight1 @@ -1155,17 +1159,7 @@ Params for bucket 8 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc2.weight9 module.decoder.layers.43.mlp.experts.linear_fc2.weight14 module.decoder.layers.43.mlp.experts.linear_fc2.weight6 - module.decoder.layers.44.mlp.experts.linear_fc1.weight4 - module.decoder.layers.43.mlp.experts.linear_fc2.weight11 - module.decoder.layers.43.mlp.experts.linear_fc2.weight3 - module.decoder.layers.43.mlp.experts.linear_fc2.weight0 Params for bucket 9 (40894464 elements, 40894464 padded size): - module.decoder.layers.43.mlp.experts.linear_fc1.weight15 - module.decoder.layers.43.mlp.experts.linear_fc1.weight13 - module.decoder.layers.43.mlp.experts.linear_fc1.weight11 - module.decoder.layers.43.mlp.experts.linear_fc1.weight9 - module.decoder.layers.43.mlp.experts.linear_fc1.weight7 - module.decoder.layers.43.mlp.experts.linear_fc1.weight5 module.decoder.layers.43.mlp.experts.linear_fc1.weight12 module.decoder.layers.43.mlp.experts.linear_fc1.weight14 module.decoder.layers.43.mlp.experts.linear_fc1.weight10 @@ -1173,31 +1167,37 @@ Params for bucket 9 (40894464 elements, 40894464 padded size): module.decoder.layers.43.mlp.experts.linear_fc1.weight6 module.decoder.layers.43.mlp.experts.linear_fc1.weight4 module.decoder.layers.43.mlp.experts.linear_fc1.weight3 + module.decoder.layers.43.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight13 + module.decoder.layers.43.mlp.experts.linear_fc1.weight11 + module.decoder.layers.43.mlp.experts.linear_fc1.weight9 + module.decoder.layers.43.mlp.experts.linear_fc1.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight5 Params for bucket 10 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc2.weight13 module.decoder.layers.42.mlp.experts.linear_fc2.weight5 + module.decoder.layers.42.mlp.experts.linear_fc2.weight0 + module.decoder.layers.42.mlp.experts.linear_fc1.weight14 module.decoder.layers.42.mlp.experts.linear_fc2.weight10 module.decoder.layers.42.mlp.experts.linear_fc2.weight2 - module.decoder.layers.43.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc2.weight15 module.decoder.layers.42.mlp.experts.linear_fc2.weight7 + module.decoder.layers.43.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc2.weight12 module.decoder.layers.42.mlp.experts.linear_fc2.weight4 - module.decoder.layers.42.mlp.experts.linear_fc1.weight15 - module.decoder.layers.43.mlp.experts.linear_fc1.weight1 module.decoder.layers.42.mlp.experts.linear_fc2.weight9 - module.decoder.layers.43.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight14 + module.decoder.layers.43.mlp.experts.linear_fc1.weight1 module.decoder.layers.42.mlp.experts.linear_fc2.weight6 + module.decoder.layers.42.mlp.experts.linear_fc1.weight15 + module.decoder.layers.43.mlp.experts.linear_fc1.weight0 module.decoder.layers.42.mlp.experts.linear_fc2.weight11 module.decoder.layers.42.mlp.experts.linear_fc2.weight3 - module.decoder.layers.42.mlp.experts.linear_fc2.weight0 - module.decoder.layers.42.mlp.experts.linear_fc1.weight14 module.decoder.layers.42.mlp.experts.linear_fc2.weight8 module.decoder.layers.42.mlp.experts.linear_fc2.weight1 Params for bucket 11 (40894464 elements, 40894464 padded size): - module.decoder.layers.42.mlp.experts.linear_fc1.weight12 module.decoder.layers.42.mlp.experts.linear_fc1.weight10 + module.decoder.layers.42.mlp.experts.linear_fc1.weight12 module.decoder.layers.42.mlp.experts.linear_fc1.weight8 module.decoder.layers.42.mlp.experts.linear_fc1.weight6 module.decoder.layers.42.mlp.experts.linear_fc1.weight4 @@ -1210,14 +1210,12 @@ Params for bucket 11 (40894464 elements, 40894464 padded size): module.decoder.layers.42.mlp.experts.linear_fc1.weight2 module.decoder.layers.42.mlp.experts.linear_fc1.weight1 Params for bucket 12 (40894464 elements, 40894464 padded size): - module.decoder.layers.42.mlp.experts.linear_fc1.weight0 - module.decoder.layers.41.mlp.experts.linear_fc2.weight8 - module.decoder.layers.41.mlp.experts.linear_fc1.weight15 module.decoder.layers.41.mlp.experts.linear_fc2.weight13 module.decoder.layers.41.mlp.experts.linear_fc2.weight5 module.decoder.layers.41.mlp.experts.linear_fc1.weight12 module.decoder.layers.41.mlp.experts.linear_fc2.weight10 module.decoder.layers.41.mlp.experts.linear_fc2.weight1 + module.decoder.layers.42.mlp.experts.linear_fc1.weight0 module.decoder.layers.41.mlp.experts.linear_fc2.weight15 module.decoder.layers.41.mlp.experts.linear_fc2.weight7 module.decoder.layers.41.mlp.experts.linear_fc2.weight2 @@ -1231,68 +1229,62 @@ Params for bucket 12 (40894464 elements, 40894464 padded size): module.decoder.layers.41.mlp.experts.linear_fc1.weight13 module.decoder.layers.41.mlp.experts.linear_fc2.weight11 module.decoder.layers.41.mlp.experts.linear_fc2.weight3 + module.decoder.layers.41.mlp.experts.linear_fc2.weight8 + module.decoder.layers.41.mlp.experts.linear_fc1.weight15 Params for bucket 13 (40894464 elements, 40894464 padded size): - module.decoder.layers.41.mlp.experts.linear_fc1.weight11 - module.decoder.layers.41.mlp.experts.linear_fc1.weight9 - module.decoder.layers.41.mlp.experts.linear_fc1.weight7 - module.decoder.layers.41.mlp.experts.linear_fc1.weight5 - module.decoder.layers.41.mlp.experts.linear_fc1.weight2 - module.decoder.layers.41.mlp.experts.linear_fc1.weight1 - module.decoder.layers.40.mlp.experts.linear_fc2.weight14 module.decoder.layers.41.mlp.experts.linear_fc1.weight10 module.decoder.layers.41.mlp.experts.linear_fc1.weight8 module.decoder.layers.41.mlp.experts.linear_fc1.weight6 module.decoder.layers.41.mlp.experts.linear_fc1.weight4 module.decoder.layers.41.mlp.experts.linear_fc1.weight3 module.decoder.layers.41.mlp.experts.linear_fc1.weight0 + module.decoder.layers.40.mlp.experts.linear_fc2.weight14 + module.decoder.layers.41.mlp.experts.linear_fc1.weight11 + module.decoder.layers.41.mlp.experts.linear_fc1.weight9 + module.decoder.layers.41.mlp.experts.linear_fc1.weight7 + module.decoder.layers.41.mlp.experts.linear_fc1.weight5 + module.decoder.layers.41.mlp.experts.linear_fc1.weight2 + module.decoder.layers.41.mlp.experts.linear_fc1.weight1 module.decoder.layers.40.mlp.experts.linear_fc2.weight15 Params for bucket 14 (40894464 elements, 40894464 padded size): module.decoder.layers.40.mlp.experts.linear_fc2.weight6 module.decoder.layers.40.mlp.experts.linear_fc2.weight11 module.decoder.layers.40.mlp.experts.linear_fc2.weight3 - module.decoder.layers.40.mlp.experts.linear_fc1.weight13 + module.decoder.layers.40.mlp.experts.linear_fc2.weight0 + module.decoder.layers.40.mlp.experts.linear_fc1.weight14 module.decoder.layers.40.mlp.experts.linear_fc2.weight8 module.decoder.layers.40.mlp.experts.linear_fc2.weight1 - module.decoder.layers.40.mlp.experts.linear_fc1.weight10 + module.decoder.layers.40.mlp.experts.linear_fc1.weight11 module.decoder.layers.40.mlp.experts.linear_fc2.weight13 module.decoder.layers.40.mlp.experts.linear_fc2.weight5 - module.decoder.layers.40.mlp.experts.linear_fc1.weight15 module.decoder.layers.40.mlp.experts.linear_fc2.weight10 module.decoder.layers.40.mlp.experts.linear_fc2.weight2 - module.decoder.layers.40.mlp.experts.linear_fc1.weight12 + module.decoder.layers.40.mlp.experts.linear_fc1.weight13 module.decoder.layers.40.mlp.experts.linear_fc2.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight10 module.decoder.layers.40.mlp.experts.linear_fc2.weight12 module.decoder.layers.40.mlp.experts.linear_fc2.weight4 - module.decoder.layers.40.mlp.experts.linear_fc2.weight0 - module.decoder.layers.40.mlp.experts.linear_fc1.weight14 + module.decoder.layers.40.mlp.experts.linear_fc1.weight15 module.decoder.layers.40.mlp.experts.linear_fc2.weight9 - module.decoder.layers.40.mlp.experts.linear_fc1.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight12 Params for bucket 15 (40894464 elements, 40894464 padded size): - module.decoder.layers.40.mlp.experts.linear_fc1.weight8 - module.decoder.layers.40.mlp.experts.linear_fc1.weight6 - module.decoder.layers.40.mlp.experts.linear_fc1.weight4 - module.decoder.layers.40.mlp.experts.linear_fc1.weight3 - module.decoder.layers.40.mlp.experts.linear_fc1.weight0 - module.decoder.layers.39.mlp.experts.linear_fc2.weight14 - module.decoder.layers.39.mlp.experts.linear_fc2.weight12 - module.decoder.layers.39.mlp.experts.linear_fc2.weight10 module.decoder.layers.40.mlp.experts.linear_fc1.weight9 - module.decoder.layers.40.mlp.experts.linear_fc1.weight5 module.decoder.layers.40.mlp.experts.linear_fc1.weight7 + module.decoder.layers.40.mlp.experts.linear_fc1.weight5 module.decoder.layers.40.mlp.experts.linear_fc1.weight2 module.decoder.layers.40.mlp.experts.linear_fc1.weight1 module.decoder.layers.39.mlp.experts.linear_fc2.weight15 module.decoder.layers.39.mlp.experts.linear_fc2.weight13 module.decoder.layers.39.mlp.experts.linear_fc2.weight11 + module.decoder.layers.40.mlp.experts.linear_fc1.weight6 + module.decoder.layers.40.mlp.experts.linear_fc1.weight8 + module.decoder.layers.40.mlp.experts.linear_fc1.weight4 + module.decoder.layers.40.mlp.experts.linear_fc1.weight3 + module.decoder.layers.40.mlp.experts.linear_fc1.weight0 + module.decoder.layers.39.mlp.experts.linear_fc2.weight14 + module.decoder.layers.39.mlp.experts.linear_fc2.weight12 + module.decoder.layers.39.mlp.experts.linear_fc2.weight10 Params for bucket 16 (40894464 elements, 40894464 padded size): - module.decoder.layers.39.mlp.experts.linear_fc2.weight6 - module.decoder.layers.39.mlp.experts.linear_fc2.weight8 - module.decoder.layers.39.mlp.experts.linear_fc2.weight4 - module.decoder.layers.39.mlp.experts.linear_fc2.weight1 - module.decoder.layers.39.mlp.experts.linear_fc1.weight15 - module.decoder.layers.39.mlp.experts.linear_fc1.weight13 - module.decoder.layers.39.mlp.experts.linear_fc1.weight11 - module.decoder.layers.39.mlp.experts.linear_fc1.weight9 module.decoder.layers.39.mlp.experts.linear_fc2.weight9 module.decoder.layers.39.mlp.experts.linear_fc2.weight7 module.decoder.layers.39.mlp.experts.linear_fc2.weight5 @@ -1303,7 +1295,24 @@ Params for bucket 16 (40894464 elements, 40894464 padded size): module.decoder.layers.39.mlp.experts.linear_fc1.weight12 module.decoder.layers.39.mlp.experts.linear_fc1.weight10 module.decoder.layers.39.mlp.experts.linear_fc1.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight8 + module.decoder.layers.39.mlp.experts.linear_fc2.weight6 + module.decoder.layers.39.mlp.experts.linear_fc2.weight4 + module.decoder.layers.39.mlp.experts.linear_fc2.weight1 + module.decoder.layers.39.mlp.experts.linear_fc1.weight15 + module.decoder.layers.39.mlp.experts.linear_fc1.weight13 + module.decoder.layers.39.mlp.experts.linear_fc1.weight11 + module.decoder.layers.39.mlp.experts.linear_fc1.weight9 Params for bucket 17 (40894464 elements, 40894464 padded size): + module.decoder.layers.39.mlp.experts.linear_fc1.weight3 + module.decoder.layers.39.mlp.experts.linear_fc1.weight6 + module.decoder.layers.39.mlp.experts.linear_fc1.weight4 + module.decoder.layers.39.mlp.experts.linear_fc1.weight0 + module.decoder.layers.38.mlp.experts.linear_fc2.weight15 + module.decoder.layers.38.mlp.experts.linear_fc2.weight13 + module.decoder.layers.38.mlp.experts.linear_fc2.weight11 + module.decoder.layers.38.mlp.experts.linear_fc2.weight9 + module.decoder.layers.38.mlp.experts.linear_fc2.weight7 module.decoder.layers.39.mlp.experts.linear_fc1.weight7 module.decoder.layers.39.mlp.experts.linear_fc1.weight5 module.decoder.layers.39.mlp.experts.linear_fc1.weight2 @@ -1313,26 +1322,17 @@ Params for bucket 17 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc2.weight10 module.decoder.layers.38.mlp.experts.linear_fc2.weight8 module.decoder.layers.38.mlp.experts.linear_fc2.weight6 - module.decoder.layers.39.mlp.experts.linear_fc1.weight3 - module.decoder.layers.39.mlp.experts.linear_fc1.weight4 - module.decoder.layers.39.mlp.experts.linear_fc1.weight6 - module.decoder.layers.39.mlp.experts.linear_fc1.weight0 - module.decoder.layers.38.mlp.experts.linear_fc2.weight15 - module.decoder.layers.38.mlp.experts.linear_fc2.weight13 - module.decoder.layers.38.mlp.experts.linear_fc2.weight11 - module.decoder.layers.38.mlp.experts.linear_fc2.weight9 - module.decoder.layers.38.mlp.experts.linear_fc2.weight7 Params for bucket 18 (40894464 elements, 40894464 padded size): - module.decoder.layers.38.mlp.experts.linear_fc2.weight4 - module.decoder.layers.38.mlp.experts.linear_fc2.weight1 - module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight5 + module.decoder.layers.38.mlp.experts.linear_fc2.weight3 module.decoder.layers.38.mlp.experts.linear_fc1.weight15 module.decoder.layers.38.mlp.experts.linear_fc1.weight13 module.decoder.layers.38.mlp.experts.linear_fc1.weight11 module.decoder.layers.38.mlp.experts.linear_fc1.weight9 module.decoder.layers.38.mlp.experts.linear_fc1.weight7 - module.decoder.layers.38.mlp.experts.linear_fc2.weight5 - module.decoder.layers.38.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc2.weight4 + module.decoder.layers.38.mlp.experts.linear_fc2.weight2 + module.decoder.layers.38.mlp.experts.linear_fc2.weight1 module.decoder.layers.38.mlp.experts.linear_fc2.weight0 module.decoder.layers.38.mlp.experts.linear_fc1.weight14 module.decoder.layers.38.mlp.experts.linear_fc1.weight12 @@ -1340,10 +1340,6 @@ Params for bucket 18 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc1.weight8 module.decoder.layers.38.mlp.experts.linear_fc1.weight6 Params for bucket 19 (40894464 elements, 40894464 padded size): - module.decoder.layers.38.mlp.experts.linear_fc1.weight2 - module.decoder.layers.37.mlp.experts.linear_fc2.weight8 - module.decoder.layers.37.mlp.experts.linear_fc2.weight13 - module.decoder.layers.37.mlp.experts.linear_fc2.weight5 module.decoder.layers.38.mlp.experts.linear_fc1.weight1 module.decoder.layers.37.mlp.experts.linear_fc2.weight10 module.decoder.layers.37.mlp.experts.linear_fc2.weight2 @@ -1360,10 +1356,14 @@ Params for bucket 19 (40894464 elements, 40894464 padded size): module.decoder.layers.38.mlp.experts.linear_fc1.weight4 module.decoder.layers.37.mlp.experts.linear_fc2.weight11 module.decoder.layers.37.mlp.experts.linear_fc2.weight3 + module.decoder.layers.38.mlp.experts.linear_fc1.weight2 + module.decoder.layers.37.mlp.experts.linear_fc2.weight8 + module.decoder.layers.37.mlp.experts.linear_fc2.weight13 + module.decoder.layers.37.mlp.experts.linear_fc2.weight5 Params for bucket 20 (40894464 elements, 40894464 padded size): module.decoder.layers.37.mlp.experts.linear_fc2.weight1 - module.decoder.layers.37.mlp.experts.linear_fc1.weight15 module.decoder.layers.37.mlp.experts.linear_fc1.weight13 + module.decoder.layers.37.mlp.experts.linear_fc1.weight15 module.decoder.layers.37.mlp.experts.linear_fc1.weight11 module.decoder.layers.37.mlp.experts.linear_fc1.weight9 module.decoder.layers.37.mlp.experts.linear_fc1.weight7 @@ -1413,33 +1413,27 @@ Params for bucket 22 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc1.weight2 Params for bucket 23 (40894464 elements, 40894464 padded size): module.decoder.layers.36.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc2.weight12 + module.decoder.layers.35.mlp.experts.linear_fc2.weight4 + module.decoder.layers.35.mlp.experts.linear_fc2.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc2.weight9 module.decoder.layers.35.mlp.experts.linear_fc2.weight14 module.decoder.layers.35.mlp.experts.linear_fc2.weight6 module.decoder.layers.35.mlp.experts.linear_fc2.weight11 module.decoder.layers.35.mlp.experts.linear_fc2.weight3 - module.decoder.layers.35.mlp.experts.linear_fc2.weight0 - module.decoder.layers.35.mlp.experts.linear_fc1.weight14 + module.decoder.layers.35.mlp.experts.linear_fc1.weight13 module.decoder.layers.35.mlp.experts.linear_fc2.weight8 module.decoder.layers.35.mlp.experts.linear_fc2.weight1 module.decoder.layers.35.mlp.experts.linear_fc2.weight13 module.decoder.layers.35.mlp.experts.linear_fc2.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight15 module.decoder.layers.35.mlp.experts.linear_fc2.weight10 module.decoder.layers.35.mlp.experts.linear_fc2.weight2 - module.decoder.layers.35.mlp.experts.linear_fc1.weight13 + module.decoder.layers.36.mlp.experts.linear_fc1.weight1 module.decoder.layers.35.mlp.experts.linear_fc2.weight15 module.decoder.layers.35.mlp.experts.linear_fc2.weight7 - module.decoder.layers.35.mlp.experts.linear_fc2.weight12 - module.decoder.layers.35.mlp.experts.linear_fc2.weight4 - module.decoder.layers.35.mlp.experts.linear_fc1.weight15 - module.decoder.layers.36.mlp.experts.linear_fc1.weight1 - module.decoder.layers.35.mlp.experts.linear_fc2.weight9 Params for bucket 24 (40894464 elements, 40894464 padded size): - module.decoder.layers.35.mlp.experts.linear_fc1.weight9 - module.decoder.layers.35.mlp.experts.linear_fc1.weight11 - module.decoder.layers.35.mlp.experts.linear_fc1.weight7 - module.decoder.layers.35.mlp.experts.linear_fc1.weight5 - module.decoder.layers.35.mlp.experts.linear_fc1.weight2 - module.decoder.layers.35.mlp.experts.linear_fc1.weight1 module.decoder.layers.35.mlp.experts.linear_fc1.weight12 module.decoder.layers.35.mlp.experts.linear_fc1.weight10 module.decoder.layers.35.mlp.experts.linear_fc1.weight8 @@ -1447,7 +1441,22 @@ Params for bucket 24 (40894464 elements, 40894464 padded size): module.decoder.layers.35.mlp.experts.linear_fc1.weight4 module.decoder.layers.35.mlp.experts.linear_fc1.weight3 module.decoder.layers.35.mlp.experts.linear_fc1.weight0 + module.decoder.layers.35.mlp.experts.linear_fc1.weight11 + module.decoder.layers.35.mlp.experts.linear_fc1.weight9 + module.decoder.layers.35.mlp.experts.linear_fc1.weight7 + module.decoder.layers.35.mlp.experts.linear_fc1.weight5 + module.decoder.layers.35.mlp.experts.linear_fc1.weight2 + module.decoder.layers.35.mlp.experts.linear_fc1.weight1 Params for bucket 25 (40894464 elements, 40894464 padded size): + module.decoder.layers.34.mlp.experts.linear_fc2.weight11 + module.decoder.layers.34.mlp.experts.linear_fc2.weight3 + module.decoder.layers.34.mlp.experts.linear_fc2.weight8 + module.decoder.layers.34.mlp.experts.linear_fc1.weight15 + module.decoder.layers.34.mlp.experts.linear_fc2.weight13 + module.decoder.layers.34.mlp.experts.linear_fc2.weight5 + module.decoder.layers.34.mlp.experts.linear_fc1.weight12 + module.decoder.layers.34.mlp.experts.linear_fc2.weight10 + module.decoder.layers.34.mlp.experts.linear_fc2.weight1 module.decoder.layers.34.mlp.experts.linear_fc2.weight15 module.decoder.layers.34.mlp.experts.linear_fc2.weight7 module.decoder.layers.34.mlp.experts.linear_fc2.weight2 @@ -1460,15 +1469,6 @@ Params for bucket 25 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc2.weight14 module.decoder.layers.34.mlp.experts.linear_fc2.weight6 module.decoder.layers.34.mlp.experts.linear_fc1.weight13 - module.decoder.layers.34.mlp.experts.linear_fc2.weight11 - module.decoder.layers.34.mlp.experts.linear_fc2.weight3 - module.decoder.layers.34.mlp.experts.linear_fc2.weight8 - module.decoder.layers.34.mlp.experts.linear_fc1.weight15 - module.decoder.layers.34.mlp.experts.linear_fc2.weight13 - module.decoder.layers.34.mlp.experts.linear_fc2.weight5 - module.decoder.layers.34.mlp.experts.linear_fc1.weight12 - module.decoder.layers.34.mlp.experts.linear_fc2.weight10 - module.decoder.layers.34.mlp.experts.linear_fc2.weight1 Params for bucket 26 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc1.weight10 module.decoder.layers.34.mlp.experts.linear_fc1.weight8 @@ -1476,16 +1476,21 @@ Params for bucket 26 (40894464 elements, 40894464 padded size): module.decoder.layers.34.mlp.experts.linear_fc1.weight4 module.decoder.layers.34.mlp.experts.linear_fc1.weight3 module.decoder.layers.34.mlp.experts.linear_fc1.weight0 - module.decoder.layers.33.mlp.experts.linear_fc2.weight14 - module.decoder.layers.33.mlp.experts.linear_fc2.weight12 - module.decoder.layers.34.mlp.experts.linear_fc1.weight9 + module.decoder.layers.33.mlp.experts.linear_fc2.weight15 + module.decoder.layers.33.mlp.experts.linear_fc2.weight13 module.decoder.layers.34.mlp.experts.linear_fc1.weight7 + module.decoder.layers.34.mlp.experts.linear_fc1.weight9 module.decoder.layers.34.mlp.experts.linear_fc1.weight5 module.decoder.layers.34.mlp.experts.linear_fc1.weight2 module.decoder.layers.34.mlp.experts.linear_fc1.weight1 - module.decoder.layers.33.mlp.experts.linear_fc2.weight15 - module.decoder.layers.33.mlp.experts.linear_fc2.weight13 + module.decoder.layers.33.mlp.experts.linear_fc2.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight12 Params for bucket 27 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc2.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight14 + module.decoder.layers.33.mlp.experts.linear_fc2.weight4 + module.decoder.layers.33.mlp.experts.linear_fc1.weight11 + module.decoder.layers.33.mlp.experts.linear_fc2.weight9 module.decoder.layers.33.mlp.experts.linear_fc2.weight6 module.decoder.layers.33.mlp.experts.linear_fc1.weight13 module.decoder.layers.33.mlp.experts.linear_fc2.weight11 @@ -1500,33 +1505,28 @@ Params for bucket 27 (40894464 elements, 40894464 padded size): module.decoder.layers.33.mlp.experts.linear_fc2.weight10 module.decoder.layers.33.mlp.experts.linear_fc2.weight2 module.decoder.layers.33.mlp.experts.linear_fc1.weight9 - module.decoder.layers.33.mlp.experts.linear_fc2.weight7 - module.decoder.layers.33.mlp.experts.linear_fc1.weight14 - module.decoder.layers.33.mlp.experts.linear_fc2.weight4 - module.decoder.layers.33.mlp.experts.linear_fc1.weight11 - module.decoder.layers.33.mlp.experts.linear_fc2.weight9 Params for bucket 28 (40894464 elements, 40894464 padded size): + module.decoder.layers.33.mlp.experts.linear_fc1.weight8 module.decoder.layers.33.mlp.experts.linear_fc1.weight4 - module.decoder.layers.33.mlp.experts.linear_fc1.weight7 module.decoder.layers.33.mlp.experts.linear_fc1.weight6 - module.decoder.layers.33.mlp.experts.linear_fc1.weight2 - module.decoder.layers.33.mlp.experts.linear_fc1.weight1 + module.decoder.layers.33.mlp.experts.linear_fc1.weight3 + module.decoder.layers.33.mlp.experts.linear_fc1.weight0 module.decoder.layers.32.mlp.experts.linear_fc2.weight14 module.decoder.layers.32.mlp.experts.linear_fc2.weight12 module.decoder.layers.32.mlp.experts.linear_fc2.weight10 module.decoder.layers.32.mlp.experts.linear_fc2.weight8 - module.decoder.layers.33.mlp.experts.linear_fc1.weight8 module.decoder.layers.33.mlp.experts.linear_fc1.weight5 - module.decoder.layers.33.mlp.experts.linear_fc1.weight3 - module.decoder.layers.33.mlp.experts.linear_fc1.weight0 + module.decoder.layers.33.mlp.experts.linear_fc1.weight7 + module.decoder.layers.33.mlp.experts.linear_fc1.weight2 + module.decoder.layers.33.mlp.experts.linear_fc1.weight1 module.decoder.layers.32.mlp.experts.linear_fc2.weight15 module.decoder.layers.32.mlp.experts.linear_fc2.weight13 module.decoder.layers.32.mlp.experts.linear_fc2.weight11 module.decoder.layers.32.mlp.experts.linear_fc2.weight9 Params for bucket 29 (40894464 elements, 40894464 padded size): - module.decoder.layers.32.mlp.experts.linear_fc2.weight3 + module.decoder.layers.32.mlp.experts.linear_fc2.weight4 module.decoder.layers.32.mlp.experts.linear_fc2.weight6 - module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight2 module.decoder.layers.32.mlp.experts.linear_fc2.weight1 module.decoder.layers.32.mlp.experts.linear_fc1.weight15 module.decoder.layers.32.mlp.experts.linear_fc1.weight13 @@ -1534,71 +1534,71 @@ Params for bucket 29 (40894464 elements, 40894464 padded size): module.decoder.layers.32.mlp.experts.linear_fc1.weight9 module.decoder.layers.32.mlp.experts.linear_fc1.weight7 module.decoder.layers.32.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc2.weight4 - module.decoder.layers.32.mlp.experts.linear_fc2.weight2 + module.decoder.layers.32.mlp.experts.linear_fc2.weight5 + module.decoder.layers.32.mlp.experts.linear_fc2.weight3 module.decoder.layers.32.mlp.experts.linear_fc2.weight0 module.decoder.layers.32.mlp.experts.linear_fc1.weight14 module.decoder.layers.32.mlp.experts.linear_fc1.weight12 module.decoder.layers.32.mlp.experts.linear_fc1.weight10 module.decoder.layers.32.mlp.experts.linear_fc1.weight8 Params for bucket 30 (40894464 elements, 40894464 padded size): + module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight8 + module.decoder.layers.32.mlp.experts.linear_fc1.weight6 module.decoder.layers.31.mlp.experts.linear_fc2.weight13 module.decoder.layers.31.mlp.experts.linear_fc2.weight5 - module.decoder.layers.32.mlp.experts.linear_fc1.weight4 + module.decoder.layers.32.mlp.experts.linear_fc1.weight1 module.decoder.layers.31.mlp.experts.linear_fc2.weight10 - module.decoder.layers.32.mlp.experts.linear_fc1.weight2 + module.decoder.layers.32.mlp.experts.linear_fc1.weight0 module.decoder.layers.31.mlp.experts.linear_fc2.weight15 module.decoder.layers.31.mlp.experts.linear_fc2.weight7 - module.decoder.layers.32.mlp.experts.linear_fc1.weight6 + module.decoder.layers.32.mlp.experts.linear_fc1.weight5 module.decoder.layers.31.mlp.experts.linear_fc2.weight12 module.decoder.layers.31.mlp.experts.linear_fc2.weight4 - module.decoder.layers.32.mlp.experts.linear_fc1.weight1 + module.decoder.layers.32.mlp.experts.linear_fc1.weight3 module.decoder.layers.31.mlp.experts.linear_fc2.weight9 - module.decoder.layers.32.mlp.experts.linear_fc1.weight0 module.decoder.layers.31.mlp.experts.linear_fc2.weight14 module.decoder.layers.31.mlp.experts.linear_fc2.weight6 - module.decoder.layers.32.mlp.experts.linear_fc1.weight5 + module.decoder.layers.32.mlp.experts.linear_fc1.weight4 module.decoder.layers.31.mlp.experts.linear_fc2.weight11 - module.decoder.layers.32.mlp.experts.linear_fc1.weight3 - module.decoder.layers.31.mlp.experts.linear_fc2.weight8 Params for bucket 31 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc2.weight3 - module.decoder.layers.31.mlp.experts.linear_fc2.weight0 - module.decoder.layers.31.mlp.experts.linear_fc1.weight14 - module.decoder.layers.31.mlp.experts.linear_fc1.weight12 - module.decoder.layers.31.mlp.experts.linear_fc1.weight10 - module.decoder.layers.31.mlp.experts.linear_fc1.weight8 - module.decoder.layers.31.mlp.experts.linear_fc1.weight6 - module.decoder.layers.31.mlp.experts.linear_fc2.weight1 module.decoder.layers.31.mlp.experts.linear_fc1.weight15 module.decoder.layers.31.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc2.weight1 module.decoder.layers.31.mlp.experts.linear_fc1.weight13 module.decoder.layers.31.mlp.experts.linear_fc1.weight11 module.decoder.layers.31.mlp.experts.linear_fc1.weight9 module.decoder.layers.31.mlp.experts.linear_fc1.weight7 module.decoder.layers.31.mlp.experts.linear_fc1.weight5 + module.decoder.layers.31.mlp.experts.linear_fc2.weight3 + module.decoder.layers.31.mlp.experts.linear_fc2.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight14 + module.decoder.layers.31.mlp.experts.linear_fc1.weight12 + module.decoder.layers.31.mlp.experts.linear_fc1.weight10 + module.decoder.layers.31.mlp.experts.linear_fc1.weight8 + module.decoder.layers.31.mlp.experts.linear_fc1.weight6 Params for bucket 32 (40894464 elements, 40894464 padded size): - module.decoder.layers.31.mlp.experts.linear_fc1.weight3 - module.decoder.layers.30.mlp.experts.linear_fc2.weight10 - module.decoder.layers.30.mlp.experts.linear_fc2.weight2 - module.decoder.layers.30.mlp.experts.linear_fc2.weight15 - module.decoder.layers.30.mlp.experts.linear_fc2.weight7 - module.decoder.layers.31.mlp.experts.linear_fc1.weight4 - module.decoder.layers.30.mlp.experts.linear_fc2.weight12 - module.decoder.layers.30.mlp.experts.linear_fc2.weight4 - module.decoder.layers.31.mlp.experts.linear_fc1.weight2 - module.decoder.layers.30.mlp.experts.linear_fc2.weight9 module.decoder.layers.30.mlp.experts.linear_fc2.weight14 module.decoder.layers.30.mlp.experts.linear_fc2.weight6 - module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.31.mlp.experts.linear_fc1.weight4 module.decoder.layers.30.mlp.experts.linear_fc2.weight11 module.decoder.layers.30.mlp.experts.linear_fc2.weight3 module.decoder.layers.30.mlp.experts.linear_fc2.weight0 - module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.31.mlp.experts.linear_fc1.weight2 module.decoder.layers.30.mlp.experts.linear_fc2.weight8 module.decoder.layers.30.mlp.experts.linear_fc2.weight1 module.decoder.layers.30.mlp.experts.linear_fc2.weight13 module.decoder.layers.30.mlp.experts.linear_fc2.weight5 + module.decoder.layers.31.mlp.experts.linear_fc1.weight1 + module.decoder.layers.30.mlp.experts.linear_fc2.weight10 + module.decoder.layers.30.mlp.experts.linear_fc2.weight2 + module.decoder.layers.31.mlp.experts.linear_fc1.weight0 + module.decoder.layers.30.mlp.experts.linear_fc2.weight15 + module.decoder.layers.30.mlp.experts.linear_fc2.weight7 + module.decoder.layers.30.mlp.experts.linear_fc2.weight12 + module.decoder.layers.30.mlp.experts.linear_fc2.weight4 + module.decoder.layers.31.mlp.experts.linear_fc1.weight3 + module.decoder.layers.30.mlp.experts.linear_fc2.weight9 Params for bucket 33 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc1.weight15 module.decoder.layers.30.mlp.experts.linear_fc1.weight13 @@ -1606,35 +1606,35 @@ Params for bucket 33 (40894464 elements, 40894464 padded size): module.decoder.layers.30.mlp.experts.linear_fc1.weight9 module.decoder.layers.30.mlp.experts.linear_fc1.weight7 module.decoder.layers.30.mlp.experts.linear_fc1.weight5 - module.decoder.layers.30.mlp.experts.linear_fc1.weight14 module.decoder.layers.30.mlp.experts.linear_fc1.weight12 + module.decoder.layers.30.mlp.experts.linear_fc1.weight14 module.decoder.layers.30.mlp.experts.linear_fc1.weight10 module.decoder.layers.30.mlp.experts.linear_fc1.weight8 module.decoder.layers.30.mlp.experts.linear_fc1.weight6 module.decoder.layers.30.mlp.experts.linear_fc1.weight4 module.decoder.layers.30.mlp.experts.linear_fc1.weight3 Params for bucket 34 (40894464 elements, 40894464 padded size): - module.decoder.layers.29.mlp.experts.linear_fc2.weight12 - module.decoder.layers.29.mlp.experts.linear_fc2.weight4 - module.decoder.layers.29.mlp.experts.linear_fc2.weight9 - module.decoder.layers.29.mlp.experts.linear_fc2.weight0 + module.decoder.layers.30.mlp.experts.linear_fc1.weight2 module.decoder.layers.29.mlp.experts.linear_fc2.weight14 module.decoder.layers.29.mlp.experts.linear_fc2.weight6 module.decoder.layers.29.mlp.experts.linear_fc2.weight11 module.decoder.layers.29.mlp.experts.linear_fc2.weight3 - module.decoder.layers.30.mlp.experts.linear_fc1.weight2 + module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.29.mlp.experts.linear_fc2.weight8 module.decoder.layers.29.mlp.experts.linear_fc1.weight15 + module.decoder.layers.30.mlp.experts.linear_fc1.weight0 module.decoder.layers.29.mlp.experts.linear_fc2.weight13 module.decoder.layers.29.mlp.experts.linear_fc2.weight5 - module.decoder.layers.30.mlp.experts.linear_fc1.weight1 module.decoder.layers.29.mlp.experts.linear_fc2.weight10 module.decoder.layers.29.mlp.experts.linear_fc2.weight1 - module.decoder.layers.30.mlp.experts.linear_fc1.weight0 module.decoder.layers.29.mlp.experts.linear_fc2.weight15 module.decoder.layers.29.mlp.experts.linear_fc2.weight7 module.decoder.layers.29.mlp.experts.linear_fc2.weight2 module.decoder.layers.29.mlp.experts.linear_fc1.weight14 + module.decoder.layers.29.mlp.experts.linear_fc2.weight12 + module.decoder.layers.29.mlp.experts.linear_fc2.weight4 + module.decoder.layers.29.mlp.experts.linear_fc2.weight9 + module.decoder.layers.29.mlp.experts.linear_fc2.weight0 Params for bucket 35 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc1.weight13 module.decoder.layers.29.mlp.experts.linear_fc1.weight11 @@ -1650,9 +1650,12 @@ Params for bucket 35 (40894464 elements, 40894464 padded size): module.decoder.layers.29.mlp.experts.linear_fc1.weight4 module.decoder.layers.29.mlp.experts.linear_fc1.weight3 Params for bucket 36 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc2.weight11 + module.decoder.layers.28.mlp.experts.linear_fc2.weight3 + module.decoder.layers.28.mlp.experts.linear_fc2.weight0 + module.decoder.layers.28.mlp.experts.linear_fc1.weight14 module.decoder.layers.28.mlp.experts.linear_fc2.weight8 module.decoder.layers.28.mlp.experts.linear_fc2.weight1 - module.decoder.layers.29.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight13 module.decoder.layers.28.mlp.experts.linear_fc2.weight5 module.decoder.layers.28.mlp.experts.linear_fc2.weight10 @@ -1665,13 +1668,16 @@ Params for bucket 36 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc1.weight15 module.decoder.layers.28.mlp.experts.linear_fc2.weight9 module.decoder.layers.28.mlp.experts.linear_fc1.weight12 + module.decoder.layers.29.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc2.weight14 module.decoder.layers.28.mlp.experts.linear_fc2.weight6 - module.decoder.layers.28.mlp.experts.linear_fc2.weight11 - module.decoder.layers.28.mlp.experts.linear_fc2.weight3 - module.decoder.layers.28.mlp.experts.linear_fc2.weight0 - module.decoder.layers.28.mlp.experts.linear_fc1.weight14 Params for bucket 37 (40894464 elements, 40894464 padded size): + module.decoder.layers.28.mlp.experts.linear_fc1.weight10 + module.decoder.layers.28.mlp.experts.linear_fc1.weight8 + module.decoder.layers.28.mlp.experts.linear_fc1.weight6 + module.decoder.layers.28.mlp.experts.linear_fc1.weight4 + module.decoder.layers.28.mlp.experts.linear_fc1.weight3 + module.decoder.layers.28.mlp.experts.linear_fc1.weight0 module.decoder.layers.28.mlp.experts.linear_fc1.weight11 module.decoder.layers.28.mlp.experts.linear_fc1.weight9 module.decoder.layers.28.mlp.experts.linear_fc1.weight7 @@ -1679,80 +1685,65 @@ Params for bucket 37 (40894464 elements, 40894464 padded size): module.decoder.layers.28.mlp.experts.linear_fc1.weight2 module.decoder.layers.28.mlp.experts.linear_fc1.weight1 module.decoder.layers.27.mlp.experts.linear_fc2.weight15 - module.decoder.layers.28.mlp.experts.linear_fc1.weight8 - module.decoder.layers.28.mlp.experts.linear_fc1.weight10 - module.decoder.layers.28.mlp.experts.linear_fc1.weight6 - module.decoder.layers.28.mlp.experts.linear_fc1.weight4 - module.decoder.layers.28.mlp.experts.linear_fc1.weight3 - module.decoder.layers.28.mlp.experts.linear_fc1.weight0 module.decoder.layers.27.mlp.experts.linear_fc2.weight14 Params for bucket 38 (40894464 elements, 40894464 padded size): module.decoder.layers.27.mlp.experts.linear_fc2.weight11 module.decoder.layers.27.mlp.experts.linear_fc2.weight3 - module.decoder.layers.27.mlp.experts.linear_fc1.weight10 module.decoder.layers.27.mlp.experts.linear_fc2.weight8 - module.decoder.layers.27.mlp.experts.linear_fc1.weight15 + module.decoder.layers.27.mlp.experts.linear_fc2.weight2 + module.decoder.layers.27.mlp.experts.linear_fc1.weight14 module.decoder.layers.27.mlp.experts.linear_fc2.weight13 module.decoder.layers.27.mlp.experts.linear_fc2.weight5 - module.decoder.layers.27.mlp.experts.linear_fc1.weight12 + module.decoder.layers.27.mlp.experts.linear_fc1.weight11 module.decoder.layers.27.mlp.experts.linear_fc2.weight10 module.decoder.layers.27.mlp.experts.linear_fc2.weight1 module.decoder.layers.27.mlp.experts.linear_fc2.weight7 - module.decoder.layers.27.mlp.experts.linear_fc2.weight2 - module.decoder.layers.27.mlp.experts.linear_fc1.weight14 + module.decoder.layers.27.mlp.experts.linear_fc1.weight13 module.decoder.layers.27.mlp.experts.linear_fc2.weight12 module.decoder.layers.27.mlp.experts.linear_fc2.weight4 - module.decoder.layers.27.mlp.experts.linear_fc1.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight10 module.decoder.layers.27.mlp.experts.linear_fc2.weight9 module.decoder.layers.27.mlp.experts.linear_fc2.weight0 + module.decoder.layers.27.mlp.experts.linear_fc1.weight15 module.decoder.layers.27.mlp.experts.linear_fc2.weight6 - module.decoder.layers.27.mlp.experts.linear_fc1.weight13 + module.decoder.layers.27.mlp.experts.linear_fc1.weight12 Params for bucket 39 (40894464 elements, 40894464 padded size): - module.decoder.layers.27.mlp.experts.linear_fc1.weight8 - module.decoder.layers.27.mlp.experts.linear_fc1.weight6 - module.decoder.layers.27.mlp.experts.linear_fc1.weight4 - module.decoder.layers.27.mlp.experts.linear_fc1.weight3 - module.decoder.layers.27.mlp.experts.linear_fc1.weight0 - module.decoder.layers.26.mlp.experts.linear_fc2.weight15 - module.decoder.layers.26.mlp.experts.linear_fc2.weight13 - module.decoder.layers.26.mlp.experts.linear_fc2.weight11 module.decoder.layers.27.mlp.experts.linear_fc1.weight9 module.decoder.layers.27.mlp.experts.linear_fc1.weight7 module.decoder.layers.27.mlp.experts.linear_fc1.weight5 module.decoder.layers.27.mlp.experts.linear_fc1.weight2 module.decoder.layers.27.mlp.experts.linear_fc1.weight1 - module.decoder.layers.26.mlp.experts.linear_fc2.weight14 - module.decoder.layers.26.mlp.experts.linear_fc2.weight12 - module.decoder.layers.26.mlp.experts.linear_fc2.weight10 -Params for bucket 40 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc2.weight9 + module.decoder.layers.26.mlp.experts.linear_fc2.weight15 + module.decoder.layers.26.mlp.experts.linear_fc2.weight13 + module.decoder.layers.26.mlp.experts.linear_fc2.weight11 + module.decoder.layers.27.mlp.experts.linear_fc1.weight6 + module.decoder.layers.27.mlp.experts.linear_fc1.weight8 + module.decoder.layers.27.mlp.experts.linear_fc1.weight4 + module.decoder.layers.27.mlp.experts.linear_fc1.weight3 + module.decoder.layers.27.mlp.experts.linear_fc1.weight0 + module.decoder.layers.26.mlp.experts.linear_fc2.weight14 + module.decoder.layers.26.mlp.experts.linear_fc2.weight12 + module.decoder.layers.26.mlp.experts.linear_fc2.weight10 +Params for bucket 40 (40894464 elements, 40894464 padded size): + module.decoder.layers.26.mlp.experts.linear_fc2.weight9 module.decoder.layers.26.mlp.experts.linear_fc2.weight7 module.decoder.layers.26.mlp.experts.linear_fc2.weight5 module.decoder.layers.26.mlp.experts.linear_fc2.weight3 - module.decoder.layers.26.mlp.experts.linear_fc1.weight15 - module.decoder.layers.26.mlp.experts.linear_fc1.weight13 - module.decoder.layers.26.mlp.experts.linear_fc1.weight11 - module.decoder.layers.26.mlp.experts.linear_fc1.weight9 - module.decoder.layers.26.mlp.experts.linear_fc2.weight8 - module.decoder.layers.26.mlp.experts.linear_fc2.weight6 - module.decoder.layers.26.mlp.experts.linear_fc2.weight4 - module.decoder.layers.26.mlp.experts.linear_fc2.weight2 - module.decoder.layers.26.mlp.experts.linear_fc2.weight1 module.decoder.layers.26.mlp.experts.linear_fc2.weight0 module.decoder.layers.26.mlp.experts.linear_fc1.weight14 module.decoder.layers.26.mlp.experts.linear_fc1.weight12 module.decoder.layers.26.mlp.experts.linear_fc1.weight10 module.decoder.layers.26.mlp.experts.linear_fc1.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight8 + module.decoder.layers.26.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc2.weight4 + module.decoder.layers.26.mlp.experts.linear_fc2.weight2 + module.decoder.layers.26.mlp.experts.linear_fc2.weight1 + module.decoder.layers.26.mlp.experts.linear_fc1.weight15 + module.decoder.layers.26.mlp.experts.linear_fc1.weight13 + module.decoder.layers.26.mlp.experts.linear_fc1.weight11 + module.decoder.layers.26.mlp.experts.linear_fc1.weight9 Params for bucket 41 (40894464 elements, 40894464 padded size): - module.decoder.layers.26.mlp.experts.linear_fc1.weight7 - module.decoder.layers.26.mlp.experts.linear_fc1.weight5 - module.decoder.layers.26.mlp.experts.linear_fc1.weight2 - module.decoder.layers.26.mlp.experts.linear_fc1.weight1 - module.decoder.layers.25.mlp.experts.linear_fc2.weight15 - module.decoder.layers.25.mlp.experts.linear_fc2.weight13 - module.decoder.layers.25.mlp.experts.linear_fc2.weight11 - module.decoder.layers.25.mlp.experts.linear_fc2.weight9 - module.decoder.layers.25.mlp.experts.linear_fc2.weight7 module.decoder.layers.26.mlp.experts.linear_fc1.weight4 module.decoder.layers.26.mlp.experts.linear_fc1.weight3 module.decoder.layers.26.mlp.experts.linear_fc1.weight6 @@ -1762,7 +1753,24 @@ Params for bucket 41 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc2.weight10 module.decoder.layers.25.mlp.experts.linear_fc2.weight8 module.decoder.layers.25.mlp.experts.linear_fc2.weight6 + module.decoder.layers.26.mlp.experts.linear_fc1.weight7 + module.decoder.layers.26.mlp.experts.linear_fc1.weight5 + module.decoder.layers.26.mlp.experts.linear_fc1.weight2 + module.decoder.layers.26.mlp.experts.linear_fc1.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight15 + module.decoder.layers.25.mlp.experts.linear_fc2.weight13 + module.decoder.layers.25.mlp.experts.linear_fc2.weight11 + module.decoder.layers.25.mlp.experts.linear_fc2.weight9 + module.decoder.layers.25.mlp.experts.linear_fc2.weight7 Params for bucket 42 (40894464 elements, 40894464 padded size): + module.decoder.layers.25.mlp.experts.linear_fc2.weight1 + module.decoder.layers.25.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc2.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight15 + module.decoder.layers.25.mlp.experts.linear_fc1.weight13 + module.decoder.layers.25.mlp.experts.linear_fc1.weight11 + module.decoder.layers.25.mlp.experts.linear_fc1.weight9 + module.decoder.layers.25.mlp.experts.linear_fc1.weight7 module.decoder.layers.25.mlp.experts.linear_fc2.weight5 module.decoder.layers.25.mlp.experts.linear_fc2.weight3 module.decoder.layers.25.mlp.experts.linear_fc2.weight0 @@ -1771,43 +1779,28 @@ Params for bucket 42 (40894464 elements, 40894464 padded size): module.decoder.layers.25.mlp.experts.linear_fc1.weight10 module.decoder.layers.25.mlp.experts.linear_fc1.weight8 module.decoder.layers.25.mlp.experts.linear_fc1.weight6 - module.decoder.layers.25.mlp.experts.linear_fc2.weight2 - module.decoder.layers.25.mlp.experts.linear_fc2.weight1 - module.decoder.layers.25.mlp.experts.linear_fc2.weight4 - module.decoder.layers.25.mlp.experts.linear_fc1.weight15 - module.decoder.layers.25.mlp.experts.linear_fc1.weight13 - module.decoder.layers.25.mlp.experts.linear_fc1.weight11 - module.decoder.layers.25.mlp.experts.linear_fc1.weight9 - module.decoder.layers.25.mlp.experts.linear_fc1.weight7 Params for bucket 43 (40894464 elements, 40894464 padded size): - module.decoder.layers.25.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc2.weight13 + module.decoder.layers.24.mlp.experts.linear_fc2.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight10 + module.decoder.layers.24.mlp.experts.linear_fc2.weight2 + module.decoder.layers.25.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc2.weight15 + module.decoder.layers.24.mlp.experts.linear_fc2.weight7 module.decoder.layers.24.mlp.experts.linear_fc2.weight12 module.decoder.layers.24.mlp.experts.linear_fc2.weight4 - module.decoder.layers.25.mlp.experts.linear_fc1.weight5 + module.decoder.layers.25.mlp.experts.linear_fc1.weight1 module.decoder.layers.24.mlp.experts.linear_fc2.weight9 - module.decoder.layers.25.mlp.experts.linear_fc1.weight3 + module.decoder.layers.25.mlp.experts.linear_fc1.weight0 module.decoder.layers.24.mlp.experts.linear_fc2.weight14 module.decoder.layers.24.mlp.experts.linear_fc2.weight6 + module.decoder.layers.25.mlp.experts.linear_fc1.weight5 module.decoder.layers.24.mlp.experts.linear_fc2.weight11 module.decoder.layers.24.mlp.experts.linear_fc2.weight3 - module.decoder.layers.25.mlp.experts.linear_fc1.weight4 + module.decoder.layers.25.mlp.experts.linear_fc1.weight3 module.decoder.layers.24.mlp.experts.linear_fc2.weight8 - module.decoder.layers.25.mlp.experts.linear_fc1.weight2 - module.decoder.layers.24.mlp.experts.linear_fc2.weight13 - module.decoder.layers.24.mlp.experts.linear_fc2.weight5 - module.decoder.layers.24.mlp.experts.linear_fc2.weight10 - module.decoder.layers.24.mlp.experts.linear_fc2.weight2 - module.decoder.layers.25.mlp.experts.linear_fc1.weight1 - module.decoder.layers.24.mlp.experts.linear_fc2.weight15 - module.decoder.layers.24.mlp.experts.linear_fc2.weight7 Params for bucket 44 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc2.weight1 - module.decoder.layers.24.mlp.experts.linear_fc1.weight13 - module.decoder.layers.24.mlp.experts.linear_fc1.weight15 - module.decoder.layers.24.mlp.experts.linear_fc1.weight11 - module.decoder.layers.24.mlp.experts.linear_fc1.weight9 - module.decoder.layers.24.mlp.experts.linear_fc1.weight7 - module.decoder.layers.24.mlp.experts.linear_fc1.weight5 module.decoder.layers.24.mlp.experts.linear_fc1.weight14 module.decoder.layers.24.mlp.experts.linear_fc2.weight0 module.decoder.layers.24.mlp.experts.linear_fc1.weight12 @@ -1815,29 +1808,42 @@ Params for bucket 44 (40894464 elements, 40894464 padded size): module.decoder.layers.24.mlp.experts.linear_fc1.weight8 module.decoder.layers.24.mlp.experts.linear_fc1.weight6 module.decoder.layers.24.mlp.experts.linear_fc1.weight4 + module.decoder.layers.24.mlp.experts.linear_fc2.weight1 + module.decoder.layers.24.mlp.experts.linear_fc1.weight13 + module.decoder.layers.24.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight11 + module.decoder.layers.24.mlp.experts.linear_fc1.weight9 + module.decoder.layers.24.mlp.experts.linear_fc1.weight7 + module.decoder.layers.24.mlp.experts.linear_fc1.weight5 Params for bucket 45 (40894464 elements, 40894464 padded size): - module.decoder.layers.24.mlp.experts.linear_fc1.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight11 module.decoder.layers.23.mlp.experts.linear_fc2.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight0 - module.decoder.layers.24.mlp.experts.linear_fc1.weight0 + module.decoder.layers.24.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc2.weight8 module.decoder.layers.23.mlp.experts.linear_fc2.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight13 module.decoder.layers.23.mlp.experts.linear_fc2.weight5 - module.decoder.layers.24.mlp.experts.linear_fc1.weight3 + module.decoder.layers.23.mlp.experts.linear_fc1.weight15 + module.decoder.layers.24.mlp.experts.linear_fc1.weight1 module.decoder.layers.23.mlp.experts.linear_fc2.weight10 module.decoder.layers.23.mlp.experts.linear_fc2.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight0 module.decoder.layers.23.mlp.experts.linear_fc2.weight15 module.decoder.layers.23.mlp.experts.linear_fc2.weight7 module.decoder.layers.23.mlp.experts.linear_fc2.weight12 module.decoder.layers.23.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight15 - module.decoder.layers.24.mlp.experts.linear_fc1.weight2 + module.decoder.layers.24.mlp.experts.linear_fc1.weight3 module.decoder.layers.23.mlp.experts.linear_fc2.weight9 module.decoder.layers.23.mlp.experts.linear_fc2.weight14 module.decoder.layers.23.mlp.experts.linear_fc2.weight6 Params for bucket 46 (40894464 elements, 40894464 padded size): + module.decoder.layers.23.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight11 + module.decoder.layers.23.mlp.experts.linear_fc1.weight9 + module.decoder.layers.23.mlp.experts.linear_fc1.weight7 + module.decoder.layers.23.mlp.experts.linear_fc1.weight5 + module.decoder.layers.23.mlp.experts.linear_fc1.weight2 module.decoder.layers.23.mlp.experts.linear_fc1.weight14 module.decoder.layers.23.mlp.experts.linear_fc1.weight12 module.decoder.layers.23.mlp.experts.linear_fc1.weight10 @@ -1845,35 +1851,35 @@ Params for bucket 46 (40894464 elements, 40894464 padded size): module.decoder.layers.23.mlp.experts.linear_fc1.weight6 module.decoder.layers.23.mlp.experts.linear_fc1.weight4 module.decoder.layers.23.mlp.experts.linear_fc1.weight3 - module.decoder.layers.23.mlp.experts.linear_fc1.weight11 - module.decoder.layers.23.mlp.experts.linear_fc1.weight13 - module.decoder.layers.23.mlp.experts.linear_fc1.weight9 - module.decoder.layers.23.mlp.experts.linear_fc1.weight7 - module.decoder.layers.23.mlp.experts.linear_fc1.weight5 - module.decoder.layers.23.mlp.experts.linear_fc1.weight2 Params for bucket 47 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc2.weight8 + module.decoder.layers.22.mlp.experts.linear_fc1.weight15 + module.decoder.layers.22.mlp.experts.linear_fc2.weight13 + module.decoder.layers.22.mlp.experts.linear_fc2.weight5 + module.decoder.layers.22.mlp.experts.linear_fc2.weight10 + module.decoder.layers.22.mlp.experts.linear_fc2.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight15 module.decoder.layers.22.mlp.experts.linear_fc2.weight7 module.decoder.layers.22.mlp.experts.linear_fc2.weight2 module.decoder.layers.22.mlp.experts.linear_fc1.weight14 - module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight12 module.decoder.layers.22.mlp.experts.linear_fc2.weight4 - module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight9 module.decoder.layers.22.mlp.experts.linear_fc2.weight0 + module.decoder.layers.23.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc2.weight14 module.decoder.layers.22.mlp.experts.linear_fc2.weight6 module.decoder.layers.22.mlp.experts.linear_fc1.weight13 + module.decoder.layers.23.mlp.experts.linear_fc1.weight0 module.decoder.layers.22.mlp.experts.linear_fc2.weight11 module.decoder.layers.22.mlp.experts.linear_fc2.weight3 - module.decoder.layers.22.mlp.experts.linear_fc2.weight8 - module.decoder.layers.22.mlp.experts.linear_fc1.weight15 - module.decoder.layers.22.mlp.experts.linear_fc2.weight13 - module.decoder.layers.22.mlp.experts.linear_fc2.weight5 - module.decoder.layers.22.mlp.experts.linear_fc2.weight10 - module.decoder.layers.22.mlp.experts.linear_fc2.weight1 Params for bucket 48 (40894464 elements, 40894464 padded size): + module.decoder.layers.22.mlp.experts.linear_fc1.weight9 + module.decoder.layers.22.mlp.experts.linear_fc1.weight11 + module.decoder.layers.22.mlp.experts.linear_fc1.weight7 + module.decoder.layers.22.mlp.experts.linear_fc1.weight5 + module.decoder.layers.22.mlp.experts.linear_fc1.weight2 + module.decoder.layers.22.mlp.experts.linear_fc1.weight1 module.decoder.layers.22.mlp.experts.linear_fc1.weight12 module.decoder.layers.22.mlp.experts.linear_fc1.weight10 module.decoder.layers.22.mlp.experts.linear_fc1.weight8 @@ -1881,137 +1887,131 @@ Params for bucket 48 (40894464 elements, 40894464 padded size): module.decoder.layers.22.mlp.experts.linear_fc1.weight4 module.decoder.layers.22.mlp.experts.linear_fc1.weight3 module.decoder.layers.22.mlp.experts.linear_fc1.weight0 - module.decoder.layers.22.mlp.experts.linear_fc1.weight11 - module.decoder.layers.22.mlp.experts.linear_fc1.weight9 - module.decoder.layers.22.mlp.experts.linear_fc1.weight7 - module.decoder.layers.22.mlp.experts.linear_fc1.weight5 - module.decoder.layers.22.mlp.experts.linear_fc1.weight2 - module.decoder.layers.22.mlp.experts.linear_fc1.weight1 Params for bucket 49 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc2.weight9 - module.decoder.layers.21.mlp.experts.linear_fc1.weight12 module.decoder.layers.21.mlp.experts.linear_fc2.weight14 module.decoder.layers.21.mlp.experts.linear_fc2.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight13 module.decoder.layers.21.mlp.experts.linear_fc2.weight11 module.decoder.layers.21.mlp.experts.linear_fc2.weight3 module.decoder.layers.21.mlp.experts.linear_fc2.weight0 - module.decoder.layers.21.mlp.experts.linear_fc1.weight14 module.decoder.layers.21.mlp.experts.linear_fc2.weight8 module.decoder.layers.21.mlp.experts.linear_fc2.weight1 - module.decoder.layers.21.mlp.experts.linear_fc1.weight11 + module.decoder.layers.21.mlp.experts.linear_fc1.weight15 module.decoder.layers.21.mlp.experts.linear_fc2.weight13 module.decoder.layers.21.mlp.experts.linear_fc2.weight5 + module.decoder.layers.21.mlp.experts.linear_fc1.weight12 module.decoder.layers.21.mlp.experts.linear_fc2.weight10 module.decoder.layers.21.mlp.experts.linear_fc2.weight2 - module.decoder.layers.21.mlp.experts.linear_fc1.weight13 module.decoder.layers.21.mlp.experts.linear_fc2.weight15 module.decoder.layers.21.mlp.experts.linear_fc2.weight7 + module.decoder.layers.21.mlp.experts.linear_fc1.weight14 module.decoder.layers.21.mlp.experts.linear_fc2.weight12 module.decoder.layers.21.mlp.experts.linear_fc2.weight4 - module.decoder.layers.21.mlp.experts.linear_fc1.weight15 + module.decoder.layers.21.mlp.experts.linear_fc1.weight11 Params for bucket 50 (40894464 elements, 40894464 padded size): module.decoder.layers.21.mlp.experts.linear_fc1.weight10 - module.decoder.layers.21.mlp.experts.linear_fc1.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight6 + module.decoder.layers.21.mlp.experts.linear_fc1.weight9 module.decoder.layers.21.mlp.experts.linear_fc1.weight4 module.decoder.layers.21.mlp.experts.linear_fc1.weight3 module.decoder.layers.21.mlp.experts.linear_fc1.weight0 - module.decoder.layers.20.mlp.experts.linear_fc2.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight14 + module.decoder.layers.20.mlp.experts.linear_fc2.weight12 module.decoder.layers.21.mlp.experts.linear_fc1.weight7 - module.decoder.layers.21.mlp.experts.linear_fc1.weight9 + module.decoder.layers.21.mlp.experts.linear_fc1.weight8 module.decoder.layers.21.mlp.experts.linear_fc1.weight5 module.decoder.layers.21.mlp.experts.linear_fc1.weight2 module.decoder.layers.21.mlp.experts.linear_fc1.weight1 + module.decoder.layers.20.mlp.experts.linear_fc2.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight13 - module.decoder.layers.20.mlp.experts.linear_fc2.weight12 Params for bucket 51 (40894464 elements, 40894464 padded size): - module.decoder.layers.20.mlp.experts.linear_fc2.weight9 + module.decoder.layers.20.mlp.experts.linear_fc2.weight4 + module.decoder.layers.20.mlp.experts.linear_fc1.weight11 module.decoder.layers.20.mlp.experts.linear_fc2.weight0 + module.decoder.layers.20.mlp.experts.linear_fc2.weight10 module.decoder.layers.20.mlp.experts.linear_fc2.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight13 - module.decoder.layers.20.mlp.experts.linear_fc2.weight11 module.decoder.layers.20.mlp.experts.linear_fc2.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight10 - module.decoder.layers.20.mlp.experts.linear_fc2.weight8 + module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight9 module.decoder.layers.20.mlp.experts.linear_fc1.weight15 module.decoder.layers.20.mlp.experts.linear_fc2.weight5 module.decoder.layers.20.mlp.experts.linear_fc1.weight12 - module.decoder.layers.20.mlp.experts.linear_fc2.weight10 module.decoder.layers.20.mlp.experts.linear_fc2.weight1 module.decoder.layers.20.mlp.experts.linear_fc1.weight9 - module.decoder.layers.20.mlp.experts.linear_fc2.weight7 + module.decoder.layers.20.mlp.experts.linear_fc2.weight11 + module.decoder.layers.20.mlp.experts.linear_fc2.weight8 module.decoder.layers.20.mlp.experts.linear_fc2.weight2 module.decoder.layers.20.mlp.experts.linear_fc1.weight14 - module.decoder.layers.20.mlp.experts.linear_fc2.weight4 - module.decoder.layers.20.mlp.experts.linear_fc1.weight11 Params for bucket 52 (40894464 elements, 40894464 padded size): + module.decoder.layers.20.mlp.experts.linear_fc1.weight7 + module.decoder.layers.20.mlp.experts.linear_fc1.weight5 + module.decoder.layers.20.mlp.experts.linear_fc1.weight2 + module.decoder.layers.20.mlp.experts.linear_fc1.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight15 + module.decoder.layers.19.mlp.experts.linear_fc2.weight13 + module.decoder.layers.19.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc2.weight9 module.decoder.layers.20.mlp.experts.linear_fc1.weight8 module.decoder.layers.20.mlp.experts.linear_fc1.weight6 module.decoder.layers.20.mlp.experts.linear_fc1.weight4 module.decoder.layers.20.mlp.experts.linear_fc1.weight3 module.decoder.layers.20.mlp.experts.linear_fc1.weight0 - module.decoder.layers.19.mlp.experts.linear_fc2.weight15 - module.decoder.layers.19.mlp.experts.linear_fc2.weight13 - module.decoder.layers.19.mlp.experts.linear_fc2.weight11 - module.decoder.layers.19.mlp.experts.linear_fc2.weight9 - module.decoder.layers.20.mlp.experts.linear_fc1.weight5 - module.decoder.layers.20.mlp.experts.linear_fc1.weight7 - module.decoder.layers.20.mlp.experts.linear_fc1.weight2 - module.decoder.layers.20.mlp.experts.linear_fc1.weight1 module.decoder.layers.19.mlp.experts.linear_fc2.weight14 module.decoder.layers.19.mlp.experts.linear_fc2.weight12 module.decoder.layers.19.mlp.experts.linear_fc2.weight10 module.decoder.layers.19.mlp.experts.linear_fc2.weight8 Params for bucket 53 (40894464 elements, 40894464 padded size): module.decoder.layers.19.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc2.weight0 + module.decoder.layers.19.mlp.experts.linear_fc2.weight3 module.decoder.layers.19.mlp.experts.linear_fc1.weight15 module.decoder.layers.19.mlp.experts.linear_fc1.weight13 module.decoder.layers.19.mlp.experts.linear_fc1.weight11 module.decoder.layers.19.mlp.experts.linear_fc1.weight9 module.decoder.layers.19.mlp.experts.linear_fc1.weight7 - module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.19.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc2.weight6 module.decoder.layers.19.mlp.experts.linear_fc2.weight2 module.decoder.layers.19.mlp.experts.linear_fc2.weight1 + module.decoder.layers.19.mlp.experts.linear_fc2.weight0 module.decoder.layers.19.mlp.experts.linear_fc1.weight14 module.decoder.layers.19.mlp.experts.linear_fc1.weight12 module.decoder.layers.19.mlp.experts.linear_fc1.weight10 module.decoder.layers.19.mlp.experts.linear_fc1.weight8 Params for bucket 54 (40894464 elements, 40894464 padded size): - module.decoder.layers.19.mlp.experts.linear_fc1.weight5 + module.decoder.layers.18.mlp.experts.linear_fc2.weight15 + module.decoder.layers.18.mlp.experts.linear_fc2.weight7 + module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.18.mlp.experts.linear_fc2.weight12 + module.decoder.layers.18.mlp.experts.linear_fc2.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight9 + module.decoder.layers.19.mlp.experts.linear_fc1.weight6 module.decoder.layers.18.mlp.experts.linear_fc2.weight14 module.decoder.layers.18.mlp.experts.linear_fc2.weight6 - module.decoder.layers.19.mlp.experts.linear_fc1.weight3 + module.decoder.layers.19.mlp.experts.linear_fc1.weight1 module.decoder.layers.18.mlp.experts.linear_fc2.weight11 + module.decoder.layers.19.mlp.experts.linear_fc1.weight0 module.decoder.layers.18.mlp.experts.linear_fc2.weight8 - module.decoder.layers.19.mlp.experts.linear_fc1.weight4 + module.decoder.layers.19.mlp.experts.linear_fc1.weight5 module.decoder.layers.18.mlp.experts.linear_fc2.weight13 module.decoder.layers.18.mlp.experts.linear_fc2.weight5 - module.decoder.layers.19.mlp.experts.linear_fc1.weight2 + module.decoder.layers.19.mlp.experts.linear_fc1.weight3 module.decoder.layers.18.mlp.experts.linear_fc2.weight10 - module.decoder.layers.19.mlp.experts.linear_fc1.weight6 - module.decoder.layers.18.mlp.experts.linear_fc2.weight15 - module.decoder.layers.18.mlp.experts.linear_fc2.weight7 - module.decoder.layers.19.mlp.experts.linear_fc1.weight1 - module.decoder.layers.18.mlp.experts.linear_fc2.weight12 - module.decoder.layers.18.mlp.experts.linear_fc2.weight4 - module.decoder.layers.19.mlp.experts.linear_fc1.weight0 - module.decoder.layers.18.mlp.experts.linear_fc2.weight9 Params for bucket 55 (40894464 elements, 40894464 padded size): - module.decoder.layers.18.mlp.experts.linear_fc2.weight2 + module.decoder.layers.18.mlp.experts.linear_fc2.weight3 module.decoder.layers.18.mlp.experts.linear_fc1.weight15 - module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight13 module.decoder.layers.18.mlp.experts.linear_fc1.weight11 module.decoder.layers.18.mlp.experts.linear_fc1.weight9 module.decoder.layers.18.mlp.experts.linear_fc1.weight7 module.decoder.layers.18.mlp.experts.linear_fc1.weight5 - module.decoder.layers.18.mlp.experts.linear_fc2.weight3 + module.decoder.layers.18.mlp.experts.linear_fc2.weight2 module.decoder.layers.18.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc2.weight1 module.decoder.layers.18.mlp.experts.linear_fc1.weight14 module.decoder.layers.18.mlp.experts.linear_fc1.weight12 module.decoder.layers.18.mlp.experts.linear_fc1.weight10 @@ -2019,69 +2019,63 @@ Params for bucket 55 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight6 Params for bucket 56 (40894464 elements, 40894464 padded size): module.decoder.layers.18.mlp.experts.linear_fc1.weight2 - module.decoder.layers.17.mlp.experts.linear_fc2.weight8 - module.decoder.layers.17.mlp.experts.linear_fc2.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight13 - module.decoder.layers.17.mlp.experts.linear_fc2.weight5 - module.decoder.layers.18.mlp.experts.linear_fc1.weight1 - module.decoder.layers.17.mlp.experts.linear_fc2.weight10 - module.decoder.layers.17.mlp.experts.linear_fc2.weight2 - module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight15 module.decoder.layers.17.mlp.experts.linear_fc2.weight7 module.decoder.layers.17.mlp.experts.linear_fc2.weight12 module.decoder.layers.17.mlp.experts.linear_fc2.weight4 - module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.18.mlp.experts.linear_fc1.weight1 module.decoder.layers.17.mlp.experts.linear_fc2.weight9 + module.decoder.layers.18.mlp.experts.linear_fc1.weight0 module.decoder.layers.17.mlp.experts.linear_fc2.weight14 module.decoder.layers.17.mlp.experts.linear_fc2.weight6 - module.decoder.layers.18.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc2.weight11 module.decoder.layers.17.mlp.experts.linear_fc2.weight3 module.decoder.layers.17.mlp.experts.linear_fc2.weight0 + module.decoder.layers.18.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc2.weight8 + module.decoder.layers.17.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc2.weight13 + module.decoder.layers.17.mlp.experts.linear_fc2.weight5 + module.decoder.layers.18.mlp.experts.linear_fc1.weight4 + module.decoder.layers.17.mlp.experts.linear_fc2.weight10 + module.decoder.layers.17.mlp.experts.linear_fc2.weight2 Params for bucket 57 (40894464 elements, 40894464 padded size): - module.decoder.layers.17.mlp.experts.linear_fc1.weight15 - module.decoder.layers.17.mlp.experts.linear_fc1.weight13 - module.decoder.layers.17.mlp.experts.linear_fc1.weight11 - module.decoder.layers.17.mlp.experts.linear_fc1.weight9 - module.decoder.layers.17.mlp.experts.linear_fc1.weight7 - module.decoder.layers.17.mlp.experts.linear_fc1.weight5 - module.decoder.layers.17.mlp.experts.linear_fc1.weight12 module.decoder.layers.17.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight12 module.decoder.layers.17.mlp.experts.linear_fc1.weight10 module.decoder.layers.17.mlp.experts.linear_fc1.weight8 module.decoder.layers.17.mlp.experts.linear_fc1.weight6 module.decoder.layers.17.mlp.experts.linear_fc1.weight4 module.decoder.layers.17.mlp.experts.linear_fc1.weight3 + module.decoder.layers.17.mlp.experts.linear_fc1.weight15 + module.decoder.layers.17.mlp.experts.linear_fc1.weight13 + module.decoder.layers.17.mlp.experts.linear_fc1.weight11 + module.decoder.layers.17.mlp.experts.linear_fc1.weight9 + module.decoder.layers.17.mlp.experts.linear_fc1.weight7 + module.decoder.layers.17.mlp.experts.linear_fc1.weight5 Params for bucket 58 (40894464 elements, 40894464 padded size): - module.decoder.layers.16.mlp.experts.linear_fc2.weight13 - module.decoder.layers.16.mlp.experts.linear_fc2.weight5 + module.decoder.layers.16.mlp.experts.linear_fc2.weight14 + module.decoder.layers.16.mlp.experts.linear_fc2.weight6 + module.decoder.layers.17.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight11 + module.decoder.layers.16.mlp.experts.linear_fc2.weight3 module.decoder.layers.16.mlp.experts.linear_fc2.weight0 module.decoder.layers.16.mlp.experts.linear_fc1.weight14 + module.decoder.layers.17.mlp.experts.linear_fc1.weight0 + module.decoder.layers.16.mlp.experts.linear_fc2.weight8 + module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc2.weight13 + module.decoder.layers.16.mlp.experts.linear_fc2.weight5 module.decoder.layers.16.mlp.experts.linear_fc2.weight10 module.decoder.layers.16.mlp.experts.linear_fc2.weight2 - module.decoder.layers.17.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc2.weight15 module.decoder.layers.16.mlp.experts.linear_fc2.weight7 module.decoder.layers.16.mlp.experts.linear_fc2.weight12 module.decoder.layers.16.mlp.experts.linear_fc2.weight4 - module.decoder.layers.17.mlp.experts.linear_fc1.weight1 - module.decoder.layers.16.mlp.experts.linear_fc2.weight9 - module.decoder.layers.16.mlp.experts.linear_fc2.weight14 - module.decoder.layers.17.mlp.experts.linear_fc1.weight0 - module.decoder.layers.16.mlp.experts.linear_fc2.weight6 module.decoder.layers.16.mlp.experts.linear_fc1.weight15 - module.decoder.layers.16.mlp.experts.linear_fc2.weight11 - module.decoder.layers.16.mlp.experts.linear_fc2.weight3 - module.decoder.layers.16.mlp.experts.linear_fc2.weight8 - module.decoder.layers.16.mlp.experts.linear_fc2.weight1 + module.decoder.layers.17.mlp.experts.linear_fc1.weight2 + module.decoder.layers.16.mlp.experts.linear_fc2.weight9 Params for bucket 59 (40894464 elements, 40894464 padded size): - module.decoder.layers.16.mlp.experts.linear_fc1.weight10 - module.decoder.layers.16.mlp.experts.linear_fc1.weight12 - module.decoder.layers.16.mlp.experts.linear_fc1.weight8 - module.decoder.layers.16.mlp.experts.linear_fc1.weight6 - module.decoder.layers.16.mlp.experts.linear_fc1.weight4 - module.decoder.layers.16.mlp.experts.linear_fc1.weight3 module.decoder.layers.16.mlp.experts.linear_fc1.weight13 module.decoder.layers.16.mlp.experts.linear_fc1.weight11 module.decoder.layers.16.mlp.experts.linear_fc1.weight9 @@ -2089,36 +2083,35 @@ Params for bucket 59 (40894464 elements, 40894464 padded size): module.decoder.layers.16.mlp.experts.linear_fc1.weight5 module.decoder.layers.16.mlp.experts.linear_fc1.weight2 module.decoder.layers.16.mlp.experts.linear_fc1.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight12 + module.decoder.layers.16.mlp.experts.linear_fc1.weight10 + module.decoder.layers.16.mlp.experts.linear_fc1.weight8 + module.decoder.layers.16.mlp.experts.linear_fc1.weight6 + module.decoder.layers.16.mlp.experts.linear_fc1.weight4 + module.decoder.layers.16.mlp.experts.linear_fc1.weight3 Params for bucket 60 (40894464 elements, 40894464 padded size): - module.decoder.layers.15.mlp.experts.linear_fc2.weight14 - module.decoder.layers.15.mlp.experts.linear_fc2.weight6 - module.decoder.layers.15.mlp.experts.linear_fc1.weight13 - module.decoder.layers.15.mlp.experts.linear_fc2.weight11 - module.decoder.layers.15.mlp.experts.linear_fc2.weight3 - module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight8 - module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight14 module.decoder.layers.15.mlp.experts.linear_fc2.weight13 module.decoder.layers.15.mlp.experts.linear_fc2.weight5 - module.decoder.layers.15.mlp.experts.linear_fc1.weight12 module.decoder.layers.15.mlp.experts.linear_fc2.weight10 module.decoder.layers.15.mlp.experts.linear_fc2.weight1 + module.decoder.layers.16.mlp.experts.linear_fc1.weight0 module.decoder.layers.15.mlp.experts.linear_fc2.weight15 module.decoder.layers.15.mlp.experts.linear_fc2.weight7 - module.decoder.layers.15.mlp.experts.linear_fc2.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight13 module.decoder.layers.15.mlp.experts.linear_fc2.weight12 module.decoder.layers.15.mlp.experts.linear_fc2.weight4 module.decoder.layers.15.mlp.experts.linear_fc2.weight9 module.decoder.layers.15.mlp.experts.linear_fc2.weight0 + module.decoder.layers.15.mlp.experts.linear_fc1.weight15 + module.decoder.layers.15.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc2.weight6 + module.decoder.layers.15.mlp.experts.linear_fc1.weight12 + module.decoder.layers.15.mlp.experts.linear_fc2.weight11 + module.decoder.layers.15.mlp.experts.linear_fc2.weight3 Params for bucket 61 (40894464 elements, 40894464 padded size): - module.decoder.layers.15.mlp.experts.linear_fc1.weight11 - module.decoder.layers.15.mlp.experts.linear_fc1.weight7 - module.decoder.layers.15.mlp.experts.linear_fc1.weight9 - module.decoder.layers.15.mlp.experts.linear_fc1.weight5 - module.decoder.layers.15.mlp.experts.linear_fc1.weight2 - module.decoder.layers.15.mlp.experts.linear_fc1.weight1 - module.decoder.layers.14.mlp.experts.linear_fc2.weight15 module.decoder.layers.15.mlp.experts.linear_fc1.weight10 module.decoder.layers.15.mlp.experts.linear_fc1.weight8 module.decoder.layers.15.mlp.experts.linear_fc1.weight6 @@ -2126,7 +2119,17 @@ Params for bucket 61 (40894464 elements, 40894464 padded size): module.decoder.layers.15.mlp.experts.linear_fc1.weight3 module.decoder.layers.15.mlp.experts.linear_fc1.weight0 module.decoder.layers.14.mlp.experts.linear_fc2.weight14 + module.decoder.layers.15.mlp.experts.linear_fc1.weight11 + module.decoder.layers.15.mlp.experts.linear_fc1.weight9 + module.decoder.layers.15.mlp.experts.linear_fc1.weight7 + module.decoder.layers.15.mlp.experts.linear_fc1.weight5 + module.decoder.layers.15.mlp.experts.linear_fc1.weight2 + module.decoder.layers.15.mlp.experts.linear_fc1.weight1 + module.decoder.layers.14.mlp.experts.linear_fc2.weight15 Params for bucket 62 (40894464 elements, 40894464 padded size): + module.decoder.layers.14.mlp.experts.linear_fc2.weight8 + module.decoder.layers.14.mlp.experts.linear_fc2.weight1 + module.decoder.layers.14.mlp.experts.linear_fc1.weight15 module.decoder.layers.14.mlp.experts.linear_fc2.weight13 module.decoder.layers.14.mlp.experts.linear_fc2.weight5 module.decoder.layers.14.mlp.experts.linear_fc1.weight12 @@ -2144,75 +2147,72 @@ Params for bucket 62 (40894464 elements, 40894464 padded size): module.decoder.layers.14.mlp.experts.linear_fc2.weight3 module.decoder.layers.14.mlp.experts.linear_fc2.weight0 module.decoder.layers.14.mlp.experts.linear_fc1.weight10 - module.decoder.layers.14.mlp.experts.linear_fc2.weight8 - module.decoder.layers.14.mlp.experts.linear_fc2.weight1 - module.decoder.layers.14.mlp.experts.linear_fc1.weight15 Params for bucket 63 (40894464 elements, 40894464 padded size): - module.decoder.layers.14.mlp.experts.linear_fc1.weight8 - module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight9 module.decoder.layers.14.mlp.experts.linear_fc1.weight5 - module.decoder.layers.14.mlp.experts.linear_fc1.weight4 + module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight2 + module.decoder.layers.14.mlp.experts.linear_fc1.weight1 module.decoder.layers.14.mlp.experts.linear_fc1.weight0 module.decoder.layers.13.mlp.experts.linear_fc2.weight15 module.decoder.layers.13.mlp.experts.linear_fc2.weight13 module.decoder.layers.13.mlp.experts.linear_fc2.weight11 - module.decoder.layers.14.mlp.experts.linear_fc1.weight9 - module.decoder.layers.14.mlp.experts.linear_fc1.weight7 + module.decoder.layers.14.mlp.experts.linear_fc1.weight6 + module.decoder.layers.14.mlp.experts.linear_fc1.weight8 + module.decoder.layers.14.mlp.experts.linear_fc1.weight4 module.decoder.layers.14.mlp.experts.linear_fc1.weight3 - module.decoder.layers.14.mlp.experts.linear_fc1.weight2 - module.decoder.layers.14.mlp.experts.linear_fc1.weight1 module.decoder.layers.13.mlp.experts.linear_fc2.weight14 module.decoder.layers.13.mlp.experts.linear_fc2.weight12 module.decoder.layers.13.mlp.experts.linear_fc2.weight10 Params for bucket 64 (40894464 elements, 40894464 padded size): module.decoder.layers.13.mlp.experts.linear_fc2.weight9 + module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight7 - module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight3 - module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight0 module.decoder.layers.13.mlp.experts.linear_fc1.weight14 module.decoder.layers.13.mlp.experts.linear_fc1.weight12 module.decoder.layers.13.mlp.experts.linear_fc1.weight10 module.decoder.layers.13.mlp.experts.linear_fc1.weight8 + module.decoder.layers.13.mlp.experts.linear_fc2.weight6 module.decoder.layers.13.mlp.experts.linear_fc2.weight8 - module.decoder.layers.13.mlp.experts.linear_fc2.weight5 module.decoder.layers.13.mlp.experts.linear_fc2.weight4 + module.decoder.layers.13.mlp.experts.linear_fc2.weight2 module.decoder.layers.13.mlp.experts.linear_fc2.weight1 module.decoder.layers.13.mlp.experts.linear_fc1.weight15 module.decoder.layers.13.mlp.experts.linear_fc1.weight13 module.decoder.layers.13.mlp.experts.linear_fc1.weight11 module.decoder.layers.13.mlp.experts.linear_fc1.weight9 Params for bucket 65 (40894464 elements, 40894464 padded size): + module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight3 module.decoder.layers.13.mlp.experts.linear_fc1.weight6 - module.decoder.layers.13.mlp.experts.linear_fc1.weight4 module.decoder.layers.13.mlp.experts.linear_fc1.weight0 - module.decoder.layers.12.mlp.experts.linear_fc2.weight14 - module.decoder.layers.12.mlp.experts.linear_fc2.weight12 - module.decoder.layers.12.mlp.experts.linear_fc2.weight10 - module.decoder.layers.12.mlp.experts.linear_fc2.weight8 - module.decoder.layers.12.mlp.experts.linear_fc2.weight6 - module.decoder.layers.13.mlp.experts.linear_fc1.weight7 - module.decoder.layers.13.mlp.experts.linear_fc1.weight5 - module.decoder.layers.13.mlp.experts.linear_fc1.weight2 - module.decoder.layers.13.mlp.experts.linear_fc1.weight1 module.decoder.layers.12.mlp.experts.linear_fc2.weight15 module.decoder.layers.12.mlp.experts.linear_fc2.weight13 module.decoder.layers.12.mlp.experts.linear_fc2.weight11 module.decoder.layers.12.mlp.experts.linear_fc2.weight9 module.decoder.layers.12.mlp.experts.linear_fc2.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight7 + module.decoder.layers.13.mlp.experts.linear_fc1.weight5 + module.decoder.layers.13.mlp.experts.linear_fc1.weight2 + module.decoder.layers.13.mlp.experts.linear_fc1.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight14 + module.decoder.layers.12.mlp.experts.linear_fc2.weight12 + module.decoder.layers.12.mlp.experts.linear_fc2.weight10 + module.decoder.layers.12.mlp.experts.linear_fc2.weight8 + module.decoder.layers.12.mlp.experts.linear_fc2.weight6 Params for bucket 66 (40894464 elements, 40894464 padded size): - module.decoder.layers.12.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc2.weight2 - module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight5 + module.decoder.layers.12.mlp.experts.linear_fc2.weight3 module.decoder.layers.12.mlp.experts.linear_fc1.weight15 module.decoder.layers.12.mlp.experts.linear_fc1.weight13 module.decoder.layers.12.mlp.experts.linear_fc1.weight11 module.decoder.layers.12.mlp.experts.linear_fc1.weight9 module.decoder.layers.12.mlp.experts.linear_fc1.weight7 - module.decoder.layers.12.mlp.experts.linear_fc2.weight5 - module.decoder.layers.12.mlp.experts.linear_fc2.weight3 + module.decoder.layers.12.mlp.experts.linear_fc2.weight1 + module.decoder.layers.12.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc2.weight2 module.decoder.layers.12.mlp.experts.linear_fc2.weight0 module.decoder.layers.12.mlp.experts.linear_fc1.weight14 module.decoder.layers.12.mlp.experts.linear_fc1.weight12 @@ -2220,11 +2220,6 @@ Params for bucket 66 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight8 module.decoder.layers.12.mlp.experts.linear_fc1.weight6 Params for bucket 67 (40894464 elements, 40894464 padded size): - module.decoder.layers.12.mlp.experts.linear_fc1.weight5 - module.decoder.layers.11.mlp.experts.linear_fc2.weight12 - module.decoder.layers.11.mlp.experts.linear_fc2.weight4 - module.decoder.layers.12.mlp.experts.linear_fc1.weight3 - module.decoder.layers.11.mlp.experts.linear_fc2.weight9 module.decoder.layers.11.mlp.experts.linear_fc2.weight14 module.decoder.layers.11.mlp.experts.linear_fc2.weight6 module.decoder.layers.12.mlp.experts.linear_fc1.weight4 @@ -2240,10 +2235,15 @@ Params for bucket 67 (40894464 elements, 40894464 padded size): module.decoder.layers.12.mlp.experts.linear_fc1.weight0 module.decoder.layers.11.mlp.experts.linear_fc2.weight15 module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.12.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.12.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 Params for bucket 68 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc2.weight1 - module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 module.decoder.layers.11.mlp.experts.linear_fc1.weight11 module.decoder.layers.11.mlp.experts.linear_fc1.weight9 module.decoder.layers.11.mlp.experts.linear_fc1.weight7 @@ -2256,28 +2256,34 @@ Params for bucket 68 (40894464 elements, 40894464 padded size): module.decoder.layers.11.mlp.experts.linear_fc1.weight6 module.decoder.layers.11.mlp.experts.linear_fc1.weight4 Params for bucket 69 (40894464 elements, 40894464 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight15 module.decoder.layers.10.mlp.experts.linear_fc2.weight7 module.decoder.layers.10.mlp.experts.linear_fc2.weight12 module.decoder.layers.10.mlp.experts.linear_fc2.weight4 - module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 module.decoder.layers.10.mlp.experts.linear_fc2.weight14 module.decoder.layers.10.mlp.experts.linear_fc2.weight6 - module.decoder.layers.11.mlp.experts.linear_fc1.weight1 module.decoder.layers.10.mlp.experts.linear_fc2.weight11 module.decoder.layers.10.mlp.experts.linear_fc2.weight3 - module.decoder.layers.10.mlp.experts.linear_fc2.weight0 - module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight8 - module.decoder.layers.10.mlp.experts.linear_fc2.weight1 - module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 module.decoder.layers.10.mlp.experts.linear_fc2.weight13 module.decoder.layers.10.mlp.experts.linear_fc2.weight5 - module.decoder.layers.11.mlp.experts.linear_fc1.weight3 module.decoder.layers.10.mlp.experts.linear_fc2.weight10 - module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 Params for bucket 70 (40894464 elements, 40894464 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 module.decoder.layers.10.mlp.experts.linear_fc1.weight14 module.decoder.layers.10.mlp.experts.linear_fc1.weight10 module.decoder.layers.10.mlp.experts.linear_fc1.weight12 @@ -2285,20 +2291,12 @@ Params for bucket 70 (40894464 elements, 40894464 padded size): module.decoder.layers.10.mlp.experts.linear_fc1.weight6 module.decoder.layers.10.mlp.experts.linear_fc1.weight4 module.decoder.layers.10.mlp.experts.linear_fc1.weight3 - module.decoder.layers.10.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight11 - module.decoder.layers.10.mlp.experts.linear_fc1.weight9 - module.decoder.layers.10.mlp.experts.linear_fc1.weight7 - module.decoder.layers.10.mlp.experts.linear_fc1.weight5 - module.decoder.layers.10.mlp.experts.linear_fc1.weight2 Params for bucket 71 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight10 module.decoder.layers.9.mlp.experts.linear_fc2.weight2 module.decoder.layers.9.mlp.experts.linear_fc1.weight13 - module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight15 module.decoder.layers.9.mlp.experts.linear_fc2.weight7 - module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight12 module.decoder.layers.9.mlp.experts.linear_fc2.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight15 @@ -2309,31 +2307,27 @@ Params for bucket 71 (40894464 elements, 40894464 padded size): module.decoder.layers.9.mlp.experts.linear_fc2.weight3 module.decoder.layers.9.mlp.experts.linear_fc2.weight0 module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc2.weight8 module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 module.decoder.layers.9.mlp.experts.linear_fc2.weight13 module.decoder.layers.9.mlp.experts.linear_fc2.weight5 Params for bucket 72 (40894464 elements, 40894464 padded size): - module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 module.decoder.layers.9.mlp.experts.linear_fc1.weight7 module.decoder.layers.9.mlp.experts.linear_fc1.weight5 module.decoder.layers.9.mlp.experts.linear_fc1.weight2 module.decoder.layers.9.mlp.experts.linear_fc1.weight1 module.decoder.layers.9.mlp.experts.linear_fc1.weight12 - module.decoder.layers.9.mlp.experts.linear_fc1.weight10 module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 module.decoder.layers.9.mlp.experts.linear_fc1.weight6 module.decoder.layers.9.mlp.experts.linear_fc1.weight4 module.decoder.layers.9.mlp.experts.linear_fc1.weight3 module.decoder.layers.9.mlp.experts.linear_fc1.weight0 Params for bucket 73 (40894464 elements, 40894464 padded size): - module.decoder.layers.8.mlp.experts.linear_fc2.weight10 - module.decoder.layers.8.mlp.experts.linear_fc2.weight1 - module.decoder.layers.8.mlp.experts.linear_fc2.weight15 - module.decoder.layers.8.mlp.experts.linear_fc2.weight7 - module.decoder.layers.8.mlp.experts.linear_fc2.weight2 - module.decoder.layers.8.mlp.experts.linear_fc1.weight14 module.decoder.layers.8.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc2.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight11 @@ -2344,11 +2338,17 @@ Params for bucket 73 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight13 module.decoder.layers.8.mlp.experts.linear_fc2.weight11 module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 module.decoder.layers.8.mlp.experts.linear_fc2.weight8 module.decoder.layers.8.mlp.experts.linear_fc1.weight15 module.decoder.layers.8.mlp.experts.linear_fc2.weight13 module.decoder.layers.8.mlp.experts.linear_fc2.weight5 module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 Params for bucket 74 (40894464 elements, 40894464 padded size): module.decoder.layers.8.mlp.experts.linear_fc1.weight9 module.decoder.layers.8.mlp.experts.linear_fc1.weight7 @@ -2358,52 +2358,60 @@ Params for bucket 74 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc2.weight14 module.decoder.layers.7.mlp.experts.linear_fc2.weight12 module.decoder.layers.8.mlp.experts.linear_fc1.weight10 - module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 module.decoder.layers.8.mlp.experts.linear_fc1.weight4 module.decoder.layers.8.mlp.experts.linear_fc1.weight3 module.decoder.layers.8.mlp.experts.linear_fc1.weight0 module.decoder.layers.7.mlp.experts.linear_fc2.weight15 module.decoder.layers.7.mlp.experts.linear_fc2.weight13 Params for bucket 75 (40894464 elements, 40894464 padded size): - module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight9 - module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight14 - module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 module.decoder.layers.7.mlp.experts.linear_fc2.weight11 module.decoder.layers.7.mlp.experts.linear_fc2.weight3 - module.decoder.layers.7.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc2.weight1 - module.decoder.layers.7.mlp.experts.linear_fc1.weight13 - module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc2.weight0 module.decoder.layers.7.mlp.experts.linear_fc1.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight10 - module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 module.decoder.layers.7.mlp.experts.linear_fc1.weight15 - module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight12 Params for bucket 76 (40894464 elements, 40894464 padded size): module.decoder.layers.7.mlp.experts.linear_fc1.weight7 module.decoder.layers.7.mlp.experts.linear_fc1.weight5 module.decoder.layers.7.mlp.experts.linear_fc1.weight2 module.decoder.layers.7.mlp.experts.linear_fc1.weight1 - module.decoder.layers.6.mlp.experts.linear_fc2.weight14 - module.decoder.layers.6.mlp.experts.linear_fc2.weight12 - module.decoder.layers.6.mlp.experts.linear_fc2.weight10 - module.decoder.layers.6.mlp.experts.linear_fc2.weight8 - module.decoder.layers.7.mlp.experts.linear_fc1.weight8 - module.decoder.layers.7.mlp.experts.linear_fc1.weight4 - module.decoder.layers.7.mlp.experts.linear_fc1.weight6 - module.decoder.layers.7.mlp.experts.linear_fc1.weight3 - module.decoder.layers.7.mlp.experts.linear_fc1.weight0 module.decoder.layers.6.mlp.experts.linear_fc2.weight15 module.decoder.layers.6.mlp.experts.linear_fc2.weight13 module.decoder.layers.6.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 Params for bucket 77 (40894464 elements, 40894464 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 module.decoder.layers.6.mlp.experts.linear_fc2.weight6 module.decoder.layers.6.mlp.experts.linear_fc2.weight4 module.decoder.layers.6.mlp.experts.linear_fc2.weight2 @@ -2413,34 +2421,26 @@ Params for bucket 77 (40894464 elements, 40894464 padded size): module.decoder.layers.6.mlp.experts.linear_fc1.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight9 module.decoder.layers.6.mlp.experts.linear_fc1.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc2.weight3 - module.decoder.layers.6.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc2.weight0 - module.decoder.layers.6.mlp.experts.linear_fc1.weight14 - module.decoder.layers.6.mlp.experts.linear_fc1.weight12 - module.decoder.layers.6.mlp.experts.linear_fc1.weight10 - module.decoder.layers.6.mlp.experts.linear_fc1.weight8 Params for bucket 78 (40894464 elements, 40894464 padded size): - module.decoder.layers.6.mlp.experts.linear_fc1.weight1 - module.decoder.layers.5.mlp.experts.linear_fc2.weight15 - module.decoder.layers.5.mlp.experts.linear_fc2.weight7 - module.decoder.layers.6.mlp.experts.linear_fc1.weight0 - module.decoder.layers.5.mlp.experts.linear_fc2.weight12 - module.decoder.layers.5.mlp.experts.linear_fc2.weight4 - module.decoder.layers.6.mlp.experts.linear_fc1.weight5 - module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 module.decoder.layers.5.mlp.experts.linear_fc2.weight14 module.decoder.layers.5.mlp.experts.linear_fc2.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight4 - module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 module.decoder.layers.6.mlp.experts.linear_fc1.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 module.decoder.layers.5.mlp.experts.linear_fc2.weight13 module.decoder.layers.5.mlp.experts.linear_fc2.weight5 - module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc2.weight3 module.decoder.layers.5.mlp.experts.linear_fc2.weight0 @@ -2449,70 +2449,70 @@ Params for bucket 79 (40894464 elements, 40894464 padded size): module.decoder.layers.5.mlp.experts.linear_fc1.weight10 module.decoder.layers.5.mlp.experts.linear_fc1.weight8 module.decoder.layers.5.mlp.experts.linear_fc1.weight6 - module.decoder.layers.5.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight15 module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 module.decoder.layers.5.mlp.experts.linear_fc1.weight13 module.decoder.layers.5.mlp.experts.linear_fc1.weight11 module.decoder.layers.5.mlp.experts.linear_fc1.weight9 module.decoder.layers.5.mlp.experts.linear_fc1.weight7 module.decoder.layers.5.mlp.experts.linear_fc1.weight5 Params for bucket 80 (40894464 elements, 40894464 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 module.decoder.layers.4.mlp.experts.linear_fc2.weight12 module.decoder.layers.4.mlp.experts.linear_fc2.weight4 - module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc2.weight9 - module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 module.decoder.layers.4.mlp.experts.linear_fc2.weight14 module.decoder.layers.4.mlp.experts.linear_fc2.weight6 module.decoder.layers.4.mlp.experts.linear_fc2.weight11 module.decoder.layers.4.mlp.experts.linear_fc2.weight3 module.decoder.layers.4.mlp.experts.linear_fc2.weight0 - module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 module.decoder.layers.4.mlp.experts.linear_fc2.weight8 module.decoder.layers.4.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc2.weight13 - module.decoder.layers.4.mlp.experts.linear_fc2.weight5 - module.decoder.layers.5.mlp.experts.linear_fc1.weight4 - module.decoder.layers.4.mlp.experts.linear_fc2.weight10 - module.decoder.layers.4.mlp.experts.linear_fc2.weight2 - module.decoder.layers.5.mlp.experts.linear_fc1.weight2 - module.decoder.layers.4.mlp.experts.linear_fc2.weight15 - module.decoder.layers.4.mlp.experts.linear_fc2.weight7 Params for bucket 81 (40894464 elements, 40894464 padded size): - module.decoder.layers.4.mlp.experts.linear_fc1.weight15 - module.decoder.layers.4.mlp.experts.linear_fc1.weight11 - module.decoder.layers.4.mlp.experts.linear_fc1.weight13 - module.decoder.layers.4.mlp.experts.linear_fc1.weight9 - module.decoder.layers.4.mlp.experts.linear_fc1.weight7 - module.decoder.layers.4.mlp.experts.linear_fc1.weight5 - module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 module.decoder.layers.4.mlp.experts.linear_fc1.weight10 module.decoder.layers.4.mlp.experts.linear_fc1.weight8 module.decoder.layers.4.mlp.experts.linear_fc1.weight6 module.decoder.layers.4.mlp.experts.linear_fc1.weight4 module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 Params for bucket 82 (40894464 elements, 40894464 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 module.decoder.layers.3.mlp.experts.linear_fc2.weight13 module.decoder.layers.3.mlp.experts.linear_fc2.weight5 module.decoder.layers.3.mlp.experts.linear_fc2.weight10 module.decoder.layers.3.mlp.experts.linear_fc2.weight1 - module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight15 module.decoder.layers.3.mlp.experts.linear_fc2.weight7 module.decoder.layers.3.mlp.experts.linear_fc2.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight12 module.decoder.layers.3.mlp.experts.linear_fc2.weight4 - module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight9 module.decoder.layers.3.mlp.experts.linear_fc2.weight0 - module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc2.weight14 module.decoder.layers.3.mlp.experts.linear_fc2.weight6 module.decoder.layers.3.mlp.experts.linear_fc2.weight11 module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 module.decoder.layers.3.mlp.experts.linear_fc2.weight8 module.decoder.layers.3.mlp.experts.linear_fc1.weight15 Params for bucket 83 (40894464 elements, 40894464 padded size): @@ -2530,27 +2530,27 @@ Params for bucket 83 (40894464 elements, 40894464 padded size): module.decoder.layers.3.mlp.experts.linear_fc1.weight2 module.decoder.layers.3.mlp.experts.linear_fc1.weight1 Params for bucket 84 (40894464 elements, 40894464 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight6 - module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 module.decoder.layers.2.mlp.experts.linear_fc2.weight11 module.decoder.layers.2.mlp.experts.linear_fc2.weight3 - module.decoder.layers.2.mlp.experts.linear_fc2.weight0 module.decoder.layers.2.mlp.experts.linear_fc2.weight8 module.decoder.layers.2.mlp.experts.linear_fc2.weight1 - module.decoder.layers.2.mlp.experts.linear_fc1.weight15 - module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 module.decoder.layers.2.mlp.experts.linear_fc2.weight13 module.decoder.layers.2.mlp.experts.linear_fc2.weight5 - module.decoder.layers.2.mlp.experts.linear_fc1.weight12 - module.decoder.layers.2.mlp.experts.linear_fc2.weight10 - module.decoder.layers.2.mlp.experts.linear_fc2.weight2 - module.decoder.layers.2.mlp.experts.linear_fc2.weight15 - module.decoder.layers.2.mlp.experts.linear_fc2.weight7 - module.decoder.layers.2.mlp.experts.linear_fc1.weight14 - module.decoder.layers.2.mlp.experts.linear_fc2.weight12 - module.decoder.layers.2.mlp.experts.linear_fc2.weight4 Params for bucket 85 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight10 module.decoder.layers.2.mlp.experts.linear_fc1.weight8 @@ -2567,35 +2567,27 @@ Params for bucket 85 (40894464 elements, 40894464 padded size): module.decoder.layers.2.mlp.experts.linear_fc1.weight0 module.decoder.layers.1.mlp.experts.linear_fc2.weight15 Params for bucket 86 (40894464 elements, 40894464 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 module.decoder.layers.1.mlp.experts.linear_fc2.weight13 module.decoder.layers.1.mlp.experts.linear_fc2.weight5 - module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 module.decoder.layers.1.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc2.weight2 - module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 module.decoder.layers.1.mlp.experts.linear_fc2.weight7 module.decoder.layers.1.mlp.experts.linear_fc2.weight12 module.decoder.layers.1.mlp.experts.linear_fc2.weight4 - module.decoder.layers.1.mlp.experts.linear_fc2.weight1 - module.decoder.layers.1.mlp.experts.linear_fc2.weight0 - module.decoder.layers.1.mlp.experts.linear_fc1.weight14 - module.decoder.layers.1.mlp.experts.linear_fc2.weight9 - module.decoder.layers.1.mlp.experts.linear_fc1.weight11 - module.decoder.layers.1.mlp.experts.linear_fc2.weight6 - module.decoder.layers.1.mlp.experts.linear_fc2.weight11 - module.decoder.layers.1.mlp.experts.linear_fc2.weight3 module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 Params for bucket 87 (40894464 elements, 40894464 padded size): - module.decoder.layers.1.mlp.experts.linear_fc1.weight8 - module.decoder.layers.1.mlp.experts.linear_fc1.weight6 - module.decoder.layers.1.mlp.experts.linear_fc1.weight4 - module.decoder.layers.1.mlp.experts.linear_fc1.weight3 - module.decoder.layers.1.mlp.experts.linear_fc1.weight0 - module.decoder.layers.0.mlp.experts.linear_fc2.weight14 - module.decoder.layers.0.mlp.experts.linear_fc2.weight12 - module.decoder.layers.0.mlp.experts.linear_fc2.weight10 module.decoder.layers.1.mlp.experts.linear_fc1.weight9 module.decoder.layers.1.mlp.experts.linear_fc1.weight7 module.decoder.layers.1.mlp.experts.linear_fc1.weight5 @@ -2604,20 +2596,28 @@ Params for bucket 87 (40894464 elements, 40894464 padded size): module.decoder.layers.0.mlp.experts.linear_fc2.weight15 module.decoder.layers.0.mlp.experts.linear_fc2.weight13 module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 Params for bucket 88 (40894464 elements, 40894464 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 module.decoder.layers.0.mlp.experts.linear_fc2.weight8 module.decoder.layers.0.mlp.experts.linear_fc2.weight6 module.decoder.layers.0.mlp.experts.linear_fc2.weight4 module.decoder.layers.0.mlp.experts.linear_fc2.weight2 module.decoder.layers.0.mlp.experts.linear_fc2.weight1 - module.decoder.layers.0.mlp.experts.linear_fc1.weight15 - module.decoder.layers.0.mlp.experts.linear_fc1.weight13 - module.decoder.layers.0.mlp.experts.linear_fc1.weight11 - module.decoder.layers.0.mlp.experts.linear_fc1.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight9 - module.decoder.layers.0.mlp.experts.linear_fc2.weight5 - module.decoder.layers.0.mlp.experts.linear_fc2.weight7 - module.decoder.layers.0.mlp.experts.linear_fc2.weight3 module.decoder.layers.0.mlp.experts.linear_fc2.weight0 module.decoder.layers.0.mlp.experts.linear_fc1.weight14 module.decoder.layers.0.mlp.experts.linear_fc1.weight12 @@ -2628,13 +2628,13 @@ Params for bucket 89 (25165824 elements, 25165824 padded size): module.decoder.layers.0.mlp.experts.linear_fc1.weight5 module.decoder.layers.0.mlp.experts.linear_fc1.weight2 module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc1.weight3 module.decoder.layers.0.mlp.experts.linear_fc1.weight4 - module.decoder.layers.0.mlp.experts.linear_fc1.weight6 module.decoder.layers.0.mlp.experts.linear_fc1.weight0 -INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine -[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 16:53:49 +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-26 18:05:56 > building train, validation, and test datasets ... > datasets target sizes (minimum size): train: 96 @@ -2642,7 +2642,7 @@ INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine test: 0 INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] > building train, validation, and test datasets for GPT ... -INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=4096, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers @@ -2655,14 +2655,14 @@ INFO:megatron.core.datasets.gpt_dataset: Load the sample index from 466b63c71b42 INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from 466b63c71b42dae59ebdc25db6336051-GPTDataset-train-shuffle_index.npy INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 29073 > finished creating GPT datasets ... -[after dataloaders are built] datetime: 2026-08-26 16:53:49 +[after dataloaders are built] datetime: 2026-08-26 18:05:56 done with setup ... training ... (min, max) time across ranks (ms): - model-and-optimizer-setup ......................: (6223.34, 6270.27) - train/valid/test-data-iterators-setup ..........: (58.52, 70.35) + model-and-optimizer-setup ......................: (6174.08, 6222.50) + train/valid/test-data-iterators-setup ..........: (60.37, 70.26) Overwriting rerun_state_machine.current_iteration from -1 to 0... -[before the start of training step] datetime: 2026-08-26 16:53:49 +[before the start of training step] datetime: 2026-08-26 18:05:56 INFO:moonep.api:MoonEP comm buffer sizes: config : R=8, S=4096, H=2048, E=128, K=8, NvS=36832, NvS_padded=36864 sm_count : num_sms=32, num_sms_dedup=148 @@ -2671,7 +2671,7 @@ INFO:moonep.api:MoonEP comm buffer sizes: local_temp_buffer_size : 1.26 MiB total_size : 147.26 MiB WARNING:megatron.core.rerun_state_machine:Result validation enabled - [2026-08-26 16:54:16] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26759.8 | throughput per GPU (TFLOP/s/GPU): 3.3 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095192E+01 | loss scale: 1.0 | grad norm: 54.494 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 + [2026-08-26 18:06:23] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 26483.6 | throughput per GPU (TFLOP/s/GPU): 3.4 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.095192E+01 | loss scale: 1.0 | grad norm: 54.496 | number of skipped iterations: 0 | number of nan iterations: 0 |Number of parameters in transformer block in billions: 29.90 Number of parameters in embedding layers in billions: 0.21 Total number of parameters in billions: 30.10 @@ -3275,15 +3275,15 @@ WARNING:megatron.core.utils:CUDA out of memory. Tried to allocate 144.00 MiB. GP [rank7]: return torch.where(keep, x, x.new_zeros(())) [rank7]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ [rank7]: torch.OutOfMemoryError: CUDA out of memory. Tried to allocate 144.00 MiB. GPU 7 has a total capacity of 178.35 GiB of which 112.81 MiB is free. Including non-PyTorch memory, this process has 178.23 GiB memory in use. Of the allocated memory 150.05 GiB is allocated by PyTorch, and 1.43 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables) -W0826 16:54:17.532000 282417 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers -W0826 16:54:17.533000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282513 closing signal SIGTERM -W0826 16:54:17.537000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282514 closing signal SIGTERM -W0826 16:54:17.545000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282515 closing signal SIGTERM -W0826 16:54:17.546000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282516 closing signal SIGTERM -W0826 16:54:17.549000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282517 closing signal SIGTERM -W0826 16:54:17.558000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282518 closing signal SIGTERM -W0826 16:54:17.567000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282519 closing signal SIGTERM -W0826 16:54:17.569000 282417 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 282520 closing signal SIGTERM +W0826 18:06:24.546000 454848 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0826 18:06:24.547000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454944 closing signal SIGTERM +W0826 18:06:24.554000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454945 closing signal SIGTERM +W0826 18:06:24.559000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454946 closing signal SIGTERM +W0826 18:06:24.574000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454947 closing signal SIGTERM +W0826 18:06:24.581000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454948 closing signal SIGTERM +W0826 18:06:24.594000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454949 closing signal SIGTERM +W0826 18:06:24.603000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454950 closing signal SIGTERM +W0826 18:06:24.645000 454848 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 454951 closing signal SIGTERM Traceback (most recent call last): File "/usr/local/bin/torchrun", line 8, in sys.exit(main()) @@ -3311,4 +3311,4 @@ Traceback (most recent call last): time.sleep(monitor_interval) File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) -torch.distributed.elastic.multiprocessing.api.SignalException: Process 282417 got signal: 15 +torch.distributed.elastic.multiprocessing.api.SignalException: Process 454848 got signal: 15 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv index dff4792c7f9c30e67bfd62827782f563e5465a46..876e9048c13adc821ba74ea48fd2c716fefd26aa 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -1,7 +1,5 @@ model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir QWEN3_235B moonep natural none OOM 182589 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN3_30B moonep natural none OOM 169195 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN3_30B moonep natural none OOM 175959 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep natural none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 DSV2_LITE_L24 moonep forced none OK 157151 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 -QWEN2_57B moonep natural none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 -QWEN2_57B moonep forced none OK 141555 1 0 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv index d9cc0cc5446eb259dc543f04f5868d04f5929c7d..17ac23c03bcf2c15cb45c2aa893e0ed1f8dd4c09 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -1,5 +1,4 @@ model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir QWEN3_235B moonep DOES_NOT_FIT 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN3_30B moonep DOES_NOT_FIT 169195 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 +QWEN3_30B moonep DOES_NOT_FIT 175959 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 DSV2_LITE_L24 moonep SELECTED none 0 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN2_57B moonep SELECTED none 0 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 diff --git a/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/moonep/tierbprobe/tierbprobe_gbs8/results.tsv index 4089426a7ae8546e9ed78d2e35dd0954f94974bc..49174012fc7dfb1d6930b85b0d7f31716d9e4e07 100644 --- a/moonep/tierbprobe/tierbprobe_gbs8/results.tsv +++ b/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -1,7 +1,5 @@ config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir QWEN3_235B moonep matched 16 8192 8 none none 1 OOM NA NA 182589 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone/rep1 -QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 169195 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 -DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 798.26 793.60 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 -DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 796.77 790.80 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1 -QWEN2_57B moonep matched 8 4096 8 none none 1 OK 413.31 366.10 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone/rep1 -QWEN2_57B moonep matched 8 4096 8 none none+forcebal 1 OK 408.20 357.00 141555 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN2_57B__moonep__matched__seq4096__gbs8__capnone__forcebal/rep1 +QWEN3_30B moonep matched 16 4096 8 none none 1 OOM NA NA 175959 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none 1 OK 799.34 793.30 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone/rep1 +DSV2_LITE_L24 moonep matched 8 8192 8 none none+forcebal 1 OK 792.51 789.10 157151 /workspace/tierb_study_20260826_144644/moonep/tierbprobe/tierbprobe_gbs8/DSV2_LITE_L24__moonep__matched__seq8192__gbs8__capnone__forcebal/rep1